xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' into feat/TD-27463-3.0

This commit is contained in:
kailixu 2023-12-18 14:20:10 +08:00
parent b54f816c2a 93f3a3135f
commit 9fa90fc953
126 changed files with 10372 additions and 5736 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
deps/arm/dm_static/libdmodule.a vendored
View File

Binary file not shown.

BIN
deps/darwin/arm/dm_static/libdmodule.a vendored
View File

Binary file not shown.

BIN
deps/darwin/x64/dm_static/libdmodule.a vendored
View File

Binary file not shown.

BIN
deps/x86/dm_static/libdmodule.a vendored
View File

Binary file not shown.

2
docs/en/07-develop/01-connect/index.md
View File

@ -177,7 +177,7 @@ Just need to add the reference to [TDengine.Connector](https://www.nuget.org/pac
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="TDengine.Connector" Version="3.0.0" /> <PackageReference Include="TDengine.Connector" Version="3.1.0" />
</ItemGroup> </ItemGroup>
</Project> </Project>

3
docs/en/07-develop/04-query-data/_cs_async.mdx
View File

@ -1,3 +0,0 @@
```csharp
{{#include docs/examples/csharp/asyncQuery/Program.cs}}
```

4
docs/en/07-develop/04-query-data/index.mdx
View File

@ -15,7 +15,6 @@ import CQuery from "./_c.mdx";
import PhpQuery from "./_php.mdx"; import PhpQuery from "./_php.mdx";
import PyAsync from "./_py_async.mdx"; import PyAsync from "./_py_async.mdx";
import NodeAsync from "./_js_async.mdx"; import NodeAsync from "./_js_async.mdx";
import CsAsync from "./_cs_async.mdx";
import CAsync from "./_c_async.mdx"; import CAsync from "./_c_async.mdx";
## Introduction ## Introduction
@ -174,9 +173,6 @@ Please note that async query can only be used with a native connection.
<TabItem label="Python" value="python"> <TabItem label="Python" value="python">
<PyAsync /> <PyAsync />
</TabItem> </TabItem>
<TabItem label="C#" value="csharp">
<CsAsync />
</TabItem>
<TabItem label="C" value="c"> <TabItem label="C" value="c">
<CAsync /> <CAsync />
</TabItem> </TabItem>

52
docs/en/07-develop/07-tmq.mdx
View File

@ -248,23 +248,23 @@ function close()
<TabItem value="C#" label="C#"> <TabItem value="C#" label="C#">
```csharp ```csharp
class ConsumerBuilder<TValue>
ConsumerBuilder(IEnumerable<KeyValuePair<string, string>> config) ConsumerBuilder(IEnumerable<KeyValuePair<string, string>> config)
virtual IConsumer Build() public IConsumer<TValue> Build()
Consumer(ConsumerBuilder builder)
void Subscribe(IEnumerable<string> topics) void Subscribe(IEnumerable<string> topics)
void Subscribe(string topic) void Subscribe(string topic)
ConsumeResult Consume(int millisecondsTimeout) ConsumeResult<TValue> Consume(int millisecondsTimeout)
List<string> Subscription() List<string> Subscription()
void Unsubscribe() void Unsubscribe()
void Commit(ConsumeResult consumerResult) List<TopicPartitionOffset> Commit()
void Close() void Close()
``` ```
@ -500,25 +500,19 @@ let consumer = taos.consumer({
<TabItem value="C#" label="C#"> <TabItem value="C#" label="C#">
```csharp ```csharp
using TDengineTMQ; var cfg = new Dictionary<string, string>()
{
// Create consumer groups on demand (GourpID) and enable automatic commits (EnableAutoCommit), { "group.id", "group1" },
// an automatic commit interval (AutoCommitIntervalMs), and a username (TDConnectUser) and password (TDConnectPasswd) { "auto.offset.reset", "latest" },
var cfg = new ConsumerConfig { "td.connect.ip", "127.0.0.1" },
{ { "td.connect.user", "root" },
EnableAutoCommit = "true" { "td.connect.pass", "taosdata" },
AutoCommitIntervalMs = "1000" { "td.connect.port", "6030" },
GourpId = "TDengine-TMQ-C#", { "client.id", "tmq_example" },
TDConnectUser = "root", { "enable.auto.commit", "true" },
TDConnectPasswd = "taosdata", { "msg.with.table.name", "false" },
AutoOffsetReset = "latest" };
MsgWithTableName = "true", var consumer = new ConsumerBuilder<Dictionary<string, object>>(cfg).Build();
TDConnectIp = "127.0.0.1",
TDConnectPort = "6030"
};
var consumer = new ConsumerBuilder(cfg).Build();
``` ```
</TabItem> </TabItem>
@ -747,10 +741,12 @@ while(true){
## Consume data ## Consume data
while (true) while (true)
{ {
var consumerRes = consumer.Consume(100); using (var result = consumer.Consume(500))
// process ConsumeResult {
ProcessMsg(consumerRes); if (result == null) continue;
consumer.Commit(consumerRes); ProcessMsg(result);
consumer.Commit();
}
} }
``` ```

6
docs/en/12-taos-sql/05-insert.md
View File

@ -158,8 +158,8 @@ Automatically creating table and the table name is specified through the `tbname
```sql ```sql
INSERT INTO meters(tbname, location, groupId, ts, current, phase) INSERT INTO meters(tbname, location, groupId, ts, current, phase)
values('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:34.630', 10.2, 219, 0.32) values('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:34.630', 10.2, 0.32)
values('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:35.779', 10.15, 217, 0.33) ('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:35.779', 10.15, 0.33)
values('d31002', NULL, 2, '2021-07-13 14:06:34.255', 10.15, 217, 0.33) ('d31002', NULL, 2, '2021-07-13 14:06:34.255', 10.15, 0.33)
``` ```

1317
docs/en/14-reference/03-connector/09-csharp.mdx
View File

File diff suppressed because it is too large Load Diff

3
docs/examples/csharp/.gitignore vendored
View File

@ -1,5 +1,5 @@
.vs .vs
asyncQuery/bin .idea
connect/bin connect/bin
influxdbLine/bin influxdbLine/bin
optsJSON/bin optsJSON/bin
@ -12,7 +12,6 @@ wsConnect/bin
wsInsert/bin wsInsert/bin
wsQuery/bin wsQuery/bin
wsStmt/bin wsStmt/bin
asyncQuery/obj
connect/obj connect/obj
influxdbLine/obj influxdbLine/obj
optsJSON/obj optsJSON/obj

111
docs/examples/csharp/asyncQuery/Program.cs
View File

@ -1,111 +0,0 @@
using System;
using System.Collections.Generic;
using TDengineDriver;
using TDengineDriver.Impl;
using System.Runtime.InteropServices;
namespace TDengineExample
{
public class AsyncQueryExample
{
static void Main()
{
IntPtr conn = GetConnection();
try
{
QueryAsyncCallback queryAsyncCallback = new QueryAsyncCallback(QueryCallback);
TDengine.QueryAsync(conn, "select * from meters", queryAsyncCallback, IntPtr.Zero);
Thread.Sleep(2000);
}
finally
{
TDengine.Close(conn);
}
}
static void QueryCallback(IntPtr param, IntPtr taosRes, int code)
{
if (code == 0 && taosRes != IntPtr.Zero)
{
FetchRawBlockAsyncCallback fetchRowAsyncCallback = new FetchRawBlockAsyncCallback(FetchRawBlockCallback);
TDengine.FetchRawBlockAsync(taosRes, fetchRowAsyncCallback, param);
}
else
{
throw new Exception($"async query data failed,code:{code},reason:{TDengine.Error(taosRes)}");
}
}
// Iteratively call this interface until "numOfRows" is no greater than 0.
static void FetchRawBlockCallback(IntPtr param, IntPtr taosRes, int numOfRows)
{
if (numOfRows > 0)
{
Console.WriteLine($"{numOfRows} rows async retrieved");
IntPtr pdata = TDengine.GetRawBlock(taosRes);
List<TDengineMeta> metaList = TDengine.FetchFields(taosRes);
List<object> dataList = LibTaos.ReadRawBlock(pdata, metaList, numOfRows);
for (int i = 0; i < dataList.Count; i++)
{
if (i != 0 && (i + 1) % metaList.Count == 0)
{
Console.WriteLine("{0}\t|", dataList[i]);
}
else
{
Console.Write("{0}\t|", dataList[i]);
}
}
Console.WriteLine("");
TDengine.FetchRawBlockAsync(taosRes, FetchRawBlockCallback, param);
}
else
{
if (numOfRows == 0)
{
Console.WriteLine("async retrieve complete.");
}
else
{
throw new Exception($"FetchRawBlockCallback callback error, error code {numOfRows}");
}
TDengine.FreeResult(taosRes);
}
}
static IntPtr GetConnection()
{
string host = "localhost";
short port = 6030;
string username = "root";
string password = "taosdata";
string dbname = "power";
var conn = TDengine.Connect(host, username, password, dbname, port);
if (conn == IntPtr.Zero)
{
throw new Exception("Connect to TDengine failed");
}
else
{
Console.WriteLine("Connect to TDengine success");
}
return conn;
}
}
}
// //output:
// // Connect to TDengine success
// // 8 rows async retrieved
// // 1538548685500 | 11.8 | 221 | 0.28 | california.losangeles | 2 |
// // 1538548696600 | 13.4 | 223 | 0.29 | california.losangeles | 2 |
// // 1538548685000 | 10.8 | 223 | 0.29 | california.losangeles | 3 |
// // 1538548686500 | 11.5 | 221 | 0.35 | california.losangeles | 3 |
// // 1538548685000 | 10.3 | 219 | 0.31 | california.sanfrancisco | 2 |
// // 1538548695000 | 12.6 | 218 | 0.33 | california.sanfrancisco | 2 |
// // 1538548696800 | 12.3 | 221 | 0.31 | california.sanfrancisco | 2 |
// // 1538548696650 | 10.3 | 218 | 0.25 | california.sanfrancisco | 3 |
// // async retrieve complete.

15
docs/examples/csharp/asyncQuery/asyncquery.csproj
View File

@ -1,15 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net6.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<StartupObject>TDengineExample.AsyncQueryExample</StartupObject>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="TDengine.Connector" Version="3.0.*" />
</ItemGroup>
</Project>

21
docs/examples/csharp/connect/Program.cs
View File

@ -1,4 +1,5 @@
using TDengineDriver; using TDengine.Driver;
using TDengine.Driver.Client;
namespace TDengineExample namespace TDengineExample
{ {
@ -7,23 +8,11 @@ namespace TDengineExample
{ {
static void Main(String[] args) static void Main(String[] args)
{ {
string host = "localhost"; var builder = new ConnectionStringBuilder("host=localhost;port=6030;username=root;password=taosdata");
short port = 6030; using (var client = DbDriver.Open(builder))
string username = "root";
string password = "taosdata";
string dbname = "";
var conn = TDengine.Connect(host, username, password, dbname, port);
if (conn == IntPtr.Zero)
{ {
throw new Exception("Connect to TDengine failed"); Console.WriteLine("connected");
} }
else
{
Console.WriteLine("Connect to TDengine success");
}
TDengine.Close(conn);
TDengine.Cleanup();
} }
} }
} }

2
docs/examples/csharp/connect/connect.csproj
View File

@ -9,7 +9,7 @@
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="TDengine.Connector" Version="3.0.*" /> <PackageReference Include="TDengine.Connector" Version="3.1.*" />
</ItemGroup> </ItemGroup>
</Project> </Project>

6
docs/examples/csharp/csharp.sln
View File

@ -3,8 +3,6 @@ Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 16 # Visual Studio Version 16
VisualStudioVersion = 16.0.30114.105 VisualStudioVersion = 16.0.30114.105
MinimumVisualStudioVersion = 10.0.40219.1 MinimumVisualStudioVersion = 10.0.40219.1
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "asyncquery", "asyncQuery\asyncquery.csproj", "{E2A5F00C-14E7-40E1-A2DE-6AB2975616D3}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "connect", "connect\connect.csproj", "{CCC5042D-93FC-4AE0-B2F6-7E692FD476B7}" Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "connect", "connect\connect.csproj", "{CCC5042D-93FC-4AE0-B2F6-7E692FD476B7}"
EndProject EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "influxdbline", "influxdbLine\influxdbline.csproj", "{6A24FB80-1E3C-4E2D-A5AB-914FA583874D}" Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "influxdbline", "influxdbLine\influxdbline.csproj", "{6A24FB80-1E3C-4E2D-A5AB-914FA583874D}"
@ -38,10 +36,6 @@ Global
HideSolutionNode = FALSE HideSolutionNode = FALSE
EndGlobalSection EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution GlobalSection(ProjectConfigurationPlatforms) = postSolution
{E2A5F00C-14E7-40E1-A2DE-6AB2975616D3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{E2A5F00C-14E7-40E1-A2DE-6AB2975616D3}.Debug|Any CPU.Build.0 = Debug|Any CPU
{E2A5F00C-14E7-40E1-A2DE-6AB2975616D3}.Release|Any CPU.ActiveCfg = Release|Any CPU
{E2A5F00C-14E7-40E1-A2DE-6AB2975616D3}.Release|Any CPU.Build.0 = Release|Any CPU
{CCC5042D-93FC-4AE0-B2F6-7E692FD476B7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {CCC5042D-93FC-4AE0-B2F6-7E692FD476B7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{CCC5042D-93FC-4AE0-B2F6-7E692FD476B7}.Debug|Any CPU.Build.0 = Debug|Any CPU {CCC5042D-93FC-4AE0-B2F6-7E692FD476B7}.Debug|Any CPU.Build.0 = Debug|Any CPU
{CCC5042D-93FC-4AE0-B2F6-7E692FD476B7}.Release|Any CPU.ActiveCfg = Release|Any CPU {CCC5042D-93FC-4AE0-B2F6-7E692FD476B7}.Release|Any CPU.ActiveCfg = Release|Any CPU

68
docs/examples/csharp/influxdbLine/Program.cs
View File

@ -1,4 +1,5 @@
using TDengineDriver; using TDengine.Driver;
using TDengine.Driver.Client;
namespace TDengineExample namespace TDengineExample
{ {
@ -6,60 +7,23 @@ namespace TDengineExample
{ {
static void Main() static void Main()
{ {
IntPtr conn = GetConnection(); var builder =
PrepareDatabase(conn); new ConnectionStringBuilder("host=localhost;port=6030;username=root;password=taosdata");
string[] lines = { using (var client = DbDriver.Open(builder))
"meters,location=California.LosAngeles,groupid=2 current=11.8,voltage=221,phase=0.28 1648432611249",
"meters,location=California.LosAngeles,groupid=2 current=13.4,voltage=223,phase=0.29 1648432611250",
"meters,location=California.LosAngeles,groupid=3 current=10.8,voltage=223,phase=0.29 1648432611249",
"meters,location=California.LosAngeles,groupid=3 current=11.3,voltage=221,phase=0.35 1648432611250"
};
IntPtr res = TDengine.SchemalessInsert(conn, lines, lines.Length, (int)TDengineSchemalessProtocol.TSDB_SML_LINE_PROTOCOL, (int)TDengineSchemalessPrecision.TSDB_SML_TIMESTAMP_MILLI_SECONDS);
if (TDengine.ErrorNo(res) != 0)
{ {
throw new Exception("SchemalessInsert failed since " + TDengine.Error(res)); client.Exec("CREATE DATABASE test WAL_RETENTION_PERIOD 3600");
} client.Exec("use test");
else string[] lines = {
{ "meters,location=California.LosAngeles,groupid=2 current=11.8,voltage=221,phase=0.28 1648432611249",
int affectedRows = TDengine.AffectRows(res); "meters,location=California.LosAngeles,groupid=2 current=13.4,voltage=223,phase=0.29 1648432611250",
Console.WriteLine($"SchemalessInsert success, affected {affectedRows} rows"); "meters,location=California.LosAngeles,groupid=3 current=10.8,voltage=223,phase=0.29 1648432611249",
} "meters,location=California.LosAngeles,groupid=3 current=11.3,voltage=221,phase=0.35 1648432611250"
TDengine.FreeResult(res); };
client.SchemalessInsert(lines,
} TDengineSchemalessProtocol.TSDB_SML_LINE_PROTOCOL,
static IntPtr GetConnection() TDengineSchemalessPrecision.TSDB_SML_TIMESTAMP_MILLI_SECONDS, 0, ReqId.GetReqId());
{
string host = "localhost";
short port = 6030;
string username = "root";
string password = "taosdata";
string dbname = "";
var conn = TDengine.Connect(host, username, password, dbname, port);
if (conn == IntPtr.Zero)
{
throw new Exception("Connect to TDengine failed");
}
else
{
Console.WriteLine("Connect to TDengine success");
}
return conn;
}
static void PrepareDatabase(IntPtr conn)
{
IntPtr res = TDengine.Query(conn, "CREATE DATABASE test WAL_RETENTION_PERIOD 3600");
if (TDengine.ErrorNo(res) != 0)
{
throw new Exception("failed to create database, reason: " + TDengine.Error(res));
}
res = TDengine.Query(conn, "USE test");
if (TDengine.ErrorNo(res) != 0)
{
throw new Exception("failed to change database, reason: " + TDengine.Error(res));
} }
} }
} }
} }

2
docs/examples/csharp/influxdbLine/influxdbline.csproj
View File

@ -9,7 +9,7 @@
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="TDengine.Connector" Version="3.0.*" /> <PackageReference Include="TDengine.Connector" Version="3.1.*" />
</ItemGroup> </ItemGroup>
</Project> </Project>

75
docs/examples/csharp/optsJSON/Program.cs
View File

@ -1,69 +1,28 @@
using TDengineDriver; using TDengine.Driver;
using TDengine.Driver.Client;
namespace TDengineExample namespace TDengineExample
{ {
internal class OptsJsonExample internal class OptsJsonExample
{ {
static void Main() public static void Main(string[] args)
{ {
IntPtr conn = GetConnection(); var builder =
try new ConnectionStringBuilder("host=localhost;port=6030;username=root;password=taosdata");
using (var client = DbDriver.Open(builder))
{ {
PrepareDatabase(conn); client.Exec("CREATE DATABASE test WAL_RETENTION_PERIOD 3600");
string[] lines = { "[{\"metric\": \"meters.current\", \"timestamp\": 1648432611249, \"value\": 10.3, \"tags\": {\"location\": \"California.SanFrancisco\", \"groupid\": 2}}," + client.Exec("use test");
" {\"metric\": \"meters.voltage\", \"timestamp\": 1648432611249, \"value\": 219, \"tags\": {\"location\": \"California.LosAngeles\", \"groupid\": 1}}, " + string[] lines =
"{\"metric\": \"meters.current\", \"timestamp\": 1648432611250, \"value\": 12.6, \"tags\": {\"location\": \"California.SanFrancisco\", \"groupid\": 2}}," +
" {\"metric\": \"meters.voltage\", \"timestamp\": 1648432611250, \"value\": 221, \"tags\": {\"location\": \"California.LosAngeles\", \"groupid\": 1}}]"
};
IntPtr res = TDengine.SchemalessInsert(conn, lines, 1, (int)TDengineSchemalessProtocol.TSDB_SML_JSON_PROTOCOL, (int)TDengineSchemalessPrecision.TSDB_SML_TIMESTAMP_NOT_CONFIGURED);
if (TDengine.ErrorNo(res) != 0)
{ {
throw new Exception("SchemalessInsert failed since " + TDengine.Error(res)); "[{\"metric\": \"meters.current\", \"timestamp\": 1648432611249, \"value\": 10.3, \"tags\": {\"location\": \"California.SanFrancisco\", \"groupid\": 2}}," +
} " {\"metric\": \"meters.voltage\", \"timestamp\": 1648432611249, \"value\": 219, \"tags\": {\"location\": \"California.LosAngeles\", \"groupid\": 1}}, " +
else "{\"metric\": \"meters.current\", \"timestamp\": 1648432611250, \"value\": 12.6, \"tags\": {\"location\": \"California.SanFrancisco\", \"groupid\": 2}}," +
{ " {\"metric\": \"meters.voltage\", \"timestamp\": 1648432611250, \"value\": 221, \"tags\": {\"location\": \"California.LosAngeles\", \"groupid\": 1}}]"
int affectedRows = TDengine.AffectRows(res); };
Console.WriteLine($"SchemalessInsert success, affected {affectedRows} rows"); client.SchemalessInsert(lines, TDengineSchemalessProtocol.TSDB_SML_JSON_PROTOCOL,
} TDengineSchemalessPrecision.TSDB_SML_TIMESTAMP_MILLI_SECONDS, 0, ReqId.GetReqId());
TDengine.FreeResult(res);
}
finally
{
TDengine.Close(conn);
}
}
static IntPtr GetConnection()
{
string host = "localhost";
short port = 6030;
string username = "root";
string password = "taosdata";
string dbname = "";
var conn = TDengine.Connect(host, username, password, dbname, port);
if (conn == IntPtr.Zero)
{
throw new Exception("Connect to TDengine failed");
}
else
{
Console.WriteLine("Connect to TDengine success");
}
return conn;
}
static void PrepareDatabase(IntPtr conn)
{
IntPtr res = TDengine.Query(conn, "CREATE DATABASE test WAL_RETENTION_PERIOD 3600");
if (TDengine.ErrorNo(res) != 0)
{
throw new Exception("failed to create database, reason: " + TDengine.Error(res));
}
res = TDengine.Query(conn, "USE test");
if (TDengine.ErrorNo(res) != 0)
{
throw new Exception("failed to change database, reason: " + TDengine.Error(res));
} }
} }
} }
} }

2
docs/examples/csharp/optsJSON/optsJSON.csproj
View File

@ -9,7 +9,7 @@
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="TDengine.Connector" Version="3.0.*" /> <PackageReference Include="TDengine.Connector" Version="3.1.*" />
</ItemGroup> </ItemGroup>
</Project> </Project>

81
docs/examples/csharp/optsTelnet/Program.cs
View File

@ -1,72 +1,31 @@
using TDengineDriver; using TDengine.Driver;
using TDengine.Driver.Client;
namespace TDengineExample namespace TDengineExample
{ {
internal class OptsTelnetExample internal class OptsTelnetExample
{ {
static void Main() public static void Main(string[] args)
{ {
IntPtr conn = GetConnection(); var builder =
try new ConnectionStringBuilder("host=localhost;port=6030;username=root;password=taosdata");
using (var client = DbDriver.Open(builder))
{ {
PrepareDatabase(conn); client.Exec("CREATE DATABASE test WAL_RETENTION_PERIOD 3600");
client.Exec("USE test");
string[] lines = { string[] lines = {
"meters.current 1648432611249 10.3 location=California.SanFrancisco groupid=2", "meters.current 1648432611249 10.3 location=California.SanFrancisco groupid=2",
"meters.current 1648432611250 12.6 location=California.SanFrancisco groupid=2", "meters.current 1648432611250 12.6 location=California.SanFrancisco groupid=2",
"meters.current 1648432611249 10.8 location=California.LosAngeles groupid=3", "meters.current 1648432611249 10.8 location=California.LosAngeles groupid=3",
"meters.current 1648432611250 11.3 location=California.LosAngeles groupid=3", "meters.current 1648432611250 11.3 location=California.LosAngeles groupid=3",
"meters.voltage 1648432611249 219 location=California.SanFrancisco groupid=2", "meters.voltage 1648432611249 219 location=California.SanFrancisco groupid=2",
"meters.voltage 1648432611250 218 location=California.SanFrancisco groupid=2", "meters.voltage 1648432611250 218 location=California.SanFrancisco groupid=2",
"meters.voltage 1648432611249 221 location=California.LosAngeles groupid=3", "meters.voltage 1648432611249 221 location=California.LosAngeles groupid=3",
"meters.voltage 1648432611250 217 location=California.LosAngeles groupid=3", "meters.voltage 1648432611250 217 location=California.LosAngeles groupid=3",
}; };
IntPtr res = TDengine.SchemalessInsert(conn, lines, lines.Length, (int)TDengineSchemalessProtocol.TSDB_SML_TELNET_PROTOCOL, (int)TDengineSchemalessPrecision.TSDB_SML_TIMESTAMP_NOT_CONFIGURED); client.SchemalessInsert(lines,
if (TDengine.ErrorNo(res) != 0) TDengineSchemalessProtocol.TSDB_SML_TELNET_PROTOCOL,
{ TDengineSchemalessPrecision.TSDB_SML_TIMESTAMP_MILLI_SECONDS, 0, ReqId.GetReqId());
throw new Exception("SchemalessInsert failed since " + TDengine.Error(res));
}
else
{
int affectedRows = TDengine.AffectRows(res);
Console.WriteLine($"SchemalessInsert success, affected {affectedRows} rows");
}
TDengine.FreeResult(res);
}
catch
{
TDengine.Close(conn);
}
}
static IntPtr GetConnection()
{
string host = "localhost";
short port = 6030;
string username = "root";
string password = "taosdata";
string dbname = "";
var conn = TDengine.Connect(host, username, password, dbname, port);
if (conn == IntPtr.Zero)
{
throw new Exception("Connect to TDengine failed");
}
else
{
Console.WriteLine("Connect to TDengine success");
}
return conn;
}
static void PrepareDatabase(IntPtr conn)
{
IntPtr res = TDengine.Query(conn, "CREATE DATABASE test WAL_RETENTION_PERIOD 3600");
if (TDengine.ErrorNo(res) != 0)
{
throw new Exception("failed to create database, reason: " + TDengine.Error(res));
}
res = TDengine.Query(conn, "USE test");
if (TDengine.ErrorNo(res) != 0)
{
throw new Exception("failed to change database, reason: " + TDengine.Error(res));
} }
} }
} }

2
docs/examples/csharp/optsTelnet/optstelnet.csproj
View File

@ -9,7 +9,7 @@
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="TDengine.Connector" Version="3.0.*" /> <PackageReference Include="TDengine.Connector" Version="3.1.*" />
</ItemGroup> </ItemGroup>
</Project> </Project>

87
docs/examples/csharp/query/Program.cs
View File

@ -1,80 +1,35 @@
using TDengineDriver; using System.Text;
using TDengineDriver.Impl; using TDengine.Driver;
using System.Runtime.InteropServices; using TDengine.Driver.Client;
namespace TDengineExample namespace TDengineExample
{ {
internal class QueryExample internal class QueryExample
{ {
static void Main() public static void Main(string[] args)
{ {
IntPtr conn = GetConnection(); var builder = new ConnectionStringBuilder("host=localhost;port=6030;username=root;password=taosdata");
try using (var client = DbDriver.Open(builder))
{ {
// run query try
IntPtr res = TDengine.Query(conn, "SELECT * FROM meters LIMIT 2");
if (TDengine.ErrorNo(res) != 0)
{ {
throw new Exception("Failed to query since: " + TDengine.Error(res)); client.Exec("use power");
} string query = "SELECT * FROM meters";
using (var rows = client.Query(query))
// get filed count
int fieldCount = TDengine.FieldCount(res);
Console.WriteLine("fieldCount=" + fieldCount);
// print column names
List<TDengineMeta> metas = LibTaos.GetMeta(res);
for (int i = 0; i < metas.Count; i++)
{
Console.Write(metas[i].name + "\t");
}
Console.WriteLine();
// print values
List<Object> resData = LibTaos.GetData(res);
for (int i = 0; i < resData.Count; i++)
{
Console.Write($"|{resData[i].ToString()} \t");
if (((i + 1) % metas.Count == 0))
{ {
Console.WriteLine(""); while (rows.Read())
{
Console.WriteLine(
$"{((DateTime)rows.GetValue(0)):yyyy-MM-dd HH:mm:ss.fff}, {rows.GetValue(1)}, {rows.GetValue(2)}, {rows.GetValue(3)}, {rows.GetValue(4)}, {Encoding.UTF8.GetString((byte[])rows.GetValue(5))}");
}
} }
} }
Console.WriteLine(); catch (Exception e)
{
// Free result after use Console.WriteLine(e.ToString());
TDengine.FreeResult(res); throw;
}
} }
finally
{
TDengine.Close(conn);
}
}
static IntPtr GetConnection()
{
string host = "localhost";
short port = 6030;
string username = "root";
string password = "taosdata";
string dbname = "power";
var conn = TDengine.Connect(host, username, password, dbname, port);
if (conn == IntPtr.Zero)
{
throw new Exception("Connect to TDengine failed");
}
else
{
Console.WriteLine("Connect to TDengine success");
}
return conn;
} }
} }
} }
// output:
// Connect to TDengine success
// fieldCount=6
// ts current voltage phase location groupid
// 1648432611249 10.3 219 0.31 California.SanFrancisco 2
// 1648432611749 12.6 218 0.33 California.SanFrancisco 2

2
docs/examples/csharp/query/query.csproj
View File

@ -9,7 +9,7 @@
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="TDengine.Connector" Version="3.0.*" /> <PackageReference Include="TDengine.Connector" Version="3.1.*" />
</ItemGroup> </ItemGroup>
</Project> </Project>

92
docs/examples/csharp/sqlInsert/Program.cs
View File

@ -1,69 +1,47 @@
using TDengineDriver; using System.Text;
using TDengine.Driver;
using TDengine.Driver.Client;
namespace TDengineExample namespace TDengineExample
{ {
internal class SQLInsertExample internal class SQLInsertExample
{ {
static void Main() public static void Main(string[] args)
{ {
IntPtr conn = GetConnection(); var builder = new ConnectionStringBuilder("host=localhost;port=6030;username=root;password=taosdata");
try using (var client = DbDriver.Open(builder))
{ {
IntPtr res = TDengine.Query(conn, "CREATE DATABASE power WAL_RETENTION_PERIOD 3600"); try
CheckRes(conn, res, "failed to create database"); {
res = TDengine.Query(conn, "USE power"); client.Exec("create database power");
CheckRes(conn, res, "failed to change database"); client.Exec("CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (groupId INT, location BINARY(24))");
res = TDengine.Query(conn, "CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)"); string insertQuery =
CheckRes(conn, res, "failed to create stable"); "INSERT INTO " +
var sql = "INSERT INTO d1001 USING meters TAGS('California.SanFrancisco', 2) VALUES ('2018-10-03 14:38:05.000', 10.30000, 219, 0.31000) ('2018-10-03 14:38:15.000', 12.60000, 218, 0.33000) ('2018-10-03 14:38:16.800', 12.30000, 221, 0.31000) " + "power.d1001 USING power.meters TAGS(2,'California.SanFrancisco') " +
"d1002 USING power.meters TAGS('California.SanFrancisco', 3) VALUES('2018-10-03 14:38:16.650', 10.30000, 218, 0.25000) " + "VALUES " +
"d1003 USING power.meters TAGS('California.LosAngeles', 2) VALUES('2018-10-03 14:38:05.500', 11.80000, 221, 0.28000)('2018-10-03 14:38:16.600', 13.40000, 223, 0.29000) " + "('2023-10-03 14:38:05.000', 10.30000, 219, 0.31000) " +
"d1004 USING power.meters TAGS('California.LosAngeles', 3) VALUES('2018-10-03 14:38:05.000', 10.80000, 223, 0.29000)('2018-10-03 14:38:06.500', 11.50000, 221, 0.35000)"; "('2023-10-03 14:38:15.000', 12.60000, 218, 0.33000) " +
res = TDengine.Query(conn, sql); "('2023-10-03 14:38:16.800', 12.30000, 221, 0.31000) " +
CheckRes(conn, res, "failed to insert data"); "power.d1002 USING power.meters TAGS(3, 'California.SanFrancisco') " +
int affectedRows = TDengine.AffectRows(res); "VALUES " +
Console.WriteLine("affectedRows " + affectedRows); "('2023-10-03 14:38:16.650', 10.30000, 218, 0.25000) " +
TDengine.FreeResult(res); "power.d1003 USING power.meters TAGS(2,'California.LosAngeles') " +
} "VALUES " +
finally "('2023-10-03 14:38:05.500', 11.80000, 221, 0.28000) " +
{ "('2023-10-03 14:38:16.600', 13.40000, 223, 0.29000) " +
TDengine.Close(conn); "power.d1004 USING power.meters TAGS(3,'California.LosAngeles') " +
} "VALUES " +
"('2023-10-03 14:38:05.000', 10.80000, 223, 0.29000) " +
} "('2023-10-03 14:38:06.500', 11.50000, 221, 0.35000)";
client.Exec(insertQuery);
static IntPtr GetConnection() }
{ catch (Exception e)
string host = "localhost"; {
short port = 6030; Console.WriteLine(e.ToString());
string username = "root"; throw;
string password = "taosdata"; }
string dbname = "";
var conn = TDengine.Connect(host, username, password, dbname, port);
if (conn == IntPtr.Zero)
{
throw new Exception("Connect to TDengine failed");
}
else
{
Console.WriteLine("Connect to TDengine success");
}
return conn;
}
static void CheckRes(IntPtr conn, IntPtr res, String errorMsg)
{
if (TDengine.ErrorNo(res) != 0)
{
throw new Exception($"{errorMsg} since: {TDengine.Error(res)}");
} }
} }
} }
} }
// output:
// Connect to TDengine success
// affectedRows 8

2
docs/examples/csharp/sqlInsert/sqlinsert.csproj
View File

@ -9,7 +9,7 @@
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="TDengine.Connector" Version="3.0.*" /> <PackageReference Include="TDengine.Connector" Version="3.1.*" />
</ItemGroup> </ItemGroup>
</Project> </Project>

119
docs/examples/csharp/stmtInsert/Program.cs
View File

@ -1,109 +1,38 @@
using TDengineDriver; using TDengine.Driver;
using TDengine.Driver.Client;
namespace TDengineExample namespace TDengineExample
{ {
internal class StmtInsertExample internal class StmtInsertExample
{ {
private static IntPtr conn; public static void Main(string[] args)
private static IntPtr stmt;
static void Main()
{ {
conn = GetConnection(); var builder = new ConnectionStringBuilder("host=localhost;port=6030;username=root;password=taosdata");
try using (var client = DbDriver.Open(builder))
{ {
PrepareSTable(); try
// 1. init and prepare
stmt = TDengine.StmtInit(conn);
if (stmt == IntPtr.Zero)
{ {
throw new Exception("failed to init stmt."); client.Exec($"create database power");
client.Exec(
"CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (groupId INT, location BINARY(24))");
using (var stmt = client.StmtInit())
{
stmt.Prepare(
"Insert into power.d1001 using power.meters tags(2,'California.SanFrancisco') values(?,?,?,?)");
var ts = new DateTime(2023, 10, 03, 14, 38, 05, 000);
stmt.BindRow(new object[] { ts, (float)10.30000, (int)219, (float)0.31000 });
stmt.AddBatch();
stmt.Exec();
var affected = stmt.Affected();
Console.WriteLine($"affected rows: {affected}");
}
} }
int res = TDengine.StmtPrepare(stmt, "INSERT INTO ? USING meters TAGS(?, ?) VALUES(?, ?, ?, ?)"); catch (Exception e)
CheckStmtRes(res, "failed to prepare stmt");
// 2. bind table name and tags
TAOS_MULTI_BIND[] tags = new TAOS_MULTI_BIND[2] { TaosMultiBind.MultiBindBinary(new string[] { "California.SanFrancisco" }), TaosMultiBind.MultiBindInt(new int?[] { 2 }) };
res = TDengine.StmtSetTbnameTags(stmt, "d1001", tags);
CheckStmtRes(res, "failed to bind table name and tags");
// 3. bind values
TAOS_MULTI_BIND[] values = new TAOS_MULTI_BIND[4] {
TaosMultiBind.MultiBindTimestamp(new long[2] { 1648432611249, 1648432611749}),
TaosMultiBind.MultiBindFloat(new float?[2] { 10.3f, 12.6f}),
TaosMultiBind.MultiBindInt(new int?[2] { 219, 218}),
TaosMultiBind.MultiBindFloat(new float?[2]{ 0.31f, 0.33f})
};
res = TDengine.StmtBindParamBatch(stmt, values);
CheckStmtRes(res, "failed to bind params");
// 4. add batch
res = TDengine.StmtAddBatch(stmt);
CheckStmtRes(res, "failed to add batch");
// 5. execute
res = TDengine.StmtExecute(stmt);
CheckStmtRes(res, "failed to execute");
// 6. free
TaosMultiBind.FreeTaosBind(tags);
TaosMultiBind.FreeTaosBind(values);
}
finally
{
TDengine.Close(conn);
}
}
static IntPtr GetConnection()
{
string host = "localhost";
short port = 6030;
string username = "root";
string password = "taosdata";
string dbname = "";
var conn = TDengine.Connect(host, username, password, dbname, port);
if (conn == IntPtr.Zero)
{
throw new Exception("Connect to TDengine failed");
}
else
{
Console.WriteLine("Connect to TDengine success");
}
return conn;
}
static void PrepareSTable()
{
IntPtr res = TDengine.Query(conn, "CREATE DATABASE power WAL_RETENTION_PERIOD 3600");
CheckResPtr(res, "failed to create database");
res = TDengine.Query(conn, "USE power");
CheckResPtr(res, "failed to change database");
res = TDengine.Query(conn, "CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)");
CheckResPtr(res, "failed to create stable");
}
static void CheckStmtRes(int res, string errorMsg)
{
if (res != 0)
{
Console.WriteLine(errorMsg + ", " + TDengine.StmtErrorStr(stmt));
int code = TDengine.StmtClose(stmt);
if (code != 0)
{ {
throw new Exception($"failed to close stmt, {code} reason: {TDengine.StmtErrorStr(stmt)} "); Console.WriteLine(e);
throw;
} }
} }
} }
static void CheckResPtr(IntPtr res, string errorMsg)
{
if (TDengine.ErrorNo(res) != 0)
{
throw new Exception(errorMsg + " since:" + TDengine.Error(res));
}
}
} }
} }

2
docs/examples/csharp/stmtInsert/stmtinsert.csproj
View File

@ -9,7 +9,7 @@
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="TDengine.Connector" Version="3.0.*" /> <PackageReference Include="TDengine.Connector" Version="3.1.*" />
</ItemGroup> </ItemGroup>
</Project> </Project>

127
docs/examples/csharp/subscribe/Program.cs
View File

@ -1,95 +1,72 @@
using System; 
using TDengineTMQ; using TDengine.Driver;
using TDengineDriver; using TDengine.Driver.Client;
using System.Runtime.InteropServices; using TDengine.TMQ;
namespace TMQExample namespace TMQExample
{ {
internal class SubscribeDemo internal class SubscribeDemo
{ {
static void Main(string[] args) public static void Main(string[] args)
{ {
IntPtr conn = GetConnection(); var builder = new ConnectionStringBuilder("host=localhost;port=6030;username=root;password=taosdata");
string topic = "topic_example"; using (var client = DbDriver.Open(builder))
//create topic
IntPtr res = TDengine.Query(conn, $"create topic if not exists {topic} as select * from meters");
if (TDengine.ErrorNo(res) != 0 )
{ {
throw new Exception($"create topic failed, reason:{TDengine.Error(res)}"); try
}
var cfg = new ConsumerConfig
{
GourpId = "group_1",
TDConnectUser = "root",
TDConnectPasswd = "taosdata",
MsgWithTableName = "true",
TDConnectIp = "127.0.0.1",
};
// create consumer
var consumer = new ConsumerBuilder(cfg)
.Build();
// subscribe
consumer.Subscribe(topic);
// consume
for (int i = 0; i < 5; i++)
{
var consumeRes = consumer.Consume(300);
// print consumeResult
foreach (KeyValuePair<TopicPartition, TaosResult> kv in consumeRes.Message)
{ {
Console.WriteLine("topic partitions:\n{0}", kv.Key.ToString()); client.Exec("CREATE DATABASE power");
client.Exec("USE power");
kv.Value.Metas.ForEach(meta => client.Exec(
"CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (groupId INT, location BINARY(24))");
client.Exec("CREATE TOPIC topic_meters as SELECT * from power.meters");
var cfg = new Dictionary<string, string>()
{ {
Console.Write("{0} {1}({2}) \t|", meta.name, meta.TypeName(), meta.size); { "group.id", "group1" },
}); { "auto.offset.reset", "latest" },
Console.WriteLine(""); { "td.connect.ip", "127.0.0.1" },
kv.Value.Datas.ForEach(data => { "td.connect.user", "root" },
{ "td.connect.pass", "taosdata" },
{ "td.connect.port", "6030" },
{ "client.id", "tmq_example" },
{ "enable.auto.commit", "true" },
{ "msg.with.table.name", "false" },
};
var consumer = new ConsumerBuilder<Dictionary<string, object>>(cfg).Build();
consumer.Subscribe(new List<string>() { "topic_meters" });
Task.Run(InsertData);
while (true)
{ {
Console.WriteLine(data.ToString()); using (var cr = consumer.Consume(500))
}); {
if (cr == null) continue;
foreach (var message in cr.Message)
{
Console.WriteLine(
$"message {{{((DateTime)message.Value["ts"]).ToString("yyyy-MM-dd HH:mm:ss.fff")}, " +
$"{message.Value["current"]}, {message.Value["voltage"]}, {message.Value["phase"]}}}");
}
}
}
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
throw;
} }
consumer.Commit(consumeRes);
Console.WriteLine("\n================ {0} done ", i);
} }
// retrieve topic list
List<string> topics = consumer.Subscription();
topics.ForEach(t => Console.WriteLine("topic name:{0}", t));
// unsubscribe
consumer.Unsubscribe();
// close consumer after use.Otherwise will lead memory leak.
consumer.Close();
TDengine.Close(conn);
} }
static IntPtr GetConnection() static void InsertData()
{ {
string host = "localhost"; var builder = new ConnectionStringBuilder("host=localhost;port=6030;username=root;password=taosdata");
short port = 6030; using (var client = DbDriver.Open(builder))
string username = "root";
string password = "taosdata";
string dbname = "power";
var conn = TDengine.Connect(host, username, password, dbname, port);
if (conn == IntPtr.Zero)
{ {
throw new Exception("Connect to TDengine failed"); while (true)
{
client.Exec("INSERT into power.d1001 using power.meters tags(2,'California.SanFrancisco') values(now,11.5,219,0.30)");
Task.Delay(1000).Wait();
}
} }
else
{
Console.WriteLine("Connect to TDengine success");
}
return conn;
} }
} }

2
docs/examples/csharp/subscribe/subscribe.csproj
View File

@ -9,7 +9,7 @@
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="TDengine.Connector" Version="3.0.*" /> <PackageReference Include="TDengine.Connector" Version="3.1.*" />
</ItemGroup> </ItemGroup>
</Project> </Project>

22
docs/examples/csharp/wsConnect/Program.cs
View File

@ -1,28 +1,18 @@
using System; using System;
using TDengineWS.Impl; using TDengine.Driver;
using TDengine.Driver.Client;
namespace Examples namespace Examples
{ {
public class WSConnExample public class WSConnExample
{ {
static int Main(string[] args) static void Main(string[] args)
{ {
string DSN = "ws://root:taosdata@127.0.0.1:6041/test"; var builder = new ConnectionStringBuilder("protocol=WebSocket;host=localhost;port=6041;useSSL=false;username=root;password=taosdata");
IntPtr wsConn = LibTaosWS.WSConnectWithDSN(DSN); using (var client = DbDriver.Open(builder))
if (wsConn == IntPtr.Zero)
{ {
Console.WriteLine("get WS connection failed"); Console.WriteLine("connected");
return -1;
} }
else
{
Console.WriteLine("Establish connect success.");
// close connection.
LibTaosWS.WSClose(wsConn);
}
return 0;
} }
} }
} }

2
docs/examples/csharp/wsConnect/wsConnect.csproj
View File

@ -6,7 +6,7 @@
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="TDengine.Connector" Version="3.0.*" GeneratePathProperty="true" /> <PackageReference Include="TDengine.Connector" Version="3.1.*" GeneratePathProperty="true" />
</ItemGroup> </ItemGroup>
<Target Name="copyDLLDependency" BeforeTargets="BeforeBuild"> <Target Name="copyDLLDependency" BeforeTargets="BeforeBuild">
<ItemGroup> <ItemGroup>

83
docs/examples/csharp/wsInsert/Program.cs
View File

@ -1,61 +1,46 @@
using System; using System;
using TDengineWS.Impl; using TDengine.Driver;
using TDengine.Driver.Client;
namespace Examples namespace Examples
{ {
public class WSInsertExample public class WSInsertExample
{ {
static int Main(string[] args) public static void Main(string[] args)
{ {
string DSN = "ws://root:taosdata@127.0.0.1:6041/test"; var builder = new ConnectionStringBuilder("protocol=WebSocket;host=localhost;port=6041;useSSL=false;username=root;password=taosdata");
IntPtr wsConn = LibTaosWS.WSConnectWithDSN(DSN); using (var client = DbDriver.Open(builder))
// Assert if connection is validate
if (wsConn == IntPtr.Zero)
{ {
Console.WriteLine("get WS connection failed"); try
return -1; {
} client.Exec("create database power");
else client.Exec("CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (groupId INT, location BINARY(24))");
{ string insertQuery =
Console.WriteLine("Establish connect success."); "INSERT INTO " +
} "power.d1001 USING power.meters TAGS(2,'California.SanFrancisco') " +
"VALUES " +
string createTable = "CREATE STABLE test.meters (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupId int);"; "('2023-10-03 14:38:05.000', 10.30000, 219, 0.31000) " +
string insert = "INSERT INTO test.d1001 USING test.meters TAGS('California.SanFrancisco', 2) VALUES ('2018-10-03 14:38:05.000', 10.30000, 219, 0.31000) ('2018-10-03 14:38:15.000', 12.60000, 218, 0.33000) ('2018-10-03 14:38:16.800', 12.30000, 221, 0.31000)" + "('2023-10-03 14:38:15.000', 12.60000, 218, 0.33000) " +
"test.d1002 USING test.meters TAGS('California.SanFrancisco', 3) VALUES('2018-10-03 14:38:16.650', 10.30000, 218, 0.25000)" + "('2023-10-03 14:38:16.800', 12.30000, 221, 0.31000) " +
"test.d1003 USING test.meters TAGS('California.LosAngeles', 2) VALUES('2018-10-03 14:38:05.500', 11.80000, 221, 0.28000)('2018-10-03 14:38:16.600', 13.40000, 223, 0.29000) " + "power.d1002 USING power.meters TAGS(3, 'California.SanFrancisco') " +
"test.d1004 USING test.meters TAGS('California.LosAngeles', 3) VALUES('2018-10-03 14:38:05.000', 10.80000, 223, 0.29000)('2018-10-03 14:38:06.500', 11.50000, 221, 0.35000)"; "VALUES " +
"('2023-10-03 14:38:16.650', 10.30000, 218, 0.25000) " +
IntPtr wsRes = LibTaosWS.WSQuery(wsConn, createTable); "power.d1003 USING power.meters TAGS(2,'California.LosAngeles') " +
ValidInsert("create table", wsRes); "VALUES " +
LibTaosWS.WSFreeResult(wsRes); "('2023-10-03 14:38:05.500', 11.80000, 221, 0.28000) " +
"('2023-10-03 14:38:16.600', 13.40000, 223, 0.29000) " +
wsRes = LibTaosWS.WSQuery(wsConn, insert); "power.d1004 USING power.meters TAGS(3,'California.LosAngeles') " +
ValidInsert("insert data", wsRes); "VALUES " +
LibTaosWS.WSFreeResult(wsRes); "('2023-10-03 14:38:05.000', 10.80000, 223, 0.29000) " +
"('2023-10-03 14:38:06.500', 11.50000, 221, 0.35000)";
// close connection. client.Exec(insertQuery);
LibTaosWS.WSClose(wsConn); }
catch (Exception e)
return 0; {
} Console.WriteLine(e.ToString());
throw;
static void ValidInsert(string desc, IntPtr wsRes) }
{
int code = LibTaosWS.WSErrorNo(wsRes);
if (code != 0)
{
Console.WriteLine($"execute SQL failed: reason: {LibTaosWS.WSErrorStr(wsRes)}, code:{code}");
}
else
{
Console.WriteLine("{0} success affect {2} rows, cost {1} nanoseconds", desc, LibTaosWS.WSTakeTiming(wsRes), LibTaosWS.WSAffectRows(wsRes));
} }
} }
} }
} }
// Establish connect success.
// create table success affect 0 rows, cost 3717542 nanoseconds
// insert data success affect 8 rows, cost 2613637 nanoseconds

2
docs/examples/csharp/wsInsert/wsInsert.csproj
View File

@ -6,7 +6,7 @@
<Nullable>enable</Nullable> <Nullable>enable</Nullable>
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="TDengine.Connector" Version="3.0.*" GeneratePathProperty="true" /> <PackageReference Include="TDengine.Connector" Version="3.1.*" GeneratePathProperty="true" />
</ItemGroup> </ItemGroup>
<Target Name="copyDLLDependency" BeforeTargets="BeforeBuild"> <Target Name="copyDLLDependency" BeforeTargets="BeforeBuild">
<ItemGroup> <ItemGroup>

89
docs/examples/csharp/wsQuery/Program.cs
View File

@ -1,79 +1,36 @@
using System; using System;
using TDengineWS.Impl; using System.Text;
using System.Collections.Generic; using TDengine.Driver;
using TDengineDriver; using TDengine.Driver.Client;
namespace Examples namespace Examples
{ {
public class WSQueryExample public class WSQueryExample
{ {
static int Main(string[] args) public static void Main(string[] args)
{ {
string DSN = "ws://root:taosdata@127.0.0.1:6041/test"; var builder = new ConnectionStringBuilder("protocol=WebSocket;host=localhost;port=6041;useSSL=false;username=root;password=taosdata");
IntPtr wsConn = LibTaosWS.WSConnectWithDSN(DSN); using (var client = DbDriver.Open(builder))
if (wsConn == IntPtr.Zero)
{ {
Console.WriteLine("get WS connection failed"); try
return -1;
}
else
{
Console.WriteLine("Establish connect success.");
}
string select = "select * from test.meters";
// optional:wsRes = LibTaosWS.WSQuery(wsConn, select);
IntPtr wsRes = LibTaosWS.WSQueryTimeout(wsConn, select, 1);
// Assert if query execute success.
int code = LibTaosWS.WSErrorNo(wsRes);
if (code != 0)
{
Console.WriteLine($"execute SQL failed: reason: {LibTaosWS.WSErrorStr(wsRes)}, code:{code}");
LibTaosWS.WSFreeResult(wsRes);
return -1;
}
// get meta data
List<TDengineMeta> metas = LibTaosWS.WSGetFields(wsRes);
// get retrieved data
List<object> dataSet = LibTaosWS.WSGetData(wsRes);
// do something with result.
foreach (var meta in metas)
{
Console.Write("{0} {1}({2}) \t|\t", meta.name, meta.TypeName(), meta.size);
}
Console.WriteLine("");
for (int i = 0; i < dataSet.Count;)
{
for (int j = 0; j < metas.Count; j++)
{ {
Console.Write("{0}\t|\t", dataSet[i]); client.Exec("use power");
i++; string query = "SELECT * FROM meters";
using (var rows = client.Query(query))
{
while (rows.Read())
{
Console.WriteLine(
$"{((DateTime)rows.GetValue(0)):yyyy-MM-dd HH:mm:ss.fff}, {rows.GetValue(1)}, {rows.GetValue(2)}, {rows.GetValue(3)}, {rows.GetValue(4)}, {Encoding.UTF8.GetString((byte[])rows.GetValue(5))}");
}
}
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
throw;
} }
Console.WriteLine("");
} }
// Free result after use.
LibTaosWS.WSFreeResult(wsRes);
// close connection.
LibTaosWS.WSClose(wsConn);
return 0;
} }
} }
} }
// Establish connect success.
// ts TIMESTAMP(8) | current FLOAT(4) | voltage INT(4) | phase FLOAT(4) | location BINARY(64) | groupid INT(4) |
// 1538548685000 | 10.8 | 223 | 0.29 | California.LosAngeles | 3 |
// 1538548686500 | 11.5 | 221 | 0.35 | California.LosAngeles | 3 |
// 1538548685500 | 11.8 | 221 | 0.28 | California.LosAngeles | 2 |
// 1538548696600 | 13.4 | 223 | 0.29 | California.LosAngeles | 2 |
// 1538548685000 | 10.3 | 219 | 0.31 | California.SanFrancisco | 2 |
// 1538548695000 | 12.6 | 218 | 0.33 | California.SanFrancisco | 2 |
// 1538548696800 | 12.3 | 221 | 0.31 | California.SanFrancisco | 2 |
// 1538548696650 | 10.3 | 218 | 0.25 | California.SanFrancisco | 3 |

2
docs/examples/csharp/wsQuery/wsQuery.csproj
View File

@ -7,7 +7,7 @@
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="TDengine.Connector" Version="3.0.*" GeneratePathProperty="true" /> <PackageReference Include="TDengine.Connector" Version="3.1.*" GeneratePathProperty="true" />
</ItemGroup> </ItemGroup>
<Target Name="copyDLLDependency" BeforeTargets="BeforeBuild"> <Target Name="copyDLLDependency" BeforeTargets="BeforeBuild">
<ItemGroup> <ItemGroup>

117
docs/examples/csharp/wsStmt/Program.cs
View File

@ -1,98 +1,41 @@
using System; using System;
using TDengineWS.Impl; using TDengine.Driver;
using TDengineDriver; using TDengine.Driver.Client;
using System.Runtime.InteropServices;
namespace Examples namespace Examples
{ {
public class WSStmtExample public class WSStmtExample
{ {
static int Main(string[] args) public static void Main(string[] args)
{ {
const string DSN = "ws://root:taosdata@127.0.0.1:6041/test"; var builder =
const string table = "meters"; new ConnectionStringBuilder(
const string database = "test"; "protocol=WebSocket;host=localhost;port=6041;useSSL=false;username=root;password=taosdata");
const string childTable = "d1005"; using (var client = DbDriver.Open(builder))
string insert = $"insert into ? using {database}.{table} tags(?,?) values(?,?,?,?)";
const int numOfTags = 2;
const int numOfColumns = 4;
// Establish connection
IntPtr wsConn = LibTaosWS.WSConnectWithDSN(DSN);
if (wsConn == IntPtr.Zero)
{ {
Console.WriteLine($"get WS connection failed"); try
return -1; {
} client.Exec($"create database power");
else client.Exec(
{ "CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (groupId INT, location BINARY(24))");
Console.WriteLine("Establish connect success..."); using (var stmt = client.StmtInit())
} {
stmt.Prepare(
// init stmt "Insert into power.d1001 using power.meters tags(2,'California.SanFrancisco') values(?,?,?,?)");
IntPtr wsStmt = LibTaosWS.WSStmtInit(wsConn); var ts = new DateTime(2023, 10, 03, 14, 38, 05, 000);
if (wsStmt != IntPtr.Zero) stmt.BindRow(new object[] { ts, (float)10.30000, (int)219, (float)0.31000 });
{ stmt.AddBatch();
int code = LibTaosWS.WSStmtPrepare(wsStmt, insert); stmt.Exec();
ValidStmtStep(code, wsStmt, "WSStmtPrepare"); var affected = stmt.Affected();
Console.WriteLine($"affected rows: {affected}");
TAOS_MULTI_BIND[] wsTags = new TAOS_MULTI_BIND[] { WSMultiBind.WSBindNchar(new string[] { "California.SanDiego" }), WSMultiBind.WSBindInt(new int?[] { 4 }) }; }
code = LibTaosWS.WSStmtSetTbnameTags(wsStmt, $"{database}.{childTable}", wsTags, numOfTags); }
ValidStmtStep(code, wsStmt, "WSStmtSetTbnameTags"); catch (Exception e)
{
TAOS_MULTI_BIND[] data = new TAOS_MULTI_BIND[4]; Console.WriteLine(e);
data[0] = WSMultiBind.WSBindTimestamp(new long[] { 1538548687000, 1538548688000, 1538548689000, 1538548690000, 1538548691000 }); throw;
data[1] = WSMultiBind.WSBindFloat(new float?[] { 10.30F, 10.40F, 10.50F, 10.60F, 10.70F }); }
data[2] = WSMultiBind.WSBindInt(new int?[] { 223, 221, 222, 220, 219 });
data[3] = WSMultiBind.WSBindFloat(new float?[] { 0.31F, 0.32F, 0.33F, 0.35F, 0.28F });
code = LibTaosWS.WSStmtBindParamBatch(wsStmt, data, numOfColumns);
ValidStmtStep(code, wsStmt, "WSStmtBindParamBatch");
code = LibTaosWS.WSStmtAddBatch(wsStmt);
ValidStmtStep(code, wsStmt, "WSStmtAddBatch");
IntPtr stmtAffectRowPtr = Marshal.AllocHGlobal(Marshal.SizeOf(typeof(Int32)));
code = LibTaosWS.WSStmtExecute(wsStmt, stmtAffectRowPtr);
ValidStmtStep(code, wsStmt, "WSStmtExecute");
Console.WriteLine("WS STMT insert {0} rows...", Marshal.ReadInt32(stmtAffectRowPtr));
Marshal.FreeHGlobal(stmtAffectRowPtr);
LibTaosWS.WSStmtClose(wsStmt);
// Free unmanaged memory
WSMultiBind.WSFreeTaosBind(wsTags);
WSMultiBind.WSFreeTaosBind(data);
//check result with SQL "SELECT * FROM test.d1005;"
}
else
{
Console.WriteLine("Init STMT failed...");
}
// close connection.
LibTaosWS.WSClose(wsConn);
return 0;
}
static void ValidStmtStep(int code, IntPtr wsStmt, string desc)
{
if (code != 0)
{
Console.WriteLine($"{desc} failed,reason: {LibTaosWS.WSErrorStr(wsStmt)}, code: {code}");
}
else
{
Console.WriteLine("{0} success...", desc);
} }
} }
} }
} }
// WSStmtPrepare success...
// WSStmtSetTbnameTags success...
// WSStmtBindParamBatch success...
// WSStmtAddBatch success...
// WSStmtExecute success...
// WS STMT insert 5 rows...

2
docs/examples/csharp/wsStmt/wsStmt.csproj
View File

@ -7,7 +7,7 @@
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="TDengine.Connector" Version="3.0.*" GeneratePathProperty="true" /> <PackageReference Include="TDengine.Connector" Version="3.1.*" GeneratePathProperty="true" />
</ItemGroup> </ItemGroup>
<Target Name="copyDLLDependency" BeforeTargets="BeforeBuild"> <Target Name="copyDLLDependency" BeforeTargets="BeforeBuild">
<ItemGroup> <ItemGroup>

2
docs/zh/07-develop/01-connect/index.md
View File

@ -176,7 +176,7 @@ npm install @tdengine/rest
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="TDengine.Connector" Version="3.0.0" /> <PackageReference Include="TDengine.Connector" Version="3.1.0" />
</ItemGroup> </ItemGroup>
</Project> </Project>

3
docs/zh/07-develop/04-query-data/_cs_async.mdx
View File

@ -1,3 +0,0 @@
```csharp
{{#include docs/examples/csharp/asyncQuery/Program.cs}}
```

4
docs/zh/07-develop/04-query-data/index.mdx
View File

@ -16,7 +16,6 @@ import CQuery from "./_c.mdx";
import PhpQuery from "./_php.mdx"; import PhpQuery from "./_php.mdx";
import PyAsync from "./_py_async.mdx"; import PyAsync from "./_py_async.mdx";
import NodeAsync from "./_js_async.mdx"; import NodeAsync from "./_js_async.mdx";
import CsAsync from "./_cs_async.mdx";
import CAsync from "./_c_async.mdx"; import CAsync from "./_c_async.mdx";
## 主要查询功能 ## 主要查询功能
@ -175,9 +174,6 @@ Query OK, 6 rows in database (0.005515s)
<TabItem label="Python" value="python"> <TabItem label="Python" value="python">
<PyAsync /> <PyAsync />
</TabItem> </TabItem>
<TabItem label="C#" value="csharp">
<CsAsync />
</TabItem>
<TabItem label="C" value="c"> <TabItem label="C" value="c">
<CAsync /> <CAsync />
</TabItem> </TabItem>

52
docs/zh/07-develop/07-tmq.mdx
View File

@ -248,23 +248,23 @@ function close()
<TabItem value="C#" label="C#"> <TabItem value="C#" label="C#">
```csharp ```csharp
class ConsumerBuilder<TValue>
ConsumerBuilder(IEnumerable<KeyValuePair<string, string>> config) ConsumerBuilder(IEnumerable<KeyValuePair<string, string>> config)
virtual IConsumer Build() public IConsumer<TValue> Build()
Consumer(ConsumerBuilder builder)
void Subscribe(IEnumerable<string> topics) void Subscribe(IEnumerable<string> topics)
void Subscribe(string topic) void Subscribe(string topic)
ConsumeResult Consume(int millisecondsTimeout) ConsumeResult<TValue> Consume(int millisecondsTimeout)
List<string> Subscription() List<string> Subscription()
void Unsubscribe() void Unsubscribe()
void Commit(ConsumeResult consumerResult) List<TopicPartitionOffset> Commit()
void Close() void Close()
``` ```
@ -501,25 +501,19 @@ let consumer = taos.consumer({
<TabItem value="C#" label="C#"> <TabItem value="C#" label="C#">
```csharp ```csharp
using TDengineTMQ; var cfg = new Dictionary<string, string>()
{
// 根据需要，设置消费组 (GourpId)、自动提交 (EnableAutoCommit)、 { "group.id", "group1" },
// 自动提交时间间隔 (AutoCommitIntervalMs)、用户名 (TDConnectUser)、密码 (TDConnectPasswd) 等参数 { "auto.offset.reset", "latest" },
var cfg = new ConsumerConfig { "td.connect.ip", "127.0.0.1" },
{ { "td.connect.user", "root" },
EnableAutoCommit = "true" { "td.connect.pass", "taosdata" },
AutoCommitIntervalMs = "1000" { "td.connect.port", "6030" },
GourpId = "TDengine-TMQ-C#", { "client.id", "tmq_example" },
TDConnectUser = "root", { "enable.auto.commit", "true" },
TDConnectPasswd = "taosdata", { "msg.with.table.name", "false" },
AutoOffsetReset = "latest" };
MsgWithTableName = "true", var consumer = new ConsumerBuilder<Dictionary<string, object>>(cfg).Build();
TDConnectIp = "127.0.0.1",
TDConnectPort = "6030"
};
var consumer = new ConsumerBuilder(cfg).Build();
``` ```
</TabItem> </TabItem>
@ -748,10 +742,12 @@ while(true){
// 消费数据 // 消费数据
while (true) while (true)
{ {
var consumerRes = consumer.Consume(100); using (var result = consumer.Consume(500))
// process ConsumeResult {
ProcessMsg(consumerRes); if (result == null) continue;
consumer.Commit(consumerRes); ProcessMsg(result);
consumer.Commit();
}
} }
``` ```

1303
docs/zh/08-connector/40-csharp.mdx
View File

File diff suppressed because it is too large Load Diff

6
docs/zh/12-taos-sql/05-insert.md
View File

@ -158,7 +158,7 @@ INSERT INTO d21001 USING meters TAGS ('California.SanFrancisco', 2) FILE '/tmp/c
自动建表, 表名通过tbname列指定 自动建表, 表名通过tbname列指定
```sql ```sql
INSERT INTO meters(tbname, location, groupId, ts, current, phase) INSERT INTO meters(tbname, location, groupId, ts, current, phase)
values('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:34.630', 10.2, 219, 0.32) values('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:34.630', 10.2, 0.32)
values('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:35.779', 10.15, 217, 0.33) ('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:35.779', 10.15, 0.33)
values('d31002', NULL, 2, '2021-07-13 14:06:34.255', 10.15, 217, 0.33) ('d31002', NULL, 2, '2021-07-13 14:06:34.255', 10.15, 0.33)
``` ```

2
include/common/systable.h
View File

@ -50,6 +50,8 @@ extern "C" {
#define TSDB_INS_TABLE_STREAM_TASKS "ins_stream_tasks" #define TSDB_INS_TABLE_STREAM_TASKS "ins_stream_tasks"
#define TSDB_INS_TABLE_USER_PRIVILEGES "ins_user_privileges" #define TSDB_INS_TABLE_USER_PRIVILEGES "ins_user_privileges"
#define TSDB_INS_TABLE_VIEWS "ins_views" #define TSDB_INS_TABLE_VIEWS "ins_views"
#define TSDB_INS_TABLE_COMPACTS "ins_compacts"
#define TSDB_INS_TABLE_COMPACT_DETAILS "ins_compact_details"
#define TSDB_PERFORMANCE_SCHEMA_DB "performance_schema" #define TSDB_PERFORMANCE_SCHEMA_DB "performance_schema"
#define TSDB_PERFS_TABLE_SMAS "perf_smas" #define TSDB_PERFS_TABLE_SMAS "perf_smas"

2
include/common/tglobal.h
View File

@ -196,6 +196,7 @@ extern int64_t tsWalFsyncDataSizeLimit;
// internal // internal
extern int32_t tsTransPullupInterval; extern int32_t tsTransPullupInterval;
extern int32_t tsCompactPullupInterval;
extern int32_t tsMqRebalanceInterval; extern int32_t tsMqRebalanceInterval;
extern int32_t tsStreamCheckpointInterval; extern int32_t tsStreamCheckpointInterval;
extern float tsSinkDataRate; extern float tsSinkDataRate;
@ -214,6 +215,7 @@ extern int32_t tsMaxStreamBackendCache;
extern int32_t tsPQSortMemThreshold; extern int32_t tsPQSortMemThreshold;
extern int32_t tsResolveFQDNRetryTime; extern int32_t tsResolveFQDNRetryTime;
extern bool tsExperimental;
// #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize) // #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize)
int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd, int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd,

56
include/common/tmsg.h
View File

@ -144,6 +144,8 @@ typedef enum _mgmt_table {
TSDB_MGMT_TABLE_STREAM_TASKS, TSDB_MGMT_TABLE_STREAM_TASKS,
TSDB_MGMT_TABLE_PRIVILEGES, TSDB_MGMT_TABLE_PRIVILEGES,
TSDB_MGMT_TABLE_VIEWS, TSDB_MGMT_TABLE_VIEWS,
TSDB_MGMT_TABLE_COMPACT,
TSDB_MGMT_TABLE_COMPACT_DETAIL,
TSDB_MGMT_TABLE_MAX, TSDB_MGMT_TABLE_MAX,
} EShowType; } EShowType;
@ -308,6 +310,7 @@ typedef enum ENodeType {
QUERY_NODE_KILL_CONNECTION_STMT, QUERY_NODE_KILL_CONNECTION_STMT,
QUERY_NODE_KILL_QUERY_STMT, QUERY_NODE_KILL_QUERY_STMT,
QUERY_NODE_KILL_TRANSACTION_STMT, QUERY_NODE_KILL_TRANSACTION_STMT,
QUERY_NODE_KILL_COMPACT_STMT,
QUERY_NODE_DELETE_STMT, QUERY_NODE_DELETE_STMT,
QUERY_NODE_INSERT_STMT, QUERY_NODE_INSERT_STMT,
QUERY_NODE_QUERY, QUERY_NODE_QUERY,
@ -354,6 +357,8 @@ typedef enum ENodeType {
QUERY_NODE_SHOW_VNODES_STMT, QUERY_NODE_SHOW_VNODES_STMT,
QUERY_NODE_SHOW_USER_PRIVILEGES_STMT, QUERY_NODE_SHOW_USER_PRIVILEGES_STMT,
QUERY_NODE_SHOW_VIEWS_STMT, QUERY_NODE_SHOW_VIEWS_STMT,
QUERY_NODE_SHOW_COMPACTS_STMT,
QUERY_NODE_SHOW_COMPACT_DETAILS_STMT,
// logic plan node // logic plan node
QUERY_NODE_LOGIC_PLAN_SCAN = 1000, QUERY_NODE_LOGIC_PLAN_SCAN = 1000,
@ -1385,6 +1390,24 @@ int32_t tSerializeSCompactDbReq(void* buf, int32_t bufLen, SCompactDbReq* pReq);
int32_t tDeserializeSCompactDbReq(void* buf, int32_t bufLen, SCompactDbReq* pReq); int32_t tDeserializeSCompactDbReq(void* buf, int32_t bufLen, SCompactDbReq* pReq);
void tFreeSCompactDbReq(SCompactDbReq* pReq); void tFreeSCompactDbReq(SCompactDbReq* pReq);
typedef struct {
int32_t compactId;
int8_t bAccepted;
} SCompactDbRsp;
int32_t tSerializeSCompactDbRsp(void* buf, int32_t bufLen, SCompactDbRsp* pRsp);
int32_t tDeserializeSCompactDbRsp(void* buf, int32_t bufLen, SCompactDbRsp* pRsp);
typedef struct {
int32_t compactId;
int32_t sqlLen;
char* sql;
} SKillCompactReq;
int32_t tSerializeSKillCompactReq(void* buf, int32_t bufLen, SKillCompactReq* pReq);
int32_t tDeserializeSKillCompactReq(void* buf, int32_t bufLen, SKillCompactReq* pReq);
void tFreeSKillCompactReq(SKillCompactReq *pReq);
typedef struct { typedef struct {
char name[TSDB_FUNC_NAME_LEN]; char name[TSDB_FUNC_NAME_LEN];
int8_t igExists; int8_t igExists;
@ -1663,6 +1686,26 @@ int32_t tSerializeSCreateVnodeReq(void* buf, int32_t bufLen, SCreateVnodeReq* pR
int32_t tDeserializeSCreateVnodeReq(void* buf, int32_t bufLen, SCreateVnodeReq* pReq); int32_t tDeserializeSCreateVnodeReq(void* buf, int32_t bufLen, SCreateVnodeReq* pReq);
int32_t tFreeSCreateVnodeReq(SCreateVnodeReq* pReq); int32_t tFreeSCreateVnodeReq(SCreateVnodeReq* pReq);
typedef struct {
int32_t compactId;
int32_t vgId;
int32_t dnodeId;
} SQueryCompactProgressReq;
int32_t tSerializeSQueryCompactProgressReq(void* buf, int32_t bufLen, SQueryCompactProgressReq* pReq);
int32_t tDeserializeSQueryCompactProgressReq(void* buf, int32_t bufLen, SQueryCompactProgressReq* pReq);
typedef struct {
int32_t compactId;
int32_t vgId;
int32_t dnodeId;
int32_t numberFileset;
int32_t finished;
} SQueryCompactProgressRsp;
int32_t tSerializeSQueryCompactProgressRsp(void* buf, int32_t bufLen, SQueryCompactProgressRsp* pReq);
int32_t tDeserializeSQueryCompactProgressRsp(void* buf, int32_t bufLen, SQueryCompactProgressRsp* pReq);
typedef struct { typedef struct {
int32_t vgId; int32_t vgId;
int32_t dnodeId; int32_t dnodeId;
@ -1690,11 +1733,21 @@ typedef struct {
char db[TSDB_DB_FNAME_LEN]; char db[TSDB_DB_FNAME_LEN];
int64_t compactStartTime; int64_t compactStartTime;
STimeWindow tw; STimeWindow tw;
int32_t compactId;
} SCompactVnodeReq; } SCompactVnodeReq;
int32_t tSerializeSCompactVnodeReq(void* buf, int32_t bufLen, SCompactVnodeReq* pReq); int32_t tSerializeSCompactVnodeReq(void* buf, int32_t bufLen, SCompactVnodeReq* pReq);
int32_t tDeserializeSCompactVnodeReq(void* buf, int32_t bufLen, SCompactVnodeReq* pReq); int32_t tDeserializeSCompactVnodeReq(void* buf, int32_t bufLen, SCompactVnodeReq* pReq);
typedef struct {
int32_t compactId;
int32_t vgId;
int32_t dnodeId;
} SVKillCompactReq;
int32_t tSerializeSVKillCompactReq(void* buf, int32_t bufLen, SVKillCompactReq* pReq);
int32_t tDeserializeSVKillCompactReq(void* buf, int32_t bufLen, SVKillCompactReq* pReq);
typedef struct { typedef struct {
int32_t vgVersion; int32_t vgVersion;
int32_t buffer; int32_t buffer;
@ -1890,8 +1943,9 @@ typedef struct {
char db[TSDB_DB_FNAME_LEN]; char db[TSDB_DB_FNAME_LEN];
char tb[TSDB_TABLE_NAME_LEN]; char tb[TSDB_TABLE_NAME_LEN];
char user[TSDB_USER_LEN]; char user[TSDB_USER_LEN];
char filterTb[TSDB_TABLE_NAME_LEN]; char filterTb[TSDB_TABLE_NAME_LEN]; // for ins_columns
int64_t showId; int64_t showId;
int64_t compactId; // for compact
} SRetrieveTableReq; } SRetrieveTableReq;
typedef struct SSysTableSchema { typedef struct SSysTableSchema {

6
include/common/tmsgdef.h
View File

@ -217,6 +217,8 @@
TD_DEF_MSG_TYPE(TDMT_MND_CREATE_VIEW, "create-view", SCMCreateViewReq, NULL) TD_DEF_MSG_TYPE(TDMT_MND_CREATE_VIEW, "create-view", SCMCreateViewReq, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_DROP_VIEW, "drop-view", SCMDropViewReq, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_VIEW, "drop-view", SCMDropViewReq, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_VIEW_META, "view-meta", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_VIEW_META, "view-meta", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_KILL_COMPACT, "kill-compact", SKillCompactReq, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_COMPACT_TIMER, "compact-tmr", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL)
TD_CLOSE_MSG_SEG(TDMT_END_MND_MSG) TD_CLOSE_MSG_SEG(TDMT_END_MND_MSG)
@ -256,7 +258,7 @@
TD_DEF_MSG_TYPE(TDMT_VND_EXEC_RSMA, "vnode-exec-rsma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_EXEC_RSMA, "vnode-exec-rsma", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_DELETE, "delete-data", SVDeleteReq, SVDeleteRsp) TD_DEF_MSG_TYPE(TDMT_VND_DELETE, "delete-data", SVDeleteReq, SVDeleteRsp)
TD_DEF_MSG_TYPE(TDMT_VND_BATCH_DEL, "batch-delete", SBatchDeleteReq, NULL) TD_DEF_MSG_TYPE(TDMT_VND_BATCH_DEL, "batch-delete", SBatchDeleteReq, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "alter-config", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "alter-config", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_ALTER_REPLICA, "alter-replica", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_ALTER_REPLICA, "alter-replica", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIRM, "alter-confirm", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIRM, "alter-confirm", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_ALTER_HASHRANGE, "alter-hashrange", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_ALTER_HASHRANGE, "alter-hashrange", NULL, NULL)
@ -267,6 +269,8 @@ TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "alter-config", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_CREATE_INDEX, "vnode-create-index", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_CREATE_INDEX, "vnode-create-index", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_DROP_INDEX, "vnode-drop-index", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_DROP_INDEX, "vnode-drop-index", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_DISABLE_WRITE, "vnode-disable-write", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_DISABLE_WRITE, "vnode-disable-write", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_QUERY_COMPACT_PROGRESS, "vnode-query-compact-progress", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_KILL_COMPACT, "kill-compact", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_MAX_MSG, "vnd-max", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_MAX_MSG, "vnd-max", NULL, NULL)
TD_CLOSE_MSG_SEG(TDMT_END_VND_MSG) TD_CLOSE_MSG_SEG(TDMT_END_VND_MSG)

352
include/common/ttokendef.h
View File

@ -189,184 +189,180 @@
#define TK_ALIVE 170 #define TK_ALIVE 170
#define TK_VIEWS 171 #define TK_VIEWS 171
#define TK_VIEW 172 #define TK_VIEW 172
#define TK_NORMAL 173 #define TK_COMPACTS 173
#define TK_CHILD 174 #define TK_NORMAL 174
#define TK_LIKE 175 #define TK_CHILD 175
#define TK_TBNAME 176 #define TK_LIKE 176
#define TK_QTAGS 177 #define TK_TBNAME 177
#define TK_AS 178 #define TK_QTAGS 178
#define TK_SYSTEM 179 #define TK_AS 179
#define TK_INDEX 180 #define TK_SYSTEM 180
#define TK_FUNCTION 181 #define TK_INDEX 181
#define TK_INTERVAL 182 #define TK_FUNCTION 182
#define TK_COUNT 183 #define TK_INTERVAL 183
#define TK_LAST_ROW 184 #define TK_COUNT 184
#define TK_META 185 #define TK_LAST_ROW 185
#define TK_ONLY 186 #define TK_META 186
#define TK_TOPIC 187 #define TK_ONLY 187
#define TK_CONSUMER 188 #define TK_TOPIC 188
#define TK_GROUP 189 #define TK_CONSUMER 189
#define TK_DESC 190 #define TK_GROUP 190
#define TK_DESCRIBE 191 #define TK_DESC 191
#define TK_RESET 192 #define TK_DESCRIBE 192
#define TK_QUERY 193 #define TK_RESET 193
#define TK_CACHE 194 #define TK_QUERY 194
#define TK_EXPLAIN 195 #define TK_CACHE 195
#define TK_ANALYZE 196 #define TK_EXPLAIN 196
#define TK_VERBOSE 197 #define TK_ANALYZE 197
#define TK_NK_BOOL 198 #define TK_VERBOSE 198
#define TK_RATIO 199 #define TK_NK_BOOL 199
#define TK_NK_FLOAT 200 #define TK_RATIO 200
#define TK_OUTPUTTYPE 201 #define TK_NK_FLOAT 201
#define TK_AGGREGATE 202 #define TK_OUTPUTTYPE 202
#define TK_BUFSIZE 203 #define TK_AGGREGATE 203
#define TK_LANGUAGE 204 #define TK_BUFSIZE 204
#define TK_REPLACE 205 #define TK_LANGUAGE 205
#define TK_STREAM 206 #define TK_REPLACE 206
#define TK_INTO 207 #define TK_STREAM 207
#define TK_PAUSE 208 #define TK_INTO 208
#define TK_RESUME 209 #define TK_PAUSE 209
#define TK_TRIGGER 210 #define TK_RESUME 210
#define TK_AT_ONCE 211 #define TK_TRIGGER 211
#define TK_WINDOW_CLOSE 212 #define TK_AT_ONCE 212
#define TK_IGNORE 213 #define TK_WINDOW_CLOSE 213
#define TK_EXPIRED 214 #define TK_IGNORE 214
#define TK_FILL_HISTORY 215 #define TK_EXPIRED 215
#define TK_UPDATE 216 #define TK_FILL_HISTORY 216
#define TK_SUBTABLE 217 #define TK_UPDATE 217
#define TK_UNTREATED 218 #define TK_SUBTABLE 218
#define TK_KILL 219 #define TK_UNTREATED 219
#define TK_CONNECTION 220 #define TK_KILL 220
#define TK_TRANSACTION 221 #define TK_CONNECTION 221
#define TK_BALANCE 222 #define TK_TRANSACTION 222
#define TK_VGROUP 223 #define TK_BALANCE 223
#define TK_LEADER 224 #define TK_VGROUP 224
#define TK_MERGE 225 #define TK_LEADER 225
#define TK_REDISTRIBUTE 226 #define TK_MERGE 226
#define TK_SPLIT 227 #define TK_REDISTRIBUTE 227
#define TK_DELETE 228 #define TK_SPLIT 228
#define TK_INSERT 229 #define TK_DELETE 229
#define TK_NULL 230 #define TK_INSERT 230
#define TK_NK_QUESTION 231 #define TK_NULL 231
#define TK_NK_ALIAS 232 #define TK_NK_QUESTION 232
#define TK_NK_ARROW 233 #define TK_NK_ALIAS 233
#define TK_ROWTS 234 #define TK_NK_ARROW 234
#define TK_QSTART 235 #define TK_ROWTS 235
#define TK_QEND 236 #define TK_QSTART 236
#define TK_QDURATION 237 #define TK_QEND 237
#define TK_WSTART 238 #define TK_QDURATION 238
#define TK_WEND 239 #define TK_WSTART 239
#define TK_WDURATION 240 #define TK_WEND 240
#define TK_IROWTS 241 #define TK_WDURATION 241
#define TK_ISFILLED 242 #define TK_IROWTS 242
#define TK_CAST 243 #define TK_ISFILLED 243
#define TK_NOW 244 #define TK_CAST 244
#define TK_TODAY 245 #define TK_NOW 245
#define TK_TIMEZONE 246 #define TK_TODAY 246
#define TK_CLIENT_VERSION 247 #define TK_TIMEZONE 247
#define TK_SERVER_VERSION 248 #define TK_CLIENT_VERSION 248
#define TK_SERVER_STATUS 249 #define TK_SERVER_VERSION 249
#define TK_CURRENT_USER 250 #define TK_SERVER_STATUS 250
#define TK_CASE 251 #define TK_CURRENT_USER 251
#define TK_WHEN 252 #define TK_CASE 252
#define TK_THEN 253 #define TK_WHEN 253
#define TK_ELSE 254 #define TK_THEN 254
#define TK_BETWEEN 255 #define TK_ELSE 255
#define TK_IS 256 #define TK_BETWEEN 256
#define TK_NK_LT 257 #define TK_IS 257
#define TK_NK_GT 258 #define TK_NK_LT 258
#define TK_NK_LE 259 #define TK_NK_GT 259
#define TK_NK_GE 260 #define TK_NK_LE 260
#define TK_NK_NE 261 #define TK_NK_GE 261
#define TK_MATCH 262 #define TK_NK_NE 262
#define TK_NMATCH 263 #define TK_MATCH 263
#define TK_CONTAINS 264 #define TK_NMATCH 264
#define TK_IN 265 #define TK_CONTAINS 265
#define TK_JOIN 266 #define TK_IN 266
#define TK_INNER 267 #define TK_JOIN 267
#define TK_SELECT 268 #define TK_INNER 268
#define TK_NK_HINT 269 #define TK_SELECT 269
#define TK_DISTINCT 270 #define TK_NK_HINT 270
#define TK_WHERE 271 #define TK_DISTINCT 271
#define TK_PARTITION 272 #define TK_WHERE 272
#define TK_BY 273 #define TK_PARTITION 273
#define TK_SESSION 274 #define TK_BY 274
#define TK_STATE_WINDOW 275 #define TK_SESSION 275
#define TK_EVENT_WINDOW 276 #define TK_STATE_WINDOW 276
#define TK_SLIDING 277 #define TK_EVENT_WINDOW 277
#define TK_FILL 278 #define TK_SLIDING 278
#define TK_VALUE 279 #define TK_FILL 279
#define TK_VALUE_F 280 #define TK_VALUE 280
#define TK_NONE 281 #define TK_VALUE_F 281
#define TK_PREV 282 #define TK_NONE 282
#define TK_NULL_F 283 #define TK_PREV 283
#define TK_LINEAR 284 #define TK_NULL_F 284
#define TK_NEXT 285 #define TK_LINEAR 285
#define TK_HAVING 286 #define TK_NEXT 286
#define TK_RANGE 287 #define TK_HAVING 287
#define TK_EVERY 288 #define TK_RANGE 288
#define TK_ORDER 289 #define TK_EVERY 289
#define TK_SLIMIT 290 #define TK_ORDER 290
#define TK_SOFFSET 291 #define TK_SLIMIT 291
#define TK_LIMIT 292 #define TK_SOFFSET 292
#define TK_OFFSET 293 #define TK_LIMIT 293
#define TK_ASC 294 #define TK_OFFSET 294
#define TK_NULLS 295 #define TK_ASC 295
#define TK_ABORT 296 #define TK_NULLS 296
#define TK_AFTER 297 #define TK_ABORT 297
#define TK_ATTACH 298 #define TK_AFTER 298
#define TK_BEFORE 299 #define TK_ATTACH 299
#define TK_BEGIN 300 #define TK_BEFORE 300
#define TK_BITAND 301 #define TK_BEGIN 301
#define TK_BITNOT 302 #define TK_BITAND 302
#define TK_BITOR 303 #define TK_BITNOT 303
#define TK_BLOCKS 304 #define TK_BITOR 304
#define TK_CHANGE 305 #define TK_BLOCKS 305
#define TK_COMMA 306 #define TK_CHANGE 306
#define TK_CONCAT 307 #define TK_COMMA 307
#define TK_CONFLICT 308 #define TK_CONCAT 308
#define TK_COPY 309 #define TK_CONFLICT 309
#define TK_DEFERRED 310 #define TK_COPY 310
#define TK_DELIMITERS 311 #define TK_DEFERRED 311
#define TK_DETACH 312 #define TK_DELIMITERS 312
#define TK_DIVIDE 313 #define TK_DETACH 313
#define TK_DOT 314 #define TK_DIVIDE 314
#define TK_EACH 315 #define TK_DOT 315
#define TK_FAIL 316 #define TK_EACH 316
#define TK_FILE 317 #define TK_FAIL 317
#define TK_FOR 318 #define TK_FILE 318
#define TK_GLOB 319 #define TK_FOR 319
#define TK_ID 320 #define TK_GLOB 320
#define TK_IMMEDIATE 321 #define TK_ID 321
#define TK_IMPORT 322 #define TK_IMMEDIATE 322
#define TK_INITIALLY 323 #define TK_IMPORT 323
#define TK_INSTEAD 324 #define TK_INITIALLY 324
#define TK_ISNULL 325 #define TK_INSTEAD 325
#define TK_KEY 326 #define TK_ISNULL 326
#define TK_MODULES 327 #define TK_KEY 327
#define TK_NK_BITNOT 328 #define TK_MODULES 328
#define TK_NK_SEMI 329 #define TK_NK_BITNOT 329
#define TK_NOTNULL 330 #define TK_NK_SEMI 330
#define TK_OF 331 #define TK_NOTNULL 331
#define TK_PLUS 332 #define TK_OF 332
#define TK_PRIVILEGE 333 #define TK_PLUS 333
#define TK_RAISE 334 #define TK_PRIVILEGE 334
#define TK_RESTRICT 335 #define TK_RAISE 335
#define TK_ROW 336 #define TK_RESTRICT 336
#define TK_SEMI 337 #define TK_ROW 337
#define TK_STAR 338 #define TK_SEMI 338
#define TK_STATEMENT 339 #define TK_STAR 339
#define TK_STRICT 340 #define TK_STATEMENT 340
#define TK_STRING 341 #define TK_STRICT 341
#define TK_TIMES 342 #define TK_STRING 342
#define TK_VALUES 343 #define TK_TIMES 343
#define TK_VARIABLE 344 #define TK_VALUES 344
#define TK_WAL 345 #define TK_VARIABLE 345
#define TK_WAL 346
#define TK_NK_SPACE 600 #define TK_NK_SPACE 600
#define TK_NK_COMMENT 601 #define TK_NK_COMMENT 601

47
include/libs/executor/storageapi.h
View File

@ -38,8 +38,8 @@ extern "C" {
#define META_READER_NOLOCK 0x1 #define META_READER_NOLOCK 0x1
#define STREAM_STATE_BUFF_HASH 1 #define STREAM_STATE_BUFF_HASH 1
#define STREAM_STATE_BUFF_SORT 2 #define STREAM_STATE_BUFF_SORT 2
typedef struct SMeta SMeta; typedef struct SMeta SMeta;
typedef TSKEY (*GetTsFun)(void*); typedef TSKEY (*GetTsFun)(void*);
@ -102,14 +102,14 @@ typedef struct SMTbCursor {
} SMTbCursor; } SMTbCursor;
typedef struct SMCtbCursor { typedef struct SMCtbCursor {
SMeta *pMeta; SMeta* pMeta;
void *pCur; void* pCur;
tb_uid_t suid; tb_uid_t suid;
void *pKey; void* pKey;
void *pVal; void* pVal;
int kLen; int kLen;
int vLen; int vLen;
int8_t paused; int8_t paused;
int lock; int lock;
} SMCtbCursor; } SMCtbCursor;
@ -153,7 +153,8 @@ typedef struct {
// clang-format off // clang-format off
/*-------------------------------------------------new api format---------------------------------------------------*/ /*-------------------------------------------------new api format---------------------------------------------------*/
typedef enum { typedef enum {
TSD_READER_NOTIFY_DURATION_START TSD_READER_NOTIFY_DURATION_START,
TSD_READER_NOTIFY_NEXT_DURATION_BLOCK,
} ETsdReaderNotifyType; } ETsdReaderNotifyType;
typedef union { typedef union {
@ -262,22 +263,23 @@ typedef struct SStoreMeta {
void* (*storeGetIndexInfo)(); void* (*storeGetIndexInfo)();
void* (*getInvertIndex)(void* pVnode); void* (*getInvertIndex)(void* pVnode);
// support filter and non-filter cases. [vnodeGetCtbIdList & vnodeGetCtbIdListByFilter] // support filter and non-filter cases. [vnodeGetCtbIdList & vnodeGetCtbIdListByFilter]
int32_t (*getChildTableList)( void* pVnode, int64_t suid, SArray* list); int32_t (*getChildTableList)(void* pVnode, int64_t suid, SArray* list);
int32_t (*storeGetTableList)(void* pVnode, int8_t type, SArray* pList); int32_t (*storeGetTableList)(void* pVnode, int8_t type, SArray* pList);
void* storeGetVersionRange; void* storeGetVersionRange;
void* storeGetLastTimestamp; void* storeGetLastTimestamp;
int32_t (*getTableSchema)(void* pVnode, int64_t uid, STSchema** pSchema, int64_t* suid); // tsdbGetTableSchema int32_t (*getTableSchema)(void* pVnode, int64_t uid, STSchema** pSchema, int64_t* suid); // tsdbGetTableSchema
int32_t (*getNumOfChildTables)( void* pVnode, int64_t uid, int64_t* numOfTables, int32_t* numOfCols); int32_t (*getNumOfChildTables)(void* pVnode, int64_t uid, int64_t* numOfTables, int32_t* numOfCols);
void (*getBasicInfo)(void* pVnode, const char** dbname, int32_t* vgId, int64_t* numOfTables, int64_t* numOfNormalTables); void (*getBasicInfo)(void* pVnode, const char** dbname, int32_t* vgId, int64_t* numOfTables,
int64_t* numOfNormalTables);
int64_t (*getNumOfRowsInMem)(void* pVnode); int64_t (*getNumOfRowsInMem)(void* pVnode);
SMCtbCursor* (*openCtbCursor)(void *pVnode, tb_uid_t uid, int lock); SMCtbCursor* (*openCtbCursor)(void* pVnode, tb_uid_t uid, int lock);
int32_t (*resumeCtbCursor)(SMCtbCursor* pCtbCur, int8_t first); int32_t (*resumeCtbCursor)(SMCtbCursor* pCtbCur, int8_t first);
void (*pauseCtbCursor)(SMCtbCursor* pCtbCur); void (*pauseCtbCursor)(SMCtbCursor* pCtbCur);
void (*closeCtbCursor)(SMCtbCursor *pCtbCur); void (*closeCtbCursor)(SMCtbCursor* pCtbCur);
tb_uid_t (*ctbCursorNext)(SMCtbCursor* pCur); tb_uid_t (*ctbCursorNext)(SMCtbCursor* pCur);
} SStoreMeta; } SStoreMeta;
typedef struct SStoreMetaReader { typedef struct SStoreMetaReader {
@ -362,14 +364,14 @@ typedef struct SStateStore {
const SSessionKey* pKey, void** pVal, int32_t* pVLen); const SSessionKey* pKey, void** pVal, int32_t* pVLen);
SUpdateInfo* (*updateInfoInit)(int64_t interval, int32_t precision, int64_t watermark, bool igUp); SUpdateInfo* (*updateInfoInit)(int64_t interval, int32_t precision, int64_t watermark, bool igUp);
TSKEY (*updateInfoFillBlockData)(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol); TSKEY (*updateInfoFillBlockData)(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol);
bool (*updateInfoIsUpdated)(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts); bool (*updateInfoIsUpdated)(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts);
bool (*updateInfoIsTableInserted)(SUpdateInfo* pInfo, int64_t tbUid); bool (*updateInfoIsTableInserted)(SUpdateInfo* pInfo, int64_t tbUid);
bool (*isIncrementalTimeStamp)(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts); bool (*isIncrementalTimeStamp)(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts);
void (*updateInfoDestroy)(SUpdateInfo* pInfo); void (*updateInfoDestroy)(SUpdateInfo* pInfo);
void (*windowSBfDelete)(SUpdateInfo *pInfo, uint64_t count); void (*windowSBfDelete)(SUpdateInfo* pInfo, uint64_t count);
void (*windowSBfAdd)(SUpdateInfo *pInfo, uint64_t count); void (*windowSBfAdd)(SUpdateInfo* pInfo, uint64_t count);
SUpdateInfo* (*updateInfoInitP)(SInterval* pInterval, int64_t watermark, bool igUp); SUpdateInfo* (*updateInfoInitP)(SInterval* pInterval, int64_t watermark, bool igUp);
void (*updateInfoAddCloseWindowSBF)(SUpdateInfo* pInfo); void (*updateInfoAddCloseWindowSBF)(SUpdateInfo* pInfo);
@ -396,6 +398,7 @@ typedef struct SStateStore {
void (*streamStateDestroy)(SStreamState* pState, bool remove); void (*streamStateDestroy)(SStreamState* pState, bool remove);
int32_t (*streamStateDeleteCheckPoint)(SStreamState* pState, TSKEY mark); int32_t (*streamStateDeleteCheckPoint)(SStreamState* pState, TSKEY mark);
void (*streamStateReloadInfo)(SStreamState* pState, TSKEY ts); void (*streamStateReloadInfo)(SStreamState* pState, TSKEY ts);
void (*streamStateCopyBackend)(SStreamState* src, SStreamState* dst);
} SStateStore; } SStateStore;
typedef struct SStorageAPI { typedef struct SStorageAPI {

1
include/libs/function/function.h
View File

@ -171,6 +171,7 @@ typedef struct {
int32_t taskId; int32_t taskId;
int64_t streamId; int64_t streamId;
int64_t streamBackendRid; int64_t streamBackendRid;
int8_t dump;
} SStreamState; } SStreamState;
typedef struct SFunctionStateStore { typedef struct SFunctionStateStore {

13
include/libs/nodes/cmdnodes.h
View File

@ -46,6 +46,10 @@ extern "C" {
#define SHOW_LOCAL_VARIABLES_RESULT_FIELD2_LEN (TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE) #define SHOW_LOCAL_VARIABLES_RESULT_FIELD2_LEN (TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE)
#define SHOW_LOCAL_VARIABLES_RESULT_FIELD3_LEN (TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE) #define SHOW_LOCAL_VARIABLES_RESULT_FIELD3_LEN (TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE)
#define COMPACT_DB_RESULT_COLS 3
#define COMPACT_DB_RESULT_FIELD1_LEN 32
#define COMPACT_DB_RESULT_FIELD3_LEN 128
#define SHOW_ALIVE_RESULT_COLS 1 #define SHOW_ALIVE_RESULT_COLS 1
#define BIT_FLAG_MASK(n) (1 << n) #define BIT_FLAG_MASK(n) (1 << n)
@ -335,6 +339,15 @@ typedef struct SShowTableTagsStmt {
SNodeList* pTags; SNodeList* pTags;
} SShowTableTagsStmt; } SShowTableTagsStmt;
typedef struct SShowCompactsStmt {
ENodeType type;
} SShowCompactsStmt;
typedef struct SShowCompactDetailsStmt {
ENodeType type;
SNode* pCompactId;
} SShowCompactDetailsStmt;
typedef enum EIndexType { INDEX_TYPE_SMA = 1, INDEX_TYPE_FULLTEXT, INDEX_TYPE_NORMAL } EIndexType; typedef enum EIndexType { INDEX_TYPE_SMA = 1, INDEX_TYPE_FULLTEXT, INDEX_TYPE_NORMAL } EIndexType;
typedef struct SIndexOptions { typedef struct SIndexOptions {

7
include/libs/stream/streamState.h
View File

@ -50,7 +50,7 @@ void streamStateSetNumber(SStreamState* pState, int32_t number);
int32_t streamStateSaveInfo(SStreamState* pState, void* pKey, int32_t keyLen, void* pVal, int32_t vLen); int32_t streamStateSaveInfo(SStreamState* pState, void* pKey, int32_t keyLen, void* pVal, int32_t vLen);
int32_t streamStateGetInfo(SStreamState* pState, void* pKey, int32_t keyLen, void** pVal, int32_t* pLen); int32_t streamStateGetInfo(SStreamState* pState, void* pKey, int32_t keyLen, void** pVal, int32_t* pLen);
//session window // session window
int32_t streamStateSessionAddIfNotExist(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal, int32_t* pVLen); int32_t streamStateSessionAddIfNotExist(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal, int32_t* pVLen);
int32_t streamStateSessionPut(SStreamState* pState, const SSessionKey* key, void* value, int32_t vLen); int32_t streamStateSessionPut(SStreamState* pState, const SSessionKey* key, void* value, int32_t vLen);
int32_t streamStateSessionGet(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen); int32_t streamStateSessionGet(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen);
@ -65,7 +65,7 @@ SStreamStateCur* streamStateSessionSeekKeyNext(SStreamState* pState, const SSess
SStreamStateCur* streamStateSessionSeekKeyCurrentPrev(SStreamState* pState, const SSessionKey* key); SStreamStateCur* streamStateSessionSeekKeyCurrentPrev(SStreamState* pState, const SSessionKey* key);
SStreamStateCur* streamStateSessionSeekKeyCurrentNext(SStreamState* pState, const SSessionKey* key); SStreamStateCur* streamStateSessionSeekKeyCurrentNext(SStreamState* pState, const SSessionKey* key);
//state window // state window
int32_t streamStateStateAddIfNotExist(SStreamState* pState, SSessionKey* key, char* pKeyData, int32_t keyDataLen, int32_t streamStateStateAddIfNotExist(SStreamState* pState, SSessionKey* key, char* pKeyData, int32_t keyDataLen,
state_key_cmpr_fn fn, void** pVal, int32_t* pVLen); state_key_cmpr_fn fn, void** pVal, int32_t* pVLen);
@ -96,6 +96,9 @@ int32_t streamStatePutParName(SStreamState* pState, int64_t groupId, const char*
int32_t streamStateGetParName(SStreamState* pState, int64_t groupId, void** pVal); int32_t streamStateGetParName(SStreamState* pState, int64_t groupId, void** pVal);
void streamStateReloadInfo(SStreamState* pState, TSKEY ts); void streamStateReloadInfo(SStreamState* pState, TSKEY ts);
void streamStateCopyBackend(SStreamState* src, SStreamState* dst);
SStreamStateCur* createStreamStateCursor(); SStreamStateCur* createStreamStateCursor();
/***compare func **/ /***compare func **/

3
include/util/taoserror.h
View File

@ -427,6 +427,9 @@ int32_t* taosGetErrno();
#define TSDB_CODE_MND_VIEW_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x04A0) #define TSDB_CODE_MND_VIEW_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x04A0)
#define TSDB_CODE_MND_VIEW_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x04A1) #define TSDB_CODE_MND_VIEW_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x04A1)
//mnode-compact
#define TSDB_CODE_MND_INVALID_COMPACT_ID TAOS_DEF_ERROR_CODE(0, 0x04B1)
// vnode // vnode
// #define TSDB_CODE_VND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0500) // 2.x // #define TSDB_CODE_VND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0500) // 2.x

115
source/client/src/clientMsgHandler.c
View File

@ -16,6 +16,7 @@
#include "catalog.h" #include "catalog.h"
#include "clientInt.h" #include "clientInt.h"
#include "clientLog.h" #include "clientLog.h"
#include "cmdnodes.h"
#include "os.h" #include "os.h"
#include "query.h" #include "query.h"
#include "systable.h" #include "systable.h"
@ -542,6 +543,118 @@ int32_t processShowVariablesRsp(void* param, SDataBuf* pMsg, int32_t code) {
return code; return code;
} }
static int32_t buildCompactDbBlock(SCompactDbRsp* pRsp, SSDataBlock** block) {
SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
pBlock->info.hasVarCol = true;
pBlock->pDataBlock = taosArrayInit(COMPACT_DB_RESULT_COLS, sizeof(SColumnInfoData));
SColumnInfoData infoData = {0};
infoData.info.type = TSDB_DATA_TYPE_VARCHAR;
infoData.info.bytes = COMPACT_DB_RESULT_FIELD1_LEN;
taosArrayPush(pBlock->pDataBlock, &infoData);
infoData.info.type = TSDB_DATA_TYPE_INT;
infoData.info.bytes = tDataTypes[TSDB_DATA_TYPE_INT].bytes;
taosArrayPush(pBlock->pDataBlock, &infoData);
infoData.info.type = TSDB_DATA_TYPE_VARCHAR;
infoData.info.bytes = COMPACT_DB_RESULT_FIELD3_LEN;
taosArrayPush(pBlock->pDataBlock, &infoData);
blockDataEnsureCapacity(pBlock, 1);
SColumnInfoData* pResultCol = taosArrayGet(pBlock->pDataBlock, 0);
SColumnInfoData* pIdCol = taosArrayGet(pBlock->pDataBlock, 1);
SColumnInfoData* pReasonCol = taosArrayGet(pBlock->pDataBlock, 2);
char result[COMPACT_DB_RESULT_FIELD1_LEN] = {0};
char reason[COMPACT_DB_RESULT_FIELD3_LEN] = {0};
if (pRsp->bAccepted) {
STR_TO_VARSTR(result, "accepted");
colDataSetVal(pResultCol, 0, result, false);
colDataSetVal(pIdCol, 0, (void*)&pRsp->compactId, false);
STR_TO_VARSTR(reason, "success");
colDataSetVal(pReasonCol, 0, reason, false);
} else {
STR_TO_VARSTR(result, "rejected");
colDataSetVal(pResultCol, 0, result, false);
colDataSetNULL(pIdCol, 0);
STR_TO_VARSTR(reason, "compaction is ongoing");
colDataSetVal(pReasonCol, 0, reason, false);
}
pBlock->info.rows = 1;
*block = pBlock;
return TSDB_CODE_SUCCESS;
}
static int32_t buildRetriveTableRspForCompactDb(SCompactDbRsp* pCompactDb, SRetrieveTableRsp** pRsp) {
SSDataBlock* pBlock = NULL;
int32_t code = buildCompactDbBlock(pCompactDb, &pBlock);
if (code) {
return code;
}
size_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock);
*pRsp = taosMemoryCalloc(1, rspSize);
if (NULL == *pRsp) {
blockDataDestroy(pBlock);
return TSDB_CODE_OUT_OF_MEMORY;
}
(*pRsp)->useconds = 0;
(*pRsp)->completed = 1;
(*pRsp)->precision = 0;
(*pRsp)->compressed = 0;
(*pRsp)->compLen = 0;
(*pRsp)->numOfRows = htobe64((int64_t)pBlock->info.rows);
(*pRsp)->numOfCols = htonl(COMPACT_DB_RESULT_COLS);
int32_t len = blockEncode(pBlock, (*pRsp)->data, COMPACT_DB_RESULT_COLS);
blockDataDestroy(pBlock);
if (len != rspSize - sizeof(SRetrieveTableRsp)) {
uError("buildRetriveTableRspForCompactDb error, len:%d != rspSize - sizeof(SRetrieveTableRsp):%" PRIu64, len,
(uint64_t)(rspSize - sizeof(SRetrieveTableRsp)));
return TSDB_CODE_TSC_INVALID_INPUT;
}
return TSDB_CODE_SUCCESS;
}
int32_t processCompactDbRsp(void* param, SDataBuf* pMsg, int32_t code) {
SRequestObj* pRequest = param;
if (code != TSDB_CODE_SUCCESS) {
setErrno(pRequest, code);
} else {
SCompactDbRsp rsp = {0};
SRetrieveTableRsp* pRes = NULL;
code = tDeserializeSCompactDbRsp(pMsg->pData, pMsg->len, &rsp);
if (TSDB_CODE_SUCCESS == code) {
code = buildRetriveTableRspForCompactDb(&rsp, &pRes);
}
if (TSDB_CODE_SUCCESS == code) {
code = setQueryResultFromRsp(&pRequest->body.resInfo, pRes, false, true);
}
if (code != 0) {
taosMemoryFree(pRes);
}
}
taosMemoryFree(pMsg->pData);
taosMemoryFree(pMsg->pEpSet);
if (pRequest->body.queryFp != NULL) {
pRequest->body.queryFp(((SSyncQueryParam *)pRequest->body.interParam)->userParam, pRequest, code);
} else {
tsem_post(&pRequest->body.rspSem);
}
return code;
}
__async_send_cb_fn_t getMsgRspHandle(int32_t msgType) { __async_send_cb_fn_t getMsgRspHandle(int32_t msgType) {
switch (msgType) { switch (msgType) {
case TDMT_MND_CONNECT: case TDMT_MND_CONNECT:
@ -558,6 +671,8 @@ __async_send_cb_fn_t getMsgRspHandle(int32_t msgType) {
return processAlterStbRsp; return processAlterStbRsp;
case TDMT_MND_SHOW_VARIABLES: case TDMT_MND_SHOW_VARIABLES:
return processShowVariablesRsp; return processShowVariablesRsp;
case TDMT_MND_COMPACT_DB:
return processCompactDbRsp;
default: default:
return genericRspCallback; return genericRspCallback;
} }

16
source/common/src/systable.c
View File

@ -337,6 +337,20 @@ static const SSysDbTableSchema userViewsSchema[] = {
// {.name = "column_list", .bytes = 2048 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, // {.name = "column_list", .bytes = 2048 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
}; };
static const SSysDbTableSchema userCompactsSchema[] = {
{.name = "compact_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
{.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "start_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false},
};
static const SSysDbTableSchema userCompactsDetailSchema[] = {
{.name = "compact_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
{.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
{.name = "dnode_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
{.name = "number_fileset", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
{.name = "finished", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
{.name = "start_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false},
};
static const SSysTableMeta infosMeta[] = { static const SSysTableMeta infosMeta[] = {
{TSDB_INS_TABLE_DNODES, dnodesSchema, tListLen(dnodesSchema), true}, {TSDB_INS_TABLE_DNODES, dnodesSchema, tListLen(dnodesSchema), true},
@ -365,6 +379,8 @@ static const SSysTableMeta infosMeta[] = {
{TSDB_INS_TABLE_VNODES, vnodesSchema, tListLen(vnodesSchema), true}, {TSDB_INS_TABLE_VNODES, vnodesSchema, tListLen(vnodesSchema), true},
{TSDB_INS_TABLE_USER_PRIVILEGES, userUserPrivilegesSchema, tListLen(userUserPrivilegesSchema), true}, {TSDB_INS_TABLE_USER_PRIVILEGES, userUserPrivilegesSchema, tListLen(userUserPrivilegesSchema), true},
{TSDB_INS_TABLE_VIEWS, userViewsSchema, tListLen(userViewsSchema), false}, {TSDB_INS_TABLE_VIEWS, userViewsSchema, tListLen(userViewsSchema), false},
{TSDB_INS_TABLE_COMPACTS, userCompactsSchema, tListLen(userCompactsSchema), false},
{TSDB_INS_TABLE_COMPACT_DETAILS, userCompactsDetailSchema, tListLen(userCompactsDetailSchema), false},
}; };
static const SSysDbTableSchema connectionsSchema[] = { static const SSysDbTableSchema connectionsSchema[] = {

17
source/common/src/tglobal.c
View File

@ -248,6 +248,7 @@ int32_t tsTtlBatchDropNum = 10000; // number of tables dropped per batch
// internal // internal
int32_t tsTransPullupInterval = 2; int32_t tsTransPullupInterval = 2;
int32_t tsCompactPullupInterval = 10;
int32_t tsMqRebalanceInterval = 2; int32_t tsMqRebalanceInterval = 2;
int32_t tsStreamCheckpointInterval = 60; int32_t tsStreamCheckpointInterval = 60;
float tsSinkDataRate = 2.0; float tsSinkDataRate = 2.0;
@ -281,6 +282,8 @@ int32_t tsS3BlockCacheSize = 16; // number of blocks
int32_t tsS3PageCacheSize = 4096; // number of pages int32_t tsS3PageCacheSize = 4096; // number of pages
int32_t tsS3UploadDelaySec = 60 * 60 * 24; int32_t tsS3UploadDelaySec = 60 * 60 * 24;
bool tsExperimental = true;
#ifndef _STORAGE #ifndef _STORAGE
int32_t taosSetTfsCfg(SConfig *pCfg) { int32_t taosSetTfsCfg(SConfig *pCfg) {
SConfigItem *pItem = cfgGetItem(pCfg, "dataDir"); SConfigItem *pItem = cfgGetItem(pCfg, "dataDir");
@ -528,6 +531,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
} }
if (cfgAddInt32(pCfg, "numOfTaskQueueThreads", tsNumOfTaskQueueThreads, 4, 1024, CFG_SCOPE_CLIENT, CFG_DYN_NONE) != 0) if (cfgAddInt32(pCfg, "numOfTaskQueueThreads", tsNumOfTaskQueueThreads, 4, 1024, CFG_SCOPE_CLIENT, CFG_DYN_NONE) != 0)
return -1; return -1;
if (cfgAddBool(pCfg, "experimental", tsExperimental, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1;
return 0; return 0;
} }
@ -707,6 +711,9 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddInt32(pCfg, "transPullupInterval", tsTransPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != if (cfgAddInt32(pCfg, "transPullupInterval", tsTransPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) !=
0) 0)
return -1; return -1;
if (cfgAddInt32(pCfg, "compactPullupInterval", tsCompactPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) !=
0)
return -1;
if (cfgAddInt32(pCfg, "mqRebalanceInterval", tsMqRebalanceInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != if (cfgAddInt32(pCfg, "mqRebalanceInterval", tsMqRebalanceInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) !=
0) 0)
return -1; return -1;
@ -792,6 +799,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
return -1; return -1;
if (cfgAddBool(pCfg, "enableWhiteList", tsEnableWhiteList, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; if (cfgAddBool(pCfg, "enableWhiteList", tsEnableWhiteList, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
if (cfgAddBool(pCfg, "experimental", tsExperimental, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1;
GRANT_CFG_ADD; GRANT_CFG_ADD;
return 0; return 0;
} }
@ -1077,6 +1086,8 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
tsTimeToGetAvailableConn = cfgGetItem(pCfg, "timeToGetAvailableConn")->i32; tsTimeToGetAvailableConn = cfgGetItem(pCfg, "timeToGetAvailableConn")->i32;
tsKeepAliveIdle = cfgGetItem(pCfg, "keepAliveIdle")->i32; tsKeepAliveIdle = cfgGetItem(pCfg, "keepAliveIdle")->i32;
tsExperimental = cfgGetItem(pCfg, "experimental")->bval;
return 0; return 0;
} }
@ -1156,6 +1167,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
tmqMaxTopicNum = cfgGetItem(pCfg, "tmqMaxTopicNum")->i32; tmqMaxTopicNum = cfgGetItem(pCfg, "tmqMaxTopicNum")->i32;
tsTransPullupInterval = cfgGetItem(pCfg, "transPullupInterval")->i32; tsTransPullupInterval = cfgGetItem(pCfg, "transPullupInterval")->i32;
tsCompactPullupInterval = cfgGetItem(pCfg, "compactPullupInterval")->i32;
tsMqRebalanceInterval = cfgGetItem(pCfg, "mqRebalanceInterval")->i32; tsMqRebalanceInterval = cfgGetItem(pCfg, "mqRebalanceInterval")->i32;
tsTtlUnit = cfgGetItem(pCfg, "ttlUnit")->i32; tsTtlUnit = cfgGetItem(pCfg, "ttlUnit")->i32;
tsTtlPushIntervalSec = cfgGetItem(pCfg, "ttlPushInterval")->i32; tsTtlPushIntervalSec = cfgGetItem(pCfg, "ttlPushInterval")->i32;
@ -1210,6 +1222,8 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
tsS3PageCacheSize = cfgGetItem(pCfg, "s3PageCacheSize")->i32; tsS3PageCacheSize = cfgGetItem(pCfg, "s3PageCacheSize")->i32;
tsS3UploadDelaySec = cfgGetItem(pCfg, "s3UploadDelaySec")->i32; tsS3UploadDelaySec = cfgGetItem(pCfg, "s3UploadDelaySec")->i32;
tsExperimental = cfgGetItem(pCfg, "experimental")->bval;
GRANT_CFG_GET; GRANT_CFG_GET;
return 0; return 0;
} }
@ -1470,6 +1484,7 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, char *name) {
{"timeseriesThreshold", &tsTimeSeriesThreshold}, {"timeseriesThreshold", &tsTimeSeriesThreshold},
{"tmqMaxTopicNum", &tmqMaxTopicNum}, {"tmqMaxTopicNum", &tmqMaxTopicNum},
{"transPullupInterval", &tsTransPullupInterval}, {"transPullupInterval", &tsTransPullupInterval},
{"compactPullupInterval", &tsCompactPullupInterval},
{"trimVDbIntervalSec", &tsTrimVDbIntervalSec}, {"trimVDbIntervalSec", &tsTrimVDbIntervalSec},
{"ttlBatchDropNum", &tsTtlBatchDropNum}, {"ttlBatchDropNum", &tsTtlBatchDropNum},
{"ttlFlushThreshold", &tsTtlFlushThreshold}, {"ttlFlushThreshold", &tsTtlFlushThreshold},
@ -1479,6 +1494,7 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, char *name) {
{"s3PageCacheSize", &tsS3PageCacheSize}, {"s3PageCacheSize", &tsS3PageCacheSize},
{"s3UploadDelaySec", &tsS3UploadDelaySec}, {"s3UploadDelaySec", &tsS3UploadDelaySec},
{"supportVnodes", &tsNumOfSupportVnodes}, {"supportVnodes", &tsNumOfSupportVnodes},
{"experimental", &tsExperimental}
}; };
if (taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true) != 0) { if (taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true) != 0) {
@ -1702,6 +1718,7 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, char *name) {
{"shellActivityTimer", &tsShellActivityTimer}, {"shellActivityTimer", &tsShellActivityTimer},
{"slowLogThreshold", &tsSlowLogThreshold}, {"slowLogThreshold", &tsSlowLogThreshold},
{"useAdapter", &tsUseAdapter}, {"useAdapter", &tsUseAdapter},
{"experimental", &tsExperimental}
}; };
if (taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true) != 0) { if (taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true) != 0) {

197
source/common/src/tmsg.c
View File

@ -1827,7 +1827,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
char *tb = taosHashIterate(pRsp->readTbs, NULL); char *tb = taosHashIterate(pRsp->readTbs, NULL);
while (tb != NULL) { while (tb != NULL) {
size_t keyLen = 0; size_t keyLen = 0;
void * key = taosHashGetKey(tb, &keyLen); void *key = taosHashGetKey(tb, &keyLen);
if (tEncodeI32(pEncoder, keyLen) < 0) return -1; if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
if (tEncodeCStr(pEncoder, key) < 0) return -1; if (tEncodeCStr(pEncoder, key) < 0) return -1;
@ -1842,7 +1842,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
tb = taosHashIterate(pRsp->writeTbs, NULL); tb = taosHashIterate(pRsp->writeTbs, NULL);
while (tb != NULL) { while (tb != NULL) {
size_t keyLen = 0; size_t keyLen = 0;
void * key = taosHashGetKey(tb, &keyLen); void *key = taosHashGetKey(tb, &keyLen);
if (tEncodeI32(pEncoder, keyLen) < 0) return -1; if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
if (tEncodeCStr(pEncoder, key) < 0) return -1; if (tEncodeCStr(pEncoder, key) < 0) return -1;
@ -1857,7 +1857,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
tb = taosHashIterate(pRsp->alterTbs, NULL); tb = taosHashIterate(pRsp->alterTbs, NULL);
while (tb != NULL) { while (tb != NULL) {
size_t keyLen = 0; size_t keyLen = 0;
void * key = taosHashGetKey(tb, &keyLen); void *key = taosHashGetKey(tb, &keyLen);
if (tEncodeI32(pEncoder, keyLen) < 0) return -1; if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
if (tEncodeCStr(pEncoder, key) < 0) return -1; if (tEncodeCStr(pEncoder, key) < 0) return -1;
@ -1872,7 +1872,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
tb = taosHashIterate(pRsp->readViews, NULL); tb = taosHashIterate(pRsp->readViews, NULL);
while (tb != NULL) { while (tb != NULL) {
size_t keyLen = 0; size_t keyLen = 0;
void * key = taosHashGetKey(tb, &keyLen); void *key = taosHashGetKey(tb, &keyLen);
if (tEncodeI32(pEncoder, keyLen) < 0) return -1; if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
if (tEncodeCStr(pEncoder, key) < 0) return -1; if (tEncodeCStr(pEncoder, key) < 0) return -1;
@ -1887,7 +1887,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
tb = taosHashIterate(pRsp->writeViews, NULL); tb = taosHashIterate(pRsp->writeViews, NULL);
while (tb != NULL) { while (tb != NULL) {
size_t keyLen = 0; size_t keyLen = 0;
void * key = taosHashGetKey(tb, &keyLen); void *key = taosHashGetKey(tb, &keyLen);
if (tEncodeI32(pEncoder, keyLen) < 0) return -1; if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
if (tEncodeCStr(pEncoder, key) < 0) return -1; if (tEncodeCStr(pEncoder, key) < 0) return -1;
@ -1902,7 +1902,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
tb = taosHashIterate(pRsp->alterViews, NULL); tb = taosHashIterate(pRsp->alterViews, NULL);
while (tb != NULL) { while (tb != NULL) {
size_t keyLen = 0; size_t keyLen = 0;
void * key = taosHashGetKey(tb, &keyLen); void *key = taosHashGetKey(tb, &keyLen);
if (tEncodeI32(pEncoder, keyLen) < 0) return -1; if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
if (tEncodeCStr(pEncoder, key) < 0) return -1; if (tEncodeCStr(pEncoder, key) < 0) return -1;
@ -1917,7 +1917,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
int32_t *useDb = taosHashIterate(pRsp->useDbs, NULL); int32_t *useDb = taosHashIterate(pRsp->useDbs, NULL);
while (useDb != NULL) { while (useDb != NULL) {
size_t keyLen = 0; size_t keyLen = 0;
void * key = taosHashGetKey(useDb, &keyLen); void *key = taosHashGetKey(useDb, &keyLen);
if (tEncodeI32(pEncoder, keyLen) < 0) return -1; if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
if (tEncodeCStr(pEncoder, key) < 0) return -1; if (tEncodeCStr(pEncoder, key) < 0) return -1;
@ -3424,6 +3424,66 @@ int32_t tDeserializeSCompactDbReq(void *buf, int32_t bufLen, SCompactDbReq *pReq
void tFreeSCompactDbReq(SCompactDbReq *pReq) { FREESQL(); } void tFreeSCompactDbReq(SCompactDbReq *pReq) { FREESQL(); }
int32_t tSerializeSCompactDbRsp(void *buf, int32_t bufLen, SCompactDbRsp *pRsp) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeI32(&encoder, pRsp->compactId) < 0) return -1;
if (tEncodeI8(&encoder, pRsp->bAccepted) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSCompactDbRsp(void *buf, int32_t bufLen, SCompactDbRsp *pRsp) {
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeI32(&decoder, &pRsp->compactId) < 0) return -1;
if (tDecodeI8(&decoder, &pRsp->bAccepted) < 0) return -1;
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
int32_t tSerializeSKillCompactReq(void *buf, int32_t bufLen, SKillCompactReq *pReq) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeI32(&encoder, pReq->compactId) < 0) return -1;
ENCODESQL();
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSKillCompactReq(void *buf, int32_t bufLen, SKillCompactReq *pReq) {
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->compactId) < 0) return -1;
DECODESQL();
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
void tFreeSKillCompactReq(SKillCompactReq *pReq) { FREESQL(); }
int32_t tSerializeSUseDbRspImp(SEncoder *pEncoder, const SUseDbRsp *pRsp) { int32_t tSerializeSUseDbRspImp(SEncoder *pEncoder, const SUseDbRsp *pRsp) {
if (tEncodeCStr(pEncoder, pRsp->db) < 0) return -1; if (tEncodeCStr(pEncoder, pRsp->db) < 0) return -1;
if (tEncodeI64(pEncoder, pRsp->uid) < 0) return -1; if (tEncodeI64(pEncoder, pRsp->uid) < 0) return -1;
@ -4259,6 +4319,7 @@ int32_t tSerializeSRetrieveTableReq(void *buf, int32_t bufLen, SRetrieveTableReq
if (tEncodeCStr(&encoder, pReq->tb) < 0) return -1; if (tEncodeCStr(&encoder, pReq->tb) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->filterTb) < 0) return -1; if (tEncodeCStr(&encoder, pReq->filterTb) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->user) < 0) return -1; if (tEncodeCStr(&encoder, pReq->user) < 0) return -1;
if (tEncodeI64(&encoder, pReq->compactId) < 0) return -1;
tEndEncode(&encoder); tEndEncode(&encoder);
int32_t tlen = encoder.pos; int32_t tlen = encoder.pos;
@ -4276,6 +4337,11 @@ int32_t tDeserializeSRetrieveTableReq(void *buf, int32_t bufLen, SRetrieveTableR
if (tDecodeCStrTo(&decoder, pReq->tb) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->tb) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->filterTb) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->filterTb) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1;
if (!tDecodeIsEnd(&decoder)) {
if (tDecodeI64(&decoder, &pReq->compactId) < 0) return -1;
} else {
pReq->compactId = -1;
}
tEndDecode(&decoder); tEndDecode(&decoder);
tDecoderClear(&decoder); tDecoderClear(&decoder);
@ -5051,6 +5117,75 @@ int32_t tFreeSCreateVnodeReq(SCreateVnodeReq *pReq) {
return 0; return 0;
} }
int32_t tSerializeSQueryCompactProgressReq(void *buf, int32_t bufLen, SQueryCompactProgressReq *pReq) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeI32(&encoder, pReq->compactId) < 0) return -1;
if (tEncodeI32(&encoder, pReq->vgId) < 0) return -1;
if (tEncodeI32(&encoder, pReq->dnodeId) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSQueryCompactProgressReq(void *buf, int32_t bufLen, SQueryCompactProgressReq *pReq) {
int32_t headLen = sizeof(SMsgHead);
SDecoder decoder = {0};
tDecoderInit(&decoder, ((uint8_t *)buf) + headLen, bufLen - headLen);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->compactId) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -1;
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
int32_t tSerializeSQueryCompactProgressRsp(void *buf, int32_t bufLen, SQueryCompactProgressRsp *pReq) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeI32(&encoder, pReq->compactId) < 0) return -1;
if (tEncodeI32(&encoder, pReq->vgId) < 0) return -1;
if (tEncodeI32(&encoder, pReq->dnodeId) < 0) return -1;
if (tEncodeI32(&encoder, pReq->numberFileset) < 0) return -1;
if (tEncodeI32(&encoder, pReq->finished) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSQueryCompactProgressRsp(void *buf, int32_t bufLen, SQueryCompactProgressRsp *pReq) {
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->compactId) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->numberFileset) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->finished) < 0) return -1;
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
int32_t tSerializeSDropVnodeReq(void *buf, int32_t bufLen, SDropVnodeReq *pReq) { int32_t tSerializeSDropVnodeReq(void *buf, int32_t bufLen, SDropVnodeReq *pReq) {
SEncoder encoder = {0}; SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen); tEncoderInit(&encoder, buf, bufLen);
@ -5139,6 +5274,8 @@ int32_t tSerializeSCompactVnodeReq(void *buf, int32_t bufLen, SCompactVnodeReq *
if (tEncodeI64(&encoder, pReq->tw.skey) < 0) return -1; if (tEncodeI64(&encoder, pReq->tw.skey) < 0) return -1;
if (tEncodeI64(&encoder, pReq->tw.ekey) < 0) return -1; if (tEncodeI64(&encoder, pReq->tw.ekey) < 0) return -1;
if (tEncodeI32(&encoder, pReq->compactId) < 0) return -1;
tEndEncode(&encoder); tEndEncode(&encoder);
int32_t tlen = encoder.pos; int32_t tlen = encoder.pos;
@ -5165,6 +5302,42 @@ int32_t tDeserializeSCompactVnodeReq(void *buf, int32_t bufLen, SCompactVnodeReq
if (tDecodeI64(&decoder, &pReq->tw.ekey) < 0) return -1; if (tDecodeI64(&decoder, &pReq->tw.ekey) < 0) return -1;
} }
if (!tDecodeIsEnd(&decoder)) {
if (tDecodeI32(&decoder, &pReq->compactId) < 0) return -1;
}
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
int32_t tSerializeSVKillCompactReq(void *buf, int32_t bufLen, SVKillCompactReq *pReq) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeI32(&encoder, pReq->compactId) < 0) return -1;
if (tEncodeI32(&encoder, pReq->vgId) < 0) return -1;
if (tEncodeI32(&encoder, pReq->dnodeId) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSVKillCompactReq(void *buf, int32_t bufLen, SVKillCompactReq *pReq) {
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->compactId) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -1;
tEndDecode(&decoder); tEndDecode(&decoder);
tDecoderClear(&decoder); tDecoderClear(&decoder);
return 0; return 0;
@ -8169,7 +8342,7 @@ int32_t tEncodeMqDataRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) {
for (int32_t i = 0; i < pRsp->blockNum; i++) { for (int32_t i = 0; i < pRsp->blockNum; i++) {
int32_t bLen = *(int32_t *)taosArrayGet(pRsp->blockDataLen, i); int32_t bLen = *(int32_t *)taosArrayGet(pRsp->blockDataLen, i);
void * data = taosArrayGetP(pRsp->blockData, i); void *data = taosArrayGetP(pRsp->blockData, i);
if (tEncodeBinary(pEncoder, (const uint8_t *)data, bLen) < 0) return -1; if (tEncodeBinary(pEncoder, (const uint8_t *)data, bLen) < 0) return -1;
if (pRsp->withSchema) { if (pRsp->withSchema) {
SSchemaWrapper *pSW = (SSchemaWrapper *)taosArrayGetP(pRsp->blockSchema, i); SSchemaWrapper *pSW = (SSchemaWrapper *)taosArrayGetP(pRsp->blockSchema, i);
@ -8202,7 +8375,7 @@ int32_t tDecodeMqDataRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) {
} }
for (int32_t i = 0; i < pRsp->blockNum; i++) { for (int32_t i = 0; i < pRsp->blockNum; i++) {
void * data; void *data;
uint64_t bLen; uint64_t bLen;
if (tDecodeBinaryAlloc(pDecoder, &data, &bLen) < 0) return -1; if (tDecodeBinaryAlloc(pDecoder, &data, &bLen) < 0) return -1;
taosArrayPush(pRsp->blockData, &data); taosArrayPush(pRsp->blockData, &data);
@ -8248,7 +8421,7 @@ int32_t tEncodeSTaosxRsp(SEncoder *pEncoder, const STaosxRsp *pRsp) {
if (tEncodeI32(pEncoder, pRsp->createTableNum) < 0) return -1; if (tEncodeI32(pEncoder, pRsp->createTableNum) < 0) return -1;
if (pRsp->createTableNum) { if (pRsp->createTableNum) {
for (int32_t i = 0; i < pRsp->createTableNum; i++) { for (int32_t i = 0; i < pRsp->createTableNum; i++) {
void * createTableReq = taosArrayGetP(pRsp->createTableReq, i); void *createTableReq = taosArrayGetP(pRsp->createTableReq, i);
int32_t createTableLen = *(int32_t *)taosArrayGet(pRsp->createTableLen, i); int32_t createTableLen = *(int32_t *)taosArrayGet(pRsp->createTableLen, i);
if (tEncodeBinary(pEncoder, createTableReq, createTableLen) < 0) return -1; if (tEncodeBinary(pEncoder, createTableReq, createTableLen) < 0) return -1;
} }
@ -8264,7 +8437,7 @@ int32_t tDecodeSTaosxRsp(SDecoder *pDecoder, STaosxRsp *pRsp) {
pRsp->createTableLen = taosArrayInit(pRsp->createTableNum, sizeof(int32_t)); pRsp->createTableLen = taosArrayInit(pRsp->createTableNum, sizeof(int32_t));
pRsp->createTableReq = taosArrayInit(pRsp->createTableNum, sizeof(void *)); pRsp->createTableReq = taosArrayInit(pRsp->createTableNum, sizeof(void *));
for (int32_t i = 0; i < pRsp->createTableNum; i++) { for (int32_t i = 0; i < pRsp->createTableNum; i++) {
void * pCreate = NULL; void *pCreate = NULL;
uint64_t len; uint64_t len;
if (tDecodeBinaryAlloc(pDecoder, &pCreate, &len) < 0) return -1; if (tDecodeBinaryAlloc(pDecoder, &pCreate, &len) < 0) return -1;
int32_t l = (int32_t)len; int32_t l = (int32_t)len;
@ -8566,7 +8739,7 @@ void tDestroySubmitTbData(SSubmitTbData *pTbData, int32_t flag) {
taosArrayDestroy(pTbData->aCol); taosArrayDestroy(pTbData->aCol);
} else { } else {
int32_t nRow = TARRAY_SIZE(pTbData->aRowP); int32_t nRow = TARRAY_SIZE(pTbData->aRowP);
SRow ** rows = (SRow **)TARRAY_DATA(pTbData->aRowP); SRow **rows = (SRow **)TARRAY_DATA(pTbData->aRowP);
for (int32_t i = 0; i < nRow; ++i) { for (int32_t i = 0; i < nRow; ++i) {
tRowDestroy(rows[i]); tRowDestroy(rows[i]);

3
source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
View File

@ -193,6 +193,8 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_VIEW, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_VIEW, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_VIEW, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_VIEW, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_VIEW_META, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_VIEW_META, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_COMPACT, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_COMPACT_PROGRESS_RSP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SCH_QUERY, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SCH_QUERY, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SCH_MERGE_QUERY, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SCH_MERGE_QUERY, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER;
@ -222,6 +224,7 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_UPDATE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_UPDATE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_RESET_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_RESET_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_HEARTBEAT, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_HEARTBEAT, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_KILL_COMPACT_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIG_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIG_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_REPLICA_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_REPLICA_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;

2
source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
View File

@ -818,6 +818,8 @@ SArray *vmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_SCH_QUERY_HEARTBEAT, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SCH_QUERY_HEARTBEAT, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_INDEX, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_INDEX, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_INDEX, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_INDEX, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_COMPACT_PROGRESS, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_KILL_COMPACT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DEPLOY, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DEPLOY, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DROP, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DROP, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;

12
source/dnode/mgmt/node_mgmt/inc/dmMgmt.h
View File

@ -97,8 +97,12 @@ int32_t dmMarkWrapper(SMgmtWrapper *pWrapper);
void dmReleaseWrapper(SMgmtWrapper *pWrapper); void dmReleaseWrapper(SMgmtWrapper *pWrapper);
int32_t dmInitVars(SDnode *pDnode); int32_t dmInitVars(SDnode *pDnode);
void dmClearVars(SDnode *pDnode); void dmClearVars(SDnode *pDnode);
int32_t dmInitModule(SDnode *pDnode, SMgmtWrapper *wrappers); #ifdef TD_MODULE_OPTIMIZE
bool dmRequireNode(SDnode *pDnode, SMgmtWrapper *pWrapper); int32_t dmInitModule(SDnode *pDnode, SMgmtWrapper *wrappers);
bool dmRequireNode(SDnode *pDnode, SMgmtWrapper *pWrapper);
#else
int32_t dmInitModule(SDnode *pDnode);
#endif
SMgmtInputOpt dmBuildMgmtInputOpt(SMgmtWrapper *pWrapper); SMgmtInputOpt dmBuildMgmtInputOpt(SMgmtWrapper *pWrapper);
void dmSetStatus(SDnode *pDnode, EDndRunStatus stype); void dmSetStatus(SDnode *pDnode, EDndRunStatus stype);
void dmProcessServerStartupStatus(SDnode *pDnode, SRpcMsg *pMsg); void dmProcessServerStartupStatus(SDnode *pDnode, SRpcMsg *pMsg);
@ -119,7 +123,11 @@ int32_t dmInitStatusClient(SDnode *pDnode);
void dmCleanupClient(SDnode *pDnode); void dmCleanupClient(SDnode *pDnode);
void dmCleanupStatusClient(SDnode *pDnode); void dmCleanupStatusClient(SDnode *pDnode);
SMsgCb dmGetMsgcb(SDnode *pDnode); SMsgCb dmGetMsgcb(SDnode *pDnode);
#ifdef TD_MODULE_OPTIMIZE
int32_t dmInitMsgHandle(SDnode *pDnode, SMgmtWrapper *wrappers); int32_t dmInitMsgHandle(SDnode *pDnode, SMgmtWrapper *wrappers);
#else
int32_t dmInitMsgHandle(SDnode *pDnode);
#endif
int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);
// dmMonitor.c // dmMonitor.c

24
source/dnode/mgmt/node_mgmt/src/dmMgmt.c
View File

@ -24,6 +24,22 @@
#include "tglobal.h" #include "tglobal.h"
#endif #endif
#ifndef TD_MODULE_OPTIMIZE
static bool dmRequireNode(SDnode *pDnode, SMgmtWrapper *pWrapper) {
SMgmtInputOpt input = dmBuildMgmtInputOpt(pWrapper);
bool required = false;
int32_t code = (*pWrapper->func.requiredFp)(&input, &required);
if (!required) {
dDebug("node:%s, does not require startup", pWrapper->name);
} else {
dDebug("node:%s, required to startup", pWrapper->name);
}
return required;
}
#endif
int32_t dmInitDnode(SDnode *pDnode) { int32_t dmInitDnode(SDnode *pDnode) {
dDebug("start to create dnode"); dDebug("start to create dnode");
int32_t code = -1; int32_t code = -1;
@ -65,11 +81,15 @@ int32_t dmInitDnode(SDnode *pDnode) {
if (pDnode->lockfile == NULL) { if (pDnode->lockfile == NULL) {
goto _OVER; goto _OVER;
} }
#ifdef TD_MODULE_OPTIMIZE
if (dmInitModule(pDnode, pDnode->wrappers) != 0) { if (dmInitModule(pDnode, pDnode->wrappers) != 0) {
goto _OVER; goto _OVER;
} }
#else
if (dmInitModule(pDnode) != 0) {
goto _OVER;
}
#endif
indexInit(tsNumOfCommitThreads); indexInit(tsNumOfCommitThreads);
streamMetaInit(); streamMetaInit();

28
source/dnode/mgmt/node_mgmt/src/dmTransport.c
View File

@ -251,6 +251,7 @@ _OVER:
dmReleaseWrapper(pWrapper); dmReleaseWrapper(pWrapper);
} }
#ifdef TD_MODULE_OPTIMIZE
int32_t dmInitMsgHandle(SDnode *pDnode, SMgmtWrapper *wrappers) { int32_t dmInitMsgHandle(SDnode *pDnode, SMgmtWrapper *wrappers) {
SDnodeTrans *pTrans = &pDnode->trans; SDnodeTrans *pTrans = &pDnode->trans;
@ -276,6 +277,33 @@ int32_t dmInitMsgHandle(SDnode *pDnode, SMgmtWrapper *wrappers) {
return 0; return 0;
} }
#else
int32_t dmInitMsgHandle(SDnode *pDnode) {
SDnodeTrans *pTrans = &pDnode->trans;
for (EDndNodeType ntype = DNODE; ntype < NODE_END; ++ntype) {
SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype];
SArray *pArray = (*pWrapper->func.getHandlesFp)();
if (pArray == NULL) return -1;
for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) {
SMgmtHandle *pMgmt = taosArrayGet(pArray, i);
SDnodeHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(pMgmt->msgType)];
if (pMgmt->needCheckVgId) {
pHandle->needCheckVgId = pMgmt->needCheckVgId;
}
if (!pMgmt->needCheckVgId) {
pHandle->defaultNtype = ntype;
}
pWrapper->msgFps[TMSG_INDEX(pMgmt->msgType)] = pMgmt->msgFp;
}
taosArrayDestroy(pArray);
}
return 0;
}
#endif
static inline int32_t dmSendReq(const SEpSet *pEpSet, SRpcMsg *pMsg) { static inline int32_t dmSendReq(const SEpSet *pEpSet, SRpcMsg *pMsg) {
SDnode *pDnode = dmInstance(); SDnode *pDnode = dmInstance();

56
source/dnode/mnode/impl/inc/mndCompact.h Normal file
View File

@ -0,0 +1,56 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_MND_COMPACT_H_
#define _TD_MND_COMPACT_H_
#include "mndInt.h"
#ifdef __cplusplus
extern "C" {
#endif
int32_t mndInitCompact(SMnode *pMnode);
void mndCleanupCompact(SMnode *pMnode);
void tFreeCompactObj(SCompactObj *pCompact);
int32_t tSerializeSCompactObj(void *buf, int32_t bufLen, const SCompactObj *pObj);
int32_t tDeserializeSCompactObj(void *buf, int32_t bufLen, SCompactObj *pObj);
SSdbRaw* mndCompactActionEncode(SCompactObj *pCompact);
SSdbRow* mndCompactActionDecode(SSdbRaw *pRaw);
int32_t mndCompactActionInsert(SSdb *pSdb, SCompactObj *pCompact);
int32_t mndCompactActionDelete(SSdb *pSdb, SCompactObj *pCompact);
int32_t mndCompactActionUpdate(SSdb *pSdb, SCompactObj *pOldCompact, SCompactObj *pNewCompact);
int32_t mndAddCompactToTran(SMnode *pMnode, STrans *pTrans, SCompactObj* pCompact, SDbObj *pDb, SCompactDbRsp *rsp);
int32_t mndRetrieveCompact(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
int32_t mndProcessKillCompactReq(SRpcMsg *pReq);
int32_t mndProcessQueryCompactRsp(SRpcMsg *pReq);
SCompactObj *mndAcquireCompact(SMnode *pMnode, int64_t compactId);
void mndReleaseCompact(SMnode *pMnode, SCompactObj *pCompact);
void mndCompactSendProgressReq(SMnode *pMnode, SCompactObj *pCompact);
#ifdef __cplusplus
}
#endif
#endif /*_TD_MND_COMPACT_H_*/

48
source/dnode/mnode/impl/inc/mndCompactDetail.h Normal file
View File

@ -0,0 +1,48 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_MND_COMPACT_DETAIL_H_
#define _TD_MND_COMPACT_DETAIL_H_
#include "mndInt.h"
#ifdef __cplusplus
extern "C" {
#endif
int32_t mndInitCompactDetail(SMnode *pMnode);
void mndCleanupCompactDetail(SMnode *pMnode);
void tFreeCompactDetailObj(SCompactDetailObj *pCompact);
int32_t tSerializeSCompactDetailObj(void *buf, int32_t bufLen, const SCompactDetailObj *pObj);
int32_t tDeserializeSCompactDetailObj(void *buf, int32_t bufLen, SCompactDetailObj *pObj);
SSdbRaw* mndCompactDetailActionEncode(SCompactDetailObj *pCompact);
SSdbRow* mndCompactDetailActionDecode(SSdbRaw *pRaw);
int32_t mndCompactDetailActionInsert(SSdb *pSdb, SCompactDetailObj *pCompact);
int32_t mndCompactDetailActionDelete(SSdb *pSdb, SCompactDetailObj *pCompact);
int32_t mndCompactDetailActionUpdate(SSdb *pSdb, SCompactDetailObj *pOldCompact, SCompactDetailObj *pNewCompact);
int32_t mndAddCompactDetailToTran(SMnode *pMnode, STrans *pTrans, SCompactObj* pCompact, SVgObj *pVgroup,
SVnodeGid *pVgid, int32_t index);
int32_t mndRetrieveCompactDetail(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
#ifdef __cplusplus
}
#endif
#endif /*_TD_MND_COMPACT_DETAIL_H_*/

19
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -748,6 +748,25 @@ int32_t tEncodeSViewObj(SEncoder* pEncoder, const SViewObj* pObj);
int32_t tDecodeSViewObj(SDecoder* pDecoder, SViewObj* pObj, int32_t sver); int32_t tDecodeSViewObj(SDecoder* pDecoder, SViewObj* pObj, int32_t sver);
void tFreeSViewObj(SViewObj* pObj); void tFreeSViewObj(SViewObj* pObj);
typedef struct {
int32_t compactDetailId;
int32_t compactId;
int32_t vgId;
int32_t dnodeId;
int32_t numberFileset;
int32_t finished;
int64_t startTime;
int32_t newNumberFileset;
int32_t newFinished;
} SCompactDetailObj;
typedef struct {
int32_t compactId;
char dbname[TSDB_TABLE_FNAME_LEN];
int64_t startTime;
SArray* compactDetail;
} SCompactObj;
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

15
source/dnode/mnode/impl/inc/mndStream.h
View File

@ -24,8 +24,9 @@ extern "C" {
typedef struct SStreamTransInfo { typedef struct SStreamTransInfo {
int64_t startTime; int64_t startTime;
int32_t transId; int64_t streamUid;
const char *name; const char *name;
int32_t transId;
} SStreamTransInfo; } SStreamTransInfo;
// time to generated the checkpoint, if now() - checkpointTs >= tsCheckpointInterval, this checkpoint will be discard // time to generated the checkpoint, if now() - checkpointTs >= tsCheckpointInterval, this checkpoint will be discard
@ -52,6 +53,14 @@ typedef struct SStreamExecInfo {
TdThreadMutex lock; TdThreadMutex lock;
} SStreamExecInfo; } SStreamExecInfo;
#define MND_STREAM_CREATE_NAME "stream-create"
#define MND_STREAM_CHECKPOINT_NAME "stream-checkpoint"
#define MND_STREAM_PAUSE_NAME "stream-pause"
#define MND_STREAM_RESUME_NAME "stream-resume"
#define MND_STREAM_DROP_NAME "stream-drop"
#define MND_STREAM_TASK_RESET_NAME "stream-task-reset"
#define MND_STREAM_TASK_UPDATE_NAME "stream-task-update"
extern SStreamExecInfo execInfo; extern SStreamExecInfo execInfo;
int32_t mndInitStream(SMnode *pMnode); int32_t mndInitStream(SMnode *pMnode);
@ -61,9 +70,9 @@ void mndReleaseStream(SMnode *pMnode, SStreamObj *pStream);
int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb); int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb);
int32_t mndPersistStream(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); int32_t mndPersistStream(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
int32_t mndStreamRegisterTrans(STrans *pTrans, const char *pName, const char *pSrcDb, const char *pDstDb); int32_t mndStreamRegisterTrans(STrans* pTrans, const char* pTransName, int64_t streamUid);
int32_t mndAddtoCheckpointWaitingList(SStreamObj *pStream, int64_t checkpointId); int32_t mndAddtoCheckpointWaitingList(SStreamObj *pStream, int64_t checkpointId);
bool streamTransConflictOtherTrans(SMnode *pMnode, const char *pSrcDb, const char *pDstDb, bool lock); bool streamTransConflictOtherTrans(SMnode *pMnode, int64_t streamUid, const char *pTransName, bool lock);
// for sma // for sma
// TODO refactor // TODO refactor

748
source/dnode/mnode/impl/src/mndCompact.c Normal file
View File

@ -0,0 +1,748 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "mndCompact.h"
#include "mndTrans.h"
#include "mndShow.h"
#include "mndDb.h"
#include "mndCompactDetail.h"
#include "mndVgroup.h"
#include "tmsgcb.h"
#include "mndDnode.h"
#include "tmisce.h"
#include "audit.h"
#include "mndPrivilege.h"
#include "mndTrans.h"
#define MND_COMPACT_VER_NUMBER 1
static int32_t mndProcessCompactTimer(SRpcMsg *pReq);
int32_t mndInitCompact(SMnode *pMnode) {
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_COMPACT, mndRetrieveCompact);
mndSetMsgHandle(pMnode, TDMT_MND_KILL_COMPACT, mndProcessKillCompactReq);
mndSetMsgHandle(pMnode, TDMT_VND_QUERY_COMPACT_PROGRESS_RSP, mndProcessQueryCompactRsp);
mndSetMsgHandle(pMnode, TDMT_MND_COMPACT_TIMER, mndProcessCompactTimer);
mndSetMsgHandle(pMnode, TDMT_VND_KILL_COMPACT_RSP, mndTransProcessRsp);
SSdbTable table = {
.sdbType = SDB_COMPACT,
.keyType = SDB_KEY_INT32,
.encodeFp = (SdbEncodeFp)mndCompactActionEncode,
.decodeFp = (SdbDecodeFp)mndCompactActionDecode,
.insertFp = (SdbInsertFp)mndCompactActionInsert,
.updateFp = (SdbUpdateFp)mndCompactActionUpdate,
.deleteFp = (SdbDeleteFp)mndCompactActionDelete,
};
return sdbSetTable(pMnode->pSdb, table);
}
void mndCleanupCompact(SMnode *pMnode) {
mDebug("mnd compact cleanup");
}
void tFreeCompactObj(SCompactObj *pCompact) {
}
int32_t tSerializeSCompactObj(void *buf, int32_t bufLen, const SCompactObj *pObj) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeI32(&encoder, pObj->compactId) < 0) return -1;
if (tEncodeCStr(&encoder, pObj->dbname) < 0) return -1;
if (tEncodeI64(&encoder, pObj->startTime) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSCompactObj(void *buf, int32_t bufLen, SCompactObj *pObj) {
int8_t ex = 0;
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeI32(&decoder, &pObj->compactId) < 0) return -1;
if (tDecodeCStrTo(&decoder, pObj->dbname) < 0) return -1;
if (tDecodeI64(&decoder, &pObj->startTime) < 0) return -1;
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
SSdbRaw *mndCompactActionEncode(SCompactObj *pCompact) {
terrno = TSDB_CODE_SUCCESS;
void *buf = NULL;
SSdbRaw *pRaw = NULL;
int32_t tlen = tSerializeSCompactObj(NULL, 0, pCompact);
if (tlen < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto OVER;
}
int32_t size = sizeof(int32_t) + tlen;
pRaw = sdbAllocRaw(SDB_COMPACT, MND_COMPACT_VER_NUMBER, size);
if (pRaw == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto OVER;
}
buf = taosMemoryMalloc(tlen);
if (buf == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto OVER;
}
tlen = tSerializeSCompactObj(buf, tlen, pCompact);
if (tlen < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto OVER;
}
int32_t dataPos = 0;
SDB_SET_INT32(pRaw, dataPos, tlen, OVER);
SDB_SET_BINARY(pRaw, dataPos, buf, tlen, OVER);
SDB_SET_DATALEN(pRaw, dataPos, OVER);
OVER:
taosMemoryFreeClear(buf);
if (terrno != TSDB_CODE_SUCCESS) {
mError("compact:%" PRId32 ", failed to encode to raw:%p since %s", pCompact->compactId, pRaw, terrstr());
sdbFreeRaw(pRaw);
return NULL;
}
mTrace("compact:%" PRId32 ", encode to raw:%p, row:%p", pCompact->compactId, pRaw, pCompact);
return pRaw;
}
SSdbRow *mndCompactActionDecode(SSdbRaw *pRaw) {
SSdbRow *pRow = NULL;
SCompactObj *pCompact = NULL;
void *buf = NULL;
terrno = TSDB_CODE_SUCCESS;
int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) {
goto OVER;
}
if (sver != MND_COMPACT_VER_NUMBER) {
terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
mError("compact read invalid ver, data ver: %d, curr ver: %d", sver, MND_COMPACT_VER_NUMBER);
goto OVER;
}
pRow = sdbAllocRow(sizeof(SCompactObj));
if (pRow == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto OVER;
}
pCompact = sdbGetRowObj(pRow);
if (pCompact == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto OVER;
}
int32_t tlen;
int32_t dataPos = 0;
SDB_GET_INT32(pRaw, dataPos, &tlen, OVER);
buf = taosMemoryMalloc(tlen + 1);
if (buf == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto OVER;
}
SDB_GET_BINARY(pRaw, dataPos, buf, tlen, OVER);
if (tDeserializeSCompactObj(buf, tlen, pCompact) < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto OVER;
}
//taosInitRWLatch(&pView->lock);
OVER:
taosMemoryFreeClear(buf);
if (terrno != TSDB_CODE_SUCCESS) {
mError("compact:%" PRId32 ", failed to decode from raw:%p since %s", pCompact->compactId, pRaw, terrstr());
taosMemoryFreeClear(pRow);
return NULL;
}
mTrace("compact:%" PRId32 ", decode from raw:%p, row:%p", pCompact->compactId, pRaw, pCompact);
return pRow;
}
int32_t mndCompactActionInsert(SSdb *pSdb, SCompactObj *pCompact) {
mTrace("compact:%" PRId32 ", perform insert action", pCompact->compactId);
return 0;
}
int32_t mndCompactActionDelete(SSdb *pSdb, SCompactObj *pCompact) {
mTrace("compact:%" PRId32 ", perform insert action", pCompact->compactId);
tFreeCompactObj(pCompact);
return 0;
}
int32_t mndCompactActionUpdate(SSdb *pSdb, SCompactObj *pOldCompact, SCompactObj *pNewCompact) {
mTrace("compact:%" PRId32 ", perform update action, old row:%p new row:%p",
pOldCompact->compactId, pOldCompact, pNewCompact);
return 0;
}
SCompactObj *mndAcquireCompact(SMnode *pMnode, int64_t compactId) {
SSdb *pSdb = pMnode->pSdb;
SCompactObj *pCompact = sdbAcquire(pSdb, SDB_COMPACT, &compactId);
if (pCompact == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) {
terrno = TSDB_CODE_SUCCESS;
}
return pCompact;
}
void mndReleaseCompact(SMnode *pMnode, SCompactObj *pCompact) {
SSdb *pSdb = pMnode->pSdb;
sdbRelease(pSdb, pCompact);
}
//compact db
int32_t mndAddCompactToTran(SMnode *pMnode, STrans *pTrans, SCompactObj* pCompact, SDbObj *pDb, SCompactDbRsp *rsp){
pCompact->compactId = tGenIdPI32();
strcpy(pCompact->dbname, pDb->name);
pCompact->startTime = taosGetTimestampMs();
SSdbRaw *pVgRaw = mndCompactActionEncode(pCompact);
if (pVgRaw == NULL) return -1;
if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) {
sdbFreeRaw(pVgRaw);
return -1;
}
(void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY);
rsp->compactId = pCompact->compactId;
return 0;
}
//retrieve compact
int32_t mndRetrieveCompact(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows){
SMnode *pMnode = pReq->info.node;
SSdb *pSdb = pMnode->pSdb;
int32_t numOfRows = 0;
SCompactObj *pCompact = NULL;
char *sep = NULL;
SDbObj *pDb = NULL;
if (strlen(pShow->db) > 0) {
sep = strchr(pShow->db, '.');
if (sep && ((0 == strcmp(sep + 1, TSDB_INFORMATION_SCHEMA_DB) || (0 == strcmp(sep + 1, TSDB_PERFORMANCE_SCHEMA_DB))))) {
sep++;
} else {
pDb = mndAcquireDb(pMnode, pShow->db);
if (pDb == NULL) return terrno;
}
}
while (numOfRows < rows) {
pShow->pIter = sdbFetch(pSdb, SDB_COMPACT, pShow->pIter, (void **)&pCompact);
if (pShow->pIter == NULL) break;
SColumnInfoData *pColInfo;
SName n;
int32_t cols = 0;
char tmpBuf[TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE] = {0};
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)&pCompact->compactId, false);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
if (pDb != NULL || !IS_SYS_DBNAME(pCompact->dbname)) {
SName name = {0};
tNameFromString(&name, pCompact->dbname, T_NAME_ACCT | T_NAME_DB);
tNameGetDbName(&name, varDataVal(tmpBuf));
} else {
strncpy(varDataVal(tmpBuf), pCompact->dbname, strlen(pCompact->dbname) + 1);
}
varDataSetLen(tmpBuf, strlen(varDataVal(tmpBuf)));
colDataSetVal(pColInfo, numOfRows, (const char *)tmpBuf, false);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)&pCompact->startTime, false);
numOfRows++;
sdbRelease(pSdb, pCompact);
}
pShow->numOfRows += numOfRows;
mndReleaseDb(pMnode, pDb);
return numOfRows;
}
//kill compact
static void *mndBuildKillCompactReq(SMnode *pMnode, SVgObj *pVgroup, int32_t *pContLen,
int32_t compactId, int32_t dnodeid) {
SVKillCompactReq req = {0};
req.compactId = compactId;
req.vgId = pVgroup->vgId;
req.dnodeId = dnodeid;
mInfo("vgId:%d, build compact vnode config req", pVgroup->vgId);
int32_t contLen = tSerializeSVKillCompactReq(NULL, 0, &req);
if (contLen < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
contLen += sizeof(SMsgHead);
void *pReq = taosMemoryMalloc(contLen);
if (pReq == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
SMsgHead *pHead = pReq;
pHead->contLen = htonl(contLen);
pHead->vgId = htonl(pVgroup->vgId);
tSerializeSVKillCompactReq((char *)pReq + sizeof(SMsgHead), contLen, &req);
*pContLen = contLen;
return pReq;
}
static int32_t mndAddKillCompactAction(SMnode *pMnode, STrans *pTrans, SVgObj *pVgroup,
int32_t compactId, int32_t dnodeid) {
STransAction action = {0};
SDnodeObj *pDnode = mndAcquireDnode(pMnode, dnodeid);
if (pDnode == NULL) return -1;
action.epSet = mndGetDnodeEpset(pDnode);
mndReleaseDnode(pMnode, pDnode);
int32_t contLen = 0;
void *pReq = mndBuildKillCompactReq(pMnode, pVgroup, &contLen, compactId, dnodeid);
if (pReq == NULL) return -1;
action.pCont = pReq;
action.contLen = contLen;
action.msgType = TDMT_VND_KILL_COMPACT;
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
taosMemoryFree(pReq);
return -1;
}
return 0;
}
static int32_t mndKillCompact(SMnode *pMnode, SRpcMsg *pReq, SCompactObj *pCompact) {
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "kill-compact");
if (pTrans == NULL) {
mError("compact:%" PRId32 ", failed to drop since %s" , pCompact->compactId, terrstr());
return -1;
}
mInfo("trans:%d, used to kill compact:%" PRId32, pTrans->id, pCompact->compactId);
SSdbRaw *pCommitRaw = mndCompactActionEncode(pCompact);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
(void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY);
void *pIter = NULL;
while (1) {
SCompactDetailObj *pDetail = NULL;
pIter = sdbFetch(pMnode->pSdb, SDB_COMPACT_DETAIL, pIter, (void **)&pDetail);
if (pIter == NULL) break;
if (pDetail->compactId == pCompact->compactId) {
SVgObj *pVgroup = mndAcquireVgroup(pMnode, pDetail->vgId);
if(pVgroup == NULL){
mError("trans:%d, failed to append redo action since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
if(mndAddKillCompactAction(pMnode, pTrans, pVgroup, pCompact->compactId, pDetail->dnodeId) != 0){
mError("trans:%d, failed to append redo action since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
mndReleaseVgroup(pMnode, pVgroup);
/*
SSdbRaw *pCommitRaw = mndCompactDetailActionEncode(pDetail);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
(void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED);
*/
}
sdbRelease(pMnode->pSdb, pDetail);
}
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
mndTransDrop(pTrans);
return 0;
}
int32_t mndProcessKillCompactReq(SRpcMsg *pReq){
SKillCompactReq killCompactReq = {0};
if (tDeserializeSKillCompactReq(pReq->pCont, pReq->contLen, &killCompactReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
mInfo("start to kill compact:%" PRId32, killCompactReq.compactId);
SMnode *pMnode = pReq->info.node;
int32_t code = -1;
SCompactObj *pCompact = mndAcquireCompact(pMnode, killCompactReq.compactId);
if(pCompact == NULL){
terrno = TSDB_CODE_MND_INVALID_COMPACT_ID;
tFreeSKillCompactReq(&killCompactReq);
return -1;
}
if (0 != mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_COMPACT_DB)) {
goto _OVER;
}
if (mndKillCompact(pMnode, pReq, pCompact) < 0) {
goto _OVER;
}
code = TSDB_CODE_ACTION_IN_PROGRESS;
char obj[10] = {0};
sprintf(obj, "%d", pCompact->compactId);
auditRecord(pReq, pMnode->clusterId, "killCompact", pCompact->dbname, obj, killCompactReq.sql, killCompactReq.sqlLen);
_OVER:
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("failed to kill compact %" PRId32 " since %s", killCompactReq.compactId, terrstr());
}
tFreeSKillCompactReq(&killCompactReq);
sdbRelease(pMnode->pSdb, pCompact);
return code;
}
//update progress
static int32_t mndUpdateCompactProgress(SMnode *pMnode, SRpcMsg *pReq, int32_t compactId, SQueryCompactProgressRsp* rsp) {
void* pIter = NULL;
while (1) {
SCompactDetailObj *pDetail = NULL;
pIter = sdbFetch(pMnode->pSdb, SDB_COMPACT_DETAIL, pIter, (void **)&pDetail);
if (pIter == NULL) break;
if (pDetail->compactId == compactId && pDetail->vgId == rsp->vgId && pDetail->dnodeId == rsp->dnodeId) {
pDetail->newNumberFileset = rsp->numberFileset;
pDetail->newFinished = rsp->finished;
sdbRelease(pMnode->pSdb, pDetail);
return 0;
}
sdbRelease(pMnode->pSdb, pDetail);
}
return -1;
}
int32_t mndProcessQueryCompactRsp(SRpcMsg *pReq){
SQueryCompactProgressRsp req = {0};
if (tDeserializeSQueryCompactProgressRsp(pReq->pCont, pReq->contLen, &req) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
mDebug("compact:%d, receive query response, vgId:%d, dnodeId:%d, numberFileset:%d, finished:%d",
req.compactId, req.vgId, req.dnodeId, req.numberFileset, req.finished);
SMnode *pMnode = pReq->info.node;
int32_t code = -1;
if(mndUpdateCompactProgress(pMnode, pReq, req.compactId, &req) != 0){
mError("compact:%d, failed to update progress, vgId:%d, dnodeId:%d, numberFileset:%d, finished:%d",
req.compactId, req.vgId, req.dnodeId, req.numberFileset, req.finished);
return -1;
}
return 0;
}
//timer
void mndCompactSendProgressReq(SMnode *pMnode, SCompactObj *pCompact){
void *pIter = NULL;
while (1) {
SCompactDetailObj *pDetail = NULL;
pIter = sdbFetch(pMnode->pSdb, SDB_COMPACT_DETAIL, pIter, (void **)&pDetail);
if (pIter == NULL) break;
if (pDetail->compactId == pCompact->compactId) {
SEpSet epSet = {0};
SDnodeObj *pDnode = mndAcquireDnode(pMnode, pDetail->dnodeId);
if(pDnode == NULL) break;
addEpIntoEpSet(&epSet, pDnode->fqdn, pDnode->port);
mndReleaseDnode(pMnode, pDnode);
SQueryCompactProgressReq req;
req.compactId = pDetail->compactId;
req.vgId = pDetail->vgId;
req.dnodeId = pDetail->dnodeId;
int32_t contLen = tSerializeSQueryCompactProgressReq(NULL, 0, &req);
if (contLen < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
continue;
}
contLen += sizeof(SMsgHead);
SMsgHead *pHead = rpcMallocCont(contLen);
if (pHead == NULL) {
sdbCancelFetch(pMnode->pSdb, pDetail);
sdbRelease(pMnode->pSdb, pDetail);
continue;
}
pHead->contLen = htonl(contLen);
pHead->vgId = htonl(pDetail->vgId);
tSerializeSQueryCompactProgressReq((char *)pHead + sizeof(SMsgHead), contLen - sizeof(SMsgHead), &req);
SRpcMsg rpcMsg = {.msgType = TDMT_VND_QUERY_COMPACT_PROGRESS,
.contLen = contLen};
//rpcMsg.pCont = rpcMallocCont(contLen);
//if (rpcMsg.pCont == NULL) {
// return;
//}
//memcpy(rpcMsg.pCont, pHead, contLen);
rpcMsg.pCont = pHead;
char detail[1024] = {0};
int32_t len = snprintf(detail, sizeof(detail), "msgType:%s numOfEps:%d inUse:%d", TMSG_INFO(TDMT_VND_QUERY_COMPACT_PROGRESS),
epSet.numOfEps, epSet.inUse);
for (int32_t i = 0; i < epSet.numOfEps; ++i) {
len += snprintf(detail + len, sizeof(detail) - len, " ep:%d-%s:%u", i, epSet.eps[i].fqdn,
epSet.eps[i].port);
}
mDebug("compact:%d, send update progress msg to %s", pDetail->compactId, detail);
tmsgSendReq(&epSet, &rpcMsg);
}
sdbRelease(pMnode->pSdb, pDetail);
}
}
static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) {
bool needSave = false;
void* pIter = NULL;
while (1) {
SCompactDetailObj *pDetail = NULL;
pIter = sdbFetch(pMnode->pSdb, SDB_COMPACT_DETAIL, pIter, (void **)&pDetail);
if (pIter == NULL) break;
if (pDetail->compactId == compactId) {
mDebug("compact:%d, check save progress, vgId:%d, dnodeId:%d, numberFileset:%d, finished:%d, "
"newNumberFileset:%d, newFinished:%d",
pDetail->compactId, pDetail->vgId, pDetail->dnodeId, pDetail->numberFileset, pDetail->finished,
pDetail->newNumberFileset, pDetail->newFinished);
if(pDetail->numberFileset < pDetail->newNumberFileset || pDetail->finished < pDetail->newFinished)
needSave = true;
}
sdbRelease(pMnode->pSdb, pDetail);
}
if(!needSave) {
mDebug("compact:%" PRId32 ", no need to save" , compactId);
return 0;
}
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, NULL, "update-compact-progress");
if (pTrans == NULL) {
mError("trans:%" PRId32 ", failed to create since %s" , pTrans->id, terrstr());
return -1;
}
mInfo("compact:%d, trans:%d, used to update compact progress.", compactId, pTrans->id);
SCompactObj *pCompact = mndAcquireCompact(pMnode, compactId);
pIter = NULL;
while (1) {
SCompactDetailObj *pDetail = NULL;
pIter = sdbFetch(pMnode->pSdb, SDB_COMPACT_DETAIL, pIter, (void **)&pDetail);
if (pIter == NULL) break;
if (pDetail->compactId == compactId) {
mInfo("compact:%d, trans:%d, check compact progress, vgId:%d, dnodeId:%d, numberFileset:%d, finished:%d, "
"newNumberFileset:%d, newFinished:%d",
pDetail->compactId, pTrans->id, pDetail->vgId, pDetail->dnodeId, pDetail->numberFileset, pDetail->finished,
pDetail->newNumberFileset, pDetail->newFinished);
pDetail->numberFileset = pDetail->newNumberFileset;
pDetail->finished = pDetail->newFinished;
SSdbRaw *pCommitRaw = mndCompactDetailActionEncode(pDetail);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
mError("compact:%d, trans:%d, failed to append commit log since %s", pDetail->compactId, pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
(void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY);
}
sdbRelease(pMnode->pSdb, pDetail);
}
bool allFinished = true;
while (1) {
SCompactDetailObj *pDetail = NULL;
pIter = sdbFetch(pMnode->pSdb, SDB_COMPACT_DETAIL, pIter, (void **)&pDetail);
if (pIter == NULL) break;
if(pDetail->compactId == compactId){
mInfo("compact:%d, trans:%d, check compact finished, vgId:%d, dnodeId:%d, numberFileset:%d, finished:%d",
pDetail->compactId, pTrans->id, pDetail->vgId, pDetail->dnodeId, pDetail->numberFileset, pDetail->finished);
if(pDetail->numberFileset == -1 && pDetail->finished == -1){
allFinished = false;
sdbRelease(pMnode->pSdb, pDetail);
break;
}
if (pDetail->numberFileset != -1 && pDetail->finished != -1 &&
pDetail->numberFileset != pDetail->finished) {
allFinished = false;
sdbRelease(pMnode->pSdb, pDetail);
break;
}
}
sdbRelease(pMnode->pSdb, pDetail);
}
if(allFinished){
mInfo("compact:%d, all finished", pCompact->compactId);
while (1) {
SCompactDetailObj *pDetail = NULL;
pIter = sdbFetch(pMnode->pSdb, SDB_COMPACT_DETAIL, pIter, (void **)&pDetail);
if (pIter == NULL) break;
if (pDetail->compactId == pCompact->compactId) {
SSdbRaw *pCommitRaw = mndCompactDetailActionEncode(pDetail);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
mError("compact:%d, trans:%d, failed to append commit log since %s", pDetail->compactId, pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
(void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED);
}
sdbRelease(pMnode->pSdb, pDetail);
}
SSdbRaw *pCommitRaw = mndCompactActionEncode(pCompact);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
mError("compact:%d, trans:%d, failed to append commit log since %s", compactId, pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
(void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED);
}
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("compact:%d, trans:%d, failed to prepare since %s", compactId, pTrans->id, terrstr());
mndTransDrop(pTrans);
sdbRelease(pMnode->pSdb, pCompact);
return -1;
}
sdbRelease(pMnode->pSdb, pCompact);
mndTransDrop(pTrans);
return 0;
}
void mndCompactPullup(SMnode *pMnode) {
SSdb *pSdb = pMnode->pSdb;
SArray *pArray = taosArrayInit(sdbGetSize(pSdb, SDB_COMPACT), sizeof(int32_t));
if (pArray == NULL) return;
void *pIter = NULL;
while (1) {
SCompactObj *pCompact = NULL;
pIter = sdbFetch(pMnode->pSdb, SDB_COMPACT, pIter, (void **)&pCompact);
if (pIter == NULL) break;
taosArrayPush(pArray, &pCompact->compactId);
sdbRelease(pSdb, pCompact);
}
for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) {
mInfo("begin to pull up");
int32_t *pCompactId = taosArrayGet(pArray, i);
SCompactObj *pCompact = mndAcquireCompact(pMnode, *pCompactId);
if (pCompact != NULL) {
mInfo("compact:%d, begin to pull up", pCompact->compactId);
mndCompactSendProgressReq(pMnode, pCompact);
mndSaveCompactProgress(pMnode, pCompact->compactId);
}
mndReleaseCompact(pMnode, pCompact);
}
taosArrayDestroy(pArray);
}
static int32_t mndProcessCompactTimer(SRpcMsg *pReq) {
mTrace("start to process compact timer");
mndCompactPullup(pReq->info.node);
return 0;
}

300
source/dnode/mnode/impl/src/mndCompactDetail.c Normal file
View File

@ -0,0 +1,300 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "mndCompactDetail.h"
#include "mndTrans.h"
#include "mndShow.h"
#include "mndDb.h"
#define MND_COMPACT_VER_NUMBER 1
int32_t mndInitCompactDetail(SMnode *pMnode) {
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_COMPACT_DETAIL, mndRetrieveCompactDetail);
SSdbTable table = {
.sdbType = SDB_COMPACT_DETAIL,
.keyType = SDB_KEY_INT64,
.encodeFp = (SdbEncodeFp)mndCompactDetailActionEncode,
.decodeFp = (SdbDecodeFp)mndCompactDetailActionDecode,
.insertFp = (SdbInsertFp)mndCompactDetailActionInsert,
.updateFp = (SdbUpdateFp)mndCompactDetailActionUpdate,
.deleteFp = (SdbDeleteFp)mndCompactDetailActionDelete,
};
return sdbSetTable(pMnode->pSdb, table);
}
void mndCleanupCompactDetail(SMnode *pMnode) {
mDebug("mnd compact detail cleanup");
}
int32_t mndRetrieveCompactDetail(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows){
SMnode *pMnode = pReq->info.node;
SSdb *pSdb = pMnode->pSdb;
int32_t numOfRows = 0;
SCompactDetailObj *pCompactDetail = NULL;
char *sep = NULL;
SDbObj *pDb = NULL;
if (strlen(pShow->db) > 0) {
sep = strchr(pShow->db, '.');
if (sep && ((0 == strcmp(sep + 1, TSDB_INFORMATION_SCHEMA_DB) || (0 == strcmp(sep + 1, TSDB_PERFORMANCE_SCHEMA_DB))))) {
sep++;
} else {
pDb = mndAcquireDb(pMnode, pShow->db);
if (pDb == NULL) return terrno;
}
}
while(numOfRows < rows){
pShow->pIter = sdbFetch(pSdb, SDB_COMPACT_DETAIL, pShow->pIter, (void **)&pCompactDetail);
if (pShow->pIter == NULL) break;
SColumnInfoData *pColInfo;
SName n;
int32_t cols = 0;
char tmpBuf[TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE] = {0};
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)&pCompactDetail->compactId, false);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)&pCompactDetail->vgId, false);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)&pCompactDetail->dnodeId, false);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)&pCompactDetail->numberFileset, false);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)&pCompactDetail->finished, false);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)&pCompactDetail->startTime, false);
numOfRows++;
sdbRelease(pSdb, pCompactDetail);
}
pShow->numOfRows += numOfRows;
mndReleaseDb(pMnode, pDb);
return numOfRows;
}
void tFreeCompactDetailObj(SCompactDetailObj *pCompact) {
}
int32_t tSerializeSCompactDetailObj(void *buf, int32_t bufLen, const SCompactDetailObj *pObj) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeI32(&encoder, pObj->compactDetailId) < 0) return -1;
if (tEncodeI32(&encoder, pObj->compactId) < 0) return -1;
if (tEncodeI32(&encoder, pObj->vgId) < 0) return -1;
if (tEncodeI32(&encoder, pObj->dnodeId) < 0) return -1;
if (tEncodeI32(&encoder, pObj->numberFileset) < 0) return -1;
if (tEncodeI32(&encoder, pObj->finished) < 0) return -1;
if (tEncodeI64(&encoder, pObj->startTime) < 0) return -1;
if (tEncodeI32(&encoder, pObj->newNumberFileset) < 0) return -1;
if (tEncodeI32(&encoder, pObj->newFinished) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSCompactDetailObj(void *buf, int32_t bufLen, SCompactDetailObj *pObj) {
int8_t ex = 0;
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeI32(&decoder, &pObj->compactDetailId) < 0) return -1;
if (tDecodeI32(&decoder, &pObj->compactId) < 0) return -1;
if (tDecodeI32(&decoder, &pObj->vgId) < 0) return -1;
if (tDecodeI32(&decoder, &pObj->dnodeId) < 0) return -1;
if (tDecodeI32(&decoder, &pObj->numberFileset) < 0) return -1;
if (tDecodeI32(&decoder, &pObj->finished) < 0) return -1;
if (tDecodeI64(&decoder, &pObj->startTime) < 0) return -1;
if (tDecodeI32(&decoder, &pObj->newNumberFileset) < 0) return -1;
if (tDecodeI32(&decoder, &pObj->newFinished) < 0) return -1;
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
SSdbRaw *mndCompactDetailActionEncode(SCompactDetailObj *pCompact) {
terrno = TSDB_CODE_SUCCESS;
void *buf = NULL;
SSdbRaw *pRaw = NULL;
int32_t tlen = tSerializeSCompactDetailObj(NULL, 0, pCompact);
if (tlen < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto OVER;
}
int32_t size = sizeof(int32_t) + tlen;
pRaw = sdbAllocRaw(SDB_COMPACT_DETAIL, MND_COMPACT_VER_NUMBER, size);
if (pRaw == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto OVER;
}
buf = taosMemoryMalloc(tlen);
if (buf == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto OVER;
}
tlen = tSerializeSCompactDetailObj(buf, tlen, pCompact);
if (tlen < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto OVER;
}
int32_t dataPos = 0;
SDB_SET_INT32(pRaw, dataPos, tlen, OVER);
SDB_SET_BINARY(pRaw, dataPos, buf, tlen, OVER);
SDB_SET_DATALEN(pRaw, dataPos, OVER);
OVER:
taosMemoryFreeClear(buf);
if (terrno != TSDB_CODE_SUCCESS) {
mError("compact detail:%" PRId32 ", failed to encode to raw:%p since %s", pCompact->compactId, pRaw, terrstr());
sdbFreeRaw(pRaw);
return NULL;
}
mTrace("compact detail:%" PRId32 ", encode to raw:%p, row:%p", pCompact->compactId, pRaw, pCompact);
return pRaw;
}
SSdbRow *mndCompactDetailActionDecode(SSdbRaw *pRaw) {
SSdbRow *pRow = NULL;
SCompactDetailObj *pCompact = NULL;
void *buf = NULL;
terrno = TSDB_CODE_SUCCESS;
int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) {
goto OVER;
}
if (sver != MND_COMPACT_VER_NUMBER) {
terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
mError("compact detail read invalid ver, data ver: %d, curr ver: %d", sver, MND_COMPACT_VER_NUMBER);
goto OVER;
}
pRow = sdbAllocRow(sizeof(SCompactObj));
if (pRow == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto OVER;
}
pCompact = sdbGetRowObj(pRow);
if (pCompact == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto OVER;
}
int32_t tlen;
int32_t dataPos = 0;
SDB_GET_INT32(pRaw, dataPos, &tlen, OVER);
buf = taosMemoryMalloc(tlen + 1);
if (buf == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto OVER;
}
SDB_GET_BINARY(pRaw, dataPos, buf, tlen, OVER);
if (tDeserializeSCompactDetailObj(buf, tlen, pCompact) < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto OVER;
}
//taosInitRWLatch(&pView->lock);
OVER:
taosMemoryFreeClear(buf);
if (terrno != TSDB_CODE_SUCCESS) {
mError("compact detail:%" PRId32 ", failed to decode from raw:%p since %s", pCompact->compactId, pRaw, terrstr());
taosMemoryFreeClear(pRow);
return NULL;
}
mTrace("compact detail:%" PRId32 ", decode from raw:%p, row:%p", pCompact->compactId, pRaw, pCompact);
return pRow;
}
int32_t mndCompactDetailActionInsert(SSdb *pSdb, SCompactDetailObj *pCompact) {
mTrace("compact detail:%" PRId32 ", perform insert action", pCompact->compactId);
return 0;
}
int32_t mndCompactDetailActionDelete(SSdb *pSdb, SCompactDetailObj *pCompact) {
mTrace("compact detail:%" PRId32 ", perform insert action", pCompact->compactId);
tFreeCompactDetailObj(pCompact);
return 0;
}
int32_t mndCompactDetailActionUpdate(SSdb *pSdb, SCompactDetailObj *pOldCompact, SCompactDetailObj *pNewCompact) {
mTrace("compact detail:%" PRId32 ", perform update action, old row:%p new row:%p",
pOldCompact->compactId, pOldCompact, pNewCompact);
pOldCompact->numberFileset = pNewCompact->numberFileset;
pOldCompact->finished = pNewCompact->finished;
return 0;
}
int32_t mndAddCompactDetailToTran(SMnode *pMnode, STrans *pTrans, SCompactObj* pCompact, SVgObj *pVgroup,
SVnodeGid *pVgid, int32_t index){
SCompactDetailObj compactDetail = {0};
compactDetail.compactDetailId = index;
compactDetail.compactId = pCompact->compactId;
compactDetail.vgId = pVgroup->vgId;
compactDetail.dnodeId = pVgid->dnodeId;
compactDetail.startTime = taosGetTimestampMs();
compactDetail.numberFileset = -1;
compactDetail.finished = -1;
compactDetail.newNumberFileset = -1;
compactDetail.newFinished = -1;
mInfo("compact:%d, add compact detail to trans, index:%d, vgId:%d, dnodeId:%d",
compactDetail.compactId, compactDetail.compactDetailId, compactDetail.vgId, compactDetail.dnodeId);
SSdbRaw *pVgRaw = mndCompactDetailActionEncode(&compactDetail);
if (pVgRaw == NULL) return -1;
if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) {
sdbFreeRaw(pVgRaw);
return -1;
}
(void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY);
return 0;
}

21
source/dnode/mnode/impl/src/mndMain.c
View File

@ -42,6 +42,8 @@
#include "mndUser.h" #include "mndUser.h"
#include "mndVgroup.h" #include "mndVgroup.h"
#include "mndView.h" #include "mndView.h"
#include "mndCompact.h"
#include "mndCompactDetail.h"
static inline int32_t mndAcquireRpc(SMnode *pMnode) { static inline int32_t mndAcquireRpc(SMnode *pMnode) {
int32_t code = 0; int32_t code = 0;
@ -112,6 +114,16 @@ static void mndPullupTrans(SMnode *pMnode) {
} }
} }
static void mndPullupCompacts(SMnode *pMnode) {
mTrace("pullup compact timer msg");
int32_t contLen = 0;
void *pReq = mndBuildTimerMsg(&contLen);
if (pReq != NULL) {
SRpcMsg rpcMsg = {.msgType = TDMT_MND_COMPACT_TIMER, .pCont = pReq, .contLen = contLen};
tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
}
}
static void mndPullupTtl(SMnode *pMnode) { static void mndPullupTtl(SMnode *pMnode) {
mTrace("pullup ttl"); mTrace("pullup ttl");
int32_t contLen = 0; int32_t contLen = 0;
@ -287,6 +299,10 @@ static void *mndThreadFp(void *param) {
mndPullupTrans(pMnode); mndPullupTrans(pMnode);
} }
if (sec % tsCompactPullupInterval == 0) {
mndPullupCompacts(pMnode);
}
if (sec % tsMqRebalanceInterval == 0) { if (sec % tsMqRebalanceInterval == 0) {
mndCalMqRebalance(pMnode); mndCalMqRebalance(pMnode);
} }
@ -462,6 +478,8 @@ static int32_t mndInitSteps(SMnode *pMnode) {
if (mndAllocStep(pMnode, "mnode-db", mndInitDb, mndCleanupDb) != 0) return -1; if (mndAllocStep(pMnode, "mnode-db", mndInitDb, mndCleanupDb) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-func", mndInitFunc, mndCleanupFunc) != 0) return -1; if (mndAllocStep(pMnode, "mnode-func", mndInitFunc, mndCleanupFunc) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-view", mndInitView, mndCleanupView) != 0) return -1; if (mndAllocStep(pMnode, "mnode-view", mndInitView, mndCleanupView) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-compact", mndInitCompact, mndCleanupCompact) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-compact-detail", mndInitCompactDetail, mndCleanupCompactDetail) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-sdb", mndOpenSdb, NULL) != 0) return -1; if (mndAllocStep(pMnode, "mnode-sdb", mndOpenSdb, NULL) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-profile", mndInitProfile, mndCleanupProfile) != 0) return -1; if (mndAllocStep(pMnode, "mnode-profile", mndInitProfile, mndCleanupProfile) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-show", mndInitShow, mndCleanupShow) != 0) return -1; if (mndAllocStep(pMnode, "mnode-show", mndInitShow, mndCleanupShow) != 0) return -1;
@ -693,7 +711,8 @@ static int32_t mndCheckMnodeState(SRpcMsg *pMsg) {
_OVER: _OVER:
if (pMsg->msgType == TDMT_MND_TMQ_TIMER || pMsg->msgType == TDMT_MND_TELEM_TIMER || if (pMsg->msgType == TDMT_MND_TMQ_TIMER || pMsg->msgType == TDMT_MND_TELEM_TIMER ||
pMsg->msgType == TDMT_MND_TRANS_TIMER || pMsg->msgType == TDMT_MND_TTL_TIMER || pMsg->msgType == TDMT_MND_TRANS_TIMER || pMsg->msgType == TDMT_MND_TTL_TIMER ||
pMsg->msgType == TDMT_MND_TRIM_DB_TIMER || pMsg->msgType == TDMT_MND_UPTIME_TIMER) { pMsg->msgType == TDMT_MND_TRIM_DB_TIMER || pMsg->msgType == TDMT_MND_UPTIME_TIMER ||
pMsg->msgType == TDMT_MND_COMPACT_TIMER) {
mTrace("timer not process since mnode restored:%d stopped:%d, sync restored:%d role:%s ", pMnode->restored, mTrace("timer not process since mnode restored:%d stopped:%d, sync restored:%d role:%s ", pMnode->restored,
pMnode->stopped, state.restored, syncStr(state.state)); pMnode->stopped, state.restored, syncStr(state.state));
return -1; return -1;

7
source/dnode/mnode/impl/src/mndShow.c
View File

@ -17,6 +17,7 @@
#include "mndShow.h" #include "mndShow.h"
#include "mndPrivilege.h" #include "mndPrivilege.h"
#include "systable.h" #include "systable.h"
#include "mndUser.h"
#define SHOW_STEP_SIZE 100 #define SHOW_STEP_SIZE 100
#define SHOW_COLS_STEP_SIZE 4096 #define SHOW_COLS_STEP_SIZE 4096
@ -118,6 +119,10 @@ static int32_t convertToRetrieveType(char *name, int32_t len) {
type = TSDB_MGMT_TABLE_PRIVILEGES; type = TSDB_MGMT_TABLE_PRIVILEGES;
} else if (strncasecmp(name, TSDB_INS_TABLE_VIEWS, len) == 0) { } else if (strncasecmp(name, TSDB_INS_TABLE_VIEWS, len) == 0) {
type = TSDB_MGMT_TABLE_VIEWS; type = TSDB_MGMT_TABLE_VIEWS;
} else if (strncasecmp(name, TSDB_INS_TABLE_COMPACTS, len) == 0) {
type = TSDB_MGMT_TABLE_COMPACT;
} else if (strncasecmp(name, TSDB_INS_TABLE_COMPACT_DETAILS, len) == 0) {
type = TSDB_MGMT_TABLE_COMPACT_DETAIL;
} else { } else {
mError("invalid show name:%s len:%d", name, len); mError("invalid show name:%s len:%d", name, len);
} }
@ -206,7 +211,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
return -1; return -1;
} }
mDebug("mndProcessRetrieveSysTableReq tb:%s", retrieveReq.tb); mDebug("process to retrieve systable req db:%s, tb:%s", retrieveReq.db, retrieveReq.tb);
if (retrieveReq.showId == 0) { if (retrieveReq.showId == 0) {
STableMetaRsp *pMeta = taosHashGet(pMnode->infosMeta, retrieveReq.tb, strlen(retrieveReq.tb)); STableMetaRsp *pMeta = taosHashGet(pMnode->infosMeta, retrieveReq.tb, strlen(retrieveReq.tb));

61
source/dnode/mnode/impl/src/mndStream.c
View File

@ -16,12 +16,9 @@
#include "mndStream.h" #include "mndStream.h"
#include "audit.h" #include "audit.h"
#include "mndDb.h" #include "mndDb.h"
#include "mndDnode.h"
#include "mndMnode.h"
#include "mndPrivilege.h" #include "mndPrivilege.h"
#include "mndScheduler.h" #include "mndScheduler.h"
#include "mndShow.h" #include "mndShow.h"
#include "mndSnode.h"
#include "mndStb.h" #include "mndStb.h"
#include "mndTrans.h" #include "mndTrans.h"
#include "mndVgroup.h" #include "mndVgroup.h"
@ -34,13 +31,6 @@
#define MND_STREAM_RESERVE_SIZE 64 #define MND_STREAM_RESERVE_SIZE 64
#define MND_STREAM_MAX_NUM 60 #define MND_STREAM_MAX_NUM 60
#define MND_STREAM_CHECKPOINT_NAME "stream-checkpoint"
#define MND_STREAM_PAUSE_NAME "stream-pause"
#define MND_STREAM_RESUME_NAME "stream-resume"
#define MND_STREAM_DROP_NAME "stream-drop"
#define MND_STREAM_TASK_RESET_NAME "stream-task-reset"
#define MND_STREAM_TASK_UPDATE_NAME "stream-task-update"
typedef struct SNodeEntry { typedef struct SNodeEntry {
int32_t nodeId; int32_t nodeId;
bool stageUpdated; // the stage has been updated due to the leader/follower change or node reboot. bool stageUpdated; // the stage has been updated due to the leader/follower change or node reboot.
@ -843,7 +833,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
goto _OVER; goto _OVER;
} }
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pReq, "create-stream"); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, MND_STREAM_CREATE_NAME);
if (pTrans == NULL) { if (pTrans == NULL) {
mError("stream:%s, failed to create since %s", createStreamReq.name, terrstr()); mError("stream:%s, failed to create since %s", createStreamReq.name, terrstr());
goto _OVER; goto _OVER;
@ -851,11 +841,12 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
mInfo("trans:%d, used to create stream:%s", pTrans->id, createStreamReq.name); mInfo("trans:%d, used to create stream:%s", pTrans->id, createStreamReq.name);
mndTransSetDbName(pTrans, createStreamReq.sourceDB, streamObj.targetDb); mndTransSetDbName(pTrans, createStreamReq.sourceDB, streamObj.targetSTbName);
if (mndTransCheckConflict(pMnode, pTrans) != 0) { if (mndTransCheckConflict(pMnode, pTrans) != 0) {
mndTransDrop(pTrans); mndTransDrop(pTrans);
goto _OVER; goto _OVER;
} }
// create stb for stream // create stb for stream
if (createStreamReq.createStb == STREAM_CREATE_STABLE_TRUE && if (createStreamReq.createStb == STREAM_CREATE_STABLE_TRUE &&
mndCreateStbForStream(pMnode, pTrans, &streamObj, pReq->info.conn.user) < 0) { mndCreateStbForStream(pMnode, pTrans, &streamObj, pReq->info.conn.user) < 0) {
@ -942,7 +933,8 @@ int64_t mndStreamGenChkpId(SMnode *pMnode) {
if (pIter == NULL) break; if (pIter == NULL) break;
maxChkpId = TMAX(maxChkpId, pStream->checkpointId); maxChkpId = TMAX(maxChkpId, pStream->checkpointId);
mDebug("stream %p checkpoint %" PRId64 "", pStream, pStream->checkpointId); mDebug("stream:%p, %s id:%" PRIx64 "checkpoint %" PRId64 "", pStream, pStream->name, pStream->uid,
pStream->checkpointId);
sdbRelease(pSdb, pStream); sdbRelease(pSdb, pStream);
} }
@ -1032,7 +1024,7 @@ static int32_t mndProcessStreamCheckpointTrans(SMnode *pMnode, SStreamObj *pStre
return -1; return -1;
} }
bool conflict = streamTransConflictOtherTrans(pMnode, pStream->sourceDb, pStream->targetDb, true); bool conflict = streamTransConflictOtherTrans(pMnode, pStream->uid, MND_STREAM_CHECKPOINT_NAME, true);
if (conflict) { if (conflict) {
mndAddtoCheckpointWaitingList(pStream, checkpointId); mndAddtoCheckpointWaitingList(pStream, checkpointId);
mWarn("checkpoint conflict with other trans in %s, ignore the checkpoint for stream:%s %" PRIx64, pStream->sourceDb, mWarn("checkpoint conflict with other trans in %s, ignore the checkpoint for stream:%s %" PRIx64, pStream->sourceDb,
@ -1040,14 +1032,14 @@ static int32_t mndProcessStreamCheckpointTrans(SMnode *pMnode, SStreamObj *pStre
return -1; return -1;
} }
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, NULL, MND_STREAM_CHECKPOINT_NAME); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, NULL, MND_STREAM_CHECKPOINT_NAME);
if (pTrans == NULL) { if (pTrans == NULL) {
return -1; return -1;
} }
mndStreamRegisterTrans(pTrans, MND_STREAM_CHECKPOINT_NAME, pStream->sourceDb, pStream->targetDb); mndStreamRegisterTrans(pTrans, MND_STREAM_CHECKPOINT_NAME, pStream->uid);
mndTransSetDbName(pTrans, pStream->sourceDb, pStream->targetDb); mndTransSetDbName(pTrans, pStream->sourceDb, pStream->targetSTbName);
if (mndTrancCheckConflict(pMnode, pTrans) != 0) { if (mndTrancCheckConflict(pMnode, pTrans) != 0) {
mError("failed to checkpoint of stream name%s, checkpointId: %" PRId64 ", reason:%s", pStream->name, checkpointId, mError("failed to checkpoint of stream name%s, checkpointId: %" PRId64 ", reason:%s", pStream->name, checkpointId,
tstrerror(TSDB_CODE_MND_TRANS_CONFLICT)); tstrerror(TSDB_CODE_MND_TRANS_CONFLICT));
@ -1384,14 +1376,14 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) {
} }
// check if it is conflict with other trans in both sourceDb and targetDb. // check if it is conflict with other trans in both sourceDb and targetDb.
bool conflict = streamTransConflictOtherTrans(pMnode, pStream->sourceDb, pStream->targetDb, true); bool conflict = streamTransConflictOtherTrans(pMnode, pStream->uid, MND_STREAM_DROP_NAME, true);
if (conflict) { if (conflict) {
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
tFreeMDropStreamReq(&dropReq); tFreeMDropStreamReq(&dropReq);
return -1; return -1;
} }
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, pReq, MND_STREAM_DROP_NAME); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq, MND_STREAM_DROP_NAME);
if (pTrans == NULL) { if (pTrans == NULL) {
mError("stream:%s, failed to drop since %s", dropReq.name, terrstr()); mError("stream:%s, failed to drop since %s", dropReq.name, terrstr());
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
@ -1401,7 +1393,7 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) {
mInfo("trans:%d used to drop stream:%s", pTrans->id, dropReq.name); mInfo("trans:%d used to drop stream:%s", pTrans->id, dropReq.name);
mndTransSetDbName(pTrans, pStream->sourceDb, pStream->targetDb); mndTransSetDbName(pTrans, pStream->sourceDb, pStream->targetSTbName);
if (mndTransCheckConflict(pMnode, pTrans) != 0) { if (mndTransCheckConflict(pMnode, pTrans) != 0) {
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
@ -1409,7 +1401,7 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) {
return -1; return -1;
} }
int32_t code = mndStreamRegisterTrans(pTrans, MND_STREAM_DROP_NAME, pStream->sourceDb, pStream->targetDb); int32_t code = mndStreamRegisterTrans(pTrans, MND_STREAM_DROP_NAME, pStream->uid);
// drop all tasks // drop all tasks
if (mndDropStreamTasks(pMnode, pTrans, pStream) < 0) { if (mndDropStreamTasks(pMnode, pTrans, pStream) < 0) {
@ -1868,7 +1860,7 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) {
} }
// check if it is conflict with other trans in both sourceDb and targetDb. // check if it is conflict with other trans in both sourceDb and targetDb.
bool conflict = streamTransConflictOtherTrans(pMnode, pStream->sourceDb, pStream->targetDb, true); bool conflict = streamTransConflictOtherTrans(pMnode, pStream->uid, MND_STREAM_PAUSE_NAME, true);
if (conflict) { if (conflict) {
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
return -1; return -1;
@ -1880,7 +1872,7 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) {
return -1; return -1;
} }
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, pReq, "pause-stream"); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq, MND_STREAM_PAUSE_NAME);
if (pTrans == NULL) { if (pTrans == NULL) {
mError("stream:%s failed to pause stream since %s", pauseReq.name, terrstr()); mError("stream:%s failed to pause stream since %s", pauseReq.name, terrstr());
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
@ -1889,14 +1881,14 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) {
mInfo("trans:%d, used to pause stream:%s", pTrans->id, pauseReq.name); mInfo("trans:%d, used to pause stream:%s", pTrans->id, pauseReq.name);
mndTransSetDbName(pTrans, pStream->sourceDb, pStream->targetDb); mndTransSetDbName(pTrans, pStream->sourceDb, pStream->targetSTbName);
if (mndTransCheckConflict(pMnode, pTrans) != 0) { if (mndTransCheckConflict(pMnode, pTrans) != 0) {
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return -1;
} }
int32_t code = mndStreamRegisterTrans(pTrans, MND_STREAM_PAUSE_NAME, pStream->sourceDb, pStream->targetDb); int32_t code = mndStreamRegisterTrans(pTrans, MND_STREAM_PAUSE_NAME, pStream->uid);
// if nodeUpdate happened, not send pause trans // if nodeUpdate happened, not send pause trans
if (mndPauseAllStreamTasks(pMnode, pTrans, pStream) < 0) { if (mndPauseAllStreamTasks(pMnode, pTrans, pStream) < 0) {
@ -2003,13 +1995,13 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) {
} }
// check if it is conflict with other trans in both sourceDb and targetDb. // check if it is conflict with other trans in both sourceDb and targetDb.
bool conflict = streamTransConflictOtherTrans(pMnode, pStream->sourceDb, pStream->targetDb, true); bool conflict = streamTransConflictOtherTrans(pMnode, pStream->uid, MND_STREAM_RESUME_NAME, true);
if (conflict) { if (conflict) {
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
return -1; return -1;
} }
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, pReq, MND_STREAM_RESUME_NAME); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq, MND_STREAM_RESUME_NAME);
if (pTrans == NULL) { if (pTrans == NULL) {
mError("stream:%s, failed to resume stream since %s", pauseReq.name, terrstr()); mError("stream:%s, failed to resume stream since %s", pauseReq.name, terrstr());
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
@ -2018,14 +2010,14 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) {
mInfo("trans:%d used to resume stream:%s", pTrans->id, pauseReq.name); mInfo("trans:%d used to resume stream:%s", pTrans->id, pauseReq.name);
mndTransSetDbName(pTrans, pStream->sourceDb, pStream->targetDb); mndTransSetDbName(pTrans, pStream->sourceDb, pStream->targetSTbName);
if (mndTransCheckConflict(pMnode, pTrans) != 0) { if (mndTransCheckConflict(pMnode, pTrans) != 0) {
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return -1;
} }
int32_t code = mndStreamRegisterTrans(pTrans, MND_STREAM_RESUME_NAME, pStream->sourceDb, pStream->targetDb); int32_t code = mndStreamRegisterTrans(pTrans, MND_STREAM_RESUME_NAME, pStream->uid);
// resume all tasks // resume all tasks
if (mndResumeAllStreamTasks(pTrans, pMnode, pStream, pauseReq.igUntreated) < 0) { if (mndResumeAllStreamTasks(pTrans, pMnode, pStream, pauseReq.igUntreated) < 0) {
@ -2662,7 +2654,7 @@ void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) {
} }
STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, const char *name, const char *pMsg) { STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, const char *name, const char *pMsg) {
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, pReq, name); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq, name);
if (pTrans == NULL) { if (pTrans == NULL) {
mError("failed to build trans:%s, reason: %s", name, tstrerror(TSDB_CODE_OUT_OF_MEMORY)); mError("failed to build trans:%s, reason: %s", name, tstrerror(TSDB_CODE_OUT_OF_MEMORY));
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -2671,7 +2663,7 @@ STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, const
mDebug("s-task:0x%" PRIx64 " start to build trans %s", pStream->uid, pMsg); mDebug("s-task:0x%" PRIx64 " start to build trans %s", pStream->uid, pMsg);
mndTransSetDbName(pTrans, pStream->sourceDb, pStream->targetDb); mndTransSetDbName(pTrans, pStream->sourceDb, pStream->targetSTbName);
if (mndTransCheckConflict(pMnode, pTrans) != 0) { if (mndTransCheckConflict(pMnode, pTrans) != 0) {
terrno = TSDB_CODE_MND_TRANS_CONFLICT; terrno = TSDB_CODE_MND_TRANS_CONFLICT;
mError("failed to build trans:%s for stream:0x%" PRIx64 " code:%s", name, pStream->uid, tstrerror(terrno)); mError("failed to build trans:%s for stream:0x%" PRIx64 " code:%s", name, pStream->uid, tstrerror(terrno));
@ -2764,7 +2756,10 @@ int32_t killActiveCheckpointTrans(SMnode *pMnode, const char *pDBName, size_t le
STrans *pTrans = mndAcquireTrans(pMnode, pTransInfo->transId); STrans *pTrans = mndAcquireTrans(pMnode, pTransInfo->transId);
if (pTrans != NULL) { if (pTrans != NULL) {
mInfo("kill checkpoint transId:%d in Db:%s", pTransInfo->transId, pDBName); char* pDupDBName = strndup(pDBName, len);
mInfo("kill checkpoint transId:%d in Db:%s", pTransInfo->transId, pDupDBName);
taosMemoryFree(pDupDBName);
mndKillTrans(pMnode, pTrans); mndKillTrans(pMnode, pTrans);
mndReleaseTrans(pMnode, pTrans); mndReleaseTrans(pMnode, pTrans);
} }
@ -2790,7 +2785,7 @@ static int32_t mndResetStatusFromCheckpoint(SMnode *pMnode, int32_t transId) {
break; break;
} }
bool conflict = streamTransConflictOtherTrans(pMnode, pStream->sourceDb, pStream->targetDb, false); bool conflict = streamTransConflictOtherTrans(pMnode, pStream->uid, MND_STREAM_TASK_RESET_NAME, false);
if (conflict) { if (conflict) {
mError("stream:%s other trans exists in DB:%s & %s failed to start reset-status trans", pStream->name, mError("stream:%s other trans exists in DB:%s & %s failed to start reset-status trans", pStream->name,
pStream->sourceDb, pStream->targetDb); pStream->sourceDb, pStream->targetDb);

42
source/dnode/mnode/impl/src/mndStreamTrans.c
View File

@ -23,21 +23,17 @@ typedef struct SKeyInfo {
static int32_t clearFinishedTrans(SMnode* pMnode); static int32_t clearFinishedTrans(SMnode* pMnode);
int32_t mndStreamRegisterTrans(STrans* pTrans, const char* pName, const char* pSrcDb, const char* pDstDb) { int32_t mndStreamRegisterTrans(STrans* pTrans, const char* pTransName, int64_t streamUid) {
SStreamTransInfo info = {.transId = pTrans->id, .startTime = taosGetTimestampMs(), .name = pName}; SStreamTransInfo info = {
taosHashPut(execInfo.transMgmt.pDBTrans, pSrcDb, strlen(pSrcDb), &info, sizeof(SStreamTransInfo)); .transId = pTrans->id, .startTime = taosGetTimestampMs(), .name = pTransName, .streamUid = streamUid};
taosHashPut(execInfo.transMgmt.pDBTrans, &streamUid, sizeof(streamUid), &info, sizeof(SStreamTransInfo));
if (strcmp(pSrcDb, pDstDb) != 0) {
taosHashPut(execInfo.transMgmt.pDBTrans, pDstDb, strlen(pDstDb), &info, sizeof(SStreamTransInfo));
}
return 0; return 0;
} }
int32_t clearFinishedTrans(SMnode* pMnode) { int32_t clearFinishedTrans(SMnode* pMnode) {
size_t keyLen = 0; size_t keyLen = 0;
SArray* pList = taosArrayInit(4, sizeof(SKeyInfo));
void* pIter = NULL; void* pIter = NULL;
SArray* pList = taosArrayInit(4, sizeof(SKeyInfo));
while ((pIter = taosHashIterate(execInfo.transMgmt.pDBTrans, pIter)) != NULL) { while ((pIter = taosHashIterate(execInfo.transMgmt.pDBTrans, pIter)) != NULL) {
SStreamTransInfo* pEntry = (SStreamTransInfo*)pIter; SStreamTransInfo* pEntry = (SStreamTransInfo*)pIter;
@ -69,7 +65,7 @@ int32_t clearFinishedTrans(SMnode* pMnode) {
return 0; return 0;
} }
bool streamTransConflictOtherTrans(SMnode* pMnode, const char* pSrcDb, const char* pDstDb, bool lock) { bool streamTransConflictOtherTrans(SMnode* pMnode, int64_t streamUid, const char* pTransName, bool lock) {
if (lock) { if (lock) {
taosThreadMutexLock(&execInfo.lock); taosThreadMutexLock(&execInfo.lock);
} }
@ -84,22 +80,26 @@ bool streamTransConflictOtherTrans(SMnode* pMnode, const char* pSrcDb, const cha
clearFinishedTrans(pMnode); clearFinishedTrans(pMnode);
SStreamTransInfo *pEntry = taosHashGet(execInfo.transMgmt.pDBTrans, pSrcDb, strlen(pSrcDb)); SStreamTransInfo *pEntry = taosHashGet(execInfo.transMgmt.pDBTrans, &streamUid, sizeof(streamUid));
if (pEntry != NULL) { if (pEntry != NULL) {
if (lock) { SStreamTransInfo tInfo = *pEntry;
taosThreadMutexUnlock(&execInfo.lock);
}
mWarn("conflict with other transId:%d in Db:%s, trans:%s", pEntry->transId, pSrcDb, pEntry->name);
return true;
}
pEntry = taosHashGet(execInfo.transMgmt.pDBTrans, pDstDb, strlen(pDstDb));
if (pEntry != NULL) {
if (lock) { if (lock) {
taosThreadMutexUnlock(&execInfo.lock); taosThreadMutexUnlock(&execInfo.lock);
} }
mWarn("conflict with other transId:%d in Db:%s, trans:%s", pEntry->transId, pSrcDb, pEntry->name);
return true; if (strcmp(tInfo.name, MND_STREAM_CHECKPOINT_NAME) == 0) {
if (strcmp(pTransName, MND_STREAM_DROP_NAME) != 0) {
mWarn("conflict with other transId:%d streamUid:%" PRIx64 ", trans:%s", tInfo.transId, tInfo.streamUid,
tInfo.name);
return true;
}
} else if ((strcmp(tInfo.name, MND_STREAM_CREATE_NAME) == 0) ||
(strcmp(tInfo.name, MND_STREAM_DROP_NAME) == 0)) {
mWarn("conflict with other transId:%d streamUid:%" PRIx64 ", trans:%s", tInfo.transId, tInfo.streamUid,
tInfo.name);
return true;
}
} }
if (lock) { if (lock) {

22
source/dnode/mnode/sdb/inc/sdb.h
View File

@ -150,7 +150,9 @@ typedef enum {
SDB_IDX = 21, SDB_IDX = 21,
SDB_VIEW = 22, SDB_VIEW = 22,
SDB_STREAM_SEQ = 23, SDB_STREAM_SEQ = 23,
SDB_MAX = 24 SDB_COMPACT = 24,
SDB_COMPACT_DETAIL = 25,
SDB_MAX = 26
} ESdbType; } ESdbType;
typedef struct SSdbRaw { typedef struct SSdbRaw {
@ -170,11 +172,11 @@ typedef struct SSdbRow {
} SSdbRow; } SSdbRow;
typedef struct SSdb { typedef struct SSdb {
SMnode * pMnode; SMnode *pMnode;
SWal * pWal; SWal *pWal;
int64_t sync; int64_t sync;
char * currDir; char *currDir;
char * tmpDir; char *tmpDir;
int64_t commitIndex; int64_t commitIndex;
int64_t commitTerm; int64_t commitTerm;
int64_t commitConfig; int64_t commitConfig;
@ -184,7 +186,7 @@ typedef struct SSdb {
int64_t tableVer[SDB_MAX]; int64_t tableVer[SDB_MAX];
int64_t maxId[SDB_MAX]; int64_t maxId[SDB_MAX];
EKeyType keyTypes[SDB_MAX]; EKeyType keyTypes[SDB_MAX];
SHashObj * hashObjs[SDB_MAX]; SHashObj *hashObjs[SDB_MAX];
TdThreadRwlock locks[SDB_MAX]; TdThreadRwlock locks[SDB_MAX];
SdbInsertFp insertFps[SDB_MAX]; SdbInsertFp insertFps[SDB_MAX];
SdbUpdateFp updateFps[SDB_MAX]; SdbUpdateFp updateFps[SDB_MAX];
@ -199,7 +201,7 @@ typedef struct SSdb {
typedef struct SSdbIter { typedef struct SSdbIter {
TdFilePtr file; TdFilePtr file;
int64_t total; int64_t total;
char * name; char *name;
} SSdbIter; } SSdbIter;
typedef struct { typedef struct {
@ -216,8 +218,8 @@ typedef struct {
typedef struct SSdbOpt { typedef struct SSdbOpt {
const char *path; const char *path;
SMnode * pMnode; SMnode *pMnode;
SWal * pWal; SWal *pWal;
int64_t sync; int64_t sync;
} SSdbOpt; } SSdbOpt;
@ -394,7 +396,7 @@ int32_t sdbGetRawSoftVer(SSdbRaw *pRaw, int8_t *sver);
int32_t sdbGetRawTotalSize(SSdbRaw *pRaw); int32_t sdbGetRawTotalSize(SSdbRaw *pRaw);
SSdbRow *sdbAllocRow(int32_t objSize); SSdbRow *sdbAllocRow(int32_t objSize);
void * sdbGetRowObj(SSdbRow *pRow); void *sdbGetRowObj(SSdbRow *pRow);
void sdbFreeRow(SSdb *pSdb, SSdbRow *pRow, bool callFunc); void sdbFreeRow(SSdb *pSdb, SSdbRow *pRow, bool callFunc);
int32_t sdbStartRead(SSdb *pSdb, SSdbIter **ppIter, int64_t *index, int64_t *term, int64_t *config); int32_t sdbStartRead(SSdb *pSdb, SSdbIter **ppIter, int64_t *index, int64_t *term, int64_t *config);

6
source/dnode/mnode/sdb/src/sdbHash.c
View File

@ -64,7 +64,11 @@ const char *sdbTableName(ESdbType type) {
return "idx"; return "idx";
case SDB_VIEW: case SDB_VIEW:
return "view"; return "view";
default: case SDB_COMPACT:
return "compact";
case SDB_COMPACT_DETAIL:
return "compact_detail";
default:
return "undefine"; return "undefine";
} }
} }

1
source/dnode/snode/src/snode.c
View File

@ -66,6 +66,7 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProcessVer
} else { } else {
sndDebug("s-task:%s state:%p", pTask->id.idStr, pTask->pState); sndDebug("s-task:%s state:%p", pTask->id.idStr, pTask->pState);
} }
int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTask->upstreamInfo.pList); int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTask->upstreamInfo.pList);
SReadHandle handle = { SReadHandle handle = {

5
source/dnode/snode/src/snodeInitApi.c
View File

@ -14,8 +14,8 @@
*/ */
#include "storageapi.h" #include "storageapi.h"
#include "tstreamUpdate.h"
#include "streamState.h" #include "streamState.h"
#include "tstreamUpdate.h"
static void initStateStoreAPI(SStateStore* pStore); static void initStateStoreAPI(SStateStore* pStore);
static void initFunctionStateStore(SFunctionStateStore* pStore); static void initFunctionStateStore(SFunctionStateStore* pStore);
@ -100,9 +100,10 @@ void initStateStoreAPI(SStateStore* pStore) {
pStore->streamStateClose = streamStateClose; pStore->streamStateClose = streamStateClose;
pStore->streamStateBegin = streamStateBegin; pStore->streamStateBegin = streamStateBegin;
pStore->streamStateCommit = streamStateCommit; pStore->streamStateCommit = streamStateCommit;
pStore->streamStateDestroy= streamStateDestroy; pStore->streamStateDestroy = streamStateDestroy;
pStore->streamStateDeleteCheckPoint = streamStateDeleteCheckPoint; pStore->streamStateDeleteCheckPoint = streamStateDeleteCheckPoint;
pStore->streamStateReloadInfo = streamStateReloadInfo; pStore->streamStateReloadInfo = streamStateReloadInfo;
pStore->streamStateCopyBackend = streamStateCopyBackend;
} }
void initFunctionStateStore(SFunctionStateStore* pStore) { void initFunctionStateStore(SFunctionStateStore* pStore) {

1
source/dnode/vnode/CMakeLists.txt
View File

@ -93,6 +93,7 @@ IF (TD_VNODE_PLUGINS)
vnode vnode
PRIVATE PRIVATE
${TD_ENTERPRISE_DIR}/src/plugins/vnode/src/tsdbCompact.c ${TD_ENTERPRISE_DIR}/src/plugins/vnode/src/tsdbCompact.c
${TD_ENTERPRISE_DIR}/src/plugins/vnode/src/tsdbCompactMonitor.c
${TD_ENTERPRISE_DIR}/src/plugins/vnode/src/vnodeCompact.c ${TD_ENTERPRISE_DIR}/src/plugins/vnode/src/vnodeCompact.c
) )
ENDIF () ENDIF ()

3
source/dnode/vnode/src/inc/tsdb.h
View File

@ -74,6 +74,7 @@ typedef struct SDiskDataBuilder SDiskDataBuilder;
typedef struct SBlkInfo SBlkInfo; typedef struct SBlkInfo SBlkInfo;
typedef struct STsdbDataIter2 STsdbDataIter2; typedef struct STsdbDataIter2 STsdbDataIter2;
typedef struct STsdbFilterInfo STsdbFilterInfo; typedef struct STsdbFilterInfo STsdbFilterInfo;
typedef struct STFileSystem STFileSystem;
#define TSDBROW_ROW_FMT ((int8_t)0x0) #define TSDBROW_ROW_FMT ((int8_t)0x0)
#define TSDBROW_COL_FMT ((int8_t)0x1) #define TSDBROW_COL_FMT ((int8_t)0x1)
@ -392,6 +393,8 @@ struct STsdb {
TdThreadMutex pgMutex; TdThreadMutex pgMutex;
struct STFileSystem *pFS; // new struct STFileSystem *pFS; // new
SRocksCache rCache; SRocksCache rCache;
// compact monitor
struct SCompMonitor *pCompMonitor;
}; };
struct TSDBKEY { struct TSDBKEY {

4
source/dnode/vnode/src/tq/tqOffset.c
View File

@ -47,6 +47,7 @@ int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname) {
} }
} }
size = htonl(size);
void* pMemBuf = taosMemoryCalloc(1, size); void* pMemBuf = taosMemoryCalloc(1, size);
if (pMemBuf == NULL) { if (pMemBuf == NULL) {
tqError("vgId:%d failed to restore offset from file, since out of memory, malloc size:%d", vgId, size); tqError("vgId:%d failed to restore offset from file, since out of memory, malloc size:%d", vgId, size);
@ -119,6 +120,7 @@ STqOffsetStore* tqOffsetOpen(STQ* pTq) {
} }
void tqOffsetClose(STqOffsetStore* pStore) { void tqOffsetClose(STqOffsetStore* pStore) {
if(pStore == NULL) return;
tqOffsetCommitFile(pStore); tqOffsetCommitFile(pStore);
taosHashCleanup(pStore->pHash); taosHashCleanup(pStore->pHash);
taosMemoryFree(pStore); taosMemoryFree(pStore);
@ -176,7 +178,7 @@ int32_t tqOffsetCommitFile(STqOffsetStore* pStore) {
void* buf = taosMemoryCalloc(1, totLen); void* buf = taosMemoryCalloc(1, totLen);
void* abuf = POINTER_SHIFT(buf, INT_BYTES); void* abuf = POINTER_SHIFT(buf, INT_BYTES);
*(int32_t*)buf = bodyLen; *(int32_t*)buf = htonl(bodyLen);
SEncoder encoder; SEncoder encoder;
tEncoderInit(&encoder, abuf, bodyLen); tEncoderInit(&encoder, abuf, bodyLen);
tEncodeSTqOffset(&encoder, pOffset); tEncodeSTqOffset(&encoder, pOffset);

93
source/dnode/vnode/src/tqCommon/tqCommon.c
View File

@ -13,9 +13,9 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "tstream.h"
#include "tmsgcb.h" #include "tmsgcb.h"
#include "tq.h" #include "tq.h"
#include "tstream.h"
typedef struct STaskUpdateEntry { typedef struct STaskUpdateEntry {
int64_t streamId; int64_t streamId;
@ -24,7 +24,7 @@ typedef struct STaskUpdateEntry {
} STaskUpdateEntry; } STaskUpdateEntry;
int32_t tqStreamTaskStartAsync(SStreamMeta* pMeta, SMsgCb* cb, bool restart) { int32_t tqStreamTaskStartAsync(SStreamMeta* pMeta, SMsgCb* cb, bool restart) {
int32_t vgId = pMeta->vgId; int32_t vgId = pMeta->vgId;
int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList);
if (numOfTasks == 0) { if (numOfTasks == 0) {
@ -42,7 +42,7 @@ int32_t tqStreamTaskStartAsync(SStreamMeta* pMeta, SMsgCb* cb, bool restart) {
tqDebug("vgId:%d start all %d stream task(s) async", vgId, numOfTasks); tqDebug("vgId:%d start all %d stream task(s) async", vgId, numOfTasks);
pRunReq->head.vgId = vgId; pRunReq->head.vgId = vgId;
pRunReq->streamId = 0; pRunReq->streamId = 0;
pRunReq->taskId = restart? STREAM_EXEC_RESTART_ALL_TASKS_ID:STREAM_EXEC_START_ALL_TASKS_ID; pRunReq->taskId = restart ? STREAM_EXEC_RESTART_ALL_TASKS_ID : STREAM_EXEC_START_ALL_TASKS_ID;
SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_RUN, .pCont = pRunReq, .contLen = sizeof(SStreamTaskRunReq)}; SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_RUN, .pCont = pRunReq, .contLen = sizeof(SStreamTaskRunReq)};
tmsgPutToQueue(cb, STREAM_QUEUE, &msg); tmsgPutToQueue(cb, STREAM_QUEUE, &msg);
@ -50,10 +50,10 @@ int32_t tqStreamTaskStartAsync(SStreamMeta* pMeta, SMsgCb* cb, bool restart) {
} }
int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pMsg, bool restored) { int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pMsg, bool restored) {
int32_t vgId = pMeta->vgId; int32_t vgId = pMeta->vgId;
char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
int32_t len = pMsg->contLen - sizeof(SMsgHead); int32_t len = pMsg->contLen - sizeof(SMsgHead);
SRpcMsg rsp = {.info = pMsg->info, .code = TSDB_CODE_SUCCESS}; SRpcMsg rsp = {.info = pMsg->info, .code = TSDB_CODE_SUCCESS};
SStreamTaskNodeUpdateMsg req = {0}; SStreamTaskNodeUpdateMsg req = {0};
@ -72,7 +72,7 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM
streamMetaWLock(pMeta); streamMetaWLock(pMeta);
// the task epset may be updated again and again, when replaying the WAL, the task may be in stop status. // the task epset may be updated again and again, when replaying the WAL, the task may be in stop status.
STaskId id = {.streamId = req.streamId, .taskId = req.taskId}; STaskId id = {.streamId = req.streamId, .taskId = req.taskId};
SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id));
if (ppTask == NULL || *ppTask == NULL) { if (ppTask == NULL || *ppTask == NULL) {
tqError("vgId:%d failed to acquire task:0x%x when handling update, it may have been dropped already", pMeta->vgId, tqError("vgId:%d failed to acquire task:0x%x when handling update, it may have been dropped already", pMeta->vgId,
@ -96,7 +96,7 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM
} }
STaskUpdateEntry entry = {.streamId = req.streamId, .taskId = req.taskId, .transId = req.transId}; STaskUpdateEntry entry = {.streamId = req.streamId, .taskId = req.taskId, .transId = req.transId};
void* exist = taosHashGet(pMeta->updateInfo.pTasks, &entry, sizeof(STaskUpdateEntry)); void* exist = taosHashGet(pMeta->updateInfo.pTasks, &entry, sizeof(STaskUpdateEntry));
if (exist != NULL) { if (exist != NULL) {
tqDebug("s-task:%s (vgId:%d) already update in trans:%d, discard the nodeEp update msg", pTask->id.idStr, vgId, tqDebug("s-task:%s (vgId:%d) already update in trans:%d, discard the nodeEp update msg", pTask->id.idStr, vgId,
req.transId); req.transId);
@ -166,7 +166,8 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM
streamMetaWUnLock(pMeta); streamMetaWUnLock(pMeta);
} else { } else {
if (!restored) { if (!restored) {
tqDebug("vgId:%d vnode restore not completed, not restart the tasks, clear the start after nodeUpdate flag", vgId); tqDebug("vgId:%d vnode restore not completed, not restart the tasks, clear the start after nodeUpdate flag",
vgId);
pMeta->startInfo.tasksWillRestart = 0; pMeta->startInfo.tasksWillRestart = 0;
streamMetaWUnLock(pMeta); streamMetaWUnLock(pMeta);
} else { } else {
@ -238,7 +239,7 @@ int32_t tqStreamTaskProcessDispatchReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
SStreamDispatchReq req = {0}; SStreamDispatchReq req = {0};
SDecoder decoder; SDecoder decoder;
tDecoderInit(&decoder, (uint8_t*)msgBody, msgLen); tDecoderInit(&decoder, (uint8_t*)msgBody, msgLen);
if (tDecodeStreamDispatchReq(&decoder, &req) < 0) { if (tDecodeStreamDispatchReq(&decoder, &req) < 0) {
tDecoderClear(&decoder); tDecoderClear(&decoder);
@ -251,7 +252,7 @@ int32_t tqStreamTaskProcessDispatchReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, req.taskId); SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, req.taskId);
if (pTask) { if (pTask) {
SRpcMsg rsp = {.info = pMsg->info, .code = 0}; SRpcMsg rsp = {.info = pMsg->info, .code = 0};
if (streamProcessDispatchMsg(pTask, &req, &rsp) != 0){ if (streamProcessDispatchMsg(pTask, &req, &rsp) != 0) {
return -1; return -1;
} }
tDeleteStreamDispatchReq(&req); tDeleteStreamDispatchReq(&req);
@ -355,8 +356,8 @@ int32_t tqStreamTaskProcessScanHistoryFinishReq(SStreamMeta* pMeta, SRpcMsg* pMs
SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, req.downstreamTaskId); SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, req.downstreamTaskId);
if (pTask == NULL) { if (pTask == NULL) {
tqError("vgId:%d process scan history finish msg, failed to find task:0x%x, it may be destroyed", tqError("vgId:%d process scan history finish msg, failed to find task:0x%x, it may be destroyed", pMeta->vgId,
pMeta->vgId, req.downstreamTaskId); req.downstreamTaskId);
return -1; return -1;
} }
@ -381,8 +382,8 @@ int32_t tqStreamTaskProcessScanHistoryFinishRsp(SStreamMeta* pMeta, SRpcMsg* pMs
SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, req.upstreamTaskId); SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, req.upstreamTaskId);
if (pTask == NULL) { if (pTask == NULL) {
tqError("vgId:%d process scan history finish rsp, failed to find task:0x%x, it may be destroyed", tqError("vgId:%d process scan history finish rsp, failed to find task:0x%x, it may be destroyed", pMeta->vgId,
pMeta->vgId, req.upstreamTaskId); req.upstreamTaskId);
return -1; return -1;
} }
@ -428,8 +429,9 @@ int32_t tqStreamTaskProcessCheckReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
// only the leader node handle the check request // only the leader node handle the check request
if (pMeta->role == NODE_ROLE_FOLLOWER) { if (pMeta->role == NODE_ROLE_FOLLOWER) {
tqError("s-task:0x%x invalid check msg from upstream:0x%x(vgId:%d), vgId:%d is follower, not handle check status msg", tqError(
taskId, req.upstreamTaskId, req.upstreamNodeId, pMeta->vgId); "s-task:0x%x invalid check msg from upstream:0x%x(vgId:%d), vgId:%d is follower, not handle check status msg",
taskId, req.upstreamTaskId, req.upstreamNodeId, pMeta->vgId);
rsp.status = TASK_DOWNSTREAM_NOT_LEADER; rsp.status = TASK_DOWNSTREAM_NOT_LEADER;
} else { } else {
SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, taskId); SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, taskId);
@ -439,13 +441,14 @@ int32_t tqStreamTaskProcessCheckReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
char* p = NULL; char* p = NULL;
streamTaskGetStatus(pTask, &p); streamTaskGetStatus(pTask, &p);
tqDebug("s-task:%s status:%s, stage:%"PRId64" recv task check req(reqId:0x%" PRIx64 ") task:0x%x (vgId:%d), check_status:%d", tqDebug("s-task:%s status:%s, stage:%" PRId64 " recv task check req(reqId:0x%" PRIx64
pTask->id.idStr, p, rsp.oldStage, rsp.reqId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status); ") task:0x%x (vgId:%d), check_status:%d",
pTask->id.idStr, p, rsp.oldStage, rsp.reqId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status);
} else { } else {
rsp.status = TASK_DOWNSTREAM_NOT_READY; rsp.status = TASK_DOWNSTREAM_NOT_READY;
tqDebug("tq recv task check(taskId:0x%" PRIx64 "-0x%x not built yet) req(reqId:0x%" PRIx64 tqDebug("tq recv task check(taskId:0x%" PRIx64 "-0x%x not built yet) req(reqId:0x%" PRIx64
") from task:0x%x (vgId:%d), rsp check_status %d", ") from task:0x%x (vgId:%d), rsp check_status %d",
req.streamId, taskId, rsp.reqId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status); req.streamId, taskId, rsp.reqId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status);
} }
} }
@ -472,7 +475,7 @@ int32_t tqStreamTaskProcessCheckRsp(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLe
tDecoderClear(&decoder); tDecoderClear(&decoder);
tqDebug("tq task:0x%x (vgId:%d) recv check rsp(reqId:0x%" PRIx64 ") from 0x%x (vgId:%d) status %d", tqDebug("tq task:0x%x (vgId:%d) recv check rsp(reqId:0x%" PRIx64 ") from 0x%x (vgId:%d) status %d",
rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.status); rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.status);
if (!isLeader) { if (!isLeader) {
streamMetaUpdateTaskDownstreamStatus(pMeta, rsp.streamId, rsp.upstreamTaskId, 0, taosGetTimestampMs(), false); streamMetaUpdateTaskDownstreamStatus(pMeta, rsp.streamId, rsp.upstreamTaskId, 0, taosGetTimestampMs(), false);
@ -485,7 +488,7 @@ int32_t tqStreamTaskProcessCheckRsp(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLe
if (pTask == NULL) { if (pTask == NULL) {
streamMetaUpdateTaskDownstreamStatus(pMeta, rsp.streamId, rsp.upstreamTaskId, 0, taosGetTimestampMs(), false); streamMetaUpdateTaskDownstreamStatus(pMeta, rsp.streamId, rsp.upstreamTaskId, 0, taosGetTimestampMs(), false);
tqError("tq failed to locate the stream task:0x%" PRIx64 "-0x%x (vgId:%d), it may have been destroyed or stopped", tqError("tq failed to locate the stream task:0x%" PRIx64 "-0x%x (vgId:%d), it may have been destroyed or stopped",
rsp.streamId, rsp.upstreamTaskId, vgId); rsp.streamId, rsp.upstreamTaskId, vgId);
terrno = TSDB_CODE_STREAM_TASK_NOT_EXIST; terrno = TSDB_CODE_STREAM_TASK_NOT_EXIST;
return -1; return -1;
} }
@ -496,10 +499,10 @@ int32_t tqStreamTaskProcessCheckRsp(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLe
} }
int32_t tqStreamTaskProcessCheckpointReadyMsg(SStreamMeta* pMeta, SRpcMsg* pMsg) { int32_t tqStreamTaskProcessCheckpointReadyMsg(SStreamMeta* pMeta, SRpcMsg* pMsg) {
int32_t vgId = pMeta->vgId; int32_t vgId = pMeta->vgId;
char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
int32_t len = pMsg->contLen - sizeof(SMsgHead); int32_t len = pMsg->contLen - sizeof(SMsgHead);
int32_t code = 0; int32_t code = 0;
SStreamCheckpointReadyMsg req = {0}; SStreamCheckpointReadyMsg req = {0};
@ -526,7 +529,8 @@ int32_t tqStreamTaskProcessCheckpointReadyMsg(SStreamMeta* pMeta, SRpcMsg* pMsg)
return code; return code;
} }
int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, int64_t sversion, char* msg, int32_t msgLen, bool isLeader, bool restored) { int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, int64_t sversion, char* msg, int32_t msgLen, bool isLeader,
bool restored) {
int32_t code = 0; int32_t code = 0;
int32_t vgId = pMeta->vgId; int32_t vgId = pMeta->vgId;
@ -538,7 +542,7 @@ int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, int64_t sversion, char*
tqDebug("vgId:%d receive new stream task deploy msg, start to build stream task", vgId); tqDebug("vgId:%d receive new stream task deploy msg, start to build stream task", vgId);
// 1.deserialize msg and build task // 1.deserialize msg and build task
int32_t size = sizeof(SStreamTask); int32_t size = sizeof(SStreamTask);
SStreamTask* pTask = taosMemoryCalloc(1, size); SStreamTask* pTask = taosMemoryCalloc(1, size);
if (pTask == NULL) { if (pTask == NULL) {
tqError("vgId:%d failed to create stream task due to out of memory, alloc size:%d", vgId, size); tqError("vgId:%d failed to create stream task due to out of memory, alloc size:%d", vgId, size);
@ -566,7 +570,8 @@ int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, int64_t sversion, char*
streamMetaWUnLock(pMeta); streamMetaWUnLock(pMeta);
if (code < 0) { if (code < 0) {
tqError("failed to add s-task:0x%x into vgId:%d meta, total:%d, code:%s", vgId, taskId, numOfTasks, tstrerror(code)); tqError("failed to add s-task:0x%x into vgId:%d meta, total:%d, code:%s", vgId, taskId, numOfTasks,
tstrerror(code));
tFreeStreamTask(pTask); tFreeStreamTask(pTask);
return code; return code;
} }
@ -603,7 +608,7 @@ int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, int64_t sversion, char*
int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen) { int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen) {
SVDropStreamTaskReq* pReq = (SVDropStreamTaskReq*)msg; SVDropStreamTaskReq* pReq = (SVDropStreamTaskReq*)msg;
int32_t vgId = pMeta->vgId; int32_t vgId = pMeta->vgId;
tqDebug("vgId:%d receive msg to drop s-task:0x%x", vgId, pReq->taskId); tqDebug("vgId:%d receive msg to drop s-task:0x%x", vgId, pReq->taskId);
SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId); SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId);
@ -628,14 +633,14 @@ int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen
if (streamMetaCommit(pMeta) < 0) { if (streamMetaCommit(pMeta) < 0) {
// persist to disk // persist to disk
} }
streamMetaWUnLock(pMeta);
streamMetaWUnLock(pMeta);
return 0; return 0;
} }
int32_t startStreamTasks(SStreamMeta* pMeta) { int32_t startStreamTasks(SStreamMeta* pMeta) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
int32_t vgId = pMeta->vgId; int32_t vgId = pMeta->vgId;
int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList);
tqDebug("vgId:%d start to check all %d stream task(s) downstream status", vgId, numOfTasks); tqDebug("vgId:%d start to check all %d stream task(s) downstream status", vgId, numOfTasks);
@ -679,7 +684,7 @@ int32_t startStreamTasks(SStreamMeta* pMeta) {
} }
EStreamTaskEvent event = (HAS_RELATED_FILLHISTORY_TASK(pTask)) ? TASK_EVENT_INIT_STREAM_SCANHIST : TASK_EVENT_INIT; EStreamTaskEvent event = (HAS_RELATED_FILLHISTORY_TASK(pTask)) ? TASK_EVENT_INIT_STREAM_SCANHIST : TASK_EVENT_INIT;
int32_t ret = streamTaskHandleEvent(pTask->status.pSM, event); int32_t ret = streamTaskHandleEvent(pTask->status.pSM, event);
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
code = ret; code = ret;
} }
@ -692,8 +697,8 @@ int32_t startStreamTasks(SStreamMeta* pMeta) {
} }
int32_t resetStreamTaskStatus(SStreamMeta* pMeta) { int32_t resetStreamTaskStatus(SStreamMeta* pMeta) {
int32_t vgId = pMeta->vgId; int32_t vgId = pMeta->vgId;
int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList);
tqDebug("vgId:%d reset all %d stream task(s) status to be uninit", vgId, numOfTasks); tqDebug("vgId:%d reset all %d stream task(s) status to be uninit", vgId, numOfTasks);
if (numOfTasks == 0) { if (numOfTasks == 0) {
@ -703,7 +708,7 @@ int32_t resetStreamTaskStatus(SStreamMeta* pMeta) {
for (int32_t i = 0; i < numOfTasks; ++i) { for (int32_t i = 0; i < numOfTasks; ++i) {
SStreamTaskId* pTaskId = taosArrayGet(pMeta->pTaskList, i); SStreamTaskId* pTaskId = taosArrayGet(pMeta->pTaskList, i);
STaskId id = {.streamId = pTaskId->streamId, .taskId = pTaskId->taskId}; STaskId id = {.streamId = pTaskId->streamId, .taskId = pTaskId->taskId};
SStreamTask** pTask = taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); SStreamTask** pTask = taosHashGet(pMeta->pTasksMap, &id, sizeof(id));
streamTaskResetStatus(*pTask); streamTaskResetStatus(*pTask);
} }
@ -716,7 +721,7 @@ static int32_t restartStreamTasks(SStreamMeta* pMeta, bool isLeader) {
int32_t code = 0; int32_t code = 0;
int64_t st = taosGetTimestampMs(); int64_t st = taosGetTimestampMs();
while(1) { while (1) {
int32_t startVal = atomic_val_compare_exchange_32(&pMeta->startInfo.taskStarting, 0, 1); int32_t startVal = atomic_val_compare_exchange_32(&pMeta->startInfo.taskStarting, 0, 1);
if (startVal == 0) { if (startVal == 0) {
break; break;
@ -739,7 +744,7 @@ static int32_t restartStreamTasks(SStreamMeta* pMeta, bool isLeader) {
streamMetaClear(pMeta); streamMetaClear(pMeta);
int64_t el = taosGetTimestampMs() - st; int64_t el = taosGetTimestampMs() - st;
tqInfo("vgId:%d close&reload state elapsed time:%.3fs", vgId, el/1000.); tqInfo("vgId:%d close&reload state elapsed time:%.3fs", vgId, el / 1000.);
code = streamMetaLoadAllTasks(pMeta); code = streamMetaLoadAllTasks(pMeta);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
@ -780,11 +785,11 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead
} }
SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, taskId); SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, taskId);
if (pTask != NULL) { // even in halt status, the data in inputQ must be processed if (pTask != NULL) { // even in halt status, the data in inputQ must be processed
char* p = NULL; char* p = NULL;
if (streamTaskReadyToRun(pTask, &p)) { if (streamTaskReadyToRun(pTask, &p)) {
tqDebug("vgId:%d s-task:%s start to process block from inputQ, next checked ver:%" PRId64, vgId, pTask->id.idStr, tqDebug("vgId:%d s-task:%s start to process block from inputQ, next checked ver:%" PRId64, vgId, pTask->id.idStr,
pTask->chkInfo.nextProcessVer); pTask->chkInfo.nextProcessVer);
streamExecTask(pTask); streamExecTask(pTask);
} else { } else {
int8_t status = streamTaskSetSchedStatusInactive(pTask); int8_t status = streamTaskSetSchedStatusInactive(pTask);
@ -800,5 +805,3 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead
return -1; return -1;
} }
} }

5
source/dnode/vnode/src/tsdb/tsdbFS2.c
View File

@ -747,6 +747,7 @@ _exit:
} }
static int32_t tsdbFSSetBlockCommit(STFileSet *fset, bool block); static int32_t tsdbFSSetBlockCommit(STFileSet *fset, bool block);
extern int32_t tsdbStopAllCompTask(STsdb *tsdb);
int32_t tsdbDisableAndCancelAllBgTask(STsdb *pTsdb) { int32_t tsdbDisableAndCancelAllBgTask(STsdb *pTsdb) {
STFileSystem *fs = pTsdb->pFS; STFileSystem *fs = pTsdb->pFS;
@ -774,6 +775,10 @@ int32_t tsdbDisableAndCancelAllBgTask(STsdb *pTsdb) {
int64_t channel; int64_t channel;
TARRAY2_FOREACH(&channelArr, channel) { vnodeAChannelDestroy(vnodeAsyncHandle[1], channel, true); } TARRAY2_FOREACH(&channelArr, channel) { vnodeAChannelDestroy(vnodeAsyncHandle[1], channel, true); }
TARRAY2_DESTROY(&channelArr, NULL); TARRAY2_DESTROY(&channelArr, NULL);
#ifdef TD_ENTERPRISE
tsdbStopAllCompTask(pTsdb);
#endif
return 0; return 0;
} }

1
source/dnode/vnode/src/tsdb/tsdbFSet2.h
View File

@ -27,7 +27,6 @@ typedef struct SSttLvl SSttLvl;
typedef TARRAY2(STFileObj *) TFileObjArray; typedef TARRAY2(STFileObj *) TFileObjArray;
typedef TARRAY2(SSttLvl *) TSttLvlArray; typedef TARRAY2(SSttLvl *) TSttLvlArray;
typedef TARRAY2(STFileOp) TFileOpArray; typedef TARRAY2(STFileOp) TFileOpArray;
typedef struct STFileSystem STFileSystem;
typedef enum { typedef enum {
TSDB_FOP_NONE = 0, TSDB_FOP_NONE = 0,

12
source/dnode/vnode/src/tsdb/tsdbOpen.c
View File

@ -16,6 +16,9 @@
#include "tsdb.h" #include "tsdb.h"
#include "tsdbFS2.h" #include "tsdbFS2.h"
extern int32_t tsdbOpenCompMonitor(STsdb *tsdb);
extern int32_t tsdbCloseCompMonitor(STsdb *tsdb);
int32_t tsdbSetKeepCfg(STsdb *pTsdb, STsdbCfg *pCfg) { int32_t tsdbSetKeepCfg(STsdb *pTsdb, STsdbCfg *pCfg) {
STsdbKeepCfg *pKeepCfg = &pTsdb->keepCfg; STsdbKeepCfg *pKeepCfg = &pTsdb->keepCfg;
pKeepCfg->precision = pCfg->precision; pKeepCfg->precision = pCfg->precision;
@ -81,6 +84,12 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee
goto _err; goto _err;
} }
#ifdef TD_ENTERPRISE
if (tsdbOpenCompMonitor(pTsdb) < 0) {
goto _err;
}
#endif
tsdbDebug("vgId:%d, tsdb is opened at %s, days:%d, keep:%d,%d,%d, keepTimeoffset:%d", TD_VID(pVnode), pTsdb->path, tsdbDebug("vgId:%d, tsdb is opened at %s, days:%d, keep:%d,%d,%d, keepTimeoffset:%d", TD_VID(pVnode), pTsdb->path,
pTsdb->keepCfg.days, pTsdb->keepCfg.keep0, pTsdb->keepCfg.keep1, pTsdb->keepCfg.keep2, pTsdb->keepCfg.days, pTsdb->keepCfg.keep0, pTsdb->keepCfg.keep1, pTsdb->keepCfg.keep2,
pTsdb->keepCfg.keepTimeOffset); pTsdb->keepCfg.keepTimeOffset);
@ -108,6 +117,9 @@ int tsdbClose(STsdb **pTsdb) {
tsdbCloseFS(&(*pTsdb)->pFS); tsdbCloseFS(&(*pTsdb)->pFS);
tsdbCloseCache(*pTsdb); tsdbCloseCache(*pTsdb);
#ifdef TD_ENTERPRISE
tsdbCloseCompMonitor(*pTsdb);
#endif
taosThreadMutexDestroy(&(*pTsdb)->mutex); taosThreadMutexDestroy(&(*pTsdb)->mutex);
taosMemoryFreeClear(*pTsdb); taosMemoryFreeClear(*pTsdb);
} }

200
source/dnode/vnode/src/tsdb/tsdbRead2.c
View File

@ -76,6 +76,8 @@ static void updateComposedBlockInfo(STsdbReader* pReader, double el, ST
static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); } static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); }
static void resetPreFilesetMemTableListIndex(SReaderStatus* pStatus);
static int32_t setColumnIdSlotList(SBlockLoadSuppInfo* pSupInfo, SColumnInfo* pCols, const int32_t* pSlotIdList, static int32_t setColumnIdSlotList(SBlockLoadSuppInfo* pSupInfo, SColumnInfo* pCols, const int32_t* pSlotIdList,
int32_t numOfCols) { int32_t numOfCols) {
pSupInfo->smaValid = true; pSupInfo->smaValid = true;
@ -2347,53 +2349,50 @@ void updateComposedBlockInfo(STsdbReader* pReader, double el, STableBlockScanInf
} }
static int32_t buildComposedDataBlock(STsdbReader* pReader) { static int32_t buildComposedDataBlock(STsdbReader* pReader) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
bool asc = ASCENDING_TRAVERSE(pReader->info.order); bool asc = ASCENDING_TRAVERSE(pReader->info.order);
int64_t st = taosGetTimestampUs(); int64_t st = taosGetTimestampUs();
int32_t step = asc ? 1 : -1; int32_t step = asc ? 1 : -1;
double el = 0; double el = 0;
SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock;
SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter);
SSttBlockReader* pSttBlockReader = pReader->status.fileIter.pSttBlockReader; SSttBlockReader* pSttBlockReader = pReader->status.fileIter.pSttBlockReader;
SBrinRecord* pRecord = &pBlockInfo->record;
SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
SBrinRecord* pRecord = NULL;
STableBlockScanInfo* pBlockScanInfo = NULL; STableBlockScanInfo* pBlockScanInfo = NULL;
if (pBlockInfo != NULL) { if (pBlockInfo == NULL) {
if (pReader->pIgnoreTables && taosHashGet(*pReader->pIgnoreTables, &pBlockInfo->uid, sizeof(pBlockInfo->uid))) { return 0;
setBlockAllDumped(pDumpInfo, pRecord->lastKey, pReader->info.order); }
return code;
}
pBlockScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, pReader->idStr); pRecord = &pBlockInfo->record;
if (pBlockScanInfo == NULL) {
goto _end;
}
pRecord = &pBlockInfo->record; if (pReader->pIgnoreTables && taosHashGet(*pReader->pIgnoreTables, &pBlockInfo->uid, sizeof(pBlockInfo->uid))) {
TSDBKEY keyInBuf = getCurrentKeyInBuf(pBlockScanInfo, pReader); setBlockAllDumped(pDumpInfo, pRecord->lastKey, pReader->info.order);
return code;
}
// it is a clean block, load it directly pBlockScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, pReader->idStr);
int64_t cap = pReader->resBlockInfo.capacity; if (pBlockScanInfo == NULL) {
if (isCleanFileDataBlock(pReader, pBlockInfo, pBlockScanInfo, keyInBuf) && (pRecord->numRow <= cap)) { goto _end;
if (asc || (pBlockScanInfo->sttKeyInfo.status == STT_FILE_NO_DATA)) { }
code = copyBlockDataToSDataBlock(pReader);
if (code) {
goto _end;
}
// record the last key value pRecord = &pBlockInfo->record;
pBlockScanInfo->lastProcKey = asc ? pRecord->lastKey : pRecord->firstKey; TSDBKEY keyInBuf = getCurrentKeyInBuf(pBlockScanInfo, pReader);
// it is a clean block, load it directly
int64_t cap = pReader->resBlockInfo.capacity;
if (isCleanFileDataBlock(pReader, pBlockInfo, pBlockScanInfo, keyInBuf) && (pRecord->numRow <= cap)) {
if (((asc && (pRecord->firstKey < keyInBuf.ts)) || (!asc && (pRecord->lastKey > keyInBuf.ts))) &&
(pBlockScanInfo->sttKeyInfo.status == STT_FILE_NO_DATA)) {
code = copyBlockDataToSDataBlock(pReader);
if (code) {
goto _end; goto _end;
} }
}
} else { // file blocks not exist // record the last key value
ASSERT(0); pBlockScanInfo->lastProcKey = asc ? pRecord->lastKey : pRecord->firstKey;
pBlockScanInfo = *pReader->status.pTableIter; goto _end;
if (pReader->pIgnoreTables &&
taosHashGet(*pReader->pIgnoreTables, &pBlockScanInfo->uid, sizeof(pBlockScanInfo->uid))) {
return code;
} }
} }
@ -2565,7 +2564,9 @@ static void prepareDurationForNextFileSet(STsdbReader* pReader) {
} }
if (pReader->status.bProcMemPreFileset) { if (pReader->status.bProcMemPreFileset) {
resetTableListIndex(&pReader->status); tsdbDebug("will start pre-fileset %d buffer processing. %s", fid, pReader->idStr);
pReader->status.procMemUidList.tableUidList = pReader->status.uidList.tableUidList;
resetPreFilesetMemTableListIndex(&pReader->status);
} }
if (!pReader->status.bProcMemPreFileset) { if (!pReader->status.bProcMemPreFileset) {
@ -2573,6 +2574,7 @@ static void prepareDurationForNextFileSet(STsdbReader* pReader) {
STsdReaderNotifyInfo info = {0}; STsdReaderNotifyInfo info = {0};
info.duration.filesetId = fid; info.duration.filesetId = fid;
pReader->notifyFn(TSD_READER_NOTIFY_DURATION_START, &info, pReader->notifyParam); pReader->notifyFn(TSD_READER_NOTIFY_DURATION_START, &info, pReader->notifyParam);
tsdbDebug("new duration %d start notification when no buffer preceeding fileset, %s", fid, pReader->idStr);
} }
} }
@ -2643,6 +2645,14 @@ static void resetTableListIndex(SReaderStatus* pStatus) {
pStatus->pTableIter = tSimpleHashGet(pStatus->pTableMap, &uid, sizeof(uid)); pStatus->pTableIter = tSimpleHashGet(pStatus->pTableMap, &uid, sizeof(uid));
} }
static void resetPreFilesetMemTableListIndex(SReaderStatus* pStatus) {
STableUidList* pList = &pStatus->procMemUidList;
pList->currentIndex = 0;
uint64_t uid = pList->tableUidList[0];
pStatus->pProcMemTableIter = tSimpleHashGet(pStatus->pTableMap, &uid, sizeof(uid));
}
static bool moveToNextTable(STableUidList* pOrderedCheckInfo, SReaderStatus* pStatus) { static bool moveToNextTable(STableUidList* pOrderedCheckInfo, SReaderStatus* pStatus) {
pOrderedCheckInfo->currentIndex += 1; pOrderedCheckInfo->currentIndex += 1;
if (pOrderedCheckInfo->currentIndex >= tSimpleHashGetSize(pStatus->pTableMap)) { if (pOrderedCheckInfo->currentIndex >= tSimpleHashGetSize(pStatus->pTableMap)) {
@ -2655,6 +2665,19 @@ static bool moveToNextTable(STableUidList* pOrderedCheckInfo, SReaderStatus* pSt
return (pStatus->pTableIter != NULL); return (pStatus->pTableIter != NULL);
} }
static bool moveToNextTableForPreFileSetMem(SReaderStatus* pStatus) {
STableUidList* pUidList = &pStatus->procMemUidList;
pUidList->currentIndex += 1;
if (pUidList->currentIndex >= tSimpleHashGetSize(pStatus->pTableMap)) {
pStatus->pProcMemTableIter = NULL;
return false;
}
uint64_t uid = pUidList->tableUidList[pUidList->currentIndex];
pStatus->pProcMemTableIter = tSimpleHashGet(pStatus->pTableMap, &uid, sizeof(uid));
return (pStatus->pProcMemTableIter != NULL);
}
static int32_t doLoadSttBlockSequentially(STsdbReader* pReader) { static int32_t doLoadSttBlockSequentially(STsdbReader* pReader) {
SReaderStatus* pStatus = &pReader->status; SReaderStatus* pStatus = &pReader->status;
SSttBlockReader* pSttBlockReader = pStatus->fileIter.pSttBlockReader; SSttBlockReader* pSttBlockReader = pStatus->fileIter.pSttBlockReader;
@ -2889,6 +2912,47 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
return (pReader->code != TSDB_CODE_SUCCESS) ? pReader->code : code; return (pReader->code != TSDB_CODE_SUCCESS) ? pReader->code : code;
} }
static int32_t buildBlockFromBufferSeqForPreFileset(STsdbReader* pReader, int64_t endKey) {
SReaderStatus* pStatus = &pReader->status;
tsdbDebug("seq load data blocks from cache that preceeds fileset %d, %s", pReader->status.pCurrentFileset->fid, pReader->idStr);
while (1) {
if (pReader->code != TSDB_CODE_SUCCESS) {
tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", strerror(pReader->code), pReader->idStr);
return pReader->code;
}
STableBlockScanInfo** pBlockScanInfo = pStatus->pProcMemTableIter;
if (pReader->pIgnoreTables &&
taosHashGet(*pReader->pIgnoreTables, &(*pBlockScanInfo)->uid, sizeof((*pBlockScanInfo)->uid))) {
bool hasNexTable = moveToNextTableForPreFileSetMem(pStatus);
if (!hasNexTable) {
return TSDB_CODE_SUCCESS;
}
continue;
}
initMemDataIterator(*pBlockScanInfo, pReader);
initDelSkylineIterator(*pBlockScanInfo, pReader->info.order, &pReader->cost);
int32_t code = buildDataBlockFromBuf(pReader, *pBlockScanInfo, endKey);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
if (pReader->resBlockInfo.pResBlock->info.rows > 0) {
return TSDB_CODE_SUCCESS;
}
// current table is exhausted, let's try next table
bool hasNexTable = moveToNextTableForPreFileSetMem(pStatus);
if (!hasNexTable) {
return TSDB_CODE_SUCCESS;
}
}
}
static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader, int64_t endKey) { static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader, int64_t endKey) {
SReaderStatus* pStatus = &pReader->status; SReaderStatus* pStatus = &pReader->status;
STableUidList* pUidList = &pStatus->uidList; STableUidList* pUidList = &pStatus->uidList;
@ -4244,6 +4308,33 @@ _err:
return code; return code;
} }
static int32_t buildFromPreFilesetBuffer(STsdbReader* pReader) {
int32_t code = TSDB_CODE_SUCCESS;
SReaderStatus* pStatus = &pReader->status;
SSDataBlock* pBlock = pReader->resBlockInfo.pResBlock;
int32_t fid = pReader->status.pCurrentFileset->fid;
STimeWindow win = {0};
tsdbFidKeyRange(fid, pReader->pTsdb->keepCfg.days, pReader->pTsdb->keepCfg.precision, &win.skey, &win.ekey);
int64_t endKey = (ASCENDING_TRAVERSE(pReader->info.order)) ? win.skey : win.ekey;
code = buildBlockFromBufferSeqForPreFileset(pReader, endKey);
if (code != TSDB_CODE_SUCCESS || pBlock->info.rows > 0) {
return code;
} else {
tsdbDebug("finished pre-fileset %d buffer processing. %s", fid, pReader->idStr);
pStatus->bProcMemPreFileset = false;
if (pReader->notifyFn) {
STsdReaderNotifyInfo info = {0};
info.duration.filesetId = fid;
pReader->notifyFn(TSD_READER_NOTIFY_DURATION_START, &info, pReader->notifyParam);
tsdbDebug("new duration %d start notification when buffer pre-fileset, %s", fid, pReader->idStr);
}
}
return code;
}
static int32_t doTsdbNextDataBlockFilesetDelimited(STsdbReader* pReader) { static int32_t doTsdbNextDataBlockFilesetDelimited(STsdbReader* pReader) {
SReaderStatus* pStatus = &pReader->status; SReaderStatus* pStatus = &pReader->status;
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
@ -4251,22 +4342,9 @@ static int32_t doTsdbNextDataBlockFilesetDelimited(STsdbReader* pReader) {
if (pStatus->loadFromFile) { if (pStatus->loadFromFile) {
if (pStatus->bProcMemPreFileset) { if (pStatus->bProcMemPreFileset) {
int32_t fid = pReader->status.pCurrentFileset->fid; code = buildFromPreFilesetBuffer(pReader);
STimeWindow win = {0};
tsdbFidKeyRange(fid, pReader->pTsdb->keepCfg.days, pReader->pTsdb->keepCfg.precision, &win.skey, &win.ekey);
int64_t endKey = (ASCENDING_TRAVERSE(pReader->info.order)) ? win.skey : win.ekey;
code = buildBlockFromBufferSequentially(pReader, endKey);
if (code != TSDB_CODE_SUCCESS || pBlock->info.rows > 0) { if (code != TSDB_CODE_SUCCESS || pBlock->info.rows > 0) {
return code; return code;
} else {
pStatus->bProcMemPreFileset = false;
if (pReader->notifyFn) {
STsdReaderNotifyInfo info = {0};
info.duration.filesetId = fid;
pReader->notifyFn(TSD_READER_NOTIFY_DURATION_START, &info, pReader->notifyParam);
}
resetTableListIndex(pStatus);
} }
} }
@ -4275,6 +4353,21 @@ static int32_t doTsdbNextDataBlockFilesetDelimited(STsdbReader* pReader) {
return code; return code;
} }
tsdbTrace("block from file rows: %"PRId64", will process pre-file set buffer: %d. %s",
pBlock->info.rows, pStatus->bProcMemFirstFileset, pReader->idStr);
if (pStatus->bProcMemPreFileset) {
if (pBlock->info.rows > 0) {
if (pReader->notifyFn) {
int32_t fid = pReader->status.pCurrentFileset->fid;
STsdReaderNotifyInfo info = {0};
info.duration.filesetId = fid;
pReader->notifyFn(TSD_READER_NOTIFY_NEXT_DURATION_BLOCK, &info, pReader->notifyParam);
}
} else {
pStatus->bProcMemPreFileset = false;
}
}
if (pBlock->info.rows <= 0) { if (pBlock->info.rows <= 0) {
resetTableListIndex(&pReader->status); resetTableListIndex(&pReader->status);
int64_t endKey = (ASCENDING_TRAVERSE(pReader->info.order)) ? INT64_MAX : INT64_MIN; int64_t endKey = (ASCENDING_TRAVERSE(pReader->info.order)) ? INT64_MAX : INT64_MIN;
@ -4720,14 +4813,13 @@ static int32_t getBucketIndex(int32_t startRow, int32_t bucketRange, int32_t num
} }
int32_t tsdbGetFileBlocksDistInfo2(STsdbReader* pReader, STableBlockDistInfo* pTableBlockInfo) { int32_t tsdbGetFileBlocksDistInfo2(STsdbReader* pReader, STableBlockDistInfo* pTableBlockInfo) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
const int32_t numOfBuckets = 20.0;
pTableBlockInfo->totalSize = 0; pTableBlockInfo->totalSize = 0;
pTableBlockInfo->totalRows = 0; pTableBlockInfo->totalRows = 0;
pTableBlockInfo->numOfVgroups = 1; pTableBlockInfo->numOfVgroups = 1;
const int32_t numOfBuckets = 20.0;
const int32_t defaultRows = 4096;
// find the start data block in file // find the start data block in file
tsdbAcquireReader(pReader); tsdbAcquireReader(pReader);
if (pReader->flag == READER_STATUS_SUSPEND) { if (pReader->flag == READER_STATUS_SUSPEND) {

4
source/dnode/vnode/src/tsdb/tsdbReadUtil.h
View File

@ -210,12 +210,16 @@ typedef struct SReaderStatus {
SArray* pLDataIterArray; SArray* pLDataIterArray;
SRowMerger merger; SRowMerger merger;
SColumnInfoData* pPrimaryTsCol; // primary time stamp output col info data SColumnInfoData* pPrimaryTsCol; // primary time stamp output col info data
// the following for preceeds fileset memory processing
// TODO: refactor into seperate struct
bool bProcMemPreFileset; bool bProcMemPreFileset;
int64_t memTableMaxKey; int64_t memTableMaxKey;
int64_t memTableMinKey; int64_t memTableMinKey;
int64_t prevFilesetStartKey; int64_t prevFilesetStartKey;
int64_t prevFilesetEndKey; int64_t prevFilesetEndKey;
bool bProcMemFirstFileset; bool bProcMemFirstFileset;
STableUidList procMemUidList;
STableBlockScanInfo** pProcMemTableIter;
} SReaderStatus; } SReaderStatus;
struct STsdbReader { struct STsdbReader {

2
source/dnode/vnode/src/vnd/vnodeAsync.c
View File

@ -591,7 +591,7 @@ int32_t vnodeACancel(SVAsync *async, int64_t taskId) {
task->prev->next = task->next; task->prev->next = task->next;
vnodeAsyncTaskDone(async, task); vnodeAsyncTaskDone(async, task);
} else { } else {
ret = 0; // task is running, should return code TSDB_CODE_BUSY ?? ret = TSDB_CODE_FAILED;
} }
} }

8
source/dnode/vnode/src/vnd/vnodeHash.c
View File

@ -24,14 +24,6 @@ struct SVHashEntry {
void* obj; void* obj;
}; };
struct SVHashTable {
uint32_t (*hash)(const void*);
int32_t (*compare)(const void*, const void*);
int32_t numEntries;
uint32_t numBuckets;
SVHashEntry** buckets;
};
static int32_t vHashRehash(SVHashTable* ht, uint32_t newNumBuckets) { static int32_t vHashRehash(SVHashTable* ht, uint32_t newNumBuckets) {
SVHashEntry** newBuckets = (SVHashEntry**)taosMemoryCalloc(newNumBuckets, sizeof(SVHashEntry*)); SVHashEntry** newBuckets = (SVHashEntry**)taosMemoryCalloc(newNumBuckets, sizeof(SVHashEntry*));
if (newBuckets == NULL) { if (newBuckets == NULL) {

9
source/dnode/vnode/src/vnd/vnodeHash.h
View File

@ -24,6 +24,15 @@ extern "C" {
typedef struct SVHashTable SVHashTable; typedef struct SVHashTable SVHashTable;
struct SVHashTable {
uint32_t (*hash)(const void*);
int32_t (*compare)(const void*, const void*);
int32_t numEntries;
uint32_t numBuckets;
struct SVHashEntry** buckets;
};
#define vHashNumEntries(ht) ((ht)->numEntries)
int32_t vHashInit(SVHashTable** ht, uint32_t (*hash)(const void*), int32_t (*compare)(const void*, const void*)); int32_t vHashInit(SVHashTable** ht, uint32_t (*hash)(const void*), int32_t (*compare)(const void*, const void*));
int32_t vHashDestroy(SVHashTable** ht); int32_t vHashDestroy(SVHashTable** ht);
int32_t vHashPut(SVHashTable* ht, void* obj); int32_t vHashPut(SVHashTable* ht, void* obj);

1
source/dnode/vnode/src/vnd/vnodeInitApi.c
View File

@ -215,6 +215,7 @@ void initStateStoreAPI(SStateStore* pStore) {
pStore->streamStateDestroy = streamStateDestroy; pStore->streamStateDestroy = streamStateDestroy;
pStore->streamStateDeleteCheckPoint = streamStateDeleteCheckPoint; pStore->streamStateDeleteCheckPoint = streamStateDeleteCheckPoint;
pStore->streamStateReloadInfo = streamStateReloadInfo; pStore->streamStateReloadInfo = streamStateReloadInfo;
pStore->streamStateCopyBackend = streamStateCopyBackend;
} }
void initMetaReaderAPI(SStoreMetaReader* pMetaReader) { void initMetaReaderAPI(SStoreMetaReader* pMetaReader) {

12
source/dnode/vnode/src/vnd/vnodeQuery.c
View File

@ -423,9 +423,9 @@ void vnodeResetLoad(SVnode *pVnode, SVnodeLoad *pLoad) {
"nBatchInsertSuccess"); "nBatchInsertSuccess");
} }
void vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId, int64_t* numOfTables, int64_t* numOfNormalTables) { void vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId, int64_t *numOfTables, int64_t *numOfNormalTables) {
SVnode* pVnodeObj = pVnode; SVnode *pVnodeObj = pVnode;
SVnodeCfg* pConf = &pVnodeObj->config; SVnodeCfg *pConf = &pVnodeObj->config;
if (dbname) { if (dbname) {
*dbname = pConf->dbname; *dbname = pConf->dbname;
@ -444,7 +444,7 @@ void vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId, int64_t* num
} }
} }
int32_t vnodeGetTableList(void* pVnode, int8_t type, SArray* pList) { int32_t vnodeGetTableList(void *pVnode, int8_t type, SArray *pList) {
if (type == TSDB_SUPER_TABLE) { if (type == TSDB_SUPER_TABLE) {
return vnodeGetStbIdList(pVnode, 0, pList); return vnodeGetStbIdList(pVnode, 0, pList);
} else { } else {
@ -694,12 +694,12 @@ void *vnodeGetIdx(void *pVnode) {
return NULL; return NULL;
} }
return metaGetIdx(((SVnode*)pVnode)->pMeta); return metaGetIdx(((SVnode *)pVnode)->pMeta);
} }
void *vnodeGetIvtIdx(void *pVnode) { void *vnodeGetIvtIdx(void *pVnode) {
if (pVnode == NULL) { if (pVnode == NULL) {
return NULL; return NULL;
} }
return metaGetIvtIdx(((SVnode*)pVnode)->pMeta); return metaGetIvtIdx(((SVnode *)pVnode)->pMeta);
} }

54
source/dnode/vnode/src/vnd/vnodeSvr.c
View File

@ -14,24 +14,22 @@
*/ */
#include "audit.h" #include "audit.h"
#include "cos.h"
#include "tencode.h" #include "tencode.h"
#include "tmsg.h" #include "tmsg.h"
#include "tstrbuild.h" #include "tstrbuild.h"
#include "vnd.h" #include "vnd.h"
#include "cos.h"
#include "vnode.h" #include "vnode.h"
#include "vnodeInt.h" #include "vnodeInt.h"
#include "audit.h"
#include "tstrbuild.h"
static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
static int32_t vnodeProcessDropStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessDropStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp, static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp,
SRpcMsg *pOriginRpc); SRpcMsg *pOriginRpc);
static int32_t vnodeProcessAlterTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessAlterTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp, static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp,
SRpcMsg *pOriginRpc); SRpcMsg *pOriginRpc);
static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
@ -45,6 +43,9 @@ static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t ver, void *pReq,
static int32_t vnodeProcessCompactVnodeReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessCompactVnodeReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
static int32_t vnodeProcessConfigChangeReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessConfigChangeReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
extern int32_t vnodeProcessKillCompactReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
extern int32_t vnodeQueryCompactProgress(SVnode *pVnode, SRpcMsg *pMsg);
static int32_t vnodePreprocessCreateTableReq(SVnode *pVnode, SDecoder *pCoder, int64_t btime, int64_t *pUid) { static int32_t vnodePreprocessCreateTableReq(SVnode *pVnode, SDecoder *pCoder, int64_t btime, int64_t *pUid) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
@ -629,6 +630,11 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t ver, SRpcMsg
case TDMT_SYNC_CONFIG_CHANGE: case TDMT_SYNC_CONFIG_CHANGE:
vnodeProcessConfigChangeReq(pVnode, ver, pReq, len, pRsp); vnodeProcessConfigChangeReq(pVnode, ver, pReq, len, pRsp);
break; break;
#ifdef TD_ENTERPRISE
case TDMT_VND_KILL_COMPACT:
vnodeProcessKillCompactReq(pVnode, ver, pReq, len, pRsp);
break;
#endif
default: default:
vError("vgId:%d, unprocessed msg, %d", TD_VID(pVnode), pMsg->msgType); vError("vgId:%d, unprocessed msg, %d", TD_VID(pVnode), pMsg->msgType);
return -1; return -1;
@ -738,6 +744,10 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) {
return vnodeGetTableCfg(pVnode, pMsg, true); return vnodeGetTableCfg(pVnode, pMsg, true);
case TDMT_VND_BATCH_META: case TDMT_VND_BATCH_META:
return vnodeGetBatchMeta(pVnode, pMsg); return vnodeGetBatchMeta(pVnode, pMsg);
#ifdef TD_ENTERPRISE
case TDMT_VND_QUERY_COMPACT_PROGRESS:
return vnodeQueryCompactProgress(pVnode, pMsg);
#endif
// case TDMT_VND_TMQ_CONSUME: // case TDMT_VND_TMQ_CONSUME:
// return tqProcessPollReq(pVnode->pTq, pMsg); // return tqProcessPollReq(pVnode->pTq, pMsg);
case TDMT_VND_TMQ_VG_WALINFO: case TDMT_VND_TMQ_VG_WALINFO:
@ -886,8 +896,8 @@ _err:
return -1; return -1;
} }
static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp, static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp,
SRpcMsg *pOriginRpc) { SRpcMsg *pOriginRpc) {
SDecoder decoder = {0}; SDecoder decoder = {0};
SEncoder encoder = {0}; SEncoder encoder = {0};
int32_t rcode = 0; int32_t rcode = 0;
@ -937,8 +947,8 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq,
goto _exit; goto _exit;
} }
if(tsEnableAudit && tsEnableAuditCreateTable){ if (tsEnableAudit && tsEnableAuditCreateTable) {
char* str = taosMemoryCalloc(1, TSDB_TABLE_FNAME_LEN); char *str = taosMemoryCalloc(1, TSDB_TABLE_FNAME_LEN);
if (str == NULL) { if (str == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
rcode = -1; rcode = -1;
@ -992,26 +1002,26 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq,
tEncoderInit(&encoder, pRsp->pCont, pRsp->contLen); tEncoderInit(&encoder, pRsp->pCont, pRsp->contLen);
tEncodeSVCreateTbBatchRsp(&encoder, &rsp); tEncodeSVCreateTbBatchRsp(&encoder, &rsp);
if(tsEnableAudit && tsEnableAuditCreateTable){ if (tsEnableAudit && tsEnableAuditCreateTable) {
int64_t clusterId = pVnode->config.syncCfg.nodeInfo[0].clusterId; int64_t clusterId = pVnode->config.syncCfg.nodeInfo[0].clusterId;
SName name = {0}; SName name = {0};
tNameFromString(&name, pVnode->config.dbname, T_NAME_ACCT | T_NAME_DB); tNameFromString(&name, pVnode->config.dbname, T_NAME_ACCT | T_NAME_DB);
SStringBuilder sb = {0}; SStringBuilder sb = {0};
for(int32_t i = 0; i < tbNames->size; i++){ for (int32_t i = 0; i < tbNames->size; i++) {
char** key = (char**)taosArrayGet(tbNames, i); char **key = (char **)taosArrayGet(tbNames, i);
taosStringBuilderAppendStringLen(&sb, *key, strlen(*key)); taosStringBuilderAppendStringLen(&sb, *key, strlen(*key));
if(i < tbNames->size - 1){ if (i < tbNames->size - 1) {
taosStringBuilderAppendChar(&sb, ','); taosStringBuilderAppendChar(&sb, ',');
} }
taosMemoryFreeClear(*key); taosMemoryFreeClear(*key);
} }
size_t len = 0; size_t len = 0;
char* keyJoined = taosStringBuilderGetResult(&sb, &len); char *keyJoined = taosStringBuilderGetResult(&sb, &len);
if(pOriginRpc->info.conn.user != NULL && strlen(pOriginRpc->info.conn.user) > 0){ if (pOriginRpc->info.conn.user != NULL && strlen(pOriginRpc->info.conn.user) > 0) {
auditAddRecord(pOriginRpc, clusterId, "createTable", name.dbname, "", keyJoined, len); auditAddRecord(pOriginRpc, clusterId, "createTable", name.dbname, "", keyJoined, len);
} }
@ -1023,7 +1033,7 @@ _exit:
pCreateReq = req.pReqs + iReq; pCreateReq = req.pReqs + iReq;
taosMemoryFree(pCreateReq->sql); taosMemoryFree(pCreateReq->sql);
taosMemoryFree(pCreateReq->comment); taosMemoryFree(pCreateReq->comment);
taosArrayDestroy(pCreateReq->ctb.tagName); taosArrayDestroy(pCreateReq->ctb.tagName);
} }
taosArrayDestroyEx(rsp.pArray, tFreeSVCreateTbRsp); taosArrayDestroyEx(rsp.pArray, tFreeSVCreateTbRsp);
taosArrayDestroy(tbUids); taosArrayDestroy(tbUids);
@ -1156,7 +1166,7 @@ _exit:
} }
static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp, static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp,
SRpcMsg *pOriginRpc) { SRpcMsg *pOriginRpc) {
SVDropTbBatchReq req = {0}; SVDropTbBatchReq req = {0};
SVDropTbBatchRsp rsp = {0}; SVDropTbBatchRsp rsp = {0};
SDecoder decoder = {0}; SDecoder decoder = {0};
@ -1235,13 +1245,13 @@ static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t ver, void *pReq, in
size_t len = 0; size_t len = 0;
char *keyJoined = taosStringBuilderGetResult(&sb, &len); char *keyJoined = taosStringBuilderGetResult(&sb, &len);
if(pOriginRpc->info.conn.user != NULL && strlen(pOriginRpc->info.conn.user) > 0){ if (pOriginRpc->info.conn.user != NULL && strlen(pOriginRpc->info.conn.user) > 0) {
auditAddRecord(pOriginRpc, clusterId, "dropTable", name.dbname, "", keyJoined, len); auditAddRecord(pOriginRpc, clusterId, "dropTable", name.dbname, "", keyJoined, len);
} }
taosStringBuilderDestroy(&sb); taosStringBuilderDestroy(&sb);
} }
_exit: _exit:
taosArrayDestroy(tbUids); taosArrayDestroy(tbUids);
tdUidStoreFree(pStore); tdUidStoreFree(pStore);

12
source/libs/audit/src/auditMain.c
View File

@ -24,6 +24,7 @@
#include "tglobal.h" #include "tglobal.h"
#include "mnode.h" #include "mnode.h"
#include "audit.h" #include "audit.h"
#include "osMemory.h"
SAudit tsAudit = {0}; SAudit tsAudit = {0};
char* tsAuditUri = "/audit"; char* tsAuditUri = "/audit";
@ -36,9 +37,18 @@ int32_t auditInit(const SAuditCfg *pCfg) {
return 0; return 0;
} }
static FORCE_INLINE void auditDeleteRecord(SAuditRecord * record) {
if (record) {
taosMemoryFree(record->detail);
taosMemoryFree(record);
}
}
void auditCleanup() { void auditCleanup() {
tsLogFp = NULL; tsLogFp = NULL;
taosArrayDestroy(tsAudit.records); taosThreadMutexLock(&tsAudit.lock);
taosArrayDestroyP(tsAudit.records, (FDelete)auditDeleteRecord);
taosThreadMutexUnlock(&tsAudit.lock);
tsAudit.records = NULL; tsAudit.records = NULL;
taosThreadMutexDestroy(&tsAudit.lock); taosThreadMutexDestroy(&tsAudit.lock);
} }

9
source/libs/executor/inc/executorInt.h
View File

@ -298,9 +298,14 @@ typedef struct STableMergeScanInfo {
SHashObj* mSkipTables; SHashObj* mSkipTables;
int64_t mergeLimit; int64_t mergeLimit;
SSortExecInfo sortExecInfo; SSortExecInfo sortExecInfo;
bool bNewFileset;
bool bOnlyRetrieveBlock;
bool filesetDelimited; bool filesetDelimited;
bool bNewFilesetEvent;
bool bNextDurationBlockEvent;
int32_t numNextDurationBlocks;
SSDataBlock* nextDurationBlocks[2];
bool rtnNextDurationBlocks;
int32_t nextDurationBlocksIdx;
} STableMergeScanInfo; } STableMergeScanInfo;
typedef struct STagScanFilterContext { typedef struct STagScanFilterContext {

177
source/libs/executor/src/scanoperator.c
View File

@ -3240,6 +3240,52 @@ static int32_t tableMergeScanDoSkipTable(STableMergeScanInfo* pInfo, SSDataBlock
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static void doGetBlockForTableMergeScan(SOperatorInfo* pOperator, bool* pFinished, bool* pSkipped) {
STableMergeScanInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStorageAPI* pAPI = &pTaskInfo->storageAPI;
SSDataBlock* pBlock = pInfo->pReaderBlock;
int32_t code = 0;
bool hasNext = false;
STsdbReader* reader = pInfo->base.dataReader;
code = pAPI->tsdReader.tsdNextDataBlock(reader, &hasNext);
if (code != 0) {
pAPI->tsdReader.tsdReaderReleaseDataBlock(reader);
qError("table merge scan fetch next data block error code: %d, %s", code, GET_TASKID(pTaskInfo));
T_LONG_JMP(pTaskInfo->env, code);
}
if (!hasNext || isTaskKilled(pTaskInfo)) {
if (isTaskKilled(pTaskInfo)) {
qInfo("table merge scan fetch next data block found task killed. %s", GET_TASKID(pTaskInfo));
pAPI->tsdReader.tsdReaderReleaseDataBlock(reader);
}
*pFinished = true;
return;
}
uint32_t status = 0;
code = loadDataBlock(pOperator, &pInfo->base, pBlock, &status);
if (code != TSDB_CODE_SUCCESS) {
qInfo("table merge scan load datablock code %d, %s", code, GET_TASKID(pTaskInfo));
T_LONG_JMP(pTaskInfo->env, code);
}
if (status == FUNC_DATA_REQUIRED_ALL_FILTEROUT) {
*pFinished = true;
return;
}
// current block is filter out according to filter condition, continue load the next block
if (status == FUNC_DATA_REQUIRED_FILTEROUT || pBlock->info.rows == 0) {
*pSkipped = true;
return;
}
return;
}
static SSDataBlock* getBlockForTableMergeScan(void* param) { static SSDataBlock* getBlockForTableMergeScan(void* param) {
STableMergeScanSortSourceParam* source = param; STableMergeScanSortSourceParam* source = param;
SOperatorInfo* pOperator = source->pOperator; SOperatorInfo* pOperator = source->pOperator;
@ -3247,7 +3293,7 @@ static SSDataBlock* getBlockForTableMergeScan(void* param) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStorageAPI* pAPI = &pTaskInfo->storageAPI; SStorageAPI* pAPI = &pTaskInfo->storageAPI;
SSDataBlock* pBlock = pInfo->pReaderBlock; SSDataBlock* pBlock = NULL;
int32_t code = 0; int32_t code = 0;
int64_t st = taosGetTimestampUs(); int64_t st = taosGetTimestampUs();
@ -3255,53 +3301,56 @@ static SSDataBlock* getBlockForTableMergeScan(void* param) {
STsdbReader* reader = pInfo->base.dataReader; STsdbReader* reader = pInfo->base.dataReader;
while (true) { while (true) {
if (!pInfo->bOnlyRetrieveBlock) { if (pInfo->rtnNextDurationBlocks) {
code = pAPI->tsdReader.tsdNextDataBlock(reader, &hasNext); qDebug("%s table merge scan return already fetched new duration blocks. index %d num of blocks %d",
if (code != 0) { GET_TASKID(pTaskInfo), pInfo->nextDurationBlocksIdx, pInfo->numNextDurationBlocks);
pAPI->tsdReader.tsdReaderReleaseDataBlock(reader); if (pInfo->nextDurationBlocksIdx < pInfo->numNextDurationBlocks) {
qError("table merge scan fetch next data block error code: %d, %s", code, GET_TASKID(pTaskInfo)); pBlock = pInfo->nextDurationBlocks[pInfo->nextDurationBlocksIdx];
T_LONG_JMP(pTaskInfo->env, code); ++pInfo->nextDurationBlocksIdx;
} } else {
for (int32_t i = 0; i < pInfo->numNextDurationBlocks; ++i) {
if (!hasNext || isTaskKilled(pTaskInfo)) { blockDataDestroy(pInfo->nextDurationBlocks[i]);
pInfo->bNewFileset = false; pInfo->nextDurationBlocks[i] = NULL;
if (isTaskKilled(pTaskInfo)) {
qInfo("table merge scan fetch next data block found task killed. %s", GET_TASKID(pTaskInfo));
pAPI->tsdReader.tsdReaderReleaseDataBlock(reader);
} }
pInfo->rtnNextDurationBlocks = false;
pInfo->nextDurationBlocksIdx = 0;
pInfo->numNextDurationBlocks = 0;
continue;
}
} else {
bool bFinished = false;
bool bSkipped = false;
doGetBlockForTableMergeScan(pOperator, &bFinished, &bSkipped);
pBlock = pInfo->pReaderBlock;
qDebug("%s table merge scan fetch block. finished %d skipped %d next-duration-block %d new-fileset %d",
GET_TASKID(pTaskInfo), bFinished, bSkipped, pInfo->bNextDurationBlockEvent, pInfo->bNewFilesetEvent);
if (bFinished) {
pInfo->bNewFilesetEvent = false;
break; break;
} }
if (pInfo->bNewFileset) { if (pInfo->bNextDurationBlockEvent || pInfo->bNewFilesetEvent) {
pInfo->bOnlyRetrieveBlock = true; if (!bSkipped) {
return NULL; pInfo->nextDurationBlocks[pInfo->numNextDurationBlocks] = createOneDataBlock(pBlock, true);
++pInfo->numNextDurationBlocks;
if (pInfo->numNextDurationBlocks > 2) {
qError("%s table merge scan prefetch %d next duration blocks. end early.", GET_TASKID(pTaskInfo), pInfo->numNextDurationBlocks);
pInfo->bNewFilesetEvent = false;
break;
}
}
if (pInfo->bNewFilesetEvent) {
pInfo->rtnNextDurationBlocks = true;
return NULL;
}
if (pInfo->bNextDurationBlockEvent) {
pInfo->bNextDurationBlockEvent = false;
continue;
}
} }
if (bSkipped) continue;
} }
// process this data block based on the probabilities
bool processThisBlock = processBlockWithProbability(&pInfo->sample);
if (!processThisBlock) {
continue;
}
uint32_t status = 0;
code = loadDataBlock(pOperator, &pInfo->base, pBlock, &status);
if (pInfo->bOnlyRetrieveBlock) {
pInfo->bOnlyRetrieveBlock = false;
}
if (code != TSDB_CODE_SUCCESS) {
qInfo("table merge scan load datablock code %d, %s", code, GET_TASKID(pTaskInfo));
T_LONG_JMP(pTaskInfo->env, code);
}
if (status == FUNC_DATA_REQUIRED_ALL_FILTEROUT) {
break;
}
// current block is filter out according to filter condition, continue load the next block
if (status == FUNC_DATA_REQUIRED_FILTEROUT || pBlock->info.rows == 0) {
continue;
}
pBlock->info.id.groupId = tableListGetTableGroupId(pInfo->base.pTableListInfo, pBlock->info.id.uid); pBlock->info.id.groupId = tableListGetTableGroupId(pInfo->base.pTableListInfo, pBlock->info.id.uid);
if (pInfo->mergeLimit != -1) { if (pInfo->mergeLimit != -1) {
@ -3310,13 +3359,13 @@ static SSDataBlock* getBlockForTableMergeScan(void* param) {
pOperator->resultInfo.totalRows += pBlock->info.rows; pOperator->resultInfo.totalRows += pBlock->info.rows;
pInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0; pInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0;
return pBlock; return pBlock;
} }
return NULL; return NULL;
} }
SArray* generateSortByTsInfo(SArray* colMatchInfo, int32_t order) { SArray* generateSortByTsInfo(SArray* colMatchInfo, int32_t order) {
int32_t tsTargetSlotId = 0; int32_t tsTargetSlotId = 0;
for (int32_t i = 0; i < taosArrayGetSize(colMatchInfo); ++i) { for (int32_t i = 0; i < taosArrayGetSize(colMatchInfo); ++i) {
@ -3348,7 +3397,13 @@ int32_t dumpQueryTableCond(const SQueryTableDataCond* src, SQueryTableDataCond*
void tableMergeScanTsdbNotifyCb(ETsdReaderNotifyType type, STsdReaderNotifyInfo* info, void* param) { void tableMergeScanTsdbNotifyCb(ETsdReaderNotifyType type, STsdReaderNotifyInfo* info, void* param) {
STableMergeScanInfo* pTmsInfo = param; STableMergeScanInfo* pTmsInfo = param;
pTmsInfo->bNewFileset = true; if (type == TSD_READER_NOTIFY_DURATION_START) {
pTmsInfo->bNewFilesetEvent = true;
} else if (type == TSD_READER_NOTIFY_NEXT_DURATION_BLOCK) {
pTmsInfo->bNextDurationBlockEvent = true;
}
qDebug("table merge scan receive notification. type %d, fileset %d", type, info->duration.filesetId);
return; return;
} }
@ -3358,7 +3413,9 @@ int32_t startDurationForGroupTableMergeScan(SOperatorInfo* pOperator) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
int32_t numOfTable = pInfo->tableEndIndex - pInfo->tableStartIndex + 1; int32_t numOfTable = pInfo->tableEndIndex - pInfo->tableStartIndex + 1;
pInfo->bNewFileset = false; qDebug("%s table merge scan start duration ", GET_TASKID(pTaskInfo));
pInfo->bNewFilesetEvent = false;
pInfo->bNextDurationBlockEvent = false;
pInfo->sortBufSize = 2048 * pInfo->bufPageSize; pInfo->sortBufSize = 2048 * pInfo->bufPageSize;
int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize;
@ -3388,6 +3445,8 @@ int32_t startDurationForGroupTableMergeScan(SOperatorInfo* pOperator) {
void stopDurationForGroupTableMergeScan(SOperatorInfo* pOperator) { void stopDurationForGroupTableMergeScan(SOperatorInfo* pOperator) {
STableMergeScanInfo* pInfo = pOperator->info; STableMergeScanInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
qDebug("%s table merge scan stop duration ", GET_TASKID(pTaskInfo));
SSortExecInfo sortExecInfo = tsortGetSortExecInfo(pInfo->pSortHandle); SSortExecInfo sortExecInfo = tsortGetSortExecInfo(pInfo->pSortHandle);
pInfo->sortExecInfo.sortMethod = sortExecInfo.sortMethod; pInfo->sortExecInfo.sortMethod = sortExecInfo.sortMethod;
@ -3405,6 +3464,7 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SReadHandle* pHandle = &pInfo->base.readHandle; SReadHandle* pHandle = &pInfo->base.readHandle;
SStorageAPI* pAPI = &pTaskInfo->storageAPI; SStorageAPI* pAPI = &pTaskInfo->storageAPI;
qDebug("%s table merge scan start group %"PRIu64, GET_TASKID(pTaskInfo), pInfo->groupId);
{ {
size_t numOfTables = tableListGetSize(pInfo->base.pTableListInfo); size_t numOfTables = tableListGetSize(pInfo->base.pTableListInfo);
@ -3452,10 +3512,19 @@ int32_t stopGroupTableMergeScan(SOperatorInfo* pOperator) {
pAPI->tsdReader.tsdReaderClose(pInfo->base.dataReader); pAPI->tsdReader.tsdReaderClose(pInfo->base.dataReader);
pInfo->base.dataReader = NULL; pInfo->base.dataReader = NULL;
} }
for (int32_t i = 0; i < pInfo->numNextDurationBlocks; ++i) {
if (pInfo->nextDurationBlocks[i]) {
blockDataDestroy(pInfo->nextDurationBlocks[i]);
pInfo->nextDurationBlocks[i] = NULL;
}
pInfo->numNextDurationBlocks = 0;
pInfo->nextDurationBlocksIdx = 0;
}
resetLimitInfoForNextGroup(&pInfo->limitInfo); resetLimitInfoForNextGroup(&pInfo->limitInfo);
taosHashCleanup(pInfo->mSkipTables); taosHashCleanup(pInfo->mSkipTables);
pInfo->mSkipTables = NULL; pInfo->mSkipTables = NULL;
qDebug("%s table merge scan stop group %"PRIu64, GET_TASKID(pTaskInfo), pInfo->groupId);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -3535,7 +3604,7 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) {
pOperator->resultInfo.totalRows += pBlock->info.rows; pOperator->resultInfo.totalRows += pBlock->info.rows;
return pBlock; return pBlock;
} else { } else {
if (pInfo->bNewFileset) { if (pInfo->bNewFilesetEvent) {
stopDurationForGroupTableMergeScan(pOperator); stopDurationForGroupTableMergeScan(pOperator);
startDurationForGroupTableMergeScan(pOperator); startDurationForGroupTableMergeScan(pOperator);
} else { } else {
@ -3566,6 +3635,13 @@ void destroyTableMergeScanOperatorInfo(void* param) {
pTableScanInfo->base.readerAPI.tsdReaderClose(pTableScanInfo->base.dataReader); pTableScanInfo->base.readerAPI.tsdReaderClose(pTableScanInfo->base.dataReader);
pTableScanInfo->base.dataReader = NULL; pTableScanInfo->base.dataReader = NULL;
for (int32_t i = 0; i < pTableScanInfo->numNextDurationBlocks; ++i) {
if (pTableScanInfo->nextDurationBlocks[i] != NULL) {
blockDataDestroy(pTableScanInfo->nextDurationBlocks[i]);
pTableScanInfo->nextDurationBlocks[i] = NULL;
}
}
taosArrayDestroy(pTableScanInfo->sortSourceParams); taosArrayDestroy(pTableScanInfo->sortSourceParams);
tsortDestroySortHandle(pTableScanInfo->pSortHandle); tsortDestroySortHandle(pTableScanInfo->pSortHandle);
pTableScanInfo->pSortHandle = NULL; pTableScanInfo->pSortHandle = NULL;
@ -3675,8 +3751,11 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
int32_t rowSize = pInfo->pResBlock->info.rowSize; int32_t rowSize = pInfo->pResBlock->info.rowSize;
uint32_t nCols = taosArrayGetSize(pInfo->pResBlock->pDataBlock); uint32_t nCols = taosArrayGetSize(pInfo->pResBlock->pDataBlock);
pInfo->bufPageSize = getProperSortPageSize(rowSize, nCols); pInfo->bufPageSize = getProperSortPageSize(rowSize, nCols);
if (!tsExperimental) {
pInfo->filesetDelimited = pTableScanNode->filesetDelimited; pInfo->filesetDelimited = false;
} else {
pInfo->filesetDelimited = pTableScanNode->filesetDelimited;
}
setOperatorInfo(pOperator, "TableMergeScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN, false, OP_NOT_OPENED, setOperatorInfo(pOperator, "TableMergeScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN, false, OP_NOT_OPENED,
pInfo, pTaskInfo); pInfo, pTaskInfo);
pOperator->exprSupp.numOfExprs = numOfCols; pOperator->exprSupp.numOfExprs = numOfCols;

14
source/libs/executor/src/streamtimewindowoperator.c
View File

@ -408,6 +408,11 @@ void destroyStreamFinalIntervalOperatorInfo(void* param) {
taosArrayDestroy(pInfo->pDelWins); taosArrayDestroy(pInfo->pDelWins);
blockDataDestroy(pInfo->pDelRes); blockDataDestroy(pInfo->pDelRes);
pInfo->stateStore.streamFileStateDestroy(pInfo->pState->pFileState); pInfo->stateStore.streamFileStateDestroy(pInfo->pState->pFileState);
if (pInfo->pState->dump == 1) {
taosMemoryFreeClear(pInfo->pState->pTdbState->pOwner);
taosMemoryFreeClear(pInfo->pState->pTdbState);
}
taosMemoryFreeClear(pInfo->pState); taosMemoryFreeClear(pInfo->pState);
nodesDestroyNode((SNode*)pInfo->pPhyNode); nodesDestroyNode((SNode*)pInfo->pPhyNode);
@ -1462,7 +1467,11 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
initBasicInfo(&pInfo->binfo, pResBlock); initBasicInfo(&pInfo->binfo, pResBlock);
pInfo->pState = taosMemoryCalloc(1, sizeof(SStreamState)); pInfo->pState = taosMemoryCalloc(1, sizeof(SStreamState));
*(pInfo->pState) = *(pTaskInfo->streamInfo.pState); qInfo("open state %p", pInfo->pState);
pAPI->stateStore.streamStateCopyBackend(pTaskInfo->streamInfo.pState, pInfo->pState);
//*(pInfo->pState) = *(pTaskInfo->streamInfo.pState);
qInfo("copy state %p to %p", pTaskInfo->streamInfo.pState, pInfo->pState);
pAPI->stateStore.streamStateSetNumber(pInfo->pState, -1); pAPI->stateStore.streamStateSetNumber(pInfo->pState, -1);
int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str, int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str,
@ -3338,7 +3347,8 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
SColumnInfoData* pKeyColInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->stateCol.slotId); SColumnInfoData* pKeyColInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->stateCol.slotId);
for (int32_t i = 0; i < rows; i += winRows) { for (int32_t i = 0; i < rows; i += winRows) {
if (pInfo->ignoreExpiredData && checkExpiredData(&pInfo->streamAggSup.stateStore, pInfo->streamAggSup.pUpdateInfo, if (pInfo->ignoreExpiredData && checkExpiredData(&pInfo->streamAggSup.stateStore, pInfo->streamAggSup.pUpdateInfo,
&pInfo->twAggSup, pSDataBlock->info.id.uid, tsCols[i]) || colDataIsNull_s(pKeyColInfo, i)) { &pInfo->twAggSup, pSDataBlock->info.id.uid, tsCols[i]) ||
colDataIsNull_s(pKeyColInfo, i)) {
i++; i++;
continue; continue;
} }

5
source/libs/executor/src/sysscanoperator.c
View File

@ -1610,7 +1610,10 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) {
const char* name = tNameGetTableName(&pInfo->name); const char* name = tNameGetTableName(&pInfo->name);
if (pInfo->showRewrite) { if (pInfo->showRewrite) {
getDBNameFromCondition(pInfo->pCondition, dbName); getDBNameFromCondition(pInfo->pCondition, dbName);
sprintf(pInfo->req.db, "%d.%s", pInfo->accountId, dbName); if (strncasecmp(name, TSDB_INS_TABLE_COMPACTS, TSDB_TABLE_FNAME_LEN) != 0 &&
strncasecmp(name, TSDB_INS_TABLE_COMPACT_DETAILS, TSDB_TABLE_FNAME_LEN) != 0) {
sprintf(pInfo->req.db, "%d.%s", pInfo->accountId, dbName);
}
} else if (strncasecmp(name, TSDB_INS_TABLE_COLS, TSDB_TABLE_FNAME_LEN) == 0) { } else if (strncasecmp(name, TSDB_INS_TABLE_COLS, TSDB_TABLE_FNAME_LEN) == 0) {
getDBNameFromCondition(pInfo->pCondition, dbName); getDBNameFromCondition(pInfo->pCondition, dbName);
if (dbName[0]) sprintf(pInfo->req.db, "%d.%s", pInfo->accountId, dbName); if (dbName[0]) sprintf(pInfo->req.db, "%d.%s", pInfo->accountId, dbName);

4
source/libs/function/src/builtinsimpl.c
View File

@ -5634,7 +5634,7 @@ int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
double factor = pData->numOfBlocks / 80.0; double factor = pData->numOfBlocks / 80.0;
int32_t numOfBuckets = sizeof(pData->blockRowsHisto) / sizeof(pData->blockRowsHisto[0]); int32_t numOfBuckets = sizeof(pData->blockRowsHisto) / sizeof(pData->blockRowsHisto[0]);
int32_t bucketRange = (pData->defMaxRows - pData->defMinRows) / numOfBuckets; int32_t bucketRange = ceil(((double) (pData->defMaxRows - pData->defMinRows)) / numOfBuckets);
for (int32_t i = 0; i < tListLen(pData->blockRowsHisto); ++i) { for (int32_t i = 0; i < tListLen(pData->blockRowsHisto); ++i) {
len = sprintf(st + VARSTR_HEADER_SIZE, "%04d |", pData->defMinRows + bucketRange * (i + 1)); len = sprintf(st + VARSTR_HEADER_SIZE, "%04d |", pData->defMinRows + bucketRange * (i + 1));
@ -5649,7 +5649,7 @@ int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
len += x; len += x;
} }
if (num > 0) { if (pData->blockRowsHisto[i] > 0) {
double v = pData->blockRowsHisto[i] * 100.0 / pData->numOfBlocks; double v = pData->blockRowsHisto[i] * 100.0 / pData->numOfBlocks;
len += sprintf(st + VARSTR_HEADER_SIZE + len, " %d (%.2f%c)", pData->blockRowsHisto[i], v, '%'); len += sprintf(st + VARSTR_HEADER_SIZE + len, " %d (%.2f%c)", pData->blockRowsHisto[i], v, '%');
} }

4
source/libs/nodes/src/nodesCodeFuncs.c
View File

@ -259,6 +259,10 @@ const char* nodesNodeName(ENodeType type) {
return "ShowLocalVariablesStmt"; return "ShowLocalVariablesStmt";
case QUERY_NODE_SHOW_TABLE_TAGS_STMT: case QUERY_NODE_SHOW_TABLE_TAGS_STMT:
return "ShowTableTagsStmt"; return "ShowTableTagsStmt";
case QUERY_NODE_SHOW_COMPACTS_STMT:
return "ShowCompactsStmt";
case QUERY_NODE_SHOW_COMPACT_DETAILS_STMT:
return "ShowCompactDetailsStmt";
case QUERY_NODE_DELETE_STMT: case QUERY_NODE_DELETE_STMT:
return "DeleteStmt"; return "DeleteStmt";
case QUERY_NODE_INSERT_STMT: case QUERY_NODE_INSERT_STMT:

13
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -458,10 +458,15 @@ SNode* nodesMakeNode(ENodeType type) {
return makeNode(type, sizeof(SShowCreateViewStmt)); return makeNode(type, sizeof(SShowCreateViewStmt));
case QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT: case QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT:
return makeNode(type, sizeof(SShowTableDistributedStmt)); return makeNode(type, sizeof(SShowTableDistributedStmt));
case QUERY_NODE_SHOW_COMPACTS_STMT:
return makeNode(type, sizeof(SShowCompactsStmt));
case QUERY_NODE_SHOW_COMPACT_DETAILS_STMT:
return makeNode(type, sizeof(SShowCompactDetailsStmt));
case QUERY_NODE_KILL_QUERY_STMT: case QUERY_NODE_KILL_QUERY_STMT:
return makeNode(type, sizeof(SKillQueryStmt)); return makeNode(type, sizeof(SKillQueryStmt));
case QUERY_NODE_KILL_TRANSACTION_STMT: case QUERY_NODE_KILL_TRANSACTION_STMT:
case QUERY_NODE_KILL_CONNECTION_STMT: case QUERY_NODE_KILL_CONNECTION_STMT:
case QUERY_NODE_KILL_COMPACT_STMT:
return makeNode(type, sizeof(SKillStmt)); return makeNode(type, sizeof(SKillStmt));
case QUERY_NODE_DELETE_STMT: case QUERY_NODE_DELETE_STMT:
return makeNode(type, sizeof(SDeleteStmt)); return makeNode(type, sizeof(SDeleteStmt));
@ -1089,6 +1094,13 @@ void nodesDestroyNode(SNode* pNode) {
nodesDestroyNode(((SShowDnodeVariablesStmt*)pNode)->pDnodeId); nodesDestroyNode(((SShowDnodeVariablesStmt*)pNode)->pDnodeId);
nodesDestroyNode(((SShowDnodeVariablesStmt*)pNode)->pLikePattern); nodesDestroyNode(((SShowDnodeVariablesStmt*)pNode)->pLikePattern);
break; break;
case QUERY_NODE_SHOW_COMPACTS_STMT:
break;
case QUERY_NODE_SHOW_COMPACT_DETAILS_STMT: {
SShowCompactDetailsStmt* pStmt = (SShowCompactDetailsStmt*)pNode;
nodesDestroyNode(pStmt->pCompactId);
break;
}
case QUERY_NODE_SHOW_CREATE_DATABASE_STMT: case QUERY_NODE_SHOW_CREATE_DATABASE_STMT:
taosMemoryFreeClear(((SShowCreateDatabaseStmt*)pNode)->pCfg); taosMemoryFreeClear(((SShowCreateDatabaseStmt*)pNode)->pCfg);
break; break;
@ -1102,6 +1114,7 @@ void nodesDestroyNode(SNode* pNode) {
case QUERY_NODE_KILL_CONNECTION_STMT: // no pointer field case QUERY_NODE_KILL_CONNECTION_STMT: // no pointer field
case QUERY_NODE_KILL_QUERY_STMT: // no pointer field case QUERY_NODE_KILL_QUERY_STMT: // no pointer field
case QUERY_NODE_KILL_TRANSACTION_STMT: // no pointer field case QUERY_NODE_KILL_TRANSACTION_STMT: // no pointer field
case QUERY_NODE_KILL_COMPACT_STMT: // no pointer field
break; break;
case QUERY_NODE_DELETE_STMT: { case QUERY_NODE_DELETE_STMT: {
SDeleteStmt* pStmt = (SDeleteStmt*)pNode; SDeleteStmt* pStmt = (SDeleteStmt*)pNode;

Some files were not shown because too many files have changed in this diff Show More