xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch 'develop' into feature/query

This commit is contained in:
hjxilinx 2020-04-16 11:20:17 +08:00
parent 53b8867aab 89b72161dd
commit 6a504d3647
119 changed files with 3126 additions and 7175 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -19,7 +19,6 @@ tests/test/
tests/taoshebei/
tests/taoscsv/
tests/taosdalipu/
tests/pytest/
tests/jenkins/
tests/hdfs/
*.iml

3
.gitmodules vendored Normal file
View File

@ -0,0 +1,3 @@
[submodule "src/connector/go"]
path = src/connector/go
url = https://github.com/taosdata/driver-go

46
.travis.yml
View File

@ -14,6 +14,15 @@ os:
- linux
# - osx
before_install:
- |-
case $TRAVIS_OS_NAME in
linux)
sudo apt -y update
sudo apt -y install python-pip python3-pip python-setuptools python3-setuptools
;;
esac
addons:
coverity_scan:
@ -50,6 +59,15 @@ script:
- |-
case $TRAVIS_OS_NAME in
linux)
# Color setting
RED='\033[0;31m'
GREEN='\033[1;32m'
GREEN_DARK='\033[0;32m'
GREEN_UNDERLINE='\033[4;32m'
NC='\033[0m'
sudo make install
cd ../tests/script
sudo ./test.sh 2>&1 | grep 'success\|failed' | tee out.txt
@ -57,16 +75,32 @@ script:
if [ "$total_success" -gt "0" ]; then
total_success=`expr $total_success - 1`
echo -e "${GREEN} ### Total $total_success TSIM case(s) succeed! ### ${NC}"
fi
echo "Total $total_success success"
total_failed=`grep failed out.txt | wc -l`
echo "Total $total_failed failed"
if [ "$total_failed" -ne "0" ]; then
echo -e "${RED} ### Total $total_failed TSIM case(s) failed! ### ${NC}"
exit $total_failed
fi
pip install --user ../../src/connector/python/linux/python2/
pip3 install --user ../../src/connector/python/linux/python3/
cd ../pytest
sudo ./simpletest.sh 2>&1 | grep 'successfully executed\|failed' | tee pytest-out.txt
total_py_success=`grep 'successfully executed' pytest-out.txt | wc -l`
if [ "$total_py_success" -gt "0" ]; then
echo -e "${GREEN} ### Total $total_py_success python case(s) succeed! ### ${NC}"
fi
total_py_failed=`grep 'failed' pytest-out.txt | wc -l`
if [ "$total_py_failed" -ne "0" ]; then
echo -e "${RED} ### Total $total_py_failed python case(s) failed! ### ${NC}"
exit $total_py_failed
fi
;;
esac
@ -81,6 +115,10 @@ matrix:
- build-essential
- cmake
- net-tools
- python-pip
- python-setuptools
- python3-pip
- python3-setuptools
# - os: osx
# addons:

4
cmake/define.inc
View File

@ -11,9 +11,9 @@ ENDIF ()
IF (TD_VPEER)
ADD_DEFINITIONS(-D_VPEER)
ADD_DEFINITIONS(-DTSDB_REPLICA_MAX_NUM=3)
#ADD_DEFINITIONS(-DTSDB_REPLICA_MAX_NUM=3)
ELSE ()
ADD_DEFINITIONS(-DTSDB_REPLICA_MAX_NUM=1)
#ADD_DEFINITIONS(-DTSDB_REPLICA_MAX_NUM=1)
ENDIF ()
IF (TD_ACCOUNT)

2
deps/CMakeLists.txt vendored
View File

@ -5,3 +5,5 @@ ADD_SUBDIRECTORY(zlib-1.2.11)
ADD_SUBDIRECTORY(pthread)
ADD_SUBDIRECTORY(regex)
ADD_SUBDIRECTORY(iconv)
ADD_SUBDIRECTORY(lz4)
ADD_SUBDIRECTORY(cJson)

4
deps/cJson/CMakeLists.txt vendored Normal file
View File

@ -0,0 +1,4 @@
aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/src SOURCE_LIST)
add_library(cJson ${SOURCE_LIST})
target_include_directories(cJson PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/inc)

0
tests/tsim/inc/cJSON.h → deps/cJson/inc/cJSON.h vendored
View File

0
tests/tsim/src/cJSON.c → deps/cJson/src/cJSON.c vendored
View File

4
deps/lz4/CMakeLists.txt vendored Normal file
View File

@ -0,0 +1,4 @@
aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/src SOURCE_LIST)
add_library(lz4 ${SOURCE_LIST})
target_include_directories(lz4 PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/inc)

0
src/thirdparty/inc/lz4.h → deps/lz4/inc/lz4.h vendored
View File

0
src/thirdparty/src/lz4.c → deps/lz4/src/lz4.c vendored
View File

4
documentation/webdocs/markdowndocs/administrator-ch.md
View File

@ -105,7 +105,7 @@ RESTful服务使用的端口号，所有的HTTP请求（TCP）都需要向该接
**shellActivityTimer**
- 默认值：3
系统在服务端保持结果集的最长时间，范围[1-120]。
系统在服务端保持结果集的最长时间，单位：秒，范围[1-120]。
**maxUsers**
- 默认值：10,000
@ -138,7 +138,7 @@ RESTful服务使用的端口号，所有的HTTP请求（TCP）都需要向该接
系统（服务端和客户端）运行日志开关：
- 131 仅输出错误和警告信息
- 135 输入错误（ERROR）、警告（WARN）、信息（Info）
- 135 输出错误（ERROR）、警告（WARN）、信息（Info）
不同应用场景的数据往往具有不同的数据特征，比如保留天数、副本数、采集频次、记录大小、采集点的数量、压缩等都可完全不同。为获得在存储上的最高效率，TDengine提供如下存储相关的系统配置参数：

4
src/CMakeLists.txt
View File

@ -3,8 +3,6 @@ PROJECT(TDengine)
# Base compile
ADD_SUBDIRECTORY(os)
ADD_SUBDIRECTORY(thirdparty)
ADD_SUBDIRECTORY(common)
ADD_SUBDIRECTORY(util)
ADD_SUBDIRECTORY(rpc)
@ -14,5 +12,7 @@ ADD_SUBDIRECTORY(kit)
ADD_SUBDIRECTORY(plugins)
ADD_SUBDIRECTORY(mnode)
ADD_SUBDIRECTORY(vnode)
ADD_SUBDIRECTORY(tsdb)
ADD_SUBDIRECTORY(wal)
ADD_SUBDIRECTORY(dnode)
#ADD_SUBDIRECTORY(connector/jdbc)

2
src/client/src/tscServer.c
View File

@ -191,7 +191,7 @@ int tscSendMsgToServer(SSqlObj *pSql) {
}
if (pSql->cmd.command < TSDB_SQL_MGMT) {
tscPrint("%p msg:%s is sent to server %d", pSql, taosMsg[pSql->cmd.msgType], pSql->ipList.port);
tscTrace("%p msg:%s is sent to server %d", pSql, taosMsg[pSql->cmd.msgType], pSql->ipList.port);
memcpy(pMsg, pSql->cmd.payload + tsRpcHeadSize, pSql->cmd.payloadLen);
SRpcMsg rpcMsg = {

1
src/connector/go Submodule

@ -0,0 +1 @@
Subproject commit 8c58c512b6acda8bcdfa48fdc7140227b5221766

368
src/connector/go/src/taosSql/connection.go
View File

@ -1,368 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package taosSql
import "C"
import (
"context"
"errors"
"database/sql/driver"
"unsafe"
"strconv"
"strings"
"time"
)
type taosConn struct {
taos unsafe.Pointer
affectedRows int
insertId int
cfg *config
status statusFlag
parseTime bool
reset bool // set when the Go SQL package calls ResetSession
}
type taosSqlResult struct {
affectedRows int64
insertId int64
}
func (res *taosSqlResult) LastInsertId() (int64, error) {
return res.insertId, nil
}
func (res *taosSqlResult) RowsAffected() (int64, error) {
return res.affectedRows, nil
}
func (mc *taosConn) Begin() (driver.Tx, error) {
taosLog.Println("taosSql not support transaction")
return nil, errors.New("taosSql not support transaction")
}
func (mc *taosConn) Close() (err error) {
if mc.taos == nil {
return errConnNoExist
}
mc.taos_close()
return nil
}
func (mc *taosConn) Prepare(query string) (driver.Stmt, error) {
if mc.taos == nil {
return nil, errInvalidConn
}
stmt := &taosSqlStmt{
mc: mc,
pSql: query,
}
// find ? count and save to stmt.paramCount
stmt.paramCount = strings.Count(query, "?")
//fmt.Printf("prepare alloc stmt:%p, sql:%s\n", stmt, query)
taosLog.Printf("prepare alloc stmt:%p, sql:%s\n", stmt, query)
return stmt, nil
}
func (mc *taosConn) interpolateParams(query string, args []driver.Value) (string, error) {
// Number of ? should be same to len(args)
if strings.Count(query, "?") != len(args) {
return "", driver.ErrSkip
}
buf := make([]byte, defaultBufSize)
buf = buf[:0] // clear buf
argPos := 0
for i := 0; i < len(query); i++ {
q := strings.IndexByte(query[i:], '?')
if q == -1 {
buf = append(buf, query[i:]...)
break
}
buf = append(buf, query[i:i+q]...)
i += q
arg := args[argPos]
argPos++
if arg == nil {
buf = append(buf, "NULL"...)
continue
}
switch v := arg.(type) {
case int64:
buf = strconv.AppendInt(buf, v, 10)
case uint64:
// Handle uint64 explicitly because our custom ConvertValue emits unsigned values
buf = strconv.AppendUint(buf, v, 10)
case float64:
buf = strconv.AppendFloat(buf, v, 'g', -1, 64)
case bool:
if v {
buf = append(buf, '1')
} else {
buf = append(buf, '0')
}
case time.Time:
if v.IsZero() {
buf = append(buf, "'0000-00-00'"...)
} else {
v := v.In(mc.cfg.loc)
v = v.Add(time.Nanosecond * 500) // To round under microsecond
year := v.Year()
year100 := year / 100
year1 := year % 100
month := v.Month()
day := v.Day()
hour := v.Hour()
minute := v.Minute()
second := v.Second()
micro := v.Nanosecond() / 1000
buf = append(buf, []byte{
'\'',
digits10[year100], digits01[year100],
digits10[year1], digits01[year1],
'-',
digits10[month], digits01[month],
'-',
digits10[day], digits01[day],
' ',
digits10[hour], digits01[hour],
':',
digits10[minute], digits01[minute],
':',
digits10[second], digits01[second],
}...)
if micro != 0 {
micro10000 := micro / 10000
micro100 := micro / 100 % 100
micro1 := micro % 100
buf = append(buf, []byte{
'.',
digits10[micro10000], digits01[micro10000],
digits10[micro100], digits01[micro100],
digits10[micro1], digits01[micro1],
}...)
}
buf = append(buf, '\'')
}
case []byte:
if v == nil {
buf = append(buf, "NULL"...)
} else {
buf = append(buf, "_binary'"...)
if mc.status&statusNoBackslashEscapes == 0 {
buf = escapeBytesBackslash(buf, v)
} else {
buf = escapeBytesQuotes(buf, v)
}
buf = append(buf, '\'')
}
case string:
//buf = append(buf, '\'')
if mc.status&statusNoBackslashEscapes == 0 {
buf = escapeStringBackslash(buf, v)
} else {
buf = escapeStringQuotes(buf, v)
}
//buf = append(buf, '\'')
default:
return "", driver.ErrSkip
}
//if len(buf)+4 > mc.maxAllowedPacket {
if len(buf)+4 > maxTaosSqlLen {
return "", driver.ErrSkip
}
}
if argPos != len(args) {
return "", driver.ErrSkip
}
return string(buf), nil
}
func (mc *taosConn) Exec(query string, args []driver.Value) (driver.Result, error) {
if mc.taos == nil {
return nil, driver.ErrBadConn
}
if len(args) != 0 {
if !mc.cfg.interpolateParams {
return nil, driver.ErrSkip
}
// try to interpolate the parameters to save extra roundtrips for preparing and closing a statement
prepared, err := mc.interpolateParams(query, args)
if err != nil {
return nil, err
}
query = prepared
}
mc.affectedRows = 0
mc.insertId = 0
_, err := mc.taosQuery(query)
if err == nil {
return &taosSqlResult{
affectedRows: int64(mc.affectedRows),
insertId: int64(mc.insertId),
}, err
}
return nil, err
}
func (mc *taosConn) Query(query string, args []driver.Value) (driver.Rows, error) {
return mc.query(query, args)
}
func (mc *taosConn) query(query string, args []driver.Value) (*textRows, error) {
if mc.taos == nil {
return nil, driver.ErrBadConn
}
if len(args) != 0 {
if !mc.cfg.interpolateParams {
return nil, driver.ErrSkip
}
// try client-side prepare to reduce roundtrip
prepared, err := mc.interpolateParams(query, args)
if err != nil {
return nil, err
}
query = prepared
}
num_fields, err := mc.taosQuery(query)
if err == nil {
// Read Result
rows := new(textRows)
rows.mc = mc
// Columns field
rows.rs.columns, err = mc.readColumns(num_fields)
return rows, err
}
return nil, err
}
// Ping implements driver.Pinger interface
func (mc *taosConn) Ping(ctx context.Context) (err error) {
if mc.taos != nil {
return nil
}
return errInvalidConn
}
// BeginTx implements driver.ConnBeginTx interface
func (mc *taosConn) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, error) {
taosLog.Println("taosSql not support transaction")
return nil, errors.New("taosSql not support transaction")
}
func (mc *taosConn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) {
if mc.taos == nil {
return nil, errInvalidConn
}
dargs, err := namedValueToValue(args)
if err != nil {
return nil, err
}
rows, err := mc.query(query, dargs)
if err != nil {
return nil, err
}
return rows, err
}
func (mc *taosConn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) {
if mc.taos == nil {
return nil, errInvalidConn
}
dargs, err := namedValueToValue(args)
if err != nil {
return nil, err
}
return mc.Exec(query, dargs)
}
func (mc *taosConn) PrepareContext(ctx context.Context, query string) (driver.Stmt, error) {
if mc.taos == nil {
return nil, errInvalidConn
}
stmt, err := mc.Prepare(query)
if err != nil {
return nil, err
}
return stmt, nil
}
func (stmt *taosSqlStmt) QueryContext(ctx context.Context, args []driver.NamedValue) (driver.Rows, error) {
if stmt.mc == nil {
return nil, errInvalidConn
}
dargs, err := namedValueToValue(args)
if err != nil {
return nil, err
}
rows, err := stmt.query(dargs)
if err != nil {
return nil, err
}
return rows, err
}
func (stmt *taosSqlStmt) ExecContext(ctx context.Context, args []driver.NamedValue) (driver.Result, error) {
if stmt.mc == nil {
return nil, errInvalidConn
}
dargs, err := namedValueToValue(args)
if err != nil {
return nil, err
}
return stmt.Exec(dargs)
}
func (mc *taosConn) CheckNamedValue(nv *driver.NamedValue) (err error) {
nv.Value, err = converter{}.ConvertValue(nv.Value)
return
}
// ResetSession implements driver.SessionResetter.
// (From Go 1.10)
func (mc *taosConn) ResetSession(ctx context.Context) error {
if mc.taos == nil {
return driver.ErrBadConn
}
mc.reset = true
return nil
}

49
src/connector/go/src/taosSql/connector.go
View File

@ -1,49 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package taosSql
import (
"context"
"database/sql/driver"
)
type connector struct {
cfg *config
}
// Connect implements driver.Connector interface.
// Connect returns a connection to the database.
func (c *connector) Connect(ctx context.Context) (driver.Conn, error) {
var err error
// New taosConn
mc := &taosConn{
cfg: c.cfg,
parseTime: c.cfg.parseTime,
}
// Connect to Server
mc.taos, err = mc.taosConnect(mc.cfg.addr, mc.cfg.user, mc.cfg.passwd, mc.cfg.dbName, mc.cfg.port)
if err != nil {
return nil, err
}
return mc, nil
}
// Driver implements driver.Connector interface.
// Driver returns &taosSQLDriver{}.
func (c *connector) Driver() driver.Driver {
return &taosSQLDriver{}
}

51
src/connector/go/src/taosSql/const.go
View File

@ -1,51 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package taosSql
const (
timeFormat = "2006-01-02 15:04:05"
maxTaosSqlLen = 65380
defaultBufSize = maxTaosSqlLen + 32
)
type fieldType byte
type fieldFlag uint16
const (
flagNotNULL fieldFlag = 1 << iota
)
type statusFlag uint16
const (
statusInTrans statusFlag = 1 << iota
statusInAutocommit
statusReserved // Not in documentation
statusMoreResultsExists
statusNoGoodIndexUsed
statusNoIndexUsed
statusCursorExists
statusLastRowSent
statusDbDropped
statusNoBackslashEscapes
statusMetadataChanged
statusQueryWasSlow
statusPsOutParams
statusInTransReadonly
statusSessionStateChanged
)

43
src/connector/go/src/taosSql/driver.go
View File

@ -1,43 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package taosSql
import (
"context"
"database/sql"
"database/sql/driver"
)
// taosSqlDriver is exported to make the driver directly accessible.
// In general the driver is used via the database/sql package.
type taosSQLDriver struct{}
// Open new Connection.
// the DSN string is formatted
func (d taosSQLDriver) Open(dsn string) (driver.Conn, error) {
cfg, err := parseDSN(dsn)
if err != nil {
return nil, err
}
c := &connector{
cfg: cfg,
}
return c.Connect(context.Background())
}
func init() {
sql.Register("taosSql", &taosSQLDriver{})
taosLogInit()
}

202
src/connector/go/src/taosSql/dsn.go
View File

@ -1,202 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package taosSql
import (
"errors"
"net/url"
"strconv"
"strings"
"time"
)
var (
errInvalidDSNUnescaped = errors.New("invalid DSN: did you forget to escape a param value?")
errInvalidDSNAddr = errors.New("invalid DSN: network address not terminated (missing closing brace)")
errInvalidDSNPort = errors.New("invalid DSN: network port is not a valid number")
errInvalidDSNNoSlash = errors.New("invalid DSN: missing the slash separating the database name")
)
// Config is a configuration parsed from a DSN string.
// If a new Config is created instead of being parsed from a DSN string,
// the NewConfig function should be used, which sets default values.
type config struct {
user string // Username
passwd string // Password (requires User)
net string // Network type
addr string // Network address (requires Net)
port int
dbName string // Database name
params map[string]string // Connection parameters
loc *time.Location // Location for time.Time values
columnsWithAlias bool // Prepend table alias to column names
interpolateParams bool // Interpolate placeholders into query string
parseTime bool // Parse time values to time.Time
}
// NewConfig creates a new Config and sets default values.
func newConfig() *config {
return &config{
loc: time.UTC,
interpolateParams: true,
parseTime: true,
}
}
// ParseDSN parses the DSN string to a Config
func parseDSN(dsn string) (cfg *config, err error) {
taosLog.Println("input dsn:", dsn)
// New config with some default values
cfg = newConfig()
// [user[:password]@][net[(addr)]]/dbname[?param1=value1&paramN=valueN]
// Find the last '/' (since the password or the net addr might contain a '/')
foundSlash := false
for i := len(dsn) - 1; i >= 0; i-- {
if dsn[i] == '/' {
foundSlash = true
var j, k int
// left part is empty if i <= 0
if i > 0 {
// [username[:password]@][protocol[(address)]]
// Find the last '@' in dsn[:i]
for j = i; j >= 0; j-- {
if dsn[j] == '@' {
// username[:password]
// Find the first ':' in dsn[:j]
for k = 0; k < j; k++ {
if dsn[k] == ':' {
cfg.passwd = dsn[k+1 : j]
break
}
}
cfg.user = dsn[:k]
break
}
}
// [protocol[(address)]]
// Find the first '(' in dsn[j+1:i]
for k = j + 1; k < i; k++ {
if dsn[k] == '(' {
// dsn[i-1] must be == ')' if an address is specified
if dsn[i-1] != ')' {
if strings.ContainsRune(dsn[k+1:i], ')') {
return nil, errInvalidDSNUnescaped
}
return nil, errInvalidDSNAddr
}
strs := strings.Split(dsn[k+1:i-1], ":")
if len(strs) == 1 {
return nil, errInvalidDSNAddr
}
cfg.addr = strs[0]
cfg.port, err = strconv.Atoi(strs[1])
if err != nil {
return nil, errInvalidDSNPort
}
break
}
}
cfg.net = dsn[j+1 : k]
}
// dbname[?param1=value1&...&paramN=valueN]
// Find the first '?' in dsn[i+1:]
for j = i + 1; j < len(dsn); j++ {
if dsn[j] == '?' {
if err = parseDSNParams(cfg, dsn[j+1:]); err != nil {
return
}
break
}
}
cfg.dbName = dsn[i+1 : j]
break
}
}
if !foundSlash && len(dsn) > 0 {
return nil, errInvalidDSNNoSlash
}
taosLog.Printf("cfg info: %+v", cfg)
return
}
// parseDSNParams parses the DSN "query string"
// Values must be url.QueryEscape'ed
func parseDSNParams(cfg *config, params string) (err error) {
for _, v := range strings.Split(params, "&") {
param := strings.SplitN(v, "=", 2)
if len(param) != 2 {
continue
}
// cfg params
switch value := param[1]; param[0] {
case "columnsWithAlias":
var isBool bool
cfg.columnsWithAlias, isBool = readBool(value)
if !isBool {
return errors.New("invalid bool value: " + value)
}
// Enable client side placeholder substitution
case "interpolateParams":
var isBool bool
cfg.interpolateParams, isBool = readBool(value)
if !isBool {
return errors.New("invalid bool value: " + value)
}
// Time Location
case "loc":
if value, err = url.QueryUnescape(value); err != nil {
return
}
cfg.loc, err = time.LoadLocation(value)
if err != nil {
return
}
// time.Time parsing
case "parseTime":
var isBool bool
cfg.parseTime, isBool = readBool(value)
if !isBool {
return errors.New("invalid bool value: " + value)
}
default:
// lazy init
if cfg.params == nil {
cfg.params = make(map[string]string)
}
if cfg.params[param[0]], err = url.QueryUnescape(value); err != nil {
return
}
}
}
return
}

200
src/connector/go/src/taosSql/result.go
View File

@ -1,200 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package taosSql
/*
#cgo CFLAGS : -I/usr/include
#cgo LDFLAGS: -L/usr/lib -ltaos
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <taos.h>
*/
import "C"
import (
"database/sql/driver"
"errors"
"fmt"
"io"
"strconv"
"time"
"unsafe"
)
/******************************************************************************
* Result *
******************************************************************************/
// Read Packets as Field Packets until EOF-Packet or an Error appears
func (mc *taosConn) readColumns(count int) ([]taosSqlField, error) {
columns := make([]taosSqlField, count)
var result unsafe.Pointer
result = C.taos_use_result(mc.taos)
if result == nil {
return nil, errors.New("invalid result")
}
pFields := (*C.struct_taosField)(C.taos_fetch_fields(result))
// TODO: Optimized rewriting !!!!
fields := (*[1 << 30]C.struct_taosField)(unsafe.Pointer(pFields))
for i := 0; i < count; i++ {
//columns[i].tableName = ms.taos.
//fmt.Println(reflect.TypeOf(fields[i].name))
var charray []byte
for j := range fields[i].name {
//fmt.Println("fields[i].name[j]: ", fields[i].name[j])
if fields[i].name[j] != 0 {
charray = append(charray, byte(fields[i].name[j]))
} else {
break
}
}
columns[i].name = string(charray)
columns[i].length = (uint32)(fields[i].bytes)
columns[i].fieldType = fieldType(fields[i]._type)
columns[i].flags = 0
// columns[i].decimals = 0
//columns[i].charSet = 0
}
return columns, nil
}
func (rows *taosSqlRows) readRow(dest []driver.Value) error {
mc := rows.mc
if rows.rs.done || mc == nil {
return io.EOF
}
var result unsafe.Pointer
result = C.taos_use_result(mc.taos)
if result == nil {
return errors.New(C.GoString(C.taos_errstr(mc.taos)))
}
//var row *unsafe.Pointer
row := C.taos_fetch_row(result)
if row == nil {
rows.rs.done = true
C.taos_free_result(result)
rows.mc = nil
return io.EOF
}
// because sizeof(void*) == sizeof(int*) == 8
// notes: sizeof(int) == 8 in go, but sizeof(int) == 4 in C.
for i := range dest {
currentRow := (unsafe.Pointer)(uintptr(*((*int)(unsafe.Pointer(uintptr(unsafe.Pointer(row)) + uintptr(i)*unsafe.Sizeof(int(0)))))))
if currentRow == nil {
dest[i] = nil
continue
}
switch rows.rs.columns[i].fieldType {
case C.TSDB_DATA_TYPE_BOOL:
if (*((*byte)(currentRow))) != 0 {
dest[i] = true
} else {
dest[i] = false
}
break
case C.TSDB_DATA_TYPE_TINYINT:
dest[i] = (int)(*((*byte)(currentRow)))
break
case C.TSDB_DATA_TYPE_SMALLINT:
dest[i] = (int16)(*((*int16)(currentRow)))
break
case C.TSDB_DATA_TYPE_INT:
dest[i] = (int)(*((*int32)(currentRow))) // notes int32 of go <----> int of C
break
case C.TSDB_DATA_TYPE_BIGINT:
dest[i] = (int64)(*((*int64)(currentRow)))
break
case C.TSDB_DATA_TYPE_FLOAT:
dest[i] = (*((*float32)(currentRow)))
break
case C.TSDB_DATA_TYPE_DOUBLE:
dest[i] = (*((*float64)(currentRow)))
break
case C.TSDB_DATA_TYPE_BINARY, C.TSDB_DATA_TYPE_NCHAR:
charLen := rows.rs.columns[i].length
var index uint32
binaryVal := make([]byte, charLen)
for index = 0; index < charLen; index++ {
binaryVal[index] = *((*byte)(unsafe.Pointer(uintptr(currentRow) + uintptr(index))))
}
dest[i] = string(binaryVal[:])
break
case C.TSDB_DATA_TYPE_TIMESTAMP:
if mc.cfg.parseTime == true {
timestamp := (int64)(*((*int64)(currentRow)))
dest[i] = timestampConvertToString(timestamp, int(C.taos_result_precision(result)))
} else {
dest[i] = (int64)(*((*int64)(currentRow)))
}
break
default:
fmt.Println("default fieldType: set dest[] to nil")
dest[i] = nil
break
}
}
return nil
}
// Read result as Field format until all rows or an Error appears
// call this func in conn mode
func (rows *textRows) readRow(dest []driver.Value) error {
return rows.taosSqlRows.readRow(dest)
}
// call thsi func in stmt mode
func (rows *binaryRows) readRow(dest []driver.Value) error {
return rows.taosSqlRows.readRow(dest)
}
func timestampConvertToString(timestamp int64, precision int) string {
var decimal, sVal, nsVal int64
if precision == 0 {
decimal = timestamp % 1000
sVal = timestamp / 1000
nsVal = decimal * 1000
} else {
decimal = timestamp % 1000000
sVal = timestamp / 1000000
nsVal = decimal * 1000000
}
date_time := time.Unix(sVal, nsVal)
//const base_format = "2006-01-02 15:04:05"
str_time := date_time.Format(timeFormat)
return (str_time + "." + strconv.Itoa(int(decimal)))
}

296
src/connector/go/src/taosSql/rows.go
View File

@ -1,296 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package taosSql
/*
#cgo CFLAGS : -I/usr/include
#cgo LDFLAGS: -L/usr/lib -ltaos
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <taos.h>
*/
import "C"
import (
"database/sql"
"database/sql/driver"
"io"
"math"
"reflect"
)
type taosSqlField struct {
tableName string
name string
length uint32
flags fieldFlag // indicate whether this field can is null
fieldType fieldType
decimals byte
charSet uint8
}
type resultSet struct {
columns []taosSqlField
columnNames []string
done bool
}
type taosSqlRows struct {
mc *taosConn
rs resultSet
}
type binaryRows struct {
taosSqlRows
}
type textRows struct {
taosSqlRows
}
func (rows *taosSqlRows) Columns() []string {
if rows.rs.columnNames != nil {
return rows.rs.columnNames
}
columns := make([]string, len(rows.rs.columns))
if rows.mc != nil && rows.mc.cfg.columnsWithAlias {
for i := range columns {
if tableName := rows.rs.columns[i].tableName; len(tableName) > 0 {
columns[i] = tableName + "." + rows.rs.columns[i].name
} else {
columns[i] = rows.rs.columns[i].name
}
}
} else {
for i := range columns {
columns[i] = rows.rs.columns[i].name
}
}
rows.rs.columnNames = columns
return columns
}
func (rows *taosSqlRows) ColumnTypeDatabaseTypeName(i int) string {
return rows.rs.columns[i].typeDatabaseName()
}
func (rows *taosSqlRows) ColumnTypeLength(i int) (length int64, ok bool) {
return int64(rows.rs.columns[i].length), true
}
func (rows *taosSqlRows) ColumnTypeNullable(i int) (nullable, ok bool) {
return rows.rs.columns[i].flags&flagNotNULL == 0, true
}
func (rows *taosSqlRows) ColumnTypePrecisionScale(i int) (int64, int64, bool) {
column := rows.rs.columns[i]
decimals := int64(column.decimals)
switch column.fieldType {
case C.TSDB_DATA_TYPE_FLOAT:
fallthrough
case C.TSDB_DATA_TYPE_DOUBLE:
if decimals == 0x1f {
return math.MaxInt64, math.MaxInt64, true
}
return math.MaxInt64, decimals, true
}
return 0, 0, false
}
func (rows *taosSqlRows) ColumnTypeScanType(i int) reflect.Type {
return rows.rs.columns[i].scanType()
}
func (rows *taosSqlRows) Close() error {
if rows.mc != nil {
result := C.taos_use_result(rows.mc.taos)
if result != nil {
C.taos_free_result(result)
}
rows.mc = nil
}
return nil
}
func (rows *taosSqlRows) HasNextResultSet() (b bool) {
if rows.mc == nil {
return false
}
return rows.mc.status&statusMoreResultsExists != 0
}
func (rows *taosSqlRows) nextResultSet() (int, error) {
if rows.mc == nil {
return 0, io.EOF
}
// Remove unread packets from stream
if !rows.rs.done {
rows.rs.done = true
}
if !rows.HasNextResultSet() {
rows.mc = nil
return 0, io.EOF
}
rows.rs = resultSet{}
return 0,nil
}
func (rows *taosSqlRows) nextNotEmptyResultSet() (int, error) {
for {
resLen, err := rows.nextResultSet()
if err != nil {
return 0, err
}
if resLen > 0 {
return resLen, nil
}
rows.rs.done = true
}
}
func (rows *binaryRows) NextResultSet() error {
resLen, err := rows.nextNotEmptyResultSet()
if err != nil {
return err
}
rows.rs.columns, err = rows.mc.readColumns(resLen)
return err
}
// stmt.Query return binary rows, and get row from this func
func (rows *binaryRows) Next(dest []driver.Value) error {
if mc := rows.mc; mc != nil {
// Fetch next row from stream
return rows.readRow(dest)
}
return io.EOF
}
func (rows *textRows) NextResultSet() (err error) {
resLen, err := rows.nextNotEmptyResultSet()
if err != nil {
return err
}
rows.rs.columns, err = rows.mc.readColumns(resLen)
return err
}
// db.Query return text rows, and get row from this func
func (rows *textRows) Next(dest []driver.Value) error {
if mc := rows.mc; mc != nil {
// Fetch next row from stream
return rows.readRow(dest)
}
return io.EOF
}
func (mf *taosSqlField) typeDatabaseName() string {
//fmt.Println("######## (mf *taosSqlField) typeDatabaseName() mf.fieldType:", mf.fieldType)
switch mf.fieldType {
case C.TSDB_DATA_TYPE_BOOL:
return "BOOL"
case C.TSDB_DATA_TYPE_TINYINT:
return "TINYINT"
case C.TSDB_DATA_TYPE_SMALLINT:
return "SMALLINT"
case C.TSDB_DATA_TYPE_INT:
return "INT"
case C.TSDB_DATA_TYPE_BIGINT:
return "BIGINT"
case C.TSDB_DATA_TYPE_FLOAT:
return "FLOAT"
case C.TSDB_DATA_TYPE_DOUBLE:
return "DOUBLE"
case C.TSDB_DATA_TYPE_BINARY:
return "BINARY"
case C.TSDB_DATA_TYPE_NCHAR:
return "NCHAR"
case C.TSDB_DATA_TYPE_TIMESTAMP:
return "TIMESTAMP"
default:
return ""
}
}
var (
scanTypeFloat32 = reflect.TypeOf(float32(0))
scanTypeFloat64 = reflect.TypeOf(float64(0))
scanTypeInt8 = reflect.TypeOf(int8(0))
scanTypeInt16 = reflect.TypeOf(int16(0))
scanTypeInt32 = reflect.TypeOf(int32(0))
scanTypeInt64 = reflect.TypeOf(int64(0))
scanTypeNullTime = reflect.TypeOf(NullTime{})
scanTypeRawBytes = reflect.TypeOf(sql.RawBytes{})
scanTypeUnknown = reflect.TypeOf(new(interface{}))
)
func (mf *taosSqlField) scanType() reflect.Type {
//fmt.Println("######## (mf *taosSqlField) scanType() mf.fieldType:", mf.fieldType)
switch mf.fieldType {
case C.TSDB_DATA_TYPE_BOOL:
return scanTypeInt8
case C.TSDB_DATA_TYPE_TINYINT:
return scanTypeInt8
case C.TSDB_DATA_TYPE_SMALLINT:
return scanTypeInt16
case C.TSDB_DATA_TYPE_INT:
return scanTypeInt32
case C.TSDB_DATA_TYPE_BIGINT:
return scanTypeInt64
case C.TSDB_DATA_TYPE_FLOAT:
return scanTypeFloat32
case C.TSDB_DATA_TYPE_DOUBLE:
return scanTypeFloat64
case C.TSDB_DATA_TYPE_BINARY:
return scanTypeRawBytes
case C.TSDB_DATA_TYPE_NCHAR:
return scanTypeRawBytes
case C.TSDB_DATA_TYPE_TIMESTAMP:
return scanTypeNullTime
default:
return scanTypeUnknown
}
}

158
src/connector/go/src/taosSql/statement.go
View File

@ -1,158 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package taosSql
import (
"database/sql/driver"
"fmt"
"reflect"
)
type taosSqlStmt struct {
mc *taosConn
id uint32
pSql string
paramCount int
}
func (stmt *taosSqlStmt) Close() error {
return nil
}
func (stmt *taosSqlStmt) NumInput() int {
return stmt.paramCount
}
func (stmt *taosSqlStmt) Exec(args []driver.Value) (driver.Result, error) {
if stmt.mc == nil || stmt.mc.taos == nil {
return nil, errInvalidConn
}
return stmt.mc.Exec(stmt.pSql, args)
}
func (stmt *taosSqlStmt) Query(args []driver.Value) (driver.Rows, error) {
if stmt.mc == nil || stmt.mc.taos == nil {
return nil, errInvalidConn
}
return stmt.query(args)
}
func (stmt *taosSqlStmt) query(args []driver.Value) (*binaryRows, error) {
mc := stmt.mc
if mc == nil || mc.taos == nil {
return nil, errInvalidConn
}
querySql := stmt.pSql
if len(args) != 0 {
if !mc.cfg.interpolateParams {
return nil, driver.ErrSkip
}
// try client-side prepare to reduce roundtrip
prepared, err := mc.interpolateParams(stmt.pSql, args)
if err != nil {
return nil, err
}
querySql = prepared
}
num_fields, err := mc.taosQuery(querySql)
if err == nil {
// Read Result
rows := new(binaryRows)
rows.mc = mc
// Columns field
rows.rs.columns, err = mc.readColumns(num_fields)
return rows, err
}
return nil, err
}
type converter struct{}
// ConvertValue mirrors the reference/default converter in database/sql/driver
// with _one_ exception. We support uint64 with their high bit and the default
// implementation does not. This function should be kept in sync with
// database/sql/driver defaultConverter.ConvertValue() except for that
// deliberate difference.
func (c converter) ConvertValue(v interface{}) (driver.Value, error) {
if driver.IsValue(v) {
return v, nil
}
if vr, ok := v.(driver.Valuer); ok {
sv, err := callValuerValue(vr)
if err != nil {
return nil, err
}
if !driver.IsValue(sv) {
return nil, fmt.Errorf("non-Value type %T returned from Value", sv)
}
return sv, nil
}
rv := reflect.ValueOf(v)
switch rv.Kind() {
case reflect.Ptr:
// indirect pointers
if rv.IsNil() {
return nil, nil
} else {
return c.ConvertValue(rv.Elem().Interface())
}
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
return rv.Int(), nil
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
return rv.Uint(), nil
case reflect.Float32, reflect.Float64:
return rv.Float(), nil
case reflect.Bool:
return rv.Bool(), nil
case reflect.Slice:
ek := rv.Type().Elem().Kind()
if ek == reflect.Uint8 {
return rv.Bytes(), nil
}
return nil, fmt.Errorf("unsupported type %T, a slice of %s", v, ek)
case reflect.String:
return rv.String(), nil
}
return nil, fmt.Errorf("unsupported type %T, a %s", v, rv.Kind())
}
var valuerReflectType = reflect.TypeOf((*driver.Valuer)(nil)).Elem()
// callValuerValue returns vr.Value(), with one exception:
// If vr.Value is an auto-generated method on a pointer type and the
// pointer is nil, it would panic at runtime in the panicwrap
// method. Treat it like nil instead.
//
// This is so people can implement driver.Value on value types and
// still use nil pointers to those types to mean nil/NULL, just like
// string/*string.
//
// This is an exact copy of the same-named unexported function from the
// database/sql package.
func callValuerValue(vr driver.Valuer) (v driver.Value, err error) {
if rv := reflect.ValueOf(vr); rv.Kind() == reflect.Ptr &&
rv.IsNil() &&
rv.Type().Elem().Implements(valuerReflectType) {
return nil, nil
}
return vr.Value()
}

120
src/connector/go/src/taosSql/taosLog.go
View File

@ -1,120 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package taosSql
import (
"bufio"
"errors"
"fmt"
"io"
"log"
"os"
"strings"
)
// Various errors the driver might return.
var (
errInvalidConn = errors.New("invalid connection")
errConnNoExist = errors.New("no existent connection ")
)
var taosLog *log.Logger
// SetLogger is used to set the logger for critical errors.
// The initial logger
func taosLogInit() {
cfgName := "/etc/taos/taos.cfg"
logNameDefault := "/var/log/taos/taosgo.log"
var logName string
// get log path from cfg file
cfgFile, err := os.OpenFile(cfgName, os.O_RDONLY, 0644)
defer cfgFile.Close()
if err != nil {
fmt.Println(err)
logName = logNameDefault
} else {
logName, err = getLogNameFromCfg(cfgFile)
if err != nil {
fmt.Println(err)
logName = logNameDefault
}
}
logFile, err := os.OpenFile(logName, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
taosLog = log.New(logFile, "", log.LstdFlags)
taosLog.SetPrefix("TAOS DRIVER ")
taosLog.SetFlags(log.LstdFlags|log.Lshortfile)
}
func getLogNameFromCfg(f *os.File) (string, error) {
// Create file buf, *Reader
r := bufio.NewReader(f)
for {
//read one line, return to slice b
b, _, err := r.ReadLine()
if err != nil {
if err == io.EOF {
break
}
panic(err)
}
// Remove space of left and right
s := strings.TrimSpace(string(b))
if strings.Index(s, "#") == 0 {
// comment line
continue
}
if len(s) == 0 {
continue
}
var ns string
// If there is a comment on the right of the line, must be remove
index := strings.Index(s, "#")
if index > 0 {
// Gets the string to the left of the comment to determine whether it is empty
ns = s[:index]
if len(ns) == 0 {
continue
}
} else {
ns = s;
}
ss := strings.Fields(ns)
if strings.Compare("logDir", ss[0]) != 0 {
continue
}
if len(ss) < 2 {
break
}
// Add a filename after the path
logName := ss[1] + "/taosgo.log"
return logName,nil
}
return "", errors.New("no config log path, use default")
}

84
src/connector/go/src/taosSql/taosSqlCgo.go
View File

@ -1,84 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package taosSql
/*
#cgo CFLAGS : -I/usr/include
#cgo LDFLAGS: -L/usr/lib -ltaos
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <taos.h>
*/
import "C"
import (
"errors"
"unsafe"
)
func (mc *taosConn) taosConnect(ip, user, pass, db string, port int) (taos unsafe.Pointer, err error) {
cuser := C.CString(user)
cpass := C.CString(pass)
cip := C.CString(ip)
cdb := C.CString(db)
defer C.free(unsafe.Pointer(cip))
defer C.free(unsafe.Pointer(cuser))
defer C.free(unsafe.Pointer(cpass))
defer C.free(unsafe.Pointer(cdb))
taosObj := C.taos_connect(cip, cuser, cpass, cdb, (C.ushort)(port))
if taosObj == nil {
return nil, errors.New("taos_connect() fail!")
}
return (unsafe.Pointer)(taosObj), nil
}
func (mc *taosConn) taosQuery(sqlstr string) (int, error) {
//taosLog.Printf("taosQuery() input sql:%s\n", sqlstr)
csqlstr := C.CString(sqlstr)
defer C.free(unsafe.Pointer(csqlstr))
code := int(C.taos_query(mc.taos, csqlstr))
if 0 != code {
mc.taos_error()
errStr := C.GoString(C.taos_errstr(mc.taos))
taosLog.Println("taos_query() failed:", errStr)
taosLog.Printf("taosQuery() input sql:%s\n", sqlstr)
return 0, errors.New(errStr)
}
// read result and save into mc struct
num_fields := int(C.taos_field_count(mc.taos))
if 0 == num_fields { // there are no select and show kinds of commands
mc.affectedRows = int(C.taos_affected_rows(mc.taos))
mc.insertId = 0
}
return num_fields, nil
}
func (mc *taosConn) taos_close() {
C.taos_close(mc.taos)
}
func (mc *taosConn) taos_error() {
// free local resouce: allocated memory/metric-meta refcnt
//var pRes unsafe.Pointer
pRes := C.taos_use_result(mc.taos)
C.taos_free_result(pRes)
}

422
src/connector/go/src/taosSql/utils.go
View File

@ -1,422 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package taosSql
/*
#cgo CFLAGS : -I/usr/include
#include <stdlib.h>
#cgo LDFLAGS: -L/usr/lib -ltaos
void taosSetAllocMode(int mode, const char* path, _Bool autoDump);
void taosDumpMemoryLeak();
*/
import "C"
import (
"database/sql/driver"
"errors"
"fmt"
"sync/atomic"
"time"
"unsafe"
)
// Returns the bool value of the input.
// The 2nd return value indicates if the input was a valid bool value
func readBool(input string) (value bool, valid bool) {
switch input {
case "1", "true", "TRUE", "True":
return true, true
case "0", "false", "FALSE", "False":
return false, true
}
// Not a valid bool value
return
}
/******************************************************************************
* Time related utils *
******************************************************************************/
// NullTime represents a time.Time that may be NULL.
// NullTime implements the Scanner interface so
// it can be used as a scan destination:
//
// var nt NullTime
// err := db.QueryRow("SELECT time FROM foo WHERE id=?", id).Scan(&nt)
// ...
// if nt.Valid {
// // use nt.Time
// } else {
// // NULL value
// }
//
// This NullTime implementation is not driver-specific
type NullTime struct {
Time time.Time
Valid bool // Valid is true if Time is not NULL
}
// Scan implements the Scanner interface.
// The value type must be time.Time or string / []byte (formatted time-string),
// otherwise Scan fails.
func (nt *NullTime) Scan(value interface{}) (err error) {
if value == nil {
nt.Time, nt.Valid = time.Time{}, false
return
}
switch v := value.(type) {
case time.Time:
nt.Time, nt.Valid = v, true
return
case []byte:
nt.Time, err = parseDateTime(string(v), time.UTC)
nt.Valid = (err == nil)
return
case string:
nt.Time, err = parseDateTime(v, time.UTC)
nt.Valid = (err == nil)
return
}
nt.Valid = false
return fmt.Errorf("Can't convert %T to time.Time", value)
}
// Value implements the driver Valuer interface.
func (nt NullTime) Value() (driver.Value, error) {
if !nt.Valid {
return nil, nil
}
return nt.Time, nil
}
func parseDateTime(str string, loc *time.Location) (t time.Time, err error) {
base := "0000-00-00 00:00:00.0000000"
switch len(str) {
case 10, 19, 21, 22, 23, 24, 25, 26: // up to "YYYY-MM-DD HH:MM:SS.MMMMMM"
if str == base[:len(str)] {
return
}
t, err = time.Parse(timeFormat[:len(str)], str)
default:
err = fmt.Errorf("invalid time string: %s", str)
return
}
// Adjust location
if err == nil && loc != time.UTC {
y, mo, d := t.Date()
h, mi, s := t.Clock()
t, err = time.Date(y, mo, d, h, mi, s, t.Nanosecond(), loc), nil
}
return
}
// zeroDateTime is used in formatBinaryDateTime to avoid an allocation
// if the DATE or DATETIME has the zero value.
// It must never be changed.
// The current behavior depends on database/sql copying the result.
var zeroDateTime = []byte("0000-00-00 00:00:00.000000")
const digits01 = "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789"
const digits10 = "0000000000111111111122222222223333333333444444444455555555556666666666777777777788888888889999999999"
/******************************************************************************
* Convert from and to bytes *
******************************************************************************/
func uint64ToBytes(n uint64) []byte {
return []byte{
byte(n),
byte(n >> 8),
byte(n >> 16),
byte(n >> 24),
byte(n >> 32),
byte(n >> 40),
byte(n >> 48),
byte(n >> 56),
}
}
func uint64ToString(n uint64) []byte {
var a [20]byte
i := 20
// U+0030 = 0
// ...
// U+0039 = 9
var q uint64
for n >= 10 {
i--
q = n / 10
a[i] = uint8(n-q*10) + 0x30
n = q
}
i--
a[i] = uint8(n) + 0x30
return a[i:]
}
// treats string value as unsigned integer representation
func stringToInt(b []byte) int {
val := 0
for i := range b {
val *= 10
val += int(b[i] - 0x30)
}
return val
}
// reserveBuffer checks cap(buf) and expand buffer to len(buf) + appendSize.
// If cap(buf) is not enough, reallocate new buffer.
func reserveBuffer(buf []byte, appendSize int) []byte {
newSize := len(buf) + appendSize
if cap(buf) < newSize {
// Grow buffer exponentially
newBuf := make([]byte, len(buf)*2+appendSize)
copy(newBuf, buf)
buf = newBuf
}
return buf[:newSize]
}
// escapeBytesBackslash escapes []byte with backslashes (\)
// This escapes the contents of a string (provided as []byte) by adding backslashes before special
// characters, and turning others into specific escape sequences, such as
// turning newlines into \n and null bytes into \0.
func escapeBytesBackslash(buf, v []byte) []byte {
pos := len(buf)
buf = reserveBuffer(buf, len(v)*2)
for _, c := range v {
switch c {
case '\x00':
buf[pos] = '\\'
buf[pos+1] = '0'
pos += 2
case '\n':
buf[pos] = '\\'
buf[pos+1] = 'n'
pos += 2
case '\r':
buf[pos] = '\\'
buf[pos+1] = 'r'
pos += 2
case '\x1a':
buf[pos] = '\\'
buf[pos+1] = 'Z'
pos += 2
case '\'':
buf[pos] = '\\'
buf[pos+1] = '\''
pos += 2
case '"':
buf[pos] = '\\'
buf[pos+1] = '"'
pos += 2
case '\\':
buf[pos] = '\\'
buf[pos+1] = '\\'
pos += 2
default:
buf[pos] = c
pos++
}
}
return buf[:pos]
}
// escapeStringBackslash is similar to escapeBytesBackslash but for string.
func escapeStringBackslash(buf []byte, v string) []byte {
pos := len(buf)
buf = reserveBuffer(buf, len(v)*2)
for i := 0; i < len(v); i++ {
c := v[i]
switch c {
case '\x00':
buf[pos] = '\\'
buf[pos+1] = '0'
pos += 2
case '\n':
buf[pos] = '\\'
buf[pos+1] = 'n'
pos += 2
case '\r':
buf[pos] = '\\'
buf[pos+1] = 'r'
pos += 2
case '\x1a':
buf[pos] = '\\'
buf[pos+1] = 'Z'
pos += 2
//case '\'':
// buf[pos] = '\\'
// buf[pos+1] = '\''
// pos += 2
case '"':
buf[pos] = '\\'
buf[pos+1] = '"'
pos += 2
case '\\':
buf[pos] = '\\'
buf[pos+1] = '\\'
pos += 2
default:
buf[pos] = c
pos++
}
}
return buf[:pos]
}
// escapeBytesQuotes escapes apostrophes in []byte by doubling them up.
// This escapes the contents of a string by doubling up any apostrophes that
// it contains. This is used when the NO_BACKSLASH_ESCAPES SQL_MODE is in
// effect on the server.
func escapeBytesQuotes(buf, v []byte) []byte {
pos := len(buf)
buf = reserveBuffer(buf, len(v)*2)
for _, c := range v {
if c == '\'' {
buf[pos] = '\''
buf[pos+1] = '\''
pos += 2
} else {
buf[pos] = c
pos++
}
}
return buf[:pos]
}
// escapeStringQuotes is similar to escapeBytesQuotes but for string.
func escapeStringQuotes(buf []byte, v string) []byte {
pos := len(buf)
buf = reserveBuffer(buf, len(v)*2)
for i := 0; i < len(v); i++ {
c := v[i]
if c == '\'' {
buf[pos] = '\''
buf[pos+1] = '\''
pos += 2
} else {
buf[pos] = c
pos++
}
}
return buf[:pos]
}
/******************************************************************************
* Sync utils *
******************************************************************************/
// noCopy may be embedded into structs which must not be copied
// after the first use.
//
// See https://github.com/golang/go/issues/8005#issuecomment-190753527
// for details.
type noCopy struct{}
// Lock is a no-op used by -copylocks checker from `go vet`.
func (*noCopy) Lock() {}
// atomicBool is a wrapper around uint32 for usage as a boolean value with
// atomic access.
type atomicBool struct {
_noCopy noCopy
value uint32
}
// IsSet returns whether the current boolean value is true
func (ab *atomicBool) IsSet() bool {
return atomic.LoadUint32(&ab.value) > 0
}
// Set sets the value of the bool regardless of the previous value
func (ab *atomicBool) Set(value bool) {
if value {
atomic.StoreUint32(&ab.value, 1)
} else {
atomic.StoreUint32(&ab.value, 0)
}
}
// TrySet sets the value of the bool and returns whether the value changed
func (ab *atomicBool) TrySet(value bool) bool {
if value {
return atomic.SwapUint32(&ab.value, 1) == 0
}
return atomic.SwapUint32(&ab.value, 0) > 0
}
// atomicError is a wrapper for atomically accessed error values
type atomicError struct {
_noCopy noCopy
value atomic.Value
}
// Set sets the error value regardless of the previous value.
// The value must not be nil
func (ae *atomicError) Set(value error) {
ae.value.Store(value)
}
// Value returns the current error value
func (ae *atomicError) Value() error {
if v := ae.value.Load(); v != nil {
// this will panic if the value doesn't implement the error interface
return v.(error)
}
return nil
}
func namedValueToValue(named []driver.NamedValue) ([]driver.Value, error) {
dargs := make([]driver.Value, len(named))
for n, param := range named {
if len(param.Name) > 0 {
// TODO: support the use of Named Parameters #561
return nil, errors.New("taosSql: driver does not support the use of Named Parameters")
}
dargs[n] = param.Value
}
return dargs, nil
}
/******************************************************************************
* Utils for C memory issues debugging *
******************************************************************************/
func SetAllocMode(mode int32, path string) {
cpath := C.CString(path)
defer C.free(unsafe.Pointer(cpath))
C.taosSetAllocMode(C.int(mode), cpath, false)
}
func DumpMemoryLeak() {
C.taosDumpMemoryLeak()
}

10
src/dnode/CMakeLists.txt
View File

@ -7,14 +7,16 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/mnode/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/vnode/tsdb/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/tsdb/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/lz4/inc)
INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/inc)
INCLUDE_DIRECTORIES(inc)
AUX_SOURCE_DIRECTORY(src SRC)
ADD_EXECUTABLE(taosd ${SRC})
TARGET_LINK_LIBRARIES(taosd mnode taos_static monitor http tsdb twal vnode)
TARGET_LINK_LIBRARIES(taosd mnode taos_static monitor http tsdb twal vnode cJson lz4)
IF (TD_ACCOUNT)
TARGET_LINK_LIBRARIES(taosd account)
@ -31,6 +33,10 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
TARGET_LINK_LIBRARIES(taosd balance sync)
ENDIF ()
IF (TD_MPEER)
TARGET_LINK_LIBRARIES(taosd mpeer sync)
ENDIF ()
SET(PREPARE_ENV_CMD "prepare_env_cmd")
SET(PREPARE_ENV_TARGET "prepare_env_target")
ADD_CUSTOM_COMMAND(OUTPUT ${PREPARE_ENV_CMD}

2
src/dnode/inc/dnodeMClient.h
View File

@ -24,6 +24,8 @@ int32_t dnodeInitMClient();
void dnodeCleanupMClient();
void dnodeSendMsgToMnode(SRpcMsg *rpcMsg);
uint32_t dnodeGetMnodeMasteIp();
void * dnodeGetMpeerInfos();
int32_t dnodeGetDnodeId();
#ifdef __cplusplus
}

1
src/dnode/inc/dnodeMgmt.h
View File

@ -23,7 +23,6 @@ extern "C" {
int32_t dnodeInitMgmt();
void dnodeCleanupMgmt();
void dnodeMgmt(SRpcMsg *rpcMsg);
void dnodeUpdateDnodeId(int32_t dnodeId);
void* dnodeGetVnode(int32_t vgId);
int32_t dnodeGetVnodeStatus(void *pVnode);

376
src/dnode/src/dnodeMClient.c
View File

@ -15,38 +15,70 @@
#define _DEFAULT_SOURCE
#include "os.h"
#include "cJSON.h"
#include "taosmsg.h"
#include "tlog.h"
#include "trpc.h"
#include "tutil.h"
#include "tsync.h"
#include "ttime.h"
#include "ttimer.h"
#include "dnode.h"
#include "dnodeMClient.h"
#include "dnodeModule.h"
#include "dnodeMgmt.h"
#include "vnode.h"
#include "mpeer.h"
#define MPEER_CONTENT_LEN 2000
static bool dnodeReadMnodeIpList();
static void dnodeSaveMnodeIpList();
static void dnodeReadDnodeInfo();
static void dnodeUpdateDnodeInfo(int32_t dnodeId);
static void dnodeProcessRspFromMnode(SRpcMsg *pMsg);
static void dnodeProcessStatusRsp(SRpcMsg *pMsg);
static void dnodeSendStatusMsg(void *handle, void *tmrId);
static void (*tsDnodeProcessMgmtRspFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *);
static void *tsDnodeMClientRpc = NULL;
static SRpcIpSet tsDnodeMnodeIpList = {0};
static SRpcIpSet tsMnodeIpList = {0};
static SDMNodeInfos tsMnodeInfos = {0};
static void *tsDnodeTmr = NULL;
static void *tsStatusTimer = NULL;
static uint32_t tsRebootTime;
static int32_t tsDnodeId = 0;
static char tsDnodeName[TSDB_NODE_NAME_LEN];
int32_t dnodeInitMClient() {
dnodeReadDnodeInfo();
tsRebootTime = taosGetTimestampSec();
tsDnodeTmr = taosTmrInit(100, 200, 60000, "DND-DM");
if (tsDnodeTmr == NULL) {
dError("failed to init dnode timer");
return -1;
}
if (!dnodeReadMnodeIpList()) {
dTrace("failed to read mnode iplist, set it from cfg file");
memset(&tsDnodeMnodeIpList, 0, sizeof(SRpcIpSet));
tsDnodeMnodeIpList.port = tsMnodeDnodePort;
tsDnodeMnodeIpList.numOfIps = 1;
tsDnodeMnodeIpList.ip[0] = inet_addr(tsMasterIp);
if (tsSecondIp[0]) {
tsDnodeMnodeIpList.numOfIps = 2;
tsDnodeMnodeIpList.ip[1] = inet_addr(tsSecondIp);
memset(&tsMnodeIpList, 0, sizeof(SRpcIpSet));
memset(&tsMnodeInfos, 0, sizeof(SDMNodeInfos));
tsMnodeIpList.port = tsMnodeDnodePort;
tsMnodeIpList.numOfIps = 1;
tsMnodeIpList.ip[0] = inet_addr(tsMasterIp);
if (strcmp(tsSecondIp, tsMasterIp) != 0) {
tsMnodeIpList.numOfIps = 2;
tsMnodeIpList.ip[1] = inet_addr(tsSecondIp);
}
} else {
tsMnodeIpList.inUse = tsMnodeInfos.inUse;
tsMnodeIpList.numOfIps = tsMnodeInfos.nodeNum;
tsMnodeIpList.port = tsMnodeInfos.nodeInfos[0].nodePort;
for (int32_t i = 0; i < tsMnodeInfos.nodeNum; i++) {
tsMnodeIpList.ip[i] = tsMnodeInfos.nodeInfos[i].nodeIp;
}
}
tsDnodeProcessMgmtRspFp[TSDB_MSG_TYPE_DM_STATUS_RSP] = dnodeProcessStatusRsp;
SRpcInit rpcInit;
memset(&rpcInit, 0, sizeof(rpcInit));
rpcInit.localIp = tsAnyIp ? "0.0.0.0" : tsPrivateIp;
@ -67,11 +99,24 @@ int32_t dnodeInitMClient() {
return -1;
}
tsDnodeProcessMgmtRspFp[TSDB_MSG_TYPE_DM_STATUS_RSP] = dnodeProcessStatusRsp;
taosTmrReset(dnodeSendStatusMsg, 500, NULL, tsDnodeTmr, &tsStatusTimer);
dPrint("mnode rpc client is opened");
return 0;
}
void dnodeCleanupMClient() {
if (tsStatusTimer != NULL) {
taosTmrStopA(&tsStatusTimer);
tsStatusTimer = NULL;
}
if (tsDnodeTmr != NULL) {
taosTmrCleanUp(tsDnodeTmr);
tsDnodeTmr = NULL;
}
if (tsDnodeMClientRpc) {
rpcClose(tsDnodeMClientRpc);
tsDnodeMClientRpc = NULL;
@ -92,31 +137,18 @@ static void dnodeProcessRspFromMnode(SRpcMsg *pMsg) {
static void dnodeProcessStatusRsp(SRpcMsg *pMsg) {
if (pMsg->code != TSDB_CODE_SUCCESS) {
dError("status rsp is received, error:%s", tstrerror(pMsg->code));
taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
return;
}
SDMStatusRsp *pStatusRsp = pMsg->pCont;
if (pStatusRsp->ipList.numOfIps <= 0) {
dError("status msg is invalid, num of ips is %d", pStatusRsp->ipList.numOfIps);
SDMNodeInfos *mpeers = &pStatusRsp->mpeers;
if (mpeers->nodeNum <= 0) {
dError("status msg is invalid, num of ips is %d", mpeers->nodeNum);
taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
return;
}
pStatusRsp->ipList.port = htons(pStatusRsp->ipList.port);
for (int32_t i = 0; i < pStatusRsp->ipList.numOfIps; ++i) {
pStatusRsp->ipList.ip[i] = htonl(pStatusRsp->ipList.ip[i]);
}
//dTrace("status msg is received, result:%s", tstrerror(pMsg->code));
if (memcmp(&(pStatusRsp->ipList), &tsDnodeMnodeIpList, sizeof(SRpcIpSet)) != 0) {
dPrint("mnode ip list is changed, numOfIps:%d inUse:%d", pStatusRsp->ipList.numOfIps, pStatusRsp->ipList.inUse);
memcpy(&tsDnodeMnodeIpList, &pStatusRsp->ipList, sizeof(SRpcIpSet));
for (int32_t i = 0; i < tsDnodeMnodeIpList.numOfIps; ++i) {
dPrint("mnode index:%d ip:%s", i, taosIpStr(tsDnodeMnodeIpList.ip[i]));
}
dnodeSaveMnodeIpList();
}
SDnodeState *pState = &pStatusRsp->dnodeState;
pState->numOfVnodes = htonl(pState->numOfVnodes);
pState->moduleStatus = htonl(pState->moduleStatus);
@ -124,75 +156,271 @@ static void dnodeProcessStatusRsp(SRpcMsg *pMsg) {
pState->dnodeId = htonl(pState->dnodeId);
dnodeProcessModuleStatus(pState->moduleStatus);
dnodeUpdateDnodeId(pState->dnodeId);
dnodeUpdateDnodeInfo(pState->dnodeId);
SRpcIpSet mgmtIpSet = {0};
mgmtIpSet.inUse = mpeers->inUse;
mgmtIpSet.numOfIps = mpeers->nodeNum;
mgmtIpSet.port = htons(mpeers->nodeInfos[0].nodePort);
for (int32_t i = 0; i < mpeers->nodeNum; i++) {
mgmtIpSet.ip[i] = htonl(mpeers->nodeInfos[i].nodeIp);
}
if (memcmp(&mgmtIpSet, &tsMnodeIpList, sizeof(SRpcIpSet)) != 0 || tsMnodeInfos.nodeNum == 0) {
memcpy(&tsMnodeIpList, &mgmtIpSet, sizeof(SRpcIpSet));
tsMnodeInfos.inUse = mpeers->inUse;
tsMnodeInfos.nodeNum = mpeers->nodeNum;
dPrint("mnode ip list is changed, numOfIps:%d inUse:%d", tsMnodeInfos.nodeNum, tsMnodeInfos.inUse);
for (int32_t i = 0; i < mpeers->nodeNum; i++) {
tsMnodeInfos.nodeInfos[i].nodeId = htonl(mpeers->nodeInfos[i].nodeId);
tsMnodeInfos.nodeInfos[i].nodeIp = htonl(mpeers->nodeInfos[i].nodeIp);
tsMnodeInfos.nodeInfos[i].nodePort = htons(mpeers->nodeInfos[i].nodePort);
strcpy(tsMnodeInfos.nodeInfos[i].nodeName, mpeers->nodeInfos[i].nodeName);
dPrint("mnode:%d, ip:%s:%u name:%s", tsMnodeInfos.nodeInfos[i].nodeId,
taosIpStr(tsMnodeInfos.nodeInfos[i].nodeIp), tsMnodeInfos.nodeInfos[i].nodePort,
tsMnodeInfos.nodeInfos[i].nodeName);
}
dnodeSaveMnodeIpList();
mpeerUpdateSync();
}
taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
}
void dnodeSendMsgToMnode(SRpcMsg *rpcMsg) {
if (tsDnodeMClientRpc) {
rpcSendRequest(tsDnodeMClientRpc, &tsDnodeMnodeIpList, rpcMsg);
rpcSendRequest(tsDnodeMClientRpc, &tsMnodeIpList, rpcMsg);
}
}
static bool dnodeReadMnodeIpList() {
char ipFile[TSDB_FILENAME_LEN] = {0};
sprintf(ipFile, "%s/iplist", tsDnodeDir);
sprintf(ipFile, "%s/mgmtIpList.json", tsDnodeDir);
FILE *fp = fopen(ipFile, "r");
if (!fp) return false;
if (!fp) {
dTrace("failed to read mnode mgmtIpList.json, file not exist");
return false;
}
bool ret = false;
int maxLen = 2000;
char *content = calloc(1, maxLen + 1);
int len = fread(content, 1, maxLen, fp);
if (len <= 0) {
free(content);
fclose(fp);
dError("failed to read mnode mgmtIpList.json, content is null");
return false;
}
cJSON* root = cJSON_Parse(content);
if (root == NULL) {
dError("failed to read mnode mgmtIpList.json, invalid json format");
goto PARSE_OVER;
}
cJSON* inUse = cJSON_GetObjectItem(root, "inUse");
if (!inUse || inUse->type != cJSON_Number) {
dError("failed to read mnode mgmtIpList.json, inUse not found");
goto PARSE_OVER;
}
tsMnodeInfos.inUse = inUse->valueint;
cJSON* nodeNum = cJSON_GetObjectItem(root, "nodeNum");
if (!nodeNum || nodeNum->type != cJSON_Number) {
dError("failed to read mnode mgmtIpList.json, nodeNum not found");
goto PARSE_OVER;
}
tsMnodeInfos.nodeNum = nodeNum->valueint;
cJSON* nodeInfos = cJSON_GetObjectItem(root, "nodeInfos");
if (!nodeInfos || nodeInfos->type != cJSON_Array) {
dError("failed to read mnode mgmtIpList.json, nodeInfos not found");
goto PARSE_OVER;
}
int size = cJSON_GetArraySize(nodeInfos);
if (size != tsMnodeInfos.nodeNum) {
dError("failed to read mnode mgmtIpList.json, nodeInfos size not matched");
goto PARSE_OVER;
}
for (int i = 0; i < size; ++i) {
cJSON* nodeInfo = cJSON_GetArrayItem(nodeInfos, i);
if (nodeInfo == NULL) continue;
cJSON *nodeId = cJSON_GetObjectItem(nodeInfo, "nodeId");
if (!nodeId || nodeId->type != cJSON_Number) {
dError("failed to read mnode mgmtIpList.json, nodeId not found");
goto PARSE_OVER;
}
tsMnodeInfos.nodeInfos[i].nodeId = nodeId->valueint;
cJSON *nodeIp = cJSON_GetObjectItem(nodeInfo, "nodeIp");
if (!nodeIp || nodeIp->type != cJSON_String || nodeIp->valuestring == NULL) {
dError("failed to read mnode mgmtIpList.json, nodeIp not found");
goto PARSE_OVER;
}
tsMnodeInfos.nodeInfos[i].nodeIp = inet_addr(nodeIp->valuestring);
cJSON *nodePort = cJSON_GetObjectItem(nodeInfo, "nodePort");
if (!nodePort || nodePort->type != cJSON_Number) {
dError("failed to read mnode mgmtIpList.json, nodePort not found");
goto PARSE_OVER;
}
tsMnodeInfos.nodeInfos[i].nodePort = (uint16_t)nodePort->valueint;
cJSON *nodeName = cJSON_GetObjectItem(nodeInfo, "nodeName");
if (!nodeIp || nodeName->type != cJSON_String || nodeName->valuestring == NULL) {
dError("failed to read mnode mgmtIpList.json, nodeName not found");
goto PARSE_OVER;
}
strncpy(tsMnodeInfos.nodeInfos[i].nodeName, nodeName->valuestring, TSDB_NODE_NAME_LEN);
}
ret = true;
dPrint("read mnode iplist successed, numOfIps:%d inUse:%d", tsMnodeInfos.nodeNum, tsMnodeInfos.inUse);
for (int32_t i = 0; i < tsMnodeInfos.nodeNum; i++) {
dPrint("mnode:%d, ip:%s:%u name:%s", tsMnodeInfos.nodeInfos[i].nodeId,
taosIpStr(tsMnodeInfos.nodeInfos[i].nodeId), tsMnodeInfos.nodeInfos[i].nodePort,
tsMnodeInfos.nodeInfos[i].nodeName);
}
PARSE_OVER:
free(content);
cJSON_Delete(root);
fclose(fp);
return ret;
}
static void dnodeSaveMnodeIpList() {
char ipFile[TSDB_FILENAME_LEN] = {0};
sprintf(ipFile, "%s/mgmtIpList.json", tsDnodeDir);
FILE *fp = fopen(ipFile, "w");
if (!fp) return;
int32_t len = 0;
int32_t maxLen = 2000;
char * content = calloc(1, maxLen + 1);
len += snprintf(content + len, maxLen - len, "{\n");
len += snprintf(content + len, maxLen - len, " \"inUse\": %d,\n", tsMnodeInfos.inUse);
len += snprintf(content + len, maxLen - len, " \"nodeNum\": %d,\n", tsMnodeInfos.nodeNum);
len += snprintf(content + len, maxLen - len, " \"nodeInfos\": [{\n");
for (int32_t i = 0; i < tsMnodeInfos.nodeNum; i++) {
len += snprintf(content + len, maxLen - len, " \"nodeId\": %d,\n", tsMnodeInfos.nodeInfos[i].nodeId);
len += snprintf(content + len, maxLen - len, " \"nodeIp\": \"%s\",\n", taosIpStr(tsMnodeInfos.nodeInfos[i].nodeIp));
len += snprintf(content + len, maxLen - len, " \"nodePort\": %u,\n", tsMnodeInfos.nodeInfos[i].nodePort);
len += snprintf(content + len, maxLen - len, " \"nodeName\": \"%s\"\n", tsMnodeInfos.nodeInfos[i].nodeName);
if (i < tsMnodeInfos.nodeNum -1) {
len += snprintf(content + len, maxLen - len, " },{\n");
} else {
len += snprintf(content + len, maxLen - len, " }]\n");
}
}
len += snprintf(content + len, maxLen - len, "}\n");
fwrite(content, 1, len, fp);
fclose(fp);
free(content);
dPrint("save mnode iplist successed");
}
uint32_t dnodeGetMnodeMasteIp() {
return tsMnodeIpList.ip[tsMnodeIpList.inUse];
}
void* dnodeGetMpeerInfos() {
return &tsMnodeInfos;
}
static void dnodeSendStatusMsg(void *handle, void *tmrId) {
if (tsDnodeTmr == NULL) {
dError("dnode timer is already released");
return;
}
if (tsStatusTimer == NULL) {
taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
dError("failed to start status timer");
return;
}
int32_t contLen = sizeof(SDMStatusMsg) + TSDB_MAX_VNODES * sizeof(SVnodeLoad);
SDMStatusMsg *pStatus = rpcMallocCont(contLen);
if (pStatus == NULL) {
taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
dError("failed to malloc status message");
return;
}
strcpy(pStatus->dnodeName, tsDnodeName);
pStatus->version = htonl(tsVersion);
pStatus->dnodeId = htonl(tsDnodeId);
pStatus->privateIp = htonl(inet_addr(tsPrivateIp));
pStatus->publicIp = htonl(inet_addr(tsPublicIp));
pStatus->lastReboot = htonl(tsRebootTime);
pStatus->numOfTotalVnodes = htons((uint16_t) tsNumOfTotalVnodes);
pStatus->numOfCores = htons((uint16_t) tsNumOfCores);
pStatus->diskAvailable = tsAvailDataDirGB;
pStatus->alternativeRole = (uint8_t) tsAlternativeRole;
vnodeBuildStatusMsg(pStatus);
contLen = sizeof(SDMStatusMsg) + pStatus->openVnodes * sizeof(SVnodeLoad);
pStatus->openVnodes = htons(pStatus->openVnodes);
SRpcMsg rpcMsg = {
.pCont = pStatus,
.contLen = contLen,
.msgType = TSDB_MSG_TYPE_DM_STATUS
};
dnodeSendMsgToMnode(&rpcMsg);
}
static void dnodeReadDnodeInfo() {
char dnodeIdFile[TSDB_FILENAME_LEN] = {0};
sprintf(dnodeIdFile, "%s/dnodeId", tsDnodeDir);
FILE *fp = fopen(dnodeIdFile, "r");
if (!fp) return;
char option[32] = {0};
int32_t value = 0;
int32_t num = 0;
num = fscanf(fp, "%s %d", option, &value);
if (num != 2) return false;
if (strcmp(option, "inUse") != 0) return false;
tsDnodeMnodeIpList.inUse = (int8_t)value;;
num = fscanf(fp, "%s %d", option, &value);
if (num != 2) return false;
if (strcmp(option, "numOfIps") != 0) return false;
tsDnodeMnodeIpList.numOfIps = (int8_t)value;
num = fscanf(fp, "%s %d", option, &value);
if (num != 2) return false;
if (strcmp(option, "port") != 0) return false;
tsDnodeMnodeIpList.port = (uint16_t)value;
for (int32_t i = 0; i < tsDnodeMnodeIpList.numOfIps; i++) {
num = fscanf(fp, "%s %d", option, &value);
if (num != 2) return false;
if (strncmp(option, "ip", 2) != 0) return false;
tsDnodeMnodeIpList.ip[i] = (uint32_t)value;
}
if (num != 2) return;
if (strcmp(option, "dnodeId") != 0) return;
tsDnodeId = value;;
fclose(fp);
dPrint("read mnode iplist successed");
for (int32_t i = 0; i < tsDnodeMnodeIpList.numOfIps; i++) {
dPrint("mnode index:%d ip:%s", i, taosIpStr(tsDnodeMnodeIpList.ip[i]));
}
return true;
dPrint("read dnodeId:%d successed", tsDnodeId);
}
static void dnodeSaveMnodeIpList() {
char ipFile[TSDB_FILENAME_LEN] = {0};
sprintf(ipFile, "%s/iplist", tsDnodeDir);
static void dnodeSaveDnodeInfo() {
char dnodeIdFile[TSDB_FILENAME_LEN] = {0};
sprintf(dnodeIdFile, "%s/dnodeId", tsDnodeDir);
FILE *fp = fopen(ipFile, "w");
FILE *fp = fopen(dnodeIdFile, "w");
if (!fp) return;
fprintf(fp, "inUse %d\n", tsDnodeMnodeIpList.inUse);
fprintf(fp, "numOfIps %d\n", tsDnodeMnodeIpList.numOfIps);
fprintf(fp, "port %u\n", tsDnodeMnodeIpList.port);
for (int32_t i = 0; i < tsDnodeMnodeIpList.numOfIps; i++) {
fprintf(fp, "ip%d %u\n", i, tsDnodeMnodeIpList.ip[i]);
}
fprintf(fp, "dnodeId %d\n", tsDnodeId);
fclose(fp);
dPrint("save mnode iplist successed");
dPrint("save dnodeId successed");
}
uint32_t dnodeGetMnodeMasteIp() {
return tsDnodeMnodeIpList.ip[0];
void dnodeUpdateDnodeInfo(int32_t dnodeId) {
if (tsDnodeId == 0) {
dPrint("dnodeId is set to %d", dnodeId);
tsDnodeId = dnodeId;
dnodeSaveDnodeInfo();
}
}
int32_t dnodeGetDnodeId() {
return tsDnodeId;
}

3
src/dnode/src/dnodeMain.c
View File

@ -159,10 +159,10 @@ static int32_t dnodeInitSystem() {
dPrint("starting to initialize TDengine ...");
if (dnodeInitStorage() != 0) return -1;
if (dnodeInitModules() != 0) return -1;
if (dnodeInitRead() != 0) return -1;
if (dnodeInitWrite() != 0) return -1;
if (dnodeInitMClient() != 0) return -1;
if (dnodeInitModules() != 0) return -1;
if (dnodeInitMnode() != 0) return -1;
if (dnodeInitMgmt() != 0) return -1;
if (dnodeInitShell() != 0) return -1;
@ -177,7 +177,6 @@ static int32_t dnodeInitSystem() {
static void dnodeCleanUpSystem() {
if (dnodeGetRunStatus() != TSDB_DNODE_RUN_STATUS_STOPPED) {
tclearModuleStatus(TSDB_MOD_MGMT);
dnodeSetRunStatus(TSDB_DNODE_RUN_STATUS_STOPPED);
dnodeCleanupShell();
dnodeCleanupMnode();

117
src/dnode/src/dnodeMgmt.c
View File

@ -21,8 +21,6 @@
#include "tlog.h"
#include "trpc.h"
#include "tsdb.h"
#include "ttime.h"
#include "ttimer.h"
#include "twal.h"
#include "dnodeMClient.h"
#include "dnodeMgmt.h"
@ -38,52 +36,23 @@ static int32_t dnodeProcessAlterVnodeMsg(SRpcMsg *pMsg);
static int32_t dnodeProcessAlterStreamMsg(SRpcMsg *pMsg);
static int32_t dnodeProcessConfigDnodeMsg(SRpcMsg *pMsg);
static int32_t (*dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *pMsg);
static void dnodeSendStatusMsg(void *handle, void *tmrId);
static void dnodeReadDnodeId();
static void *tsDnodeTmr = NULL;
static void *tsStatusTimer = NULL;
static uint32_t tsRebootTime;
static int32_t tsDnodeId = 0;
static char tsDnodeName[TSDB_DNODE_NAME_LEN];
int32_t dnodeInitMgmt() {
dnodeReadDnodeId();
dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE] = dnodeProcessCreateVnodeMsg;
dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_DROP_VNODE] = dnodeProcessDropVnodeMsg;
dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE] = dnodeProcessAlterVnodeMsg;
dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM] = dnodeProcessAlterStreamMsg;
dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE] = dnodeProcessConfigDnodeMsg;
tsRebootTime = taosGetTimestampSec();
tsDnodeTmr = taosTmrInit(100, 200, 60000, "DND-DM");
if (tsDnodeTmr == NULL) {
dError("failed to init dnode timer");
return -1;
}
int32_t code = dnodeOpenVnodes();
if (code != TSDB_CODE_SUCCESS) {
return -1;
}
taosTmrReset(dnodeSendStatusMsg, 500, NULL, tsDnodeTmr, &tsStatusTimer);
return TSDB_CODE_SUCCESS;
}
void dnodeCleanupMgmt() {
if (tsStatusTimer != NULL) {
taosTmrStopA(&tsStatusTimer);
tsStatusTimer = NULL;
}
if (tsDnodeTmr != NULL) {
taosTmrCleanUp(tsDnodeTmr);
tsDnodeTmr = NULL;
}
dnodeCloseVnodes();
}
@ -213,89 +182,3 @@ static int32_t dnodeProcessConfigDnodeMsg(SRpcMsg *pMsg) {
SMDCfgDnodeMsg *pCfg = (SMDCfgDnodeMsg *)pMsg->pCont;
return tsCfgDynamicOptions(pCfg->config);
}
static void dnodeSendStatusMsg(void *handle, void *tmrId) {
if (tsDnodeTmr == NULL) {
dError("dnode timer is already released");
return;
}
if (tsStatusTimer == NULL) {
taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
dError("failed to start status timer");
return;
}
int32_t contLen = sizeof(SDMStatusMsg) + TSDB_MAX_VNODES * sizeof(SVnodeLoad);
SDMStatusMsg *pStatus = rpcMallocCont(contLen);
if (pStatus == NULL) {
taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
dError("failed to malloc status message");
return;
}
strcpy(pStatus->dnodeName, tsDnodeName);
pStatus->version = htonl(tsVersion);
pStatus->dnodeId = htonl(tsDnodeId);
pStatus->privateIp = htonl(inet_addr(tsPrivateIp));
pStatus->publicIp = htonl(inet_addr(tsPublicIp));
pStatus->lastReboot = htonl(tsRebootTime);
pStatus->numOfTotalVnodes = htons((uint16_t) tsNumOfTotalVnodes);
pStatus->numOfCores = htons((uint16_t) tsNumOfCores);
pStatus->diskAvailable = tsAvailDataDirGB;
pStatus->alternativeRole = (uint8_t) tsAlternativeRole;
vnodeBuildStatusMsg(pStatus);
contLen = sizeof(SDMStatusMsg) + pStatus->openVnodes * sizeof(SVnodeLoad);
pStatus->openVnodes = htons(pStatus->openVnodes);
SRpcMsg rpcMsg = {
.pCont = pStatus,
.contLen = contLen,
.msgType = TSDB_MSG_TYPE_DM_STATUS
};
dnodeSendMsgToMnode(&rpcMsg);
taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
}
static void dnodeReadDnodeId() {
char dnodeIdFile[TSDB_FILENAME_LEN] = {0};
sprintf(dnodeIdFile, "%s/dnodeId", tsDnodeDir);
FILE *fp = fopen(dnodeIdFile, "r");
if (!fp) return;
char option[32] = {0};
int32_t value = 0;
int32_t num = 0;
num = fscanf(fp, "%s %d", option, &value);
if (num != 2) return;
if (strcmp(option, "dnodeId") != 0) return;
tsDnodeId = value;;
fclose(fp);
dPrint("read dnodeId:%d successed", tsDnodeId);
}
static void dnodeSaveDnodeId() {
char dnodeIdFile[TSDB_FILENAME_LEN] = {0};
sprintf(dnodeIdFile, "%s/dnodeId", tsDnodeDir);
FILE *fp = fopen(dnodeIdFile, "w");
if (!fp) return;
fprintf(fp, "dnodeId %d\n", tsDnodeId);
fclose(fp);
dPrint("save dnodeId successed");
}
void dnodeUpdateDnodeId(int32_t dnodeId) {
if (tsDnodeId == 0) {
dPrint("dnodeId is set to %d", dnodeId);
tsDnodeId = dnodeId;
dnodeSaveDnodeId();
}
}

84
src/dnode/src/dnodeRead.c
View File

@ -32,27 +32,51 @@ typedef struct {
SRpcMsg rpcMsg;
} SReadMsg;
typedef struct {
pthread_t thread; // thread
int32_t workerId; // worker ID
} SReadWorker;
typedef struct {
int32_t max; // max number of workers
int32_t min; // min number of workers
int32_t num; // current number of workers
SReadWorker *readWorker;
} SReadWorkerPool;
static void *dnodeProcessReadQueue(void *param);
static void dnodeHandleIdleReadWorker();
static void dnodeHandleIdleReadWorker(SReadWorker *);
// module global variable
static taos_qset readQset;
static int32_t threads; // number of query threads
static int32_t maxThreads;
static int32_t minThreads;
static SReadWorkerPool readPool;
static taos_qset readQset;
int32_t dnodeInitRead() {
readQset = taosOpenQset();
minThreads = 3;
maxThreads = tsNumOfCores * tsNumOfThreadsPerCore;
if (maxThreads <= minThreads * 2) maxThreads = 2 * minThreads;
readPool.min = 2;
readPool.max = tsNumOfCores * tsNumOfThreadsPerCore;
if (readPool.max <= readPool.min * 2) readPool.max = 2 * readPool.min;
readPool.readWorker = (SReadWorker *) calloc(sizeof(SReadWorker), readPool.max);
if (readPool.readWorker == NULL) return -1;
for (int i=0; i < readPool.max; ++i) {
SReadWorker *pWorker = readPool.readWorker + i;
pWorker->workerId = i;
}
dPrint("dnode read is opened");
return 0;
}
void dnodeCleanupRead() {
for (int i=0; i < readPool.max; ++i) {
SReadWorker *pWorker = readPool.readWorker + i;
if (pWorker->thread)
pthread_join(pWorker->thread, NULL);
}
taosCloseQset(readQset);
dPrint("dnode read is closed");
}
@ -116,18 +140,25 @@ void *dnodeAllocateRqueue(void *pVnode) {
taosAddIntoQset(readQset, queue, pVnode);
// spawn a thread to process queue
if (threads < maxThreads) {
pthread_t thread;
pthread_attr_t thAttr;
pthread_attr_init(&thAttr);
pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
if (readPool.num < readPool.max) {
do {
SReadWorker *pWorker = readPool.readWorker + readPool.num;
if (pthread_create(&thread, &thAttr, dnodeProcessReadQueue, readQset) != 0) {
dError("failed to create thread to process read queue, reason:%s", strerror(errno));
}
pthread_attr_t thAttr;
pthread_attr_init(&thAttr);
pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessReadQueue, pWorker) != 0) {
dError("failed to create thread to process read queue, reason:%s", strerror(errno));
}
pthread_attr_destroy(&thAttr);
readPool.num++;
dTrace("read worker:%d is launched, total:%d", pWorker->workerId, readPool.num);
} while (readPool.num < readPool.min);
}
dTrace("pVnode:%p, queue:%p is allocated", pVnode, queue);
dTrace("pVnode:%p, read queue:%p is allocated", pVnode, queue);
return queue;
}
@ -167,14 +198,14 @@ void dnodeSendRpcReadRsp(void *pVnode, SReadMsg *pRead, int32_t code) {
}
static void *dnodeProcessReadQueue(void *param) {
taos_qset qset = (taos_qset)param;
SReadMsg *pReadMsg;
int type;
void *pVnode;
SReadWorker *pWorker = param;
SReadMsg *pReadMsg;
int type;
void *pVnode;
while (1) {
if (taosReadQitemFromQset(qset, &type, (void **)&pReadMsg, (void **)&pVnode) == 0) {
dnodeHandleIdleReadWorker();
if (taosReadQitemFromQset(readQset, &type, (void **)&pReadMsg, &pVnode) == 0) {
dnodeHandleIdleReadWorker(pWorker);
continue;
}
@ -186,11 +217,12 @@ static void *dnodeProcessReadQueue(void *param) {
return NULL;
}
static void dnodeHandleIdleReadWorker() {
static void dnodeHandleIdleReadWorker(SReadWorker *pWorker) {
int32_t num = taosGetQueueNumber(readQset);
if (num == 0 || (num <= minThreads && threads > minThreads)) {
threads--;
if (num == 0 || (num <= readPool.min && readPool.num > readPool.min)) {
readPool.num--;
dTrace("read worker:%d is released, total:%d", pWorker->workerId, readPool.num);
pthread_exit(NULL);
} else {
usleep(100);

33
src/dnode/src/dnodeWrite.c
View File

@ -28,6 +28,7 @@
#include "vnode.h"
typedef struct {
taos_qall qall;
taos_qset qset; // queue set
pthread_t thread; // thread
int32_t workerId; // worker ID
@ -40,7 +41,7 @@ typedef struct {
SRpcMsg rpcMsg;
} SWriteMsg;
typedef struct _wworker_pool {
typedef struct {
int32_t max; // max number of workers
int32_t nextId; // from 0 to max-1, cyclic
SWriteWorker *writeWorker;
@ -65,6 +66,14 @@ int32_t dnodeInitWrite() {
}
void dnodeCleanupWrite() {
for (int32_t i = 0; i < wWorkerPool.max; ++i) {
SWriteWorker *pWorker = wWorkerPool.writeWorker + i;
if (pWorker->thread) {
pthread_join(pWorker->thread, NULL);
}
}
free(wWorkerPool.writeWorker);
dPrint("dnode write is closed");
}
@ -113,6 +122,7 @@ void *dnodeAllocateWqueue(void *pVnode) {
if (pWorker->qset == NULL) return NULL;
taosAddIntoQset(pWorker->qset, queue, pVnode);
pWorker->qall = taosAllocateQall();
wWorkerPool.nextId = (wWorkerPool.nextId + 1) % wWorkerPool.max;
pthread_attr_t thAttr;
@ -122,13 +132,17 @@ void *dnodeAllocateWqueue(void *pVnode) {
if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessWriteQueue, pWorker) != 0) {
dError("failed to create thread to process read queue, reason:%s", strerror(errno));
taosCloseQset(pWorker->qset);
} else {
dTrace("write worker:%d is launched", pWorker->workerId);
}
pthread_attr_destroy(&thAttr);
} else {
taosAddIntoQset(pWorker->qset, queue, pVnode);
wWorkerPool.nextId = (wWorkerPool.nextId + 1) % wWorkerPool.max;
}
dTrace("pVnode:%p, queue:%p is allocated", pVnode, queue);
dTrace("pVnode:%p, write queue:%p is allocated", pVnode, queue);
return queue;
}
@ -160,17 +174,14 @@ void dnodeSendRpcWriteRsp(void *pVnode, void *param, int32_t code) {
static void *dnodeProcessWriteQueue(void *param) {
SWriteWorker *pWorker = (SWriteWorker *)param;
taos_qall qall;
SWriteMsg *pWrite;
SWalHead *pHead;
int32_t numOfMsgs;
int type;
void *pVnode, *item;
qall = taosAllocateQall();
while (1) {
numOfMsgs = taosReadAllQitemsFromQset(pWorker->qset, qall, &pVnode);
numOfMsgs = taosReadAllQitemsFromQset(pWorker->qset, pWorker->qall, &pVnode);
if (numOfMsgs <=0) {
dnodeHandleIdleWorker(pWorker); // thread exit if no queues anymore
continue;
@ -178,7 +189,7 @@ static void *dnodeProcessWriteQueue(void *param) {
for (int32_t i = 0; i < numOfMsgs; ++i) {
pWrite = NULL;
taosGetQitem(qall, &type, &item);
taosGetQitem(pWorker->qall, &type, &item);
if (type == TAOS_QTYPE_RPC) {
pWrite = (SWriteMsg *)item;
pHead = (SWalHead *)(pWrite->pCont - sizeof(SWalHead));
@ -196,9 +207,9 @@ static void *dnodeProcessWriteQueue(void *param) {
walFsync(vnodeGetWal(pVnode));
// browse all items, and process them one by one
taosResetQitems(qall);
taosResetQitems(pWorker->qall);
for (int32_t i = 0; i < numOfMsgs; ++i) {
taosGetQitem(qall, &type, &item);
taosGetQitem(pWorker->qall, &type, &item);
if (type == TAOS_QTYPE_RPC) {
pWrite = (SWriteMsg *)item;
dnodeSendRpcWriteRsp(pVnode, item, pWrite->rpcMsg.code);
@ -209,8 +220,6 @@ static void *dnodeProcessWriteQueue(void *param) {
}
}
taosFreeQall(qall);
return NULL;
}
@ -221,8 +230,10 @@ static void dnodeHandleIdleWorker(SWriteWorker *pWorker) {
usleep(1000);
sched_yield();
} else {
taosFreeQall(pWorker->qall);
taosCloseQset(pWorker->qset);
pWorker->qset = NULL;
dTrace("write worker:%d is released", pWorker->workerId);
pthread_exit(NULL);
}
}

27
src/inc/mnode.h
View File

@ -41,28 +41,19 @@ struct _vg_obj;
struct _db_obj;
struct _acct_obj;
struct _user_obj;
struct _mnode_obj;
typedef struct {
typedef struct _mnode_obj {
int32_t mnodeId;
uint32_t privateIp;
uint32_t publicIp;
int64_t createdTime;
int64_t lostTime;
uint64_t dbVersion;
uint32_t rack;
uint16_t idc;
uint16_t slot;
int8_t role;
int8_t status;
int8_t numOfMnodes;
int32_t numOfDnodes;
char mnodeName[TSDB_DNODE_NAME_LEN + 1];
int8_t reserved[15];
int8_t reserved[14];
int8_t updateEnd[1];
int32_t refCount;
int syncFd;
void *hbTimer;
void *pSync;
uint32_t privateIp;
uint32_t publicIp;
uint16_t port;
int8_t role;
char mnodeName[TSDB_NODE_NAME_LEN + 1];
} SMnodeObj;
typedef struct _dnode_obj {
@ -81,7 +72,7 @@ typedef struct _dnode_obj {
int8_t alternativeRole; // from dnode status msg, 0-any, 1-mgmt, 2-dnode
int8_t status; // set in balance function
int32_t customScore; // config by user
char dnodeName[TSDB_DNODE_NAME_LEN + 1];
char dnodeName[TSDB_NODE_NAME_LEN + 1];
int8_t reserved[15];
int8_t updateEnd[1];
int32_t refCount;

59
src/inc/mpeer.h Normal file
View File

@ -0,0 +1,59 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_MPEER_H
#define TDENGINE_MPEER_H
#ifdef __cplusplus
extern "C" {
#endif
struct _mnode_obj;
enum _TAOS_MN_STATUS {
TAOS_MN_STATUS_OFFLINE,
TAOS_MN_STATUS_DROPPING,
TAOS_MN_STATUS_READY
};
// general implementation
int32_t mpeerInit();
void mpeerCleanup();
// special implementation
int32_t mpeerInitMnodes();
void mpeerCleanupMnodes();
int32_t mpeerAddMnode(int32_t dnodeId);
int32_t mpeerRemoveMnode(int32_t dnodeId);
void * mpeerGetMnode(int32_t mnodeId);
int32_t mpeerGetMnodesNum();
void * mpeerGetNextMnode(void *pNode, struct _mnode_obj **pMnode);
void mpeerReleaseMnode(struct _mnode_obj *pMnode);
bool mpeerIsMaster();
void mpeerGetPrivateIpList(SRpcIpSet *ipSet);
void mpeerGetPublicIpList(SRpcIpSet *ipSet);
void mpeerGetMpeerInfos(void *mpeers);
int32_t mpeerForwardReqToPeer(void *pHead);
void mpeerUpdateSync();
#ifdef __cplusplus
}
#endif
#endif

4
src/inc/taosdef.h
View File

@ -176,7 +176,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
#define TSDB_MAX_COLUMNS 256
#define TSDB_MIN_COLUMNS 2 //PRIMARY COLUMN(timestamp) + other columns
#define TSDB_DNODE_NAME_LEN 64
#define TSDB_NODE_NAME_LEN 64
#define TSDB_TABLE_NAME_LEN 192
#define TSDB_DB_NAME_LEN 32
#define TSDB_COL_NAME_LEN 64
@ -233,7 +233,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
/*
* this is defined in CMakeList.txt
*/
//#define TSDB_REPLICA_MAX_NUM 3
#define TSDB_REPLICA_MAX_NUM 3
#define TSDB_TBNAME_COLUMN_INDEX (-1)
#define TSDB_MULTI_METERMETA_MAX_NUM 100000 // maximum batch size allowed to load metermeta

17
src/inc/taosmsg.h
View File

@ -581,7 +581,7 @@ typedef struct {
typedef struct {
uint32_t version;
int32_t dnodeId;
char dnodeName[TSDB_DNODE_NAME_LEN];
char dnodeName[TSDB_NODE_NAME_LEN + 1];
uint32_t privateIp;
uint32_t publicIp;
uint32_t lastReboot; // time stamp for last reboot
@ -595,7 +595,20 @@ typedef struct {
} SDMStatusMsg;
typedef struct {
SRpcIpSet ipList;
int32_t nodeId;
uint32_t nodeIp;
uint16_t nodePort;
char nodeName[TSDB_NODE_NAME_LEN + 1];
} SDMNodeInfo;
typedef struct {
int8_t inUse;
int8_t nodeNum;
SDMNodeInfo nodeInfos[TSDB_MAX_MPEERS];
} SDMNodeInfos;
typedef struct {
SDMNodeInfos mpeers;
SDnodeState dnodeState;
SVnodeAccess vnodeAccess[];
} SDMStatusRsp;

1
src/inc/tbalance.h
View File

@ -31,6 +31,7 @@ struct _dnode_obj;
int32_t balanceInit();
void balanceCleanUp();
void balanceNotify();
void balanceReset();
int32_t balanceAllocVnodes(struct _vg_obj *pVgroup);
int32_t balanceDropDnode(struct _dnode_obj *pDnode);

1
src/inc/tcluster.h
View File

@ -37,6 +37,7 @@ int32_t clusterInit();
void clusterCleanUp();
char* clusterGetDnodeStatusStr(int32_t dnodeStatus);
bool clusterCheckModuleInDnode(struct _dnode_obj *pDnode, int moduleType);
void clusterMonitorDnodeModule();
int32_t clusterInitDnodes();
void clusterCleanupDnodes();

3
src/inc/twal.h
View File

@ -36,12 +36,13 @@ typedef struct {
typedef struct {
int8_t commitLog; // commitLog
int8_t wals; // number of WAL files;
int8_t keep; // keep the wal file when closed
} SWalCfg;
typedef void* twalh; // WAL HANDLE
typedef int (*FWalWrite)(void *ahandle, void *pHead, int type);
twalh walOpen(const char *path, const SWalCfg *pCfg);
twalh walOpen(const char *path, const SWalCfg *pCfg);
void walClose(twalh);
int walRenew(twalh);
int walWrite(twalh, SWalHead *);

2
src/mnode/CMakeLists.txt
View File

@ -13,5 +13,5 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
AUX_SOURCE_DIRECTORY(src SRC)
ADD_LIBRARY(mnode ${SRC})
TARGET_LINK_LIBRARIES(mnode trpc tutil pthread)
#TARGET_LINK_LIBRARIES(mnode trpc tutil pthread)
ENDIF ()

50
src/mnode/inc/mgmtMnode.h
View File

@ -1,50 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_MGMT_MNODE_H
#define TDENGINE_MGMT_MNODE_H
#ifdef __cplusplus
extern "C" {
#endif
enum _TSDB_MN_STATUS {
TSDB_MN_STATUS_OFFLINE,
TSDB_MN_STATUS_UNSYNCED,
TSDB_MN_STATUS_SYNCING,
TSDB_MN_STATUS_SERVING
};
enum _TSDB_MN_ROLE {
TSDB_MN_ROLE_UNDECIDED,
TSDB_MN_ROLE_SLAVE,
TSDB_MN_ROLE_MASTER
};
int32_t mgmtInitMnodes();
void mgmtCleanupMnodes();
bool mgmtInServerStatus();
bool mgmtIsMaster();
bool mgmtCheckRedirect(void *handle);
void mgmtGetMnodePrivateIpList(SRpcIpSet *ipSet);
void mgmtGetMnodePublicIpList(SRpcIpSet *ipSet);
#ifdef __cplusplus
}
#endif
#endif

38
src/mnode/inc/mgmtSdb.h
View File

@ -21,21 +21,32 @@ extern "C" {
#endif
typedef enum {
SDB_KEY_TYPE_STRING,
SDB_KEY_TYPE_AUTO
SDB_TABLE_DNODE = 0,
SDB_TABLE_MNODE = 1,
SDB_TABLE_ACCOUNT = 2,
SDB_TABLE_USER = 3,
SDB_TABLE_DB = 4,
SDB_TABLE_VGROUP = 5,
SDB_TABLE_STABLE = 6,
SDB_TABLE_CTABLE = 7,
SDB_TABLE_MAX = 8
} ESdbTable;
typedef enum {
SDB_KEY_STRING,
SDB_KEY_INT,
SDB_KEY_AUTO
} ESdbKeyType;
typedef enum {
SDB_OPER_TYPE_GLOBAL,
SDB_OPER_TYPE_LOCAL
SDB_OPER_GLOBAL,
SDB_OPER_LOCAL
} ESdbOperType;
typedef struct {
ESdbOperType type;
void * table;
void * pObj;
int64_t version;
int32_t maxRowSize;
int32_t rowSize;
void * rowData;
} SSdbOperDesc;
@ -45,6 +56,7 @@ typedef struct {
int32_t hashSessions;
int32_t maxRowSize;
int32_t refCountPos;
ESdbTable tableId;
ESdbKeyType keyType;
int32_t (*insertFp)(SSdbOperDesc *pOper);
int32_t (*deleteFp)(SSdbOperDesc *pOper);
@ -52,10 +64,22 @@ typedef struct {
int32_t (*encodeFp)(SSdbOperDesc *pOper);
int32_t (*decodeFp)(SSdbOperDesc *pDesc);
int32_t (*destroyFp)(SSdbOperDesc *pDesc);
int32_t (*restoredFp)();
} SSdbTableDesc;
typedef struct {
int64_t version;
void * wal;
pthread_mutex_t mutex;
} SSdbObject;
int32_t sdbInit();
void sdbCleanUp();
SSdbObject *sdbGetObj();
void * sdbOpenTable(SSdbTableDesc *desc);
void sdbCloseTable(void *handle);
int sdbProcessWrite(void *param, void *data, int type);
int32_t sdbInsertRow(SSdbOperDesc *pOper);
int32_t sdbDeleteRow(SSdbOperDesc *pOper);
@ -66,7 +90,7 @@ void *sdbFetchRow(void *handle, void *pNode, void **ppRow);
void sdbIncRef(void *thandle, void *pRow);
void sdbDecRef(void *thandle, void *pRow);
int64_t sdbGetNumOfRows(void *handle);
int64_t sdbGetId(void *handle);
int32_t sdbGetId(void *handle);
uint64_t sdbGetVersion();
#ifdef __cplusplus

33
src/mnode/src/mgmtDb.c
View File

@ -24,8 +24,8 @@
#include "mgmtDb.h"
#include "tcluster.h"
#include "tgrant.h"
#include "mpeer.h"
#include "mgmtShell.h"
#include "mgmtMnode.h"
#include "mgmtProfile.h"
#include "mgmtSdb.h"
#include "mgmtTable.h"
@ -88,14 +88,9 @@ static int32_t mgmtDbActionUpdate(SSdbOperDesc *pOper) {
static int32_t mgmtDbActionEncode(SSdbOperDesc *pOper) {
SDbObj *pDb = pOper->pObj;
if (pOper->maxRowSize < tsDbUpdateSize) {
return -1;
} else {
memcpy(pOper->rowData, pDb, tsDbUpdateSize);
pOper->rowSize = tsDbUpdateSize;
return TSDB_CODE_SUCCESS;
}
memcpy(pOper->rowData, pDb, tsDbUpdateSize);
pOper->rowSize = tsDbUpdateSize;
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtDbActionDecode(SSdbOperDesc *pOper) {
@ -107,22 +102,28 @@ static int32_t mgmtDbActionDecode(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtDbActionRestored() {
return 0;
}
int32_t mgmtInitDbs() {
SDbObj tObj;
tsDbUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj;
SSdbTableDesc tableDesc = {
.tableId = SDB_TABLE_DB,
.tableName = "dbs",
.hashSessions = TSDB_MAX_DBS,
.maxRowSize = tsDbUpdateSize,
.refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
.keyType = SDB_KEY_TYPE_STRING,
.keyType = SDB_KEY_STRING,
.insertFp = mgmtDbActionInsert,
.deleteFp = mgmtDbActionDelete,
.updateFp = mgmtDbActionUpdate,
.encodeFp = mgmtDbActionEncode,
.decodeFp = mgmtDbActionDecode,
.destroyFp = mgmtDbActionDestroy,
.restoredFp = mgmtDbActionRestored
};
tsDbSdb = sdbOpenTable(&tableDesc);
@ -136,7 +137,7 @@ int32_t mgmtInitDbs() {
mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_DROP_DB, mgmtProcessDropDbMsg);
mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_DB, mgmtGetDbMeta);
mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_DB, mgmtRetrieveDbs);
mTrace("db data is initialized");
return 0;
}
@ -310,7 +311,7 @@ static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) {
pDb->cfg = *pCreate;
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.type = SDB_OPER_GLOBAL,
.table = tsDbSdb,
.pObj = pDb,
.rowSize = sizeof(SDbObj)
@ -663,7 +664,7 @@ static int32_t mgmtSetDbDropping(SDbObj *pDb) {
pDb->status = true;
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.type = SDB_OPER_GLOBAL,
.table = tsDbSdb,
.pObj = pDb,
.rowSize = tsDbUpdateSize
@ -678,8 +679,6 @@ static int32_t mgmtSetDbDropping(SDbObj *pDb) {
}
static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg) {
if (mgmtCheckRedirect(pMsg->thandle)) return;
SCMCreateDbMsg *pCreate = pMsg->pCont;
pCreate->maxSessions = htonl(pCreate->maxSessions);
pCreate->cacheBlockSize = htonl(pCreate->cacheBlockSize);
@ -750,7 +749,7 @@ static int32_t mgmtAlterDb(SDbObj *pDb, SCMAlterDbMsg *pAlter) {
if (memcmp(&newCfg, &pDb->cfg, sizeof(SDbCfg)) != 0) {
pDb->cfg = newCfg;
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.type = SDB_OPER_GLOBAL,
.table = tsDbSdb,
.pObj = pDb,
.rowSize = tsDbUpdateSize
@ -808,7 +807,7 @@ static void mgmtDropDb(SQueuedMsg *pMsg) {
mPrint("db:%s, drop db from sdb", pDb->name);
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.type = SDB_OPER_GLOBAL,
.table = tsDbSdb,
.pObj = pDb
};

8
src/mnode/src/mgmtDnode.c
View File

@ -19,8 +19,8 @@
#include "tbalance.h"
#include "tcluster.h"
#include "mnode.h"
#include "mpeer.h"
#include "mgmtDClient.h"
#include "mgmtMnode.h"
#include "mgmtShell.h"
#include "mgmtDServer.h"
#include "mgmtUser.h"
@ -77,6 +77,7 @@ void * clusterGetDnode(int32_t dnodeId) { return dnodeId == 1 ? &tsDnodeObj : N
void * clusterGetDnodeByIp(uint32_t ip) { return &tsDnodeObj; }
void clusterReleaseDnode(struct _dnode_obj *pDnode) {}
void clusterUpdateDnode(struct _dnode_obj *pDnode) {}
void clusterMonitorDnodeModule() {}
#endif
@ -141,8 +142,6 @@ static void clusterProcessCfgDnodeMsgRsp(SRpcMsg *rpcMsg) {
}
void clusterProcessDnodeStatusMsg(SRpcMsg *rpcMsg) {
if (mgmtCheckRedirect(rpcMsg->handle)) return;
SDMStatusMsg *pStatus = rpcMsg->pCont;
pStatus->dnodeId = htonl(pStatus->dnodeId);
pStatus->privateIp = htonl(pStatus->privateIp);
@ -210,6 +209,7 @@ void clusterProcessDnodeStatusMsg(SRpcMsg *rpcMsg) {
mTrace("dnode:%d, from offline to online", pDnode->dnodeId);
pDnode->status = TAOS_DN_STATUS_READY;
balanceNotify();
clusterMonitorDnodeModule();
}
clusterReleaseDnode(pDnode);
@ -221,7 +221,7 @@ void clusterProcessDnodeStatusMsg(SRpcMsg *rpcMsg) {
return;
}
mgmtGetMnodePrivateIpList(&pRsp->ipList);
mpeerGetMpeerInfos(&pRsp->mpeers);
pRsp->dnodeState.dnodeId = htonl(pDnode->dnodeId);
pRsp->dnodeState.moduleStatus = htonl(pDnode->moduleStatus);

26
src/mnode/src/mgmtMain.c
View File

@ -22,11 +22,11 @@
#include "taccount.h"
#include "tbalance.h"
#include "tcluster.h"
#include "tgrant.h"
#include "mpeer.h"
#include "mgmtDb.h"
#include "mgmtDClient.h"
#include "mgmtDServer.h"
#include "tgrant.h"
#include "mgmtMnode.h"
#include "mgmtSdb.h"
#include "mgmtVgroup.h"
#include "mgmtUser.h"
@ -105,7 +105,17 @@ int32_t mgmtStartSystem() {
}
if (mgmtInitTables() < 0) {
mError("failed to init meters");
mError("failed to init tables");
return -1;
}
if (mpeerInit() < 0) {
mError("failed to init mpeers");
return -1;
}
if (sdbInit() < 0) {
mError("failed to init sdb");
return -1;
}
@ -117,11 +127,6 @@ int32_t mgmtStartSystem() {
return -1;
}
if (mgmtInitMnodes() < 0) {
mError("failed to init mnodes");
return -1;
}
if (balanceInit() < 0) {
mError("failed to init dnode balance")
}
@ -135,7 +140,7 @@ int32_t mgmtStartSystem() {
void mgmtStopSystem() {
if (mgmtIsMaster()) {
if (mpeerIsMaster()) {
mTrace("it is a master mgmt node, it could not be stopped");
return;
}
@ -147,7 +152,7 @@ void mgmtStopSystem() {
void mgmtCleanUpSystem() {
mPrint("starting to clean up mgmt");
grantCleanUp();
mgmtCleanupMnodes();
mpeerCleanup();
balanceCleanUp();
mgmtCleanUpShell();
mgmtCleanupDClient();
@ -158,6 +163,7 @@ void mgmtCleanUpSystem() {
clusterCleanUp();
mgmtCleanUpUsers();
acctCleanUp();
sdbCleanUp();
taosTmrCleanUp(tsMgmtTmr);
mPrint("mgmt is cleaned up");
}

141
src/mnode/src/mgmtMnode.c
View File

@ -17,43 +17,40 @@
#include "os.h"
#include "taoserror.h"
#include "trpc.h"
#include "mgmtMnode.h"
#include "mgmtSdb.h"
#include "tsync.h"
#include "mpeer.h"
#include "mgmtShell.h"
#include "mgmtUser.h"
static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn);
#ifndef _MPEER
static SMnodeObj tsMnodeObj = {0};
static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn);
int32_t mgmtInitMnodes() {
mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_MNODE, mgmtGetMnodeMeta);
mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_MNODE, mgmtRetrieveMnodes);
int32_t mpeerInitMnodes() {
tsMnodeObj.mnodeId = 1;
tsMnodeObj.privateIp = inet_addr(tsPrivateIp);
tsMnodeObj.publicIp = inet_addr(tsPublicIp);
tsMnodeObj.createdTime = taosGetTimestampMs();
tsMnodeObj.role = TSDB_MN_ROLE_MASTER;
tsMnodeObj.status = TSDB_MN_STATUS_SERVING;
tsMnodeObj.numOfMnodes = 1;
sprintf(tsMnodeObj.mnodeName, "%d", tsMnodeObj.mnodeId);
tsMnodeObj.role = TAOS_SYNC_ROLE_MASTER;
tsMnodeObj.port = tsMnodeDnodePort;
sprintf(tsMnodeObj.mnodeName, "m%d", tsMnodeObj.mnodeId);
return TSDB_CODE_SUCCESS;
}
void mgmtCleanupMnodes() {}
bool mgmtInServerStatus() { return tsMnodeObj.status == TSDB_MN_STATUS_SERVING; }
bool mgmtIsMaster() { return tsMnodeObj.role == TSDB_MN_ROLE_MASTER; }
bool mgmtCheckRedirect(void *thandle) { return false; }
void mpeerCleanupMnodes() {}
int32_t mpeerAddMnode(int32_t dnodeId) { return TSDB_CODE_SUCCESS; }
int32_t mpeerRemoveMnode(int32_t dnodeId) { return TSDB_CODE_SUCCESS; }
void * mpeerGetMnode(int32_t mnodeId) { return &tsMnodeObj; }
int32_t mpeerGetMnodesNum() { return 1; }
void mpeerReleaseMnode(struct _mnode_obj *pMnode) {}
bool mpeerIsMaster() { return tsMnodeObj.role == TAOS_SYNC_ROLE_MASTER; }
void mpeerUpdateSync() {}
static int32_t mgmtGetMnodesNum() {
return 1;
}
static void *mgmtGetNextMnode(void *pNode, SMnodeObj **pMnode) {
void *mpeerGetNextMnode(void *pNode, SMnodeObj **pMnode) {
if (*pMnode == NULL) {
*pMnode = &tsMnodeObj;
} else {
@ -63,22 +60,58 @@ static void *mgmtGetNextMnode(void *pNode, SMnodeObj **pMnode) {
return *pMnode;
}
char *taosGetMnodeStatusStr(int32_t mnodeStatus) {
switch (mnodeStatus) {
case TSDB_MN_STATUS_OFFLINE: return "offline";
case TSDB_MN_STATUS_UNSYNCED: return "unsynced";
case TSDB_MN_STATUS_SYNCING: return "syncing";
case TSDB_MN_STATUS_SERVING: return "serving";
default: return "undefined";
}
void mpeerGetPrivateIpList(SRpcIpSet *ipSet) {
ipSet->inUse = 0;
ipSet->numOfIps = 1;
ipSet->port = htons(tsMnodeObj.port);
ipSet->ip[0] = htonl(tsMnodeObj.privateIp);
}
char *taosGetMnodeRoleStr(int32_t mnodeRole) {
switch (mnodeRole) {
case TSDB_MN_ROLE_UNDECIDED: return "undicided";
case TSDB_MN_ROLE_SLAVE: return "slave";
case TSDB_MN_ROLE_MASTER: return "master";
default: return "undefined";
void mpeerGetPublicIpList(SRpcIpSet *ipSet) {
ipSet->inUse = 0;
ipSet->numOfIps = 1;
ipSet->port = htons(tsMnodeObj.port);
ipSet->ip[0] = htonl(tsMnodeObj.publicIp);
}
void mpeerGetMpeerInfos(void *param) {
SDMNodeInfos *mpeers = param;
mpeers->inUse = 0;
mpeers->nodeNum = 1;
mpeers->nodeInfos[0].nodeId = htonl(tsMnodeObj.mnodeId);
mpeers->nodeInfos[0].nodeIp = htonl(tsMnodeObj.privateIp);
mpeers->nodeInfos[0].nodePort = htons(tsMnodeObj.port);
strcpy(mpeers->nodeInfos[0].nodeName, tsMnodeObj.mnodeName);
}
int32_t mpeerForwardReqToPeer(void *pHead) {
return TSDB_CODE_SUCCESS;
}
#endif
int32_t mpeerInit() {
mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_MNODE, mgmtGetMnodeMeta);
mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_MNODE, mgmtRetrieveMnodes);
return mpeerInitMnodes();
}
void mpeerCleanup() {
mpeerCleanupMnodes();
}
static char *mpeerGetMnodeRoleStr(int32_t role) {
switch (role) {
case TAOS_SYNC_ROLE_OFFLINE:
return "offline";
case TAOS_SYNC_ROLE_UNSYNCED:
return "unsynced";
case TAOS_SYNC_ROLE_SLAVE:
return "slave";
case TAOS_SYNC_ROLE_MASTER:
return "master";
default:
return "undefined";
}
}
@ -115,12 +148,6 @@ static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 10;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "status");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 10;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "role");
@ -135,7 +162,7 @@ static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo
pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1];
}
pShow->numOfRows = mgmtGetMnodesNum();
pShow->numOfRows = mpeerGetMnodesNum();
pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
pShow->pNode = NULL;
mgmtReleaseUser(pUser);
@ -151,7 +178,7 @@ static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, voi
char ipstr[32];
while (numOfRows < rows) {
pShow->pNode = mgmtGetNextMnode(pShow->pNode, (SMnodeObj **)&pMnode);
pShow->pNode = mpeerGetNextMnode(pShow->pNode, &pMnode);
if (pMnode == NULL) break;
cols = 0;
@ -175,33 +202,15 @@ static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, voi
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
strcpy(pWrite, taosGetMnodeStatusStr(pMnode->status));
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
strcpy(pWrite, taosGetMnodeRoleStr(pMnode->role));
strcpy(pWrite, mpeerGetMnodeRoleStr(pMnode->role));
cols++;
numOfRows++;
mpeerReleaseMnode(pMnode);
}
pShow->numOfReads += numOfRows;
return numOfRows;
}
void mgmtGetMnodePrivateIpList(SRpcIpSet *ipSet) {
ipSet->inUse = 0;
ipSet->port = htons(tsMnodeDnodePort);
ipSet->numOfIps = 1;
ipSet->ip[0] = htonl(tsMnodeObj.privateIp);
}
void mgmtGetMnodePublicIpList(SRpcIpSet *ipSet) {
ipSet->inUse = 0;
ipSet->port = htons(tsMnodeDnodePort);
ipSet->numOfIps = 1;
ipSet->ip[0] = htonl(tsMnodeObj.publicIp);
}
#endif
}

11
src/mnode/src/mgmtProfile.c
View File

@ -19,7 +19,7 @@
#include "taccount.h"
#include "tcluster.h"
#include "mgmtDb.h"
#include "mgmtMnode.h"
#include "mpeer.h"
#include "mgmtProfile.h"
#include "mgmtShell.h"
#include "mgmtTable.h"
@ -681,8 +681,7 @@ int32_t mgmtRetrieveConns(SShowObj *pShow, char *data, int32_t rows, void *pConn
void mgmtProcessKillQueryMsg(SQueuedMsg *pMsg) {
SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
if (mgmtCheckRedirect(pMsg->thandle)) return;
SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle, NULL);
if (pUser == NULL) {
rpcRsp.code = TSDB_CODE_INVALID_USER;
@ -705,8 +704,7 @@ void mgmtProcessKillQueryMsg(SQueuedMsg *pMsg) {
void mgmtProcessKillStreamMsg(SQueuedMsg *pMsg) {
SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
if (mgmtCheckRedirect(pMsg->thandle)) return;
SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle, NULL);
if (pUser == NULL) {
rpcRsp.code = TSDB_CODE_INVALID_USER;
@ -729,8 +727,7 @@ void mgmtProcessKillStreamMsg(SQueuedMsg *pMsg) {
void mgmtProcessKillConnectionMsg(SQueuedMsg *pMsg) {
SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
if (mgmtCheckRedirect(pMsg->thandle)) return;
SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle, NULL);
if (pUser == NULL) {
rpcRsp.code = TSDB_CODE_INVALID_USER;

1011
src/mnode/src/mgmtSdb.c
View File

File diff suppressed because it is too large Load Diff

31
src/mnode/src/mgmtShell.c
View File

@ -27,7 +27,7 @@
#include "mgmtDb.h"
#include "tcluster.h"
#include "tgrant.h"
#include "mgmtMnode.h"
#include "mpeer.h"
#include "mgmtProfile.h"
#include "mgmtSdb.h"
#include "mgmtShell.h"
@ -42,7 +42,6 @@ static int mgmtShellRetriveAuth(char *user, char *spi, char *encrypt, char *sec
static bool mgmtCheckMsgReadOnly(SQueuedMsg *pMsg);
static void mgmtProcessMsgFromShell(SRpcMsg *pMsg);
static void mgmtProcessUnSupportMsg(SRpcMsg *rpcMsg);
static void mgmtProcessMsgWhileNotReady(SRpcMsg *rpcMsg);
static void mgmtProcessShowMsg(SQueuedMsg *queuedMsg);
static void mgmtProcessRetrieveMsg(SQueuedMsg *queuedMsg);
static void mgmtProcessHeartBeatMsg(SQueuedMsg *queuedMsg);
@ -142,19 +141,13 @@ static void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg) {
return;
}
if (mgmtCheckRedirect(rpcMsg->handle)) {
if (!mpeerIsMaster()) {
// rpcSendRedirectRsp(rpcMsg->handle, mgmtGetMnodeIpListForRedirect());
mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_NO_MASTER);
rpcFreeCont(rpcMsg->pCont);
return;
}
if (!mgmtInServerStatus()) {
mgmtProcessMsgWhileNotReady(rpcMsg);
rpcFreeCont(rpcMsg->pCont);
return;
}
if (grantCheck(TSDB_GRANT_TIME) != TSDB_CODE_SUCCESS) {
mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_GRANT_EXPIRED);
rpcFreeCont(rpcMsg->pCont);
@ -337,9 +330,9 @@ static void mgmtProcessHeartBeatMsg(SQueuedMsg *pMsg) {
}
if (pMsg->usePublicIp) {
mgmtGetMnodePublicIpList(&pHBRsp->ipList);
mpeerGetPublicIpList(&pHBRsp->ipList);
} else {
mgmtGetMnodePrivateIpList(&pHBRsp->ipList);
mpeerGetPrivateIpList(&pHBRsp->ipList);
}
/*
@ -423,9 +416,9 @@ static void mgmtProcessConnectMsg(SQueuedMsg *pMsg) {
pConnectRsp->superAuth = pUser->superAuth;
if (pMsg->usePublicIp) {
mgmtGetMnodePublicIpList(&pConnectRsp->ipList);
mpeerGetPublicIpList(&pConnectRsp->ipList);
} else {
mgmtGetMnodePrivateIpList(&pConnectRsp->ipList);
mpeerGetPrivateIpList(&pConnectRsp->ipList);
}
connect_over:
@ -501,18 +494,6 @@ static void mgmtProcessUnSupportMsg(SRpcMsg *rpcMsg) {
rpcSendResponse(&rpcRsp);
}
static void mgmtProcessMsgWhileNotReady(SRpcMsg *rpcMsg) {
mTrace("%s is ignored since SDB is not ready", taosMsg[rpcMsg->msgType]);
SRpcMsg rpcRsp = {
.msgType = 0,
.pCont = 0,
.contLen = 0,
.code = TSDB_CODE_NOT_READY,
.handle = rpcMsg->handle
};
rpcSendResponse(&rpcRsp);
}
void mgmtSendSimpleResp(void *thandle, int32_t code) {
SRpcMsg rpcRsp = {
.msgType = 0,

114
src/mnode/src/mgmtTable.c
View File

@ -29,7 +29,7 @@
#include "tcluster.h"
#include "mgmtDServer.h"
#include "tgrant.h"
#include "mgmtMnode.h"
#include "mpeer.h"
#include "mgmtProfile.h"
#include "mgmtSdb.h"
#include "mgmtShell.h"
@ -169,6 +169,7 @@ static int32_t mgmtChildTableActionUpdate(SSdbOperDesc *pOper) {
}
static int32_t mgmtChildTableActionEncode(SSdbOperDesc *pOper) {
const int32_t maxRowSize = sizeof(SChildTableObj) + sizeof(SSchema) * TSDB_MAX_COLUMNS;
SChildTableObj *pTable = pOper->pObj;
assert(pTable != NULL && pOper->rowData != NULL);
@ -177,7 +178,7 @@ static int32_t mgmtChildTableActionEncode(SSdbOperDesc *pOper) {
pOper->rowSize = tsChildTableUpdateSize;
} else {
int32_t schemaSize = pTable->numOfColumns * sizeof(SSchema);
if (pOper->maxRowSize < tsChildTableUpdateSize + schemaSize) {
if (maxRowSize < tsChildTableUpdateSize + schemaSize) {
return TSDB_CODE_INVALID_MSG_LEN;
}
memcpy(pOper->rowData, pTable, tsChildTableUpdateSize);
@ -219,35 +220,11 @@ static int32_t mgmtChildTableActionDecode(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtInitChildTables() {
static int32_t mgmtChildTableActionRestored() {
void *pNode = NULL;
void *pLastNode = NULL;
SChildTableObj *pTable = NULL;
SChildTableObj tObj;
tsChildTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj;
SSdbTableDesc tableDesc = {
.tableName = "ctables",
.hashSessions = tsMaxTables,
.maxRowSize = sizeof(SChildTableObj) + sizeof(SSchema) * TSDB_MAX_COLUMNS,
.refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
.keyType = SDB_KEY_TYPE_STRING,
.insertFp = mgmtChildTableActionInsert,
.deleteFp = mgmtChildTableActionDelete,
.updateFp = mgmtChildTableActionUpdate,
.encodeFp = mgmtChildTableActionEncode,
.decodeFp = mgmtChildTableActionDecode,
.destroyFp = mgmtChildTableActionDestroy,
};
tsChildTableSdb = sdbOpenTable(&tableDesc);
if (tsChildTableSdb == NULL) {
mError("failed to init child table data");
return -1;
}
pNode = NULL;
while (1) {
pLastNode = pNode;
mgmtDecTableRef(pTable);
@ -258,7 +235,7 @@ static int32_t mgmtInitChildTables() {
if (pDb == NULL) {
mError("ctable:%s, failed to get db, discard it", pTable->info.tableId);
SSdbOperDesc desc = {0};
desc.type = SDB_OPER_TYPE_LOCAL;
desc.type = SDB_OPER_LOCAL;
desc.pObj = pTable;
desc.table = tsChildTableSdb;
sdbDeleteRow(&desc);
@ -272,7 +249,7 @@ static int32_t mgmtInitChildTables() {
mError("ctable:%s, failed to get vgroup:%d sid:%d, discard it", pTable->info.tableId, pTable->vgId, pTable->sid);
pTable->vgId = 0;
SSdbOperDesc desc = {0};
desc.type = SDB_OPER_TYPE_LOCAL;
desc.type = SDB_OPER_LOCAL;
desc.pObj = pTable;
desc.table = tsChildTableSdb;
sdbDeleteRow(&desc);
@ -286,7 +263,7 @@ static int32_t mgmtInitChildTables() {
pTable->info.tableId, pDb->name, pTable->vgId, pVgroup->dbName, pTable->sid);
pTable->vgId = 0;
SSdbOperDesc desc = {0};
desc.type = SDB_OPER_TYPE_LOCAL;
desc.type = SDB_OPER_LOCAL;
desc.pObj = pTable;
desc.table = tsChildTableSdb;
sdbDeleteRow(&desc);
@ -298,7 +275,7 @@ static int32_t mgmtInitChildTables() {
mError("ctable:%s, vgroup:%d tableList is null", pTable->info.tableId, pTable->vgId);
pTable->vgId = 0;
SSdbOperDesc desc = {0};
desc.type = SDB_OPER_TYPE_LOCAL;
desc.type = SDB_OPER_LOCAL;
desc.pObj = pTable;
desc.table = tsChildTableSdb;
sdbDeleteRow(&desc);
@ -312,7 +289,7 @@ static int32_t mgmtInitChildTables() {
mError("ctable:%s, stable:%s not exist", pTable->info.tableId, pTable->superTableId);
pTable->vgId = 0;
SSdbOperDesc desc = {0};
desc.type = SDB_OPER_TYPE_LOCAL;
desc.type = SDB_OPER_LOCAL;
desc.pObj = pTable;
desc.table = tsChildTableSdb;
sdbDeleteRow(&desc);
@ -323,6 +300,35 @@ static int32_t mgmtInitChildTables() {
}
}
return 0;
}
static int32_t mgmtInitChildTables() {
SChildTableObj tObj;
tsChildTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj;
SSdbTableDesc tableDesc = {
.tableId = SDB_TABLE_CTABLE,
.tableName = "ctables",
.hashSessions = tsMaxTables,
.maxRowSize = sizeof(SChildTableObj) + sizeof(SSchema) * TSDB_MAX_COLUMNS,
.refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
.keyType = SDB_KEY_STRING,
.insertFp = mgmtChildTableActionInsert,
.deleteFp = mgmtChildTableActionDelete,
.updateFp = mgmtChildTableActionUpdate,
.encodeFp = mgmtChildTableActionEncode,
.decodeFp = mgmtChildTableActionDecode,
.destroyFp = mgmtChildTableActionDestroy,
.restoredFp = mgmtChildTableActionRestored
};
tsChildTableSdb = sdbOpenTable(&tableDesc);
if (tsChildTableSdb == NULL) {
mError("failed to init child table data");
return -1;
}
mTrace("child table is initialized");
return 0;
}
@ -369,12 +375,14 @@ static int32_t mgmtSuperTableActionUpdate(SSdbOperDesc *pOper) {
}
static int32_t mgmtSuperTableActionEncode(SSdbOperDesc *pOper) {
const int32_t maxRowSize = sizeof(SChildTableObj) + sizeof(SSchema) * TSDB_MAX_COLUMNS;
SSuperTableObj *pStable = pOper->pObj;
assert(pOper->pObj != NULL && pOper->rowData != NULL);
int32_t schemaSize = sizeof(SSchema) * (pStable->numOfColumns + pStable->numOfTags);
if (pOper->maxRowSize < tsSuperTableUpdateSize + schemaSize) {
if (maxRowSize < tsSuperTableUpdateSize + schemaSize) {
return TSDB_CODE_INVALID_MSG_LEN;
}
@ -406,22 +414,28 @@ static int32_t mgmtSuperTableActionDecode(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtSuperTableActionRestored() {
return 0;
}
static int32_t mgmtInitSuperTables() {
SSuperTableObj tObj;
tsSuperTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj;
SSdbTableDesc tableDesc = {
.tableId = SDB_TABLE_STABLE,
.tableName = "stables",
.hashSessions = TSDB_MAX_SUPER_TABLES,
.maxRowSize = tsSuperTableUpdateSize + sizeof(SSchema) * TSDB_MAX_COLUMNS,
.refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
.keyType = SDB_KEY_TYPE_STRING,
.keyType = SDB_KEY_STRING,
.insertFp = mgmtSuperTableActionInsert,
.deleteFp = mgmtSuperTableActionDelete,
.updateFp = mgmtSuperTableActionUpdate,
.encodeFp = mgmtSuperTableActionEncode,
.decodeFp = mgmtSuperTableActionDecode,
.destroyFp = mgmtSuperTableActionDestroy,
.restoredFp = mgmtSuperTableActionRestored
};
tsSuperTableSdb = sdbOpenTable(&tableDesc);
@ -657,7 +671,7 @@ static void mgmtProcessCreateSuperTableMsg(SQueuedMsg *pMsg) {
}
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.type = SDB_OPER_GLOBAL,
.table = tsSuperTableSdb,
.pObj = pStable,
.rowSize = sizeof(SSuperTableObj) + schemaSize
@ -681,7 +695,7 @@ static void mgmtProcessDropSuperTableMsg(SQueuedMsg *pMsg) {
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_OTHERS);
} else {
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.type = SDB_OPER_GLOBAL,
.table = tsSuperTableSdb,
.pObj = pStable
};
@ -732,7 +746,7 @@ static int32_t mgmtAddSuperTableTag(SSuperTableObj *pStable, SSchema schema[], i
pStable->sversion++;
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.type = SDB_OPER_GLOBAL,
.table = tsSuperTableSdb,
.pObj = pStable,
.rowSize = tsSuperTableUpdateSize
@ -763,7 +777,7 @@ static int32_t mgmtDropSuperTableTag(SSuperTableObj *pStable, char *tagName) {
pStable->schema = realloc(pStable->schema, schemaSize);
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.type = SDB_OPER_GLOBAL,
.table = tsSuperTableSdb,
.pObj = pStable,
.rowSize = tsSuperTableUpdateSize
@ -798,7 +812,7 @@ static int32_t mgmtModifySuperTableTagName(SSuperTableObj *pStable, char *oldTag
strncpy(schema->name, newTagName, TSDB_COL_NAME_LEN);
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.type = SDB_OPER_GLOBAL,
.table = tsSuperTableSdb,
.pObj = pStable,
.rowSize = tsSuperTableUpdateSize
@ -857,7 +871,7 @@ static int32_t mgmtAddSuperTableColumn(SDbObj *pDb, SSuperTableObj *pStable, SSc
}
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.type = SDB_OPER_GLOBAL,
.table = tsSuperTableSdb,
.pObj = pStable,
.rowSize = tsSuperTableUpdateSize
@ -894,7 +908,7 @@ static int32_t mgmtDropSuperTableColumn(SDbObj *pDb, SSuperTableObj *pStable, ch
}
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.type = SDB_OPER_GLOBAL,
.table = tsSuperTableSdb,
.pObj = pStable,
.rowSize = tsSuperTableUpdateSize
@ -1042,7 +1056,7 @@ void mgmtDropAllSuperTables(SDbObj *pDropDb) {
if (strncmp(pDropDb->name, pTable->info.tableId, dbNameLen) == 0) {
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_LOCAL,
.type = SDB_OPER_LOCAL,
.table = tsSuperTableSdb,
.pObj = pTable,
};
@ -1271,7 +1285,7 @@ static SChildTableObj* mgmtDoCreateChildTable(SCMCreateTableMsg *pCreate, SVgObj
}
SSdbOperDesc desc = {0};
desc.type = SDB_OPER_TYPE_GLOBAL;
desc.type = SDB_OPER_GLOBAL;
desc.pObj = pTable;
desc.table = tsChildTableSdb;
@ -1425,7 +1439,7 @@ static int32_t mgmtAddNormalTableColumn(SDbObj *pDb, SChildTableObj *pTable, SSc
}
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.type = SDB_OPER_GLOBAL,
.table = tsChildTableSdb,
.pObj = pTable,
.rowSize = tsChildTableUpdateSize
@ -1459,7 +1473,7 @@ static int32_t mgmtDropNormalTableColumn(SDbObj *pDb, SChildTableObj *pTable, ch
}
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.type = SDB_OPER_GLOBAL,
.table = tsChildTableSdb,
.pObj = pTable,
.rowSize = tsChildTableUpdateSize
@ -1601,7 +1615,7 @@ void mgmtDropAllChildTables(SDbObj *pDropDb) {
if (strncmp(pDropDb->name, pTable->info.tableId, dbNameLen) == 0) {
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_LOCAL,
.type = SDB_OPER_LOCAL,
.table = tsChildTableSdb,
.pObj = pTable,
};
@ -1630,7 +1644,7 @@ static void mgmtDropAllChildTablesInStable(SSuperTableObj *pStable) {
if (pTable->superTable == pStable) {
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_LOCAL,
.type = SDB_OPER_LOCAL,
.table = tsChildTableSdb,
.pObj = pTable,
};
@ -1659,8 +1673,6 @@ static SChildTableObj* mgmtGetTableByPos(uint32_t dnodeId, int32_t vnode, int32_
}
static void mgmtProcessTableCfgMsg(SRpcMsg *rpcMsg) {
if (mgmtCheckRedirect(rpcMsg->handle)) return;
SDMConfigTableMsg *pCfg = (SDMConfigTableMsg *) rpcMsg->pCont;
pCfg->dnode = htonl(pCfg->dnode);
pCfg->vnode = htonl(pCfg->vnode);
@ -1721,7 +1733,7 @@ static void mgmtProcessDropChildTableRsp(SRpcMsg *rpcMsg) {
}
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.type = SDB_OPER_GLOBAL,
.table = tsChildTableSdb,
.pObj = pTable
};
@ -1764,7 +1776,7 @@ static void mgmtProcessCreateChildTableRsp(SRpcMsg *rpcMsg) {
queueMsg->thandle, tstrerror(rpcMsg->code));
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.type = SDB_OPER_GLOBAL,
.table = tsChildTableSdb,
.pObj = pTable
};

49
src/mnode/src/mgmtUser.c
View File

@ -20,7 +20,7 @@
#include "tutil.h"
#include "taccount.h"
#include "tgrant.h"
#include "mgmtMnode.h"
#include "mpeer.h"
#include "mgmtSdb.h"
#include "mgmtShell.h"
#include "mgmtUser.h"
@ -70,14 +70,9 @@ static int32_t mgmtUserActionUpdate(SSdbOperDesc *pOper) {
static int32_t mgmtUserActionEncode(SSdbOperDesc *pOper) {
SUserObj *pUser = pOper->pObj;
if (pOper->maxRowSize < tsUserUpdateSize) {
return -1;
} else {
memcpy(pOper->rowData, pUser, tsUserUpdateSize);
pOper->rowSize = tsUserUpdateSize;
return TSDB_CODE_SUCCESS;
}
memcpy(pOper->rowData, pUser, tsUserUpdateSize);
pOper->rowSize = tsUserUpdateSize;
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtUserActionDecode(SSdbOperDesc *pOper) {
@ -89,22 +84,36 @@ static int32_t mgmtUserActionDecode(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtUserActionRestored() {
if (strcmp(tsMasterIp, tsPrivateIp) == 0) {
SAcctObj *pAcct = acctGetAcct("root");
mgmtCreateUser(pAcct, "root", "taosdata");
mgmtCreateUser(pAcct, "monitor", tsInternalPass);
mgmtCreateUser(pAcct, "_root", tsInternalPass);
acctReleaseAcct(pAcct);
}
return 0;
}
int32_t mgmtInitUsers() {
SUserObj tObj;
tsUserUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj;
SSdbTableDesc tableDesc = {
.tableId = SDB_TABLE_USER,
.tableName = "users",
.hashSessions = TSDB_MAX_USERS,
.maxRowSize = tsUserUpdateSize,
.refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
.keyType = SDB_KEY_TYPE_STRING,
.keyType = SDB_KEY_STRING,
.insertFp = mgmtUserActionInsert,
.deleteFp = mgmtUserActionDelete,
.updateFp = mgmtUserActionUpdate,
.encodeFp = mgmtUserActionEncode,
.decodeFp = mgmtUserActionDecode,
.destroyFp = mgmtUserActionDestroy,
.restoredFp = mgmtUserActionRestored
};
tsUserSdb = sdbOpenTable(&tableDesc);
@ -113,12 +122,6 @@ int32_t mgmtInitUsers() {
return -1;
}
SAcctObj *pAcct = acctGetAcct("root");
mgmtCreateUser(pAcct, "root", "taosdata");
mgmtCreateUser(pAcct, "monitor", tsInternalPass);
mgmtCreateUser(pAcct, "_root", tsInternalPass);
acctReleaseAcct(pAcct);
mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_CREATE_USER, mgmtProcessCreateUserMsg);
mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_ALTER_USER, mgmtProcessAlterUserMsg);
mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_DROP_USER, mgmtProcessDropUserMsg);
@ -143,7 +146,7 @@ void mgmtReleaseUser(SUserObj *pUser) {
static int32_t mgmtUpdateUser(SUserObj *pUser) {
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.type = SDB_OPER_GLOBAL,
.table = tsUserSdb,
.pObj = pUser,
.rowSize = tsUserUpdateSize
@ -191,7 +194,7 @@ int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass) {
}
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.type = SDB_OPER_GLOBAL,
.table = tsUserSdb,
.pObj = pUser,
.rowSize = sizeof(SUserObj)
@ -208,7 +211,7 @@ int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass) {
static int32_t mgmtDropUser(SUserObj *pUser) {
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.type = SDB_OPER_GLOBAL,
.table = tsUserSdb,
.pObj = pUser
};
@ -314,8 +317,6 @@ SUserObj *mgmtGetUserFromConn(void *pConn, bool *usePublicIp) {
}
static void mgmtProcessCreateUserMsg(SQueuedMsg *pMsg) {
if (mgmtCheckRedirect(pMsg->thandle)) return;
int32_t code;
SUserObj *pUser = pMsg->pUser;
@ -333,8 +334,6 @@ static void mgmtProcessCreateUserMsg(SQueuedMsg *pMsg) {
}
static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg) {
if (mgmtCheckRedirect(pMsg->thandle)) return;
int32_t code;
SUserObj *pOperUser = pMsg->pUser;
@ -427,8 +426,6 @@ static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg) {
}
static void mgmtProcessDropUserMsg(SQueuedMsg *pMsg) {
if (mgmtCheckRedirect(pMsg->thandle)) return;
int32_t code;
SUserObj *pOperUser = pMsg->pUser;
@ -488,7 +485,7 @@ void mgmtDropAllUsers(SAcctObj *pAcct) {
if (strncmp(pUser->acct, pAcct->user, acctNameLen) == 0) {
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_LOCAL,
.type = SDB_OPER_LOCAL,
.table = tsUserSdb,
.pObj = pUser,
};

35
src/mnode/src/mgmtVgroup.c
View File

@ -24,7 +24,7 @@
#include "mgmtDb.h"
#include "mgmtDClient.h"
#include "mgmtDServer.h"
#include "mgmtMnode.h"
#include "mpeer.h"
#include "mgmtProfile.h"
#include "mgmtSdb.h"
#include "mgmtShell.h"
@ -138,13 +138,9 @@ static int32_t mgmtVgroupActionUpdate(SSdbOperDesc *pOper) {
static int32_t mgmtVgroupActionEncode(SSdbOperDesc *pOper) {
SVgObj *pVgroup = pOper->pObj;
if (pOper->maxRowSize < tsVgUpdateSize) {
return -1;
} else {
memcpy(pOper->rowData, pVgroup, tsVgUpdateSize);
pOper->rowSize = tsVgUpdateSize;
return TSDB_CODE_SUCCESS;
}
memcpy(pOper->rowData, pVgroup, tsVgUpdateSize);
pOper->rowSize = tsVgUpdateSize;
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtVgroupActionDecode(SSdbOperDesc *pOper) {
@ -156,22 +152,28 @@ static int32_t mgmtVgroupActionDecode(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtVgroupActionRestored() {
return 0;
}
int32_t mgmtInitVgroups() {
SVgObj tObj;
tsVgUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj;
SSdbTableDesc tableDesc = {
.tableId = SDB_TABLE_VGROUP,
.tableName = "vgroups",
.hashSessions = TSDB_MAX_VGROUPS,
.maxRowSize = tsVgUpdateSize,
.refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
.keyType = SDB_KEY_TYPE_AUTO,
.keyType = SDB_KEY_AUTO,
.insertFp = mgmtVgroupActionInsert,
.deleteFp = mgmtVgroupActionDelete,
.updateFp = mgmtVgroupActionUpdate,
.encodeFp = mgmtVgroupActionEncode,
.decodeFp = mgmtVgroupActionDecode,
.destroyFp = mgmtVgroupActionDestroy,
.restoredFp = mgmtVgroupActionRestored,
};
tsVgroupSdb = sdbOpenTable(&tableDesc);
@ -187,6 +189,7 @@ int32_t mgmtInitVgroups() {
mgmtAddDServerMsgHandle(TSDB_MSG_TYPE_DM_CONFIG_VNODE, mgmtProcessVnodeCfgMsg);
mTrace("vgroup is initialized");
return 0;
}
@ -200,7 +203,7 @@ SVgObj *mgmtGetVgroup(int32_t vgId) {
void mgmtUpdateVgroup(SVgObj *pVgroup) {
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.type = SDB_OPER_GLOBAL,
.table = tsVgroupSdb,
.pObj = pVgroup,
.rowSize = tsVgUpdateSize
@ -244,7 +247,7 @@ void mgmtCreateVgroup(SQueuedMsg *pMsg, SDbObj *pDb) {
}
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.type = SDB_OPER_GLOBAL,
.table = tsVgroupSdb,
.pObj = pVgroup,
.rowSize = sizeof(SVgObj)
@ -276,7 +279,7 @@ void mgmtDropVgroup(SVgObj *pVgroup, void *ahandle) {
mTrace("vgroup:%d, replica:%d is deleting from sdb", pVgroup->vgId, pVgroup->numOfVnodes);
mgmtSendDropVgroupMsg(pVgroup, NULL);
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.type = SDB_OPER_GLOBAL,
.table = tsVgroupSdb,
.pObj = pVgroup
};
@ -583,7 +586,7 @@ static void mgmtProcessCreateVnodeRsp(SRpcMsg *rpcMsg) {
mgmtAddToShellQueue(newMsg);
} else {
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.type = SDB_OPER_GLOBAL,
.table = tsVgroupSdb,
.pObj = pVgroup
};
@ -646,7 +649,7 @@ static void mgmtProcessDropVnodeRsp(SRpcMsg *rpcMsg) {
if (queueMsg->received != queueMsg->expected) return;
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.type = SDB_OPER_GLOBAL,
.table = tsVgroupSdb,
.pObj = pVgroup
};
@ -663,8 +666,6 @@ static void mgmtProcessDropVnodeRsp(SRpcMsg *rpcMsg) {
}
static void mgmtProcessVnodeCfgMsg(SRpcMsg *rpcMsg) {
if (mgmtCheckRedirect(rpcMsg->handle)) return;
SDMConfigVnodeMsg *pCfg = (SDMConfigVnodeMsg *) rpcMsg->pCont;
pCfg->dnodeId = htonl(pCfg->dnodeId);
pCfg->vgId = htonl(pCfg->vgId);
@ -705,7 +706,7 @@ void mgmtDropAllVgroups(SDbObj *pDropDb) {
if (strncmp(pDropDb->name, pVgroup->dbName, dbNameLen) == 0) {
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_LOCAL,
.type = SDB_OPER_LOCAL,
.table = tsVgroupSdb,
.pObj = pVgroup,
};

2
src/plugins/http/CMakeLists.txt
View File

@ -7,6 +7,8 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/zlib-1.2.11/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/lz4/inc)
INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
INCLUDE_DIRECTORIES(inc)
AUX_SOURCE_DIRECTORY(src SRC)

264
src/plugins/http/inc/cJSON.h
View File

@ -1,264 +0,0 @@
/*
Copyright (c) 2009-2017 Dave Gamble and cJSON contributors
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
#ifndef cJSON__h
#define cJSON__h
#ifdef __cplusplus
extern "C"
{
#endif
/* project version */
#define CJSON_VERSION_MAJOR 1
#define CJSON_VERSION_MINOR 5
#define CJSON_VERSION_PATCH 9
#include <stddef.h>
#include <stdint.h>
/* cJSON Types: */
#define cJSON_Invalid (0)
#define cJSON_False (1 << 0)
#define cJSON_True (1 << 1)
#define cJSON_NULL (1 << 2)
#define cJSON_Number (1 << 3)
#define cJSON_String (1 << 4)
#define cJSON_Array (1 << 5)
#define cJSON_Object (1 << 6)
#define cJSON_Raw (1 << 7) /* raw json */
#define cJSON_IsReference 256
#define cJSON_StringIsConst 512
/* The cJSON structure: */
typedef struct cJSON
{
/* next/prev allow you to walk array/object chains. Alternatively, use GetArraySize/GetArrayItem/GetObjectItem */
struct cJSON *next;
struct cJSON *prev;
/* An array or object item will have a child pointer pointing to a chain of the items in the array/object. */
struct cJSON *child;
/* The type of the item, as above. */
int type;
/* The item's string, if type==cJSON_String and type == cJSON_Raw */
char *valuestring;
/* writing to valueint is DEPRECATED, use cJSON_SetNumberValue instead */
int64_t valueint;
/* The item's number, if type==cJSON_Number */
double valuedouble;
/* The item's name string, if this item is the child of, or is in the list of subitems of an object. */
char *string;
} cJSON;
typedef struct cJSON_Hooks
{
void *(*malloc_fn)(size_t sz);
void (*free_fn)(void *ptr);
} cJSON_Hooks;
typedef int cJSON_bool;
#if !defined(__WINDOWS__) && (defined(WIN32) || defined(WIN64) || defined(_MSC_VER) || defined(_WIN32))
#define __WINDOWS__
#endif
#ifdef __WINDOWS__
/* When compiling for windows, we specify a specific calling convention to avoid issues where we are being called from a project with a different default calling convention. For windows you have 2 define options:
CJSON_HIDE_SYMBOLS - Define this in the case where you don't want to ever dllexport symbols
CJSON_EXPORT_SYMBOLS - Define this on library build when you want to dllexport symbols (default)
CJSON_IMPORT_SYMBOLS - Define this if you want to dllimport symbol
For *nix builds that support visibility attribute, you can define similar behavior by
setting default visibility to hidden by adding
-fvisibility=hidden (for gcc)
or
-xldscope=hidden (for sun cc)
to CFLAGS
then using the CJSON_API_VISIBILITY flag to "export" the same symbols the way CJSON_EXPORT_SYMBOLS does
*/
/* export symbols by default, this is necessary for copy pasting the C and header file */
#if !defined(CJSON_HIDE_SYMBOLS) && !defined(CJSON_IMPORT_SYMBOLS) && !defined(CJSON_EXPORT_SYMBOLS)
#define CJSON_EXPORT_SYMBOLS
#endif
#if defined(CJSON_HIDE_SYMBOLS)
#define CJSON_PUBLIC(type) type __stdcall
#elif defined(CJSON_EXPORT_SYMBOLS)
#define CJSON_PUBLIC(type) __declspec(dllexport) type __stdcall
#elif defined(CJSON_IMPORT_SYMBOLS)
#define CJSON_PUBLIC(type) __declspec(dllimport) type __stdcall
#endif
#else /* !WIN32 */
#if (defined(__GNUC__) || defined(__SUNPRO_CC) || defined (__SUNPRO_C)) && defined(CJSON_API_VISIBILITY)
#define CJSON_PUBLIC(type) __attribute__((visibility("default"))) type
#else
#define CJSON_PUBLIC(type) type
#endif
#endif
/* Limits how deeply nested arrays/objects can be before cJSON rejects to parse them.
* This is to prevent stack overflows. */
#ifndef CJSON_NESTING_LIMIT
#define CJSON_NESTING_LIMIT 1000
#endif
/* returns the version of cJSON as a string */
CJSON_PUBLIC(const char*) cJSON_Version(void);
/* Supply malloc, realloc and free functions to cJSON */
CJSON_PUBLIC(void) cJSON_InitHooks(cJSON_Hooks* hooks);
/* Memory Management: the caller is always responsible to free the results from all variants of cJSON_Parse (with cJSON_Delete) and cJSON_Print (with stdlib free, cJSON_Hooks.free_fn, or cJSON_free as appropriate). The exception is cJSON_PrintPreallocated, where the caller has full responsibility of the buffer. */
/* Supply a block of JSON, and this returns a cJSON object you can interrogate. */
CJSON_PUBLIC(cJSON *) cJSON_Parse(const char *value);
/* ParseWithOpts allows you to require (and check) that the JSON is null terminated, and to retrieve the pointer to the final byte parsed. */
/* If you supply a ptr in return_parse_end and parsing fails, then return_parse_end will contain a pointer to the error so will match cJSON_GetErrorPtr(). */
CJSON_PUBLIC(cJSON *) cJSON_ParseWithOpts(const char *value, const char **return_parse_end, cJSON_bool require_null_terminated);
/* Render a cJSON entity to text for transfer/storage. */
CJSON_PUBLIC(char *) cJSON_Print(const cJSON *item);
/* Render a cJSON entity to text for transfer/storage without any formatting. */
CJSON_PUBLIC(char *) cJSON_PrintUnformatted(const cJSON *item);
/* Render a cJSON entity to text using a buffered strategy. prebuffer is a guess at the final size. guessing well reduces reallocation. fmt=0 gives unformatted, =1 gives formatted */
CJSON_PUBLIC(char *) cJSON_PrintBuffered(const cJSON *item, int prebuffer, cJSON_bool fmt);
/* Render a cJSON entity to text using a buffer already allocated in memory with given length. Returns 1 on success and 0 on failure. */
/* NOTE: cJSON is not always 100% accurate in estimating how much memory it will use, so to be safe allocate 5 bytes more than you actually need */
CJSON_PUBLIC(cJSON_bool) cJSON_PrintPreallocated(cJSON *item, char *buffer, const int length, const cJSON_bool format);
/* Delete a cJSON entity and all subentities. */
CJSON_PUBLIC(void) cJSON_Delete(cJSON *c);
/* Returns the number of items in an array (or object). */
CJSON_PUBLIC(int) cJSON_GetArraySize(const cJSON *array);
/* Retrieve item number "item" from array "array". Returns NULL if unsuccessful. */
CJSON_PUBLIC(cJSON *) cJSON_GetArrayItem(const cJSON *array, int index);
/* Get item "string" from object. Case insensitive. */
CJSON_PUBLIC(cJSON *) cJSON_GetObjectItem(const cJSON * const object, const char * const string);
CJSON_PUBLIC(cJSON *) cJSON_GetObjectItemCaseSensitive(const cJSON * const object, const char * const string);
CJSON_PUBLIC(cJSON_bool) cJSON_HasObjectItem(const cJSON *object, const char *string);
/* For analysing failed parses. This returns a pointer to the parse error. You'll probably need to look a few chars back to make sense of it. Defined when cJSON_Parse() returns 0. 0 when cJSON_Parse() succeeds. */
CJSON_PUBLIC(const char *) cJSON_GetErrorPtr(void);
/* These functions check the type of an item */
CJSON_PUBLIC(cJSON_bool) cJSON_IsInvalid(const cJSON * const item);
CJSON_PUBLIC(cJSON_bool) cJSON_IsFalse(const cJSON * const item);
CJSON_PUBLIC(cJSON_bool) cJSON_IsTrue(const cJSON * const item);
CJSON_PUBLIC(cJSON_bool) cJSON_IsBool(const cJSON * const item);
CJSON_PUBLIC(cJSON_bool) cJSON_IsNull(const cJSON * const item);
CJSON_PUBLIC(cJSON_bool) cJSON_IsNumber(const cJSON * const item);
CJSON_PUBLIC(cJSON_bool) cJSON_IsString(const cJSON * const item);
CJSON_PUBLIC(cJSON_bool) cJSON_IsArray(const cJSON * const item);
CJSON_PUBLIC(cJSON_bool) cJSON_IsObject(const cJSON * const item);
CJSON_PUBLIC(cJSON_bool) cJSON_IsRaw(const cJSON * const item);
/* These calls create a cJSON item of the appropriate type. */
CJSON_PUBLIC(cJSON *) cJSON_CreateNull(void);
CJSON_PUBLIC(cJSON *) cJSON_CreateTrue(void);
CJSON_PUBLIC(cJSON *) cJSON_CreateFalse(void);
CJSON_PUBLIC(cJSON *) cJSON_CreateBool(cJSON_bool boolean);
CJSON_PUBLIC(cJSON *) cJSON_CreateNumber(double num);
CJSON_PUBLIC(cJSON *) cJSON_CreateString(const char *string);
/* raw json */
CJSON_PUBLIC(cJSON *) cJSON_CreateRaw(const char *raw);
CJSON_PUBLIC(cJSON *) cJSON_CreateArray(void);
CJSON_PUBLIC(cJSON *) cJSON_CreateObject(void);
/* These utilities create an Array of count items. */
CJSON_PUBLIC(cJSON *) cJSON_CreateIntArray(const int *numbers, int count);
CJSON_PUBLIC(cJSON *) cJSON_CreateFloatArray(const float *numbers, int count);
CJSON_PUBLIC(cJSON *) cJSON_CreateDoubleArray(const double *numbers, int count);
CJSON_PUBLIC(cJSON *) cJSON_CreateStringArray(const char **strings, int count);
/* Append item to the specified array/object. */
CJSON_PUBLIC(void) cJSON_AddItemToArray(cJSON *array, cJSON *item);
CJSON_PUBLIC(void) cJSON_AddItemToObject(cJSON *object, const char *string, cJSON *item);
/* Use this when string is definitely const (i.e. a literal, or as good as), and will definitely survive the cJSON object.
* WARNING: When this function was used, make sure to always check that (item->type & cJSON_StringIsConst) is zero before
* writing to `item->string` */
CJSON_PUBLIC(void) cJSON_AddItemToObjectCS(cJSON *object, const char *string, cJSON *item);
/* Append reference to item to the specified array/object. Use this when you want to add an existing cJSON to a new cJSON, but don't want to corrupt your existing cJSON. */
CJSON_PUBLIC(void) cJSON_AddItemReferenceToArray(cJSON *array, cJSON *item);
CJSON_PUBLIC(void) cJSON_AddItemReferenceToObject(cJSON *object, const char *string, cJSON *item);
/* Remove/Detatch items from Arrays/Objects. */
CJSON_PUBLIC(cJSON *) cJSON_DetachItemViaPointer(cJSON *parent, cJSON * const item);
CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromArray(cJSON *array, int which);
CJSON_PUBLIC(void) cJSON_DeleteItemFromArray(cJSON *array, int which);
CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromObject(cJSON *object, const char *string);
CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromObjectCaseSensitive(cJSON *object, const char *string);
CJSON_PUBLIC(void) cJSON_DeleteItemFromObject(cJSON *object, const char *string);
CJSON_PUBLIC(void) cJSON_DeleteItemFromObjectCaseSensitive(cJSON *object, const char *string);
/* Update array items. */
CJSON_PUBLIC(void) cJSON_InsertItemInArray(cJSON *array, int which, cJSON *newitem); /* Shifts pre-existing items to the right. */
CJSON_PUBLIC(cJSON_bool) cJSON_ReplaceItemViaPointer(cJSON * const parent, cJSON * const item, cJSON * replacement);
CJSON_PUBLIC(void) cJSON_ReplaceItemInArray(cJSON *array, int which, cJSON *newitem);
CJSON_PUBLIC(void) cJSON_ReplaceItemInObject(cJSON *object,const char *string,cJSON *newitem);
CJSON_PUBLIC(void) cJSON_ReplaceItemInObjectCaseSensitive(cJSON *object,const char *string,cJSON *newitem);
/* Duplicate a cJSON item */
CJSON_PUBLIC(cJSON *) cJSON_Duplicate(const cJSON *item, cJSON_bool recurse);
/* Duplicate will create a new, identical cJSON item to the one you pass, in new memory that will
need to be released. With recurse!=0, it will duplicate any children connected to the item.
The item->next and ->prev pointers are always zero on return from Duplicate. */
/* Recursively compare two cJSON items for equality. If either a or b is NULL or invalid, they will be considered unequal.
* case_sensitive determines if object keys are treated case sensitive (1) or case insensitive (0) */
CJSON_PUBLIC(cJSON_bool) cJSON_Compare(const cJSON * const a, const cJSON * const b, const cJSON_bool case_sensitive);
CJSON_PUBLIC(void) cJSON_Minify(char *json);
/* Macros for creating things quickly. */
#define cJSON_AddNullToObject(object,name) cJSON_AddItemToObject(object, name, cJSON_CreateNull())
#define cJSON_AddTrueToObject(object,name) cJSON_AddItemToObject(object, name, cJSON_CreateTrue())
#define cJSON_AddFalseToObject(object,name) cJSON_AddItemToObject(object, name, cJSON_CreateFalse())
#define cJSON_AddBoolToObject(object,name,b) cJSON_AddItemToObject(object, name, cJSON_CreateBool(b))
#define cJSON_AddNumberToObject(object,name,n) cJSON_AddItemToObject(object, name, cJSON_CreateNumber(n))
#define cJSON_AddStringToObject(object,name,s) cJSON_AddItemToObject(object, name, cJSON_CreateString(s))
#define cJSON_AddRawToObject(object,name,s) cJSON_AddItemToObject(object, name, cJSON_CreateRaw(s))
/* When assigning an integer value, it needs to be propagated to valuedouble too. */
#define cJSON_SetIntValue(object, number) ((object) ? (object)->valueint = (object)->valuedouble = (number) : (number))
/* helper for the cJSON_SetNumberValue macro */
CJSON_PUBLIC(double) cJSON_SetNumberHelper(cJSON *object, double number);
#define cJSON_SetNumberValue(object, number) ((object != NULL) ? cJSON_SetNumberHelper(object, (double)number) : (number))
/* Macro for iterating over an array or object */
#define cJSON_ArrayForEach(element, array) for(element = (array != NULL) ? (array)->child : NULL; element != NULL; element = element->next)
/* malloc/free objects using the malloc/free functions that have been set with cJSON_InitHooks */
CJSON_PUBLIC(void *) cJSON_malloc(size_t size);
CJSON_PUBLIC(void) cJSON_free(void *object);
#ifdef __cplusplus
}
#endif
#endif

1
src/plugins/http/inc/tgHandle.h
View File

@ -33,6 +33,7 @@
#define TG_PASS_URL_POS 3
void tgInitHandle(HttpServer *pServer);
void tgCleanupHandle();
bool tgProcessRquest(struct HttpContext *pContext);

2699
src/plugins/http/src/cJSON.c
View File

File diff suppressed because it is too large Load Diff

13
src/plugins/http/src/httpServer.c
View File

@ -270,7 +270,7 @@ void httpCleanUpConnect(HttpServer *pServer) {
for (i = 0; i < pServer->numOfThreads; ++i) {
pThread = pServer->pThreads + i;
taosCloseSocket(pThread->pollFd);
//taosCloseSocket(pThread->pollFd);
while (pThread->pHead) {
httpCleanUpContext(pThread->pHead, 0);
@ -591,7 +591,6 @@ void httpAcceptHttpConnection(void *arg) {
bool httpInitConnect(HttpServer *pServer) {
int i;
pthread_attr_t thattr;
HttpThread * pThread;
pServer->pThreads = (HttpThread *)malloc(sizeof(HttpThread) * (size_t)pServer->numOfThreads);
@ -601,8 +600,6 @@ bool httpInitConnect(HttpServer *pServer) {
}
memset(pServer->pThreads, 0, sizeof(HttpThread) * (size_t)pServer->numOfThreads);
pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
pThread = pServer->pThreads;
for (i = 0; i < pServer->numOfThreads; ++i) {
sprintf(pThread->label, "%s%d", pServer->label, i);
@ -626,21 +623,27 @@ bool httpInitConnect(HttpServer *pServer) {
return false;
}
pthread_attr_t thattr;
pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
if (pthread_create(&(pThread->thread), &thattr, (void *)httpProcessHttpData, (void *)(pThread)) != 0) {
httpError("http thread:%s, failed to create HTTP process data thread, reason:%s",
pThread->label, strerror(errno));
return false;
}
pthread_attr_destroy(&thattr);
httpTrace("http thread:%p:%s, initialized", pThread, pThread->label);
pThread++;
}
pthread_attr_t thattr;
pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
if (pthread_create(&(pServer->thread), &thattr, (void *)httpAcceptHttpConnection, (void *)(pServer)) != 0) {
httpError("http server:%s, failed to create Http accept thread, reason:%s", pServer->label, strerror(errno));
return false;
}
pthread_attr_destroy(&thattr);
httpTrace("http server:%s, initialized, ip:%s:%u, numOfThreads:%d", pServer->label, pServer->serverIp,

6
src/plugins/http/src/httpSystem.c
View File

@ -54,7 +54,7 @@ static HttpServer *httpServer = NULL;
void taosInitNote(int numOfNoteLines, int maxNotes, char* lable);
int httpInitSystem() {
taos_init();
// taos_init();
httpServer = (HttpServer *)malloc(sizeof(HttpServer));
memset(httpServer, 0, sizeof(HttpServer));
@ -122,12 +122,14 @@ void httpStopSystem() {
if (httpServer != NULL) {
httpServer->online = false;
}
tgCleanupHandle();
}
void httpCleanUpSystem() {
httpPrint("http service cleanup");
httpStopSystem();
#if 0
#if 1
if (httpServer == NULL) {
return;
}

5
src/plugins/http/src/tgHandle.c
View File

@ -116,6 +116,7 @@ void tgFreeSchemas() {
}
free(tgSchemas.schemas);
tgSchemas.size = 0;
tgSchemas.schemas = NULL;
}
}
@ -290,6 +291,10 @@ void tgInitHandle(HttpServer *pServer) {
httpAddMethod(pServer, &tgDecodeMethod);
}
void tgCleanupHandle() {
tgFreeSchemas();
}
bool tgGetUserFromUrl(HttpContext *pContext) {
HttpParser *pParser = &pContext->parser;
if (pParser->path[TG_USER_URL_POS].len > TSDB_USER_LEN - 1 || pParser->path[TG_USER_URL_POS].len <= 0) {

2
src/query/CMakeLists.txt
View File

@ -5,7 +5,7 @@ INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/vnode/tsdb/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/tsdb/inc)
INCLUDE_DIRECTORIES(inc)
IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))

4
src/rpc/CMakeLists.txt
View File

@ -4,6 +4,8 @@ PROJECT(TDengine)
INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/lz4/inc)
INCLUDE_DIRECTORIES(inc)
IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
@ -22,7 +24,7 @@ ELSEIF (TD_WINDOWS_64)
ENDIF ()
ADD_LIBRARY(trpc ${SRC})
TARGET_LINK_LIBRARIES(trpc tutil)
TARGET_LINK_LIBRARIES(trpc tutil lz4)
ADD_SUBDIRECTORY(test)

34
src/rpc/inc/rpcServer.h
View File

@ -1,34 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _rpc_server_header_
#define _rpc_server_header_
#ifdef __cplusplus
extern "C" {
#endif
#include "taosdef.h"
void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle);
void taosCleanUpTcpServer(void *param);
void taosCloseTcpServerConnection(void *param);
int taosSendTcpServerData(uint32_t ip, uint16_t port, void *data, int len, void *chandle);
#ifdef __cplusplus
}
#endif
#endif

12
src/rpc/inc/rpcClient.h → src/rpc/inc/rpcTcp.h
View File

@ -13,20 +13,22 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _rpc_client_header_
#define _rpc_client_header_
#ifndef _rpc_tcp_header_
#define _rpc_tcp_header_
#ifdef __cplusplus
extern "C" {
#endif
#include "taosdef.h"
void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle);
void taosCleanUpTcpServer(void *param);
void *taosInitTcpClient(char *ip, uint16_t port, char *label, int num, void *fp, void *shandle);
void taosCleanUpTcpClient(void *chandle);
void *taosOpenTcpClientConnection(void *shandle, void *thandle, char *ip, uint16_t port);
void taosCloseTcpClientConnection(void *chandle);
int taosSendTcpClientData(uint32_t ip, uint16_t port, void *data, int len, void *chandle);
void taosCloseTcpConnection(void *chandle);
int taosSendTcpData(uint32_t ip, uint16_t port, void *data, int len, void *chandle);
#ifdef __cplusplus
}

13
src/rpc/src/rpcCache.c
View File

@ -22,18 +22,18 @@
#include "tutil.h"
#include "rpcCache.h"
typedef struct _c_hash_t {
typedef struct SConnHash {
uint32_t ip;
uint16_t port;
char connType;
struct _c_hash_t *prev;
struct _c_hash_t *next;
void * data;
struct SConnHash *prev;
struct SConnHash *next;
void *data;
uint64_t time;
} SConnHash;
typedef struct {
SConnHash ** connHashList;
SConnHash **connHashList;
mpool_h connHashMemPool;
int maxSessions;
int total;
@ -103,7 +103,8 @@ void rpcCloseConnCache(void *handle) {
if (pCache->connHashMemPool) taosMemPoolCleanUp(pCache->connHashMemPool);
tfree(pCache->connHashList);
tfree(pCache->count)
tfree(pCache->count);
tfree(pCache->lockedBy);
pthread_mutex_unlock(&pCache->mutex);

311
src/rpc/src/rpcClient.c
View File

@ -1,311 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "os.h"
#include "taosmsg.h"
#include "tlog.h"
#include "tsocket.h"
#include "tutil.h"
#include "rpcClient.h"
#include "rpcHead.h"
#ifndef EPOLLWAKEUP
#define EPOLLWAKEUP (1u << 29)
#endif
typedef struct _tcp_fd {
void *signature;
int fd; // TCP socket FD
void * thandle;
uint32_t ip;
char ipstr[20];
uint16_t port;
struct _tcp_client *pTcp;
struct _tcp_fd * prev, *next;
} STcpFd;
typedef struct _tcp_client {
pthread_t thread;
STcpFd * pHead;
pthread_mutex_t mutex;
pthread_cond_t fdReady;
int pollFd;
int numOfFds;
char label[12];
char ipstr[20];
void *shandle; // handle passed by upper layer during server initialization
void *(*processData)(SRecvInfo *pRecv);
} STcpClient;
#define maxTcpEvents 100
static void taosCleanUpTcpFdObj(STcpFd *pFdObj);
static void *taosReadTcpData(void *param);
void *taosInitTcpClient(char *ip, uint16_t port, char *label, int num, void *fp, void *shandle) {
STcpClient *pTcp;
pthread_attr_t thattr;
pTcp = (STcpClient *)malloc(sizeof(STcpClient));
memset(pTcp, 0, sizeof(STcpClient));
strcpy(pTcp->label, label);
strcpy(pTcp->ipstr, ip);
pTcp->shandle = shandle;
if (pthread_mutex_init(&(pTcp->mutex), NULL) < 0) {
tError("%s failed to init TCP mutex, reason:%s", label, strerror(errno));
return NULL;
}
if (pthread_cond_init(&(pTcp->fdReady), NULL) != 0) {
tError("%s init TCP condition variable failed, reason:%s\n", label, strerror(errno));
return NULL;
}
pTcp->pollFd = epoll_create(10); // size does not matter
if (pTcp->pollFd < 0) {
tError("%s failed to create TCP epoll", label);
return NULL;
}
pTcp->processData = fp;
pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
if (pthread_create(&(pTcp->thread), &thattr, taosReadTcpData, (void *)(pTcp)) != 0) {
tError("%s failed to create TCP read data thread, reason:%s", label, strerror(errno));
return NULL;
}
tTrace("%s TCP client is initialized, ip:%s port:%hu", label, ip, port);
return pTcp;
}
void taosCleanUpTcpClient(void *chandle) {
STcpClient *pTcp = (STcpClient *)chandle;
if (pTcp == NULL) return;
while (pTcp->pHead) {
taosCleanUpTcpFdObj(pTcp->pHead);
pTcp->pHead = pTcp->pHead->next;
}
close(pTcp->pollFd);
pthread_cancel(pTcp->thread);
pthread_join(pTcp->thread, NULL);
// tTrace (":%s, all connections are cleaned up", pTcp->label);
tfree(pTcp);
}
void *taosOpenTcpClientConnection(void *shandle, void *thandle, char *ip, uint16_t port) {
STcpClient * pTcp = (STcpClient *)shandle;
STcpFd * pFdObj;
struct epoll_event event;
struct in_addr destIp;
int fd;
fd = taosOpenTcpClientSocket(ip, port, pTcp->ipstr);
if (fd <= 0) return NULL;
pFdObj = (STcpFd *)malloc(sizeof(STcpFd));
if (pFdObj == NULL) {
tError("%s no enough resource to allocate TCP FD IDs", pTcp->label);
tclose(fd);
return NULL;
}
memset(pFdObj, 0, sizeof(STcpFd));
pFdObj->fd = fd;
strcpy(pFdObj->ipstr, ip);
inet_aton(ip, &destIp);
pFdObj->ip = destIp.s_addr;
pFdObj->port = port;
pFdObj->pTcp = pTcp;
pFdObj->thandle = thandle;
pFdObj->signature = pFdObj;
event.events = EPOLLIN | EPOLLPRI | EPOLLWAKEUP;
event.data.ptr = pFdObj;
if (epoll_ctl(pTcp->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) {
tError("%s failed to add TCP FD for epoll, error:%s", pTcp->label, strerror(errno));
tfree(pFdObj);
tclose(fd);
return NULL;
}
// notify the data process, add into the FdObj list
pthread_mutex_lock(&(pTcp->mutex));
pFdObj->next = pTcp->pHead;
if (pTcp->pHead) (pTcp->pHead)->prev = pFdObj;
pTcp->pHead = pFdObj;
pTcp->numOfFds++;
pthread_cond_signal(&pTcp->fdReady);
pthread_mutex_unlock(&(pTcp->mutex));
tTrace("%s TCP connection to %s:%hu is created, FD:%p numOfFds:%d", pTcp->label, ip, port, pFdObj, pTcp->numOfFds);
return pFdObj;
}
void taosCloseTcpClientConnection(void *chandle) {
STcpFd *pFdObj = (STcpFd *)chandle;
if (pFdObj == NULL) return;
taosCleanUpTcpFdObj(pFdObj);
}
int taosSendTcpClientData(uint32_t ip, uint16_t port, void *data, int len, void *chandle) {
STcpFd *pFdObj = (STcpFd *)chandle;
if (chandle == NULL) return -1;
return (int)send(pFdObj->fd, data, (size_t)len, 0);
}
static void taosCleanUpTcpFdObj(STcpFd *pFdObj) {
STcpClient *pTcp;
SRecvInfo recvInfo;
if (pFdObj == NULL) return;
if (pFdObj->signature != pFdObj) return;
pTcp = pFdObj->pTcp;
if (pTcp == NULL) {
tError("double free TcpFdObj!!!!");
return;
}
epoll_ctl(pTcp->pollFd, EPOLL_CTL_DEL, pFdObj->fd, NULL);
close(pFdObj->fd);
pthread_mutex_lock(&pTcp->mutex);
pTcp->numOfFds--;
if (pTcp->numOfFds < 0)
tError("%s number of TCP FDs shall never be negative, FD:%p", pTcp->label, pFdObj);
if (pFdObj->prev) {
(pFdObj->prev)->next = pFdObj->next;
} else {
pTcp->pHead = pFdObj->next;
}
if (pFdObj->next) {
(pFdObj->next)->prev = pFdObj->prev;
}
pthread_mutex_unlock(&pTcp->mutex);
recvInfo.msg = NULL;
recvInfo.msgLen = 0;
recvInfo.ip = 0;
recvInfo.port = 0;
recvInfo.shandle = pTcp->shandle;
recvInfo.thandle = pFdObj->thandle;;
recvInfo.chandle = NULL;
recvInfo.connType = RPC_CONN_TCP;
if (pFdObj->thandle) (*(pTcp->processData))(&recvInfo);
tTrace("%s TCP is cleaned up, FD:%p numOfFds:%d", pTcp->label, pFdObj, pTcp->numOfFds);
memset(pFdObj, 0, sizeof(STcpFd));
tfree(pFdObj);
}
static void *taosReadTcpData(void *param) {
STcpClient *pTcp = (STcpClient *)param;
int i, fdNum;
STcpFd *pFdObj;
struct epoll_event events[maxTcpEvents];
SRecvInfo recvInfo;
SRpcHead rpcHead;
while (1) {
pthread_mutex_lock(&pTcp->mutex);
if (pTcp->numOfFds < 1) pthread_cond_wait(&pTcp->fdReady, &pTcp->mutex);
pthread_mutex_unlock(&pTcp->mutex);
fdNum = epoll_wait(pTcp->pollFd, events, maxTcpEvents, -1);
if (fdNum < 0) continue;
for (i = 0; i < fdNum; ++i) {
pFdObj = events[i].data.ptr;
if (events[i].events & EPOLLERR) {
tTrace("%s TCP error happened on FD\n", pTcp->label);
taosCleanUpTcpFdObj(pFdObj);
continue;
}
if (events[i].events & EPOLLHUP) {
tTrace("%s TCP FD hang up\n", pTcp->label);
taosCleanUpTcpFdObj(pFdObj);
continue;
}
int headLen = taosReadMsg(pFdObj->fd, &rpcHead, sizeof(SRpcHead));
if (headLen != sizeof(SRpcHead)) {
tError("%s read error, headLen:%d", pTcp->label, headLen);
taosCleanUpTcpFdObj(pFdObj);
continue;
}
int32_t msgLen = (int32_t)htonl((uint32_t)rpcHead.msgLen);
char *buffer = (char *)malloc((size_t)msgLen + tsRpcOverhead);
if (NULL == buffer) {
tTrace("%s TCP malloc(size:%d) fail\n", pTcp->label, msgLen);
taosCleanUpTcpFdObj(pFdObj);
continue;
}
char *msg = buffer + tsRpcOverhead;
int32_t leftLen = msgLen - headLen;
int32_t retLen = taosReadMsg(pFdObj->fd, msg + headLen, leftLen);
if (leftLen != retLen) {
tError("%s read error, leftLen:%d retLen:%d", pTcp->label, leftLen, retLen);
tfree(buffer);
taosCleanUpTcpFdObj(pFdObj);
continue;
}
// tTrace("%s TCP data is received, ip:%s:%u len:%d", pTcp->label, pFdObj->ipstr, pFdObj->port, msgLen);
memcpy(msg, &rpcHead, sizeof(SRpcHead));
recvInfo.msg = msg;
recvInfo.msgLen = msgLen;
recvInfo.ip = pFdObj->ip;
recvInfo.port = pFdObj->port;
recvInfo.shandle = pTcp->shandle;
recvInfo.thandle = pFdObj->thandle;;
recvInfo.chandle = pFdObj;
recvInfo.connType = RPC_CONN_TCP;
pFdObj->thandle = (*(pTcp->processData))(&recvInfo);
if (pFdObj->thandle == NULL) taosCleanUpTcpFdObj(pFdObj);
}
}
return NULL;
}

14
src/rpc/src/rpcHaship.c
View File

@ -17,13 +17,13 @@
#include "tlog.h"
#include "tmempool.h"
typedef struct _ip_hash_t {
typedef struct SIpHash {
uint32_t ip;
uint16_t port;
int hash;
struct _ip_hash_t *prev;
struct _ip_hash_t *next;
void * data;
struct SIpHash *prev;
struct SIpHash *next;
void *data;
} SIpHash;
typedef struct {
@ -47,7 +47,7 @@ int rpcHashIp(void *handle, uint32_t ip, uint16_t port) {
void *rpcAddIpHash(void *handle, void *data, uint32_t ip, uint16_t port) {
int hash;
SIpHash * pNode;
SIpHash *pNode;
SHashObj *pObj;
pObj = (SHashObj *)handle;
@ -70,7 +70,7 @@ void *rpcAddIpHash(void *handle, void *data, uint32_t ip, uint16_t port) {
void rpcDeleteIpHash(void *handle, uint32_t ip, uint16_t port) {
int hash;
SIpHash * pNode;
SIpHash *pNode;
SHashObj *pObj;
pObj = (SHashObj *)handle;
@ -102,7 +102,7 @@ void rpcDeleteIpHash(void *handle, uint32_t ip, uint16_t port) {
void *rpcGetIpHash(void *handle, uint32_t ip, uint16_t port) {
int hash;
SIpHash * pNode;
SIpHash *pNode;
SHashObj *pObj;
pObj = (SHashObj *)handle;

29
src/rpc/src/rpcMain.c Executable file → Normal file
View File

@ -27,8 +27,7 @@
#include "taosmsg.h"
#include "rpcUdp.h"
#include "rpcCache.h"
#include "rpcClient.h"
#include "rpcServer.h"
#include "rpcTcp.h"
#include "rpcHead.h"
#include "trpc.h"
#include "hash.h"
@ -67,7 +66,7 @@ typedef struct {
void *udphandle;// returned handle from UDP initialization
void *pCache; // connection cache
pthread_mutex_t mutex;
struct _RpcConn *connList; // connection list
struct SRpcConn *connList; // connection list
} SRpcInfo;
typedef struct {
@ -88,7 +87,7 @@ typedef struct {
char msg[0]; // RpcHead starts from here
} SRpcReqContext;
typedef struct _RpcConn {
typedef struct SRpcConn {
int sid; // session ID
uint32_t ownId; // own link ID
uint32_t peerId; // peer link ID
@ -156,8 +155,8 @@ void (*taosCleanUpConn[])(void *thandle) = {
int (*taosSendData[])(uint32_t ip, uint16_t port, void *data, int len, void *chandle) = {
taosSendUdpData,
taosSendUdpData,
taosSendTcpServerData,
taosSendTcpClientData
taosSendTcpData,
taosSendTcpData
};
void *(*taosOpenConn[])(void *shandle, void *thandle, char *ip, uint16_t port) = {
@ -170,8 +169,8 @@ void *(*taosOpenConn[])(void *shandle, void *thandle, char *ip, uint16_t port) =
void (*taosCloseConn[])(void *chandle) = {
NULL,
NULL,
taosCloseTcpServerConnection,
taosCloseTcpClientConnection
taosCloseTcpConnection,
taosCloseTcpConnection
};
static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerIpStr, uint16_t peerPort, int8_t connType);
@ -287,15 +286,15 @@ void *rpcOpen(const SRpcInit *pInit) {
void rpcClose(void *param) {
SRpcInfo *pRpc = (SRpcInfo *)param;
(*taosCleanUpConn[pRpc->connType | RPC_CONN_TCP])(pRpc->tcphandle);
(*taosCleanUpConn[pRpc->connType])(pRpc->udphandle);
for (int i = 0; i < pRpc->sessions; ++i) {
if (pRpc->connList && pRpc->connList[i].user[0]) {
rpcCloseConn((void *)(pRpc->connList + i));
}
}
(*taosCleanUpConn[pRpc->connType | RPC_CONN_TCP])(pRpc->tcphandle);
(*taosCleanUpConn[pRpc->connType])(pRpc->udphandle);
taosHashCleanup(pRpc->hash);
taosTmrCleanUp(pRpc->tmrCtrl);
taosIdPoolCleanUp(pRpc->idPool);
@ -522,11 +521,15 @@ static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerIpStr, uint16_t peerPort,
static void rpcCloseConn(void *thandle) {
SRpcConn *pConn = (SRpcConn *)thandle;
SRpcInfo *pRpc = pConn->pRpc;
if (pConn->user[0] == 0) return;
rpcLockConn(pConn);
if (pConn->user[0] == 0) {
rpcUnlockConn(pConn);
return;
}
pConn->user[0] = 0;
if (taosCloseConn[pConn->connType]) (*taosCloseConn[pConn->connType])(pConn->chandle);
@ -817,7 +820,7 @@ static void rpcProcessBrokenLink(SRpcConn *pConn) {
SRpcInfo *pRpc = pConn->pRpc;
tTrace("%s %p, link is broken", pRpc->label, pConn);
pConn->chandle = NULL;
// pConn->chandle = NULL;
if (pConn->outType) {
SRpcReqContext *pContext = pConn->pContext;

515
src/rpc/src/rpcServer.c
View File

@ -1,515 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "os.h"
#include "tlog.h"
#include "tlog.h"
#include "tsocket.h"
#include "tutil.h"
#include "rpcServer.h"
#include "rpcHead.h"
#define TAOS_IPv4ADDR_LEN 16
#ifndef EPOLLWAKEUP
#define EPOLLWAKEUP (1u << 29)
#endif
typedef struct _fd_obj {
void *signature;
int fd; // TCP socket FD
void * thandle; // handle from upper layer, like TAOS
char ipstr[TAOS_IPv4ADDR_LEN];
unsigned int ip;
uint16_t port;
struct _thread_obj *pThreadObj;
struct _fd_obj * prev, *next;
} SFdObj;
typedef struct _thread_obj {
pthread_t thread;
SFdObj * pHead;
pthread_mutex_t threadMutex;
pthread_cond_t fdReady;
int pollFd;
int numOfFds;
int threadId;
char label[12];
void *shandle; // handle passed by upper layer during server initialization
void *(*processData)(SRecvInfo *pPacket);
} SThreadObj;
typedef struct {
char ip[40];
uint16_t port;
char label[12];
int numOfThreads;
void * shandle;
SThreadObj *pThreadObj;
pthread_t thread;
} SServerObj;
static void taosCleanUpFdObj(SFdObj *pFdObj);
static void taosProcessTcpData(void *param);
static void taosAcceptTcpConnection(void *arg);
void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle) {
int i;
SServerObj *pServerObj;
pthread_attr_t thattr;
SThreadObj *pThreadObj;
pServerObj = (SServerObj *)malloc(sizeof(SServerObj));
strcpy(pServerObj->ip, ip);
pServerObj->port = port;
strcpy(pServerObj->label, label);
pServerObj->numOfThreads = numOfThreads;
pServerObj->pThreadObj = (SThreadObj *)malloc(sizeof(SThreadObj) * (size_t)numOfThreads);
if (pServerObj->pThreadObj == NULL) {
tError("TCP:%s no enough memory", label);
return NULL;
}
memset(pServerObj->pThreadObj, 0, sizeof(SThreadObj) * (size_t)numOfThreads);
pThreadObj = pServerObj->pThreadObj;
for (i = 0; i < numOfThreads; ++i) {
pThreadObj->processData = fp;
strcpy(pThreadObj->label, label);
pThreadObj->shandle = shandle;
if (pthread_mutex_init(&(pThreadObj->threadMutex), NULL) < 0) {
tError("%s failed to init TCP process data mutex, reason:%s", label, strerror(errno));
return NULL;
}
if (pthread_cond_init(&(pThreadObj->fdReady), NULL) != 0) {
tError("%s init TCP condition variable failed, reason:%s\n", label, strerror(errno));
return NULL;
}
pThreadObj->pollFd = epoll_create(10); // size does not matter
if (pThreadObj->pollFd < 0) {
tError("%s failed to create TCP epoll", label);
return NULL;
}
pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
if (pthread_create(&(pThreadObj->thread), &thattr, (void *)taosProcessTcpData, (void *)(pThreadObj)) != 0) {
tError("%s failed to create TCP process data thread, reason:%s", label, strerror(errno));
return NULL;
}
pThreadObj->threadId = i;
pThreadObj++;
}
pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
if (pthread_create(&(pServerObj->thread), &thattr, (void *)taosAcceptTcpConnection, (void *)(pServerObj)) != 0) {
tError("%s failed to create TCP accept thread, reason:%s", label, strerror(errno));
return NULL;
}
/*
if ( pthread_create(&(pServerObj->thread), &thattr,
(void*)taosAcceptUDConnection, (void *)(pServerObj)) != 0 ) {
tError("%s failed to create UD accept thread, reason:%s", label,
strerror(errno));
return NULL;
}
*/
pthread_attr_destroy(&thattr);
tTrace("%s TCP server is initialized, ip:%s port:%hu numOfThreads:%d", label, ip, port, numOfThreads);
return (void *)pServerObj;
}
void taosCleanUpTcpServer(void *handle) {
int i;
SThreadObj *pThreadObj;
SServerObj *pServerObj = (SServerObj *)handle;
if (pServerObj == NULL) return;
pthread_cancel(pServerObj->thread);
pthread_join(pServerObj->thread, NULL);
for (i = 0; i < pServerObj->numOfThreads; ++i) {
pThreadObj = pServerObj->pThreadObj + i;
while (pThreadObj->pHead) {
taosCleanUpFdObj(pThreadObj->pHead);
pThreadObj->pHead = pThreadObj->pHead;
}
close(pThreadObj->pollFd);
pthread_cancel(pThreadObj->thread);
pthread_join(pThreadObj->thread, NULL);
pthread_cond_destroy(&(pThreadObj->fdReady));
pthread_mutex_destroy(&(pThreadObj->threadMutex));
}
tfree(pServerObj->pThreadObj);
tTrace("TCP:%s, TCP server is cleaned up", pServerObj->label);
tfree(pServerObj);
}
void taosCloseTcpServerConnection(void *chandle) {
SFdObj *pFdObj = (SFdObj *)chandle;
if (pFdObj == NULL) return;
taosCleanUpFdObj(pFdObj);
}
int taosSendTcpServerData(uint32_t ip, uint16_t port, void *data, int len, void *chandle) {
SFdObj *pFdObj = (SFdObj *)chandle;
if (chandle == NULL) return -1;
return (int)send(pFdObj->fd, data, (size_t)len, 0);
}
#define maxEvents 10
static void taosProcessTcpData(void *param) {
SThreadObj * pThreadObj;
int i, fdNum;
SFdObj * pFdObj;
struct epoll_event events[maxEvents];
SRecvInfo recvInfo;
pThreadObj = (SThreadObj *)param;
SRpcHead rpcHead;
while (1) {
pthread_mutex_lock(&pThreadObj->threadMutex);
if (pThreadObj->numOfFds < 1) {
pthread_cond_wait(&pThreadObj->fdReady, &pThreadObj->threadMutex);
}
pthread_mutex_unlock(&pThreadObj->threadMutex);
fdNum = epoll_wait(pThreadObj->pollFd, events, maxEvents, -1);
if (fdNum < 0) continue;
for (i = 0; i < fdNum; ++i) {
pFdObj = events[i].data.ptr;
if (events[i].events & EPOLLERR) {
tTrace("%s TCP thread:%d, error happened on FD", pThreadObj->label, pThreadObj->threadId);
taosCleanUpFdObj(pFdObj);
continue;
}
if (events[i].events & EPOLLHUP) {
tTrace("%s TCP thread:%d, FD hang up", pThreadObj->label, pThreadObj->threadId);
taosCleanUpFdObj(pFdObj);
continue;
}
int32_t headLen = taosReadMsg(pFdObj->fd, &rpcHead, sizeof(SRpcHead));
if (headLen != sizeof(SRpcHead)) {
tError("%s read error, headLen:%d, errno:%d", pThreadObj->label, headLen, errno);
taosCleanUpFdObj(pFdObj);
continue;
}
int32_t msgLen = (int32_t)htonl((uint32_t)rpcHead.msgLen);
char *buffer = malloc(msgLen + tsRpcOverhead);
if ( NULL == buffer) {
tError("%s TCP malloc(size:%d) fail\n", pThreadObj->label, msgLen);
taosCleanUpFdObj(pFdObj);
continue;
}
char *msg = buffer + tsRpcOverhead;
int32_t leftLen = msgLen - headLen;
int32_t retLen = taosReadMsg(pFdObj->fd, msg + headLen, leftLen);
if (leftLen != retLen) {
tError("%s read error, leftLen:%d retLen:%d", pThreadObj->label, leftLen, retLen);
taosCleanUpFdObj(pFdObj);
tfree(buffer);
continue;
}
// tTrace("%s TCP data is received, ip:%s:%u len:%d", pTcp->label, pFdObj->ipstr, pFdObj->port, msgLen);
memcpy(msg, &rpcHead, sizeof(SRpcHead));
recvInfo.msg = msg;
recvInfo.msgLen = msgLen;
recvInfo.ip = pFdObj->ip;
recvInfo.port = pFdObj->port;
recvInfo.shandle = pThreadObj->shandle;
recvInfo.thandle = pFdObj->thandle;;
recvInfo.chandle = pFdObj;
recvInfo.connType = RPC_CONN_TCP;
pFdObj->thandle = (*(pThreadObj->processData))(&recvInfo);
if (pFdObj->thandle == NULL) taosCleanUpFdObj(pFdObj);
}
}
}
static void taosAcceptTcpConnection(void *arg) {
int connFd = -1;
struct sockaddr_in clientAddr;
int sockFd;
int threadId = 0;
SThreadObj * pThreadObj;
SServerObj * pServerObj;
SFdObj * pFdObj;
struct epoll_event event;
pServerObj = (SServerObj *)arg;
sockFd = taosOpenTcpServerSocket(pServerObj->ip, pServerObj->port);
if (sockFd < 0) {
tError("%s failed to open TCP socket, ip:%s, port:%hu", pServerObj->label, pServerObj->ip, pServerObj->port);
return;
} else {
tTrace("%s TCP server is ready, ip:%s, port:%hu", pServerObj->label, pServerObj->ip, pServerObj->port);
}
while (1) {
socklen_t addrlen = sizeof(clientAddr);
connFd = accept(sockFd, (struct sockaddr *)&clientAddr, &addrlen);
if (connFd < 0) {
tError("%s TCP accept failure, errno:%d, reason:%s", pServerObj->label, errno, strerror(errno));
continue;
}
tTrace("%s TCP connection from ip:%s port:%hu", pServerObj->label, inet_ntoa(clientAddr.sin_addr),
htons(clientAddr.sin_port));
taosKeepTcpAlive(connFd);
// pick up the thread to handle this connection
pThreadObj = pServerObj->pThreadObj + threadId;
pFdObj = (SFdObj *)malloc(sizeof(SFdObj));
if (pFdObj == NULL) {
tError("%s no enough resource to allocate TCP FD IDs", pServerObj->label);
close(connFd);
continue;
}
memset(pFdObj, 0, sizeof(SFdObj));
pFdObj->fd = connFd;
strcpy(pFdObj->ipstr, inet_ntoa(clientAddr.sin_addr));
pFdObj->ip = clientAddr.sin_addr.s_addr;
pFdObj->port = htons(clientAddr.sin_port);
pFdObj->pThreadObj = pThreadObj;
pFdObj->signature = pFdObj;
event.events = EPOLLIN | EPOLLPRI | EPOLLWAKEUP;
event.data.ptr = pFdObj;
if (epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_ADD, connFd, &event) < 0) {
tError("%s failed to add TCP FD for epoll, error:%s", pServerObj->label, strerror(errno));
tfree(pFdObj);
close(connFd);
continue;
}
// notify the data process, add into the FdObj list
pthread_mutex_lock(&(pThreadObj->threadMutex));
pFdObj->next = pThreadObj->pHead;
if (pThreadObj->pHead) (pThreadObj->pHead)->prev = pFdObj;
pThreadObj->pHead = pFdObj;
pThreadObj->numOfFds++;
pthread_cond_signal(&pThreadObj->fdReady);
pthread_mutex_unlock(&(pThreadObj->threadMutex));
tTrace("%s TCP thread:%d, a new connection from %s:%hu, FD:%p, numOfFds:%d", pServerObj->label,
pThreadObj->threadId, pFdObj->ipstr, pFdObj->port, pFdObj, pThreadObj->numOfFds);
// pick up next thread for next connection
threadId++;
threadId = threadId % pServerObj->numOfThreads;
}
}
static void taosCleanUpFdObj(SFdObj *pFdObj) {
SThreadObj *pThreadObj;
if (pFdObj == NULL) return;
if (pFdObj->signature != pFdObj) return;
pThreadObj = pFdObj->pThreadObj;
if (pThreadObj == NULL) {
tError("FdObj double clean up!!!");
return;
}
epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_DEL, pFdObj->fd, NULL);
close(pFdObj->fd);
pthread_mutex_lock(&pThreadObj->threadMutex);
pThreadObj->numOfFds--;
if (pThreadObj->numOfFds < 0)
tError("%s TCP thread:%d, number of FDs shall never be negative", pThreadObj->label, pThreadObj->threadId);
// remove from the FdObject list
if (pFdObj->prev) {
(pFdObj->prev)->next = pFdObj->next;
} else {
pThreadObj->pHead = pFdObj->next;
}
if (pFdObj->next) {
(pFdObj->next)->prev = pFdObj->prev;
}
pthread_mutex_unlock(&pThreadObj->threadMutex);
// notify the upper layer, so it will clean the associated context
SRecvInfo recvInfo;
recvInfo.msg = NULL;
recvInfo.msgLen = 0;
recvInfo.ip = 0;
recvInfo.port = 0;
recvInfo.shandle = pThreadObj->shandle;
recvInfo.thandle = pFdObj->thandle;;
recvInfo.chandle = NULL;
recvInfo.connType = RPC_CONN_TCP;
if (pFdObj->thandle) (*(pThreadObj->processData))(&recvInfo);
tTrace("%s TCP thread:%d, FD:%p is cleaned up, numOfFds:%d", pThreadObj->label, pThreadObj->threadId,
pFdObj, pThreadObj->numOfFds);
memset(pFdObj, 0, sizeof(SFdObj));
tfree(pFdObj);
}
#if 0
static void taosAcceptUDConnection(void *arg) {
int connFd = -1;
int sockFd;
int threadId = 0;
SThreadObj * pThreadObj;
SServerObj * pServerObj;
SFdObj * pFdObj;
struct epoll_event event;
pServerObj = (SServerObj *)arg;
sockFd = taosOpenUDServerSocket(pServerObj->ip, pServerObj->port);
if (sockFd < 0) {
tError("%s failed to open UD socket, ip:%s, port:%hu", pServerObj->label, pServerObj->ip, pServerObj->port);
return;
} else {
tTrace("%s UD server is ready, ip:%s, port:%hu", pServerObj->label, pServerObj->ip, pServerObj->port);
}
while (1) {
connFd = accept(sockFd, NULL, NULL);
if (connFd < 0) {
tError("%s UD accept failure, errno:%d, reason:%s", pServerObj->label, errno, strerror(errno));
continue;
}
// pick up the thread to handle this connection
pThreadObj = pServerObj->pThreadObj + threadId;
pFdObj = (SFdObj *)malloc(sizeof(SFdObj));
if (pFdObj == NULL) {
tError("%s no enough resource to allocate TCP FD IDs", pServerObj->label);
close(connFd);
continue;
}
memset(pFdObj, 0, sizeof(SFdObj));
pFdObj->fd = connFd;
pFdObj->pThreadObj = pThreadObj;
event.events = EPOLLIN | EPOLLPRI | EPOLLWAKEUP;
event.data.ptr = pFdObj;
if (epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_ADD, connFd, &event) < 0) {
tError("%s failed to add UD FD for epoll, error:%s", pServerObj->label, strerror(errno));
tfree(pFdObj);
close(connFd);
continue;
}
// notify the data process, add into the FdObj list
pthread_mutex_lock(&(pThreadObj->threadMutex));
pFdObj->next = pThreadObj->pHead;
if (pThreadObj->pHead) (pThreadObj->pHead)->prev = pFdObj;
pThreadObj->pHead = pFdObj;
pThreadObj->numOfFds++;
pthread_cond_signal(&pThreadObj->fdReady);
pthread_mutex_unlock(&(pThreadObj->threadMutex));
tTrace("%s UD thread:%d, a new connection, numOfFds:%d", pServerObj->label, pThreadObj->threadId,
pThreadObj->numOfFds);
// pick up next thread for next connection
threadId++;
threadId = threadId % pServerObj->numOfThreads;
}
}
#endif
#if 0
void taosListTcpConnection(void *handle, char *buffer) {
SServerObj *pServerObj;
SThreadObj *pThreadObj;
SFdObj * pFdObj;
int i, numOfFds, numOfConns;
char * msg;
pServerObj = (SServerObj *)handle;
buffer[0] = 0;
msg = buffer;
numOfConns = 0;
pThreadObj = pServerObj->pThreadObj;
for (i = 0; i < pServerObj->numOfThreads; ++i) {
numOfFds = 0;
sprintf(msg, "TCP:%s Thread:%d number of connections:%d\n", pServerObj->label, pThreadObj->threadId,
pThreadObj->numOfFds);
msg = msg + strlen(msg);
pFdObj = pThreadObj->pHead;
while (pFdObj) {
sprintf(msg, " ip:%s port:%hu\n", pFdObj->ipstr, pFdObj->port);
msg = msg + strlen(msg);
numOfFds++;
numOfConns++;
pFdObj = pFdObj->next;
}
if (numOfFds != pThreadObj->numOfFds)
tError("TCP:%s thread:%d BIG error, numOfFds:%d actual numOfFds:%d", pServerObj->label, pThreadObj->threadId,
pThreadObj->numOfFds, numOfFds);
pThreadObj++;
}
sprintf(msg, "TCP:%s total connections:%d\n", pServerObj->label, numOfConns);
return;
}
#endif

499
src/rpc/src/rpcTcp.c Normal file
View File

@ -0,0 +1,499 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "os.h"
#include "tlog.h"
#include "tlog.h"
#include "tsocket.h"
#include "tutil.h"
#include "rpcHead.h"
#include "rpcTcp.h"
#ifndef EPOLLWAKEUP
#define EPOLLWAKEUP (1u << 29)
#endif
typedef struct SFdObj {
void *signature;
int fd; // TCP socket FD
void *thandle; // handle from upper layer, like TAOS
uint32_t ip;
uint16_t port;
struct SThreadObj *pThreadObj;
struct SFdObj *prev;
struct SFdObj *next;
} SFdObj;
typedef struct SThreadObj {
pthread_t thread;
SFdObj * pHead;
pthread_mutex_t mutex;
pthread_cond_t fdReady;
char ipstr[TSDB_IPv4ADDR_LEN];
int pollFd;
int numOfFds;
int threadId;
char label[12];
void *shandle; // handle passed by upper layer during server initialization
void *(*processData)(SRecvInfo *pPacket);
} SThreadObj;
typedef struct {
char ip[TSDB_IPv4ADDR_LEN];
uint16_t port;
char label[12];
int numOfThreads;
void * shandle;
SThreadObj *pThreadObj;
pthread_t thread;
} SServerObj;
static void *taosProcessTcpData(void *param);
static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, int fd);
static void taosFreeFdObj(SFdObj *pFdObj);
static void taosReportBrokenLink(SFdObj *pFdObj);
static void taosAcceptTcpConnection(void *arg);
void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle) {
SServerObj *pServerObj;
SThreadObj *pThreadObj;
pServerObj = (SServerObj *)calloc(sizeof(SServerObj), 1);
strcpy(pServerObj->ip, ip);
pServerObj->port = port;
strcpy(pServerObj->label, label);
pServerObj->numOfThreads = numOfThreads;
pServerObj->pThreadObj = (SThreadObj *)calloc(sizeof(SThreadObj), numOfThreads);
if (pServerObj->pThreadObj == NULL) {
tError("TCP:%s no enough memory", label);
free(pServerObj);
return NULL;
}
int code = 0;
pThreadObj = pServerObj->pThreadObj;
for (int i = 0; i < numOfThreads; ++i) {
pThreadObj->processData = fp;
strcpy(pThreadObj->label, label);
pThreadObj->shandle = shandle;
code = pthread_mutex_init(&(pThreadObj->mutex), NULL);
if (code < 0) {
tError("%s failed to init TCP process data mutex(%s)", label, strerror(errno));
break;;
}
code = pthread_cond_init(&(pThreadObj->fdReady), NULL);
if (code != 0) {
tError("%s init TCP condition variable failed(%s)", label, strerror(errno));
break;
}
pThreadObj->pollFd = epoll_create(10); // size does not matter
if (pThreadObj->pollFd < 0) {
tError("%s failed to create TCP epoll", label);
code = -1;
break;
}
pthread_attr_t thattr;
pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
code = pthread_create(&(pThreadObj->thread), &thattr, taosProcessTcpData, (void *)(pThreadObj));
pthread_attr_destroy(&thattr);
if (code != 0) {
tError("%s failed to create TCP process data thread(%s)", label, strerror(errno));
break;
}
pThreadObj->threadId = i;
pThreadObj++;
}
if (code == 0) {
pthread_attr_t thattr;
pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
code = pthread_create(&(pServerObj->thread), &thattr, (void *)taosAcceptTcpConnection, (void *)(pServerObj));
pthread_attr_destroy(&thattr);
if (code != 0) {
tError("%s failed to create TCP accept thread(%s)", label, strerror(errno));
}
}
if (code != 0) {
free(pServerObj->pThreadObj);
free(pServerObj);
pServerObj = NULL;
} else {
tTrace("%s TCP server is initialized, ip:%s port:%hu numOfThreads:%d", label, ip, port, numOfThreads);
}
return (void *)pServerObj;
}
void taosCleanUpTcpServer(void *handle) {
SServerObj *pServerObj = handle;
SThreadObj *pThreadObj;
if (pServerObj == NULL) return;
pthread_cancel(pServerObj->thread);
pthread_join(pServerObj->thread, NULL);
for (int i = 0; i < pServerObj->numOfThreads; ++i) {
pThreadObj = pServerObj->pThreadObj + i;
while (pThreadObj->pHead) {
SFdObj *pFdObj = pThreadObj->pHead;
pThreadObj->pHead = pFdObj->next;
taosFreeFdObj(pFdObj);
}
close(pThreadObj->pollFd);
pthread_cancel(pThreadObj->thread);
pthread_join(pThreadObj->thread, NULL);
pthread_cond_destroy(&(pThreadObj->fdReady));
pthread_mutex_destroy(&(pThreadObj->mutex));
}
tTrace("TCP:%s, TCP server is cleaned up", pServerObj->label);
tfree(pServerObj->pThreadObj);
tfree(pServerObj);
}
static void taosAcceptTcpConnection(void *arg) {
int connFd = -1;
struct sockaddr_in caddr;
int sockFd;
int threadId = 0;
SThreadObj *pThreadObj;
SServerObj *pServerObj;
pServerObj = (SServerObj *)arg;
sockFd = taosOpenTcpServerSocket(pServerObj->ip, pServerObj->port);
if (sockFd < 0) return;
tTrace("%s TCP server is ready, ip:%s:%hu", pServerObj->label, pServerObj->ip, pServerObj->port);
while (1) {
socklen_t addrlen = sizeof(caddr);
connFd = accept(sockFd, (struct sockaddr *)&caddr, &addrlen);
if (connFd < 0) {
tError("%s TCP accept failure(%s)", pServerObj->label, errno, strerror(errno));
continue;
}
tTrace("%s TCP connection from ip:%s:%hu", pServerObj->label, inet_ntoa(caddr.sin_addr), caddr.sin_port);
taosKeepTcpAlive(connFd);
// pick up the thread to handle this connection
pThreadObj = pServerObj->pThreadObj + threadId;
SFdObj *pFdObj = taosMallocFdObj(pThreadObj, connFd);
if (pFdObj) {
pFdObj->ip = caddr.sin_addr.s_addr;
pFdObj->port = caddr.sin_port;
tTrace("%s new connection from %s:%hu, FD:%p, numOfFds:%d", pServerObj->label,
inet_ntoa(caddr.sin_addr), pFdObj->port, pFdObj, pThreadObj->numOfFds);
} else {
close(connFd);
tError("%s failed to malloc FdObj(%s)", pServerObj->label, strerror(errno));
}
// pick up next thread for next connection
threadId++;
threadId = threadId % pServerObj->numOfThreads;
}
}
void *taosInitTcpClient(char *ip, uint16_t port, char *label, int num, void *fp, void *shandle) {
SThreadObj *pThreadObj;
pthread_attr_t thattr;
pThreadObj = (SThreadObj *)malloc(sizeof(SThreadObj));
memset(pThreadObj, 0, sizeof(SThreadObj));
strcpy(pThreadObj->label, label);
strcpy(pThreadObj->ipstr, ip);
pThreadObj->shandle = shandle;
if (pthread_mutex_init(&(pThreadObj->mutex), NULL) < 0) {
tError("%s failed to init TCP client mutex(%s)", label, strerror(errno));
return NULL;
}
if (pthread_cond_init(&(pThreadObj->fdReady), NULL) != 0) {
tError("%s init TCP condition variable failed(%s)", label, strerror(errno));
return NULL;
}
pThreadObj->pollFd = epoll_create(10); // size does not matter
if (pThreadObj->pollFd < 0) {
tError("%s failed to create TCP client epoll", label);
return NULL;
}
pThreadObj->processData = fp;
pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
int code = pthread_create(&(pThreadObj->thread), &thattr, taosProcessTcpData, (void *)(pThreadObj));
pthread_attr_destroy(&thattr);
if (code != 0) {
tError("%s failed to create TCP read data thread(%s)", label, strerror(errno));
return NULL;
}
tTrace("%s TCP client is initialized, ip:%s:%hu", label, ip, port);
return pThreadObj;
}
void taosCleanUpTcpClient(void *chandle) {
SThreadObj *pThreadObj = chandle;
if (pThreadObj == NULL) return;
while (pThreadObj->pHead) {
SFdObj *pFdObj = pThreadObj->pHead;
pThreadObj->pHead = pFdObj->next;
taosFreeFdObj(pFdObj);
}
close(pThreadObj->pollFd);
pthread_cancel(pThreadObj->thread);
pthread_join(pThreadObj->thread, NULL);
tTrace (":%s, all connections are cleaned up", pThreadObj->label);
tfree(pThreadObj);
}
void *taosOpenTcpClientConnection(void *shandle, void *thandle, char *ip, uint16_t port) {
SThreadObj * pThreadObj = shandle;
struct in_addr destIp;
int fd = taosOpenTcpClientSocket(ip, port, pThreadObj->ipstr);
if (fd <= 0) return NULL;
inet_aton(ip, &destIp);
SFdObj *pFdObj = taosMallocFdObj(pThreadObj, fd);
if (pFdObj) {
pFdObj->thandle = thandle;
pFdObj->port = port;
pFdObj->ip = destIp.s_addr;
tTrace("%s %p, TCP connection to %s:%hu is created, FD:%p numOfFds:%d",
pThreadObj->label, thandle, ip, port, pFdObj, pThreadObj->numOfFds);
} else {
close(fd);
tError("%s failed to malloc client FdObj(%s)", pThreadObj->label, strerror(errno));
}
return pFdObj;
}
void taosCloseTcpConnection(void *chandle) {
SFdObj *pFdObj = chandle;
if (pFdObj == NULL) return;
taosFreeFdObj(pFdObj);
}
int taosSendTcpData(uint32_t ip, uint16_t port, void *data, int len, void *chandle) {
SFdObj *pFdObj = chandle;
if (chandle == NULL) return -1;
return (int)send(pFdObj->fd, data, (size_t)len, 0);
}
static void taosReportBrokenLink(SFdObj *pFdObj) {
SThreadObj *pThreadObj = pFdObj->pThreadObj;
// notify the upper layer, so it will clean the associated context
if (pFdObj->thandle) {
SRecvInfo recvInfo;
recvInfo.msg = NULL;
recvInfo.msgLen = 0;
recvInfo.ip = 0;
recvInfo.port = 0;
recvInfo.shandle = pThreadObj->shandle;
recvInfo.thandle = pFdObj->thandle;;
recvInfo.chandle = NULL;
recvInfo.connType = RPC_CONN_TCP;
(*(pThreadObj->processData))(&recvInfo);
}
}
#define maxEvents 10
static void *taosProcessTcpData(void *param) {
SThreadObj *pThreadObj = param;
SFdObj *pFdObj;
struct epoll_event events[maxEvents];
SRecvInfo recvInfo;
SRpcHead rpcHead;
while (1) {
pthread_mutex_lock(&pThreadObj->mutex);
if (pThreadObj->numOfFds < 1) {
pthread_cond_wait(&pThreadObj->fdReady, &pThreadObj->mutex);
}
pthread_mutex_unlock(&pThreadObj->mutex);
int fdNum = epoll_wait(pThreadObj->pollFd, events, maxEvents, -1);
if (fdNum < 0) continue;
for (int i = 0; i < fdNum; ++i) {
pFdObj = events[i].data.ptr;
if (events[i].events & EPOLLERR) {
tTrace("%s %p, error happened on FD", pThreadObj->label, pFdObj->thandle);
taosReportBrokenLink(pFdObj);
continue;
}
if (events[i].events & EPOLLHUP) {
tTrace("%s %p, FD hang up", pThreadObj->label, pFdObj->thandle);
taosReportBrokenLink(pFdObj);
continue;
}
int32_t headLen = taosReadMsg(pFdObj->fd, &rpcHead, sizeof(SRpcHead));
if (headLen != sizeof(SRpcHead)) {
tError("%s %p, read error, headLen:%d", pThreadObj->label, pFdObj->thandle, headLen);
taosReportBrokenLink(pFdObj);
continue;
}
int32_t msgLen = (int32_t)htonl((uint32_t)rpcHead.msgLen);
char *buffer = malloc(msgLen + tsRpcOverhead);
if ( NULL == buffer) {
tError("%s %p, TCP malloc(size:%d) fail", pThreadObj->label, pFdObj->thandle, msgLen);
taosReportBrokenLink(pFdObj);
continue;
}
char *msg = buffer + tsRpcOverhead;
int32_t leftLen = msgLen - headLen;
int32_t retLen = taosReadMsg(pFdObj->fd, msg + headLen, leftLen);
if (leftLen != retLen) {
tError("%s %p, read error, leftLen:%d retLen:%d",
pThreadObj->label, pFdObj->thandle, leftLen, retLen);
taosReportBrokenLink(pFdObj);
tfree(buffer);
continue;
}
// tTrace("%s TCP data is received, ip:%s:%u len:%d", pThreadObj->label, pFdObj->ipstr, pFdObj->port, msgLen);
memcpy(msg, &rpcHead, sizeof(SRpcHead));
recvInfo.msg = msg;
recvInfo.msgLen = msgLen;
recvInfo.ip = pFdObj->ip;
recvInfo.port = pFdObj->port;
recvInfo.shandle = pThreadObj->shandle;
recvInfo.thandle = pFdObj->thandle;;
recvInfo.chandle = pFdObj;
recvInfo.connType = RPC_CONN_TCP;
pFdObj->thandle = (*(pThreadObj->processData))(&recvInfo);
if (pFdObj->thandle == NULL) taosFreeFdObj(pFdObj);
}
}
return NULL;
}
static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, int fd) {
struct epoll_event event;
SFdObj *pFdObj = (SFdObj *)calloc(sizeof(SFdObj), 1);
if (pFdObj == NULL) return NULL;
pFdObj->fd = fd;
pFdObj->pThreadObj = pThreadObj;
pFdObj->signature = pFdObj;
event.events = EPOLLIN | EPOLLPRI | EPOLLWAKEUP;
event.data.ptr = pFdObj;
if (epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) {
tfree(pFdObj);
return NULL;
}
// notify the data process, add into the FdObj list
pthread_mutex_lock(&(pThreadObj->mutex));
pFdObj->next = pThreadObj->pHead;
if (pThreadObj->pHead) (pThreadObj->pHead)->prev = pFdObj;
pThreadObj->pHead = pFdObj;
pThreadObj->numOfFds++;
pthread_cond_signal(&pThreadObj->fdReady);
pthread_mutex_unlock(&(pThreadObj->mutex));
return pFdObj;
}
static void taosFreeFdObj(SFdObj *pFdObj) {
if (pFdObj == NULL) return;
if (pFdObj->signature != pFdObj) return;
SThreadObj *pThreadObj = pFdObj->pThreadObj;
pthread_mutex_lock(&pThreadObj->mutex);
if (pFdObj->signature == NULL) {
pthread_mutex_unlock(&pThreadObj->mutex);
return;
}
pFdObj->signature = NULL;
close(pFdObj->fd);
epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_DEL, pFdObj->fd, NULL);
pThreadObj->numOfFds--;
if (pThreadObj->numOfFds < 0)
tError("%s %p, TCP thread:%d, number of FDs is negative!!!",
pThreadObj->label, pFdObj->thandle, pThreadObj->threadId);
// remove from the FdObject list
if (pFdObj->prev) {
(pFdObj->prev)->next = pFdObj->next;
} else {
pThreadObj->pHead = pFdObj->next;
}
if (pFdObj->next) {
(pFdObj->next)->prev = pFdObj->prev;
}
pthread_mutex_unlock(&pThreadObj->mutex);
tTrace("%s %p, FD:%p is cleaned, numOfFds:%d",
pThreadObj->label, pFdObj->thandle, pFdObj, pThreadObj->numOfFds);
tfree(pFdObj);
}

36
src/rpc/src/rpcUdp.c
View File

@ -32,7 +32,7 @@
int tsUdpDelay = 0;
typedef struct {
void * signature;
void *signature;
int index;
int fd;
uint16_t port; // peer port
@ -53,23 +53,23 @@ typedef struct {
int server;
char ip[16]; // local IP
uint16_t port; // local Port
void * shandle; // handle passed by upper layer during server initialization
void *shandle; // handle passed by upper layer during server initialization
int threads;
char label[12];
void * tmrCtrl;
void *tmrCtrl;
void *(*fp)(SRecvInfo *pPacket);
SUdpConn udpConn[];
} SUdpConnSet;
typedef struct {
void * signature;
void *signature;
uint32_t ip; // dest IP
uint16_t port; // dest Port
SUdpConn * pConn;
SUdpConn *pConn;
struct sockaddr_in destAdd;
void * msgHdr;
void *msgHdr;
int totalLen;
void * timer;
void *timer;
int emptyNum;
} SUdpBuf;
@ -78,9 +78,8 @@ static SUdpBuf *taosCreateUdpBuf(SUdpConn *pConn, uint32_t ip, uint16_t port);
static void taosProcessUdpBufTimer(void *param, void *tmrId);
void *taosInitUdpConnection(char *ip, uint16_t port, char *label, int threads, void *fp, void *shandle) {
pthread_attr_t thAttr;
SUdpConn * pConn;
SUdpConnSet * pSet;
SUdpConn *pConn;
SUdpConnSet *pSet;
int size = (int)sizeof(SUdpConnSet) + threads * (int)sizeof(SUdpConn);
pSet = (SUdpConnSet *)malloc((size_t)size);
@ -106,9 +105,6 @@ void *taosInitUdpConnection(char *ip, uint16_t port, char *label, int threads, v
}
}
pthread_attr_init(&thAttr);
pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
uint16_t ownPort;
for (int i = 0; i < threads; ++i) {
pConn = pSet->udpConn + i;
@ -146,7 +142,12 @@ void *taosInitUdpConnection(char *ip, uint16_t port, char *label, int threads, v
pConn->tmrCtrl = pSet->tmrCtrl;
}
if (pthread_create(&pConn->thread, &thAttr, taosRecvUdpData, pConn) != 0) {
pthread_attr_t thAttr;
pthread_attr_init(&thAttr);
pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
int code = pthread_create(&pConn->thread, &thAttr, taosRecvUdpData, pConn);
pthread_attr_destroy(&thAttr);
if (code != 0) {
tError("%s failed to create thread to process UDP data, reason:%s", label, strerror(errno));
taosCloseSocket(pConn->fd);
taosCleanUpUdpConnection(pSet);
@ -156,7 +157,6 @@ void *taosInitUdpConnection(char *ip, uint16_t port, char *label, int threads, v
++pSet->threads;
}
pthread_attr_destroy(&thAttr);
tTrace("%s UDP connection is initialized, ip:%s port:%hu threads:%d", label, ip, port, threads);
return pSet;
@ -164,7 +164,7 @@ void *taosInitUdpConnection(char *ip, uint16_t port, char *label, int threads, v
void taosCleanUpUdpConnection(void *handle) {
SUdpConnSet *pSet = (SUdpConnSet *)handle;
SUdpConn * pConn;
SUdpConn *pConn;
if (pSet == NULL) return;
@ -205,10 +205,10 @@ void *taosOpenUdpConnection(void *shandle, void *thandle, char *ip, uint16_t por
}
static void *taosRecvUdpData(void *param) {
SUdpConn *pConn = param;
struct sockaddr_in sourceAdd;
int dataLen;
unsigned int addLen;
SUdpConn * pConn = (SUdpConn *)param;
uint16_t port;
int minSize = sizeof(SRpcHead);
SRecvInfo recvInfo;
@ -274,7 +274,7 @@ static void *taosRecvUdpData(void *param) {
int taosSendUdpData(uint32_t ip, uint16_t port, void *data, int dataLen, void *chandle) {
SUdpConn *pConn = (SUdpConn *)chandle;
SUdpBuf * pBuf;
SUdpBuf *pBuf;
if (pConn == NULL || pConn->signature != pConn) return -1;

2
src/rpc/test/rclient.c
View File

@ -204,6 +204,8 @@ int main(int argc, char *argv[]) {
tPrint("it takes %.3f mseconds to send %d requests to server", usedTime, numOfReqs*appThreads);
tPrint("Performance: %.3f requests per second, msgSize:%d bytes", 1000.0*numOfReqs*appThreads/usedTime, msgSize);
getchar();
taosCloseLogger();
return 0;

4
src/thirdparty/CMakeLists.txt vendored
View File

@ -1,4 +0,0 @@
aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/src SOURCE_LIST)
add_library(thirdparty ${SOURCE_LIST})
target_include_directories(thirdparty PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/inc)

0
src/vnode/tsdb/CMakeLists.txt → src/tsdb/CMakeLists.txt
View File

0
src/vnode/tsdb/inc/tsdb.h → src/tsdb/inc/tsdb.h
View File

0
src/vnode/tsdb/inc/tsdbMain.h → src/tsdb/inc/tsdbMain.h
View File

0
src/vnode/tsdb/src/tsdbCache.c → src/tsdb/src/tsdbCache.c
View File

0
src/vnode/tsdb/src/tsdbCompactor.c → src/tsdb/src/tsdbCompactor.c
View File

0
src/vnode/tsdb/src/tsdbFile.c → src/tsdb/src/tsdbFile.c
View File

10
src/vnode/tsdb/src/tsdbMain.c → src/tsdb/src/tsdbMain.c
View File

@ -814,6 +814,13 @@ static SSkipListIterator **tsdbCreateTableIters(STsdbMeta *pMeta, int maxTables)
return iters;
}
static void tsdbFreeMemTable(SMemTable *pMemTable) {
if (pMemTable) {
tSkipListDestroy(pMemTable->pData);
free(pMemTable);
}
}
// Commit to file
static void *tsdbCommitData(void *arg) {
// TODO
@ -860,7 +867,8 @@ static void *tsdbCommitData(void *arg) {
// TODO: free the skiplist
for (int i = 0; i < pCfg->maxTables; i++) {
STable *pTable = pMeta->tables[i];
if (pTable && pTable->imem) { // Here has memory leak
if (pTable && pTable->imem) {
tsdbFreeMemTable(pTable->imem);
pTable->imem = NULL;
}
}

2
src/vnode/tsdb/src/tsdbMeta.c → src/tsdb/src/tsdbMeta.c
View File

@ -34,7 +34,7 @@ void *tsdbEncodeTable(STable *pTable, int *contLen) {
*contLen = tsdbEstimateTableEncodeSize(pTable);
if (*contLen < 0) return NULL;
void *ret = malloc(*contLen);
void *ret = calloc(1, *contLen);
if (ret == NULL) return NULL;
void *ptr = ret;

0
src/vnode/tsdb/src/tsdbMetaFile.c → src/tsdb/src/tsdbMetaFile.c
View File

0
src/vnode/tsdb/src/tsdbRead.c → src/tsdb/src/tsdbRead.c
View File

0
src/vnode/tsdb/tests/CMakeLists.txt → src/tsdb/tests/CMakeLists.txt
View File

0
src/vnode/tsdb/tests/tsdbTests.cpp → src/tsdb/tests/tsdbTests.cpp
View File

8
src/util/CMakeLists.txt
View File

@ -6,9 +6,11 @@ INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/lz4/inc)
AUX_SOURCE_DIRECTORY(src SRC)
ADD_LIBRARY(tutil ${SRC})
TARGET_LINK_LIBRARIES(tutil thirdparty pthread os m rt)
TARGET_LINK_LIBRARIES(tutil pthread os m rt)
FIND_PATH(ICONV_INCLUDE_EXIST iconv.h /usr/include/ /usr/local/include/)
IF (ICONV_INCLUDE_EXIST)
ADD_DEFINITIONS(-DUSE_LIBICONV)
@ -65,7 +67,7 @@ ELSEIF (TD_WINDOWS_64)
LIST(APPEND SRC ./src/tutil.c)
LIST(APPEND SRC ./src/version.c)
ADD_LIBRARY(tutil ${SRC})
TARGET_LINK_LIBRARIES(tutil thirdparty iconv regex pthread os winmm IPHLPAPI ws2_32)
TARGET_LINK_LIBRARIES(tutil iconv regex pthread os winmm IPHLPAPI ws2_32)
ELSEIF(TD_DARWIN_64)
ADD_DEFINITIONS(-DUSE_LIBICONV)
LIST(APPEND SRC ./src/hash.c)
@ -102,7 +104,7 @@ ELSEIF(TD_DARWIN_64)
LIST(APPEND SRC ./src/version.c)
LIST(APPEND SRC ./src/hash.c)
ADD_LIBRARY(tutil ${SRC})
TARGET_LINK_LIBRARIES(tutil thirdparty iconv pthread os)
TARGET_LINK_LIBRARIES(tutil iconv pthread os)
ENDIF()
#IF (TD_CLUSTER)

2
src/util/src/hash.c
View File

@ -298,7 +298,7 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool threadsafe) {
*/
static SHashNode *doCreateHashNode(const char *key, size_t keyLen, const char *pData, size_t dataSize,
uint32_t hashVal) {
size_t totalSize = dataSize + sizeof(SHashNode) + keyLen;
size_t totalSize = dataSize + sizeof(SHashNode) + keyLen + 1; // one extra byte for null
SHashNode *pNewNode = calloc(1, totalSize);
if (pNewNode == NULL) {

1
src/util/src/ihash.c
View File

@ -189,7 +189,6 @@ void taosCleanUpIntHash(void *handle) {
free(pObj->hashList);
}
memset(pObj, 0, sizeof(IHashObj));
free(pObj->lockedBy);
free(pObj);
}

17
src/util/src/tqueue.c
View File

@ -117,7 +117,7 @@ int taosWriteQitem(taos_queue param, int type, void *item) {
queue->numOfItems++;
if (queue->qset) atomic_add_fetch_32(&queue->qset->numOfItems, 1);
pTrace("item:%p is put into queue, type:%d items:%d", item, type, queue->numOfItems);
pTrace("item:%p is put into queue:%p, type:%d items:%d", item, queue, type, queue->numOfItems);
pthread_mutex_unlock(&queue->mutex);
@ -297,14 +297,16 @@ int taosReadQitemFromQset(taos_qset param, int *type, void **pitem, void **phand
STaosQset *qset = (STaosQset *)param;
STaosQnode *pNode = NULL;
int code = 0;
pthread_mutex_lock(&qset->mutex);
for(int i=0; i<qset->numOfQueues; ++i) {
pthread_mutex_lock(&qset->mutex);
//pthread_mutex_lock(&qset->mutex);
if (qset->current == NULL)
qset->current = qset->head;
STaosQueue *queue = qset->current;
if (queue) qset->current = queue->next;
pthread_mutex_unlock(&qset->mutex);
//pthread_mutex_unlock(&qset->mutex);
if (queue == NULL) break;
pthread_mutex_lock(&queue->mutex);
@ -326,6 +328,8 @@ int taosReadQitemFromQset(taos_qset param, int *type, void **pitem, void **phand
if (pNode) break;
}
pthread_mutex_unlock(&qset->mutex);
return code;
}
@ -335,13 +339,15 @@ int taosReadAllQitemsFromQset(taos_qset param, taos_qall p2, void **phandle) {
STaosQall *qall = (STaosQall *)p2;
int code = 0;
pthread_mutex_lock(&qset->mutex);
for(int i=0; i<qset->numOfQueues; ++i) {
pthread_mutex_lock(&qset->mutex);
// pthread_mutex_lock(&qset->mutex);
if (qset->current == NULL)
qset->current = qset->head;
queue = qset->current;
if (queue) qset->current = queue->next;
pthread_mutex_unlock(&qset->mutex);
// pthread_mutex_unlock(&qset->mutex);
if (queue == NULL) break;
pthread_mutex_lock(&queue->mutex);
@ -365,6 +371,7 @@ int taosReadAllQitemsFromQset(taos_qset param, taos_qall p2, void **phandle) {
if (code != 0) break;
}
pthread_mutex_unlock(&qset->mutex);
return code;
}

109
src/util/src/tsched.c
View File

@ -40,66 +40,68 @@ static void *taosProcessSchedQueue(void *param);
static void taosDumpSchedulerStatus(void *qhandle, void *tmrId);
void *taosInitScheduler(int queueSize, int numOfThreads, const char *label) {
pthread_attr_t attr;
SSchedQueue * pSched = (SSchedQueue *)malloc(sizeof(SSchedQueue));
SSchedQueue *pSched = (SSchedQueue *)calloc(sizeof(SSchedQueue), 1);
if (pSched == NULL) {
pError("%s: no enough memory for pSched, reason: %s", label, strerror(errno));
goto _error;
pError("%s: no enough memory for pSched", label);
return NULL;
}
pSched->queue = (SSchedMsg *)calloc(sizeof(SSchedMsg), queueSize);
if (pSched->queue == NULL) {
pError("%s: no enough memory for queue", label);
taosCleanUpScheduler(pSched);
return NULL;
}
pSched->qthread = calloc(sizeof(pthread_t), numOfThreads);
if (pSched->qthread == NULL) {
pError("%s: no enough memory for qthread", label);
taosCleanUpScheduler(pSched);
return NULL;
}
memset(pSched, 0, sizeof(SSchedQueue));
pSched->queueSize = queueSize;
strncpy(pSched->label, label, sizeof(pSched->label)); // fix buffer overflow
pSched->label[sizeof(pSched->label)-1] = '\0';
if (pthread_mutex_init(&pSched->queueMutex, NULL) < 0) {
pError("init %s:queueMutex failed, reason:%s", pSched->label, strerror(errno));
goto _error;
}
if (tsem_init(&pSched->emptySem, 0, (unsigned int)pSched->queueSize) != 0) {
pError("init %s:empty semaphore failed, reason:%s", pSched->label, strerror(errno));
goto _error;
}
if (tsem_init(&pSched->fullSem, 0, 0) != 0) {
pError("init %s:full semaphore failed, reason:%s", pSched->label, strerror(errno));
goto _error;
}
if ((pSched->queue = (SSchedMsg *)malloc((size_t)pSched->queueSize * sizeof(SSchedMsg))) == NULL) {
pError("%s: no enough memory for queue, reason:%s", pSched->label, strerror(errno));
goto _error;
}
memset(pSched->queue, 0, (size_t)pSched->queueSize * sizeof(SSchedMsg));
pSched->fullSlot = 0;
pSched->emptySlot = 0;
pSched->qthread = malloc(sizeof(pthread_t) * (size_t)numOfThreads);
if (pSched->qthread == NULL) {
pError("%s: no enough memory for qthread, reason: %s", pSched->label, strerror(errno));
goto _error;
if (pthread_mutex_init(&pSched->queueMutex, NULL) < 0) {
pError("init %s:queueMutex failed(%s)", label, strerror(errno));
taosCleanUpScheduler(pSched);
return NULL;
}
pthread_attr_init(&attr);
pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE);
if (tsem_init(&pSched->emptySem, 0, (unsigned int)pSched->queueSize) != 0) {
pError("init %s:empty semaphore failed(%s)", label, strerror(errno));
taosCleanUpScheduler(pSched);
return NULL;
}
if (tsem_init(&pSched->fullSem, 0, 0) != 0) {
pError("init %s:full semaphore failed(%s)", label, strerror(errno));
taosCleanUpScheduler(pSched);
return NULL;
}
for (int i = 0; i < numOfThreads; ++i) {
if (pthread_create(pSched->qthread + i, &attr, taosProcessSchedQueue, (void *)pSched) != 0) {
pError("%s: failed to create rpc thread, reason:%s", pSched->label, strerror(errno));
goto _error;
pthread_attr_t attr;
pthread_attr_init(&attr);
pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE);
int code = pthread_create(pSched->qthread + i, &attr, taosProcessSchedQueue, (void *)pSched);
pthread_attr_destroy(&attr);
if (code != 0) {
pError("%s: failed to create rpc thread(%s)", label, strerror(errno));
taosCleanUpScheduler(pSched);
return NULL;
}
++pSched->numOfThreads;
}
pTrace("%s scheduler is initialized, numOfThreads:%d", pSched->label, pSched->numOfThreads);
pTrace("%s scheduler is initialized, numOfThreads:%d", label, pSched->numOfThreads);
return (void *)pSched;
_error:
taosCleanUpScheduler(pSched);
return NULL;
}
void *taosInitSchedulerWithInfo(int queueSize, int numOfThreads, const char *label, void *tmrCtrl) {
@ -124,21 +126,21 @@ void *taosProcessSchedQueue(void *param) {
pTrace("wait %s fullSem was interrupted", pSched->label);
continue;
}
pError("wait %s fullSem failed, errno:%d, reason:%s", pSched->label, errno, strerror(errno));
pError("wait %s fullSem failed(%s)", pSched->label, strerror(errno));
}
if (pthread_mutex_lock(&pSched->queueMutex) != 0)
pError("lock %s queueMutex failed, reason:%s", pSched->label, strerror(errno));
pError("lock %s queueMutex failed(%s)", pSched->label, strerror(errno));
msg = pSched->queue[pSched->fullSlot];
memset(pSched->queue + pSched->fullSlot, 0, sizeof(SSchedMsg));
pSched->fullSlot = (pSched->fullSlot + 1) % pSched->queueSize;
if (pthread_mutex_unlock(&pSched->queueMutex) != 0)
pError("unlock %s queueMutex failed, reason:%s\n", pSched->label, strerror(errno));
pError("unlock %s queueMutex failed(%s)", pSched->label, strerror(errno));
if (tsem_post(&pSched->emptySem) != 0)
pError("post %s emptySem failed, reason:%s\n", pSched->label, strerror(errno));
pError("post %s emptySem failed(%s)", pSched->label, strerror(errno));
if (msg.fp)
(*(msg.fp))(&msg);
@ -158,22 +160,23 @@ int taosScheduleTask(void *qhandle, SSchedMsg *pMsg) {
while (tsem_wait(&pSched->emptySem) != 0) {
if (errno != EINTR) {
pError("wait %s emptySem failed, reason:%s", pSched->label, strerror(errno));
pError("wait %s emptySem failed(%s)", pSched->label, strerror(errno));
break;
}
pTrace("wait %s emptySem was interrupted", pSched->label);
}
if (pthread_mutex_lock(&pSched->queueMutex) != 0)
pError("lock %s queueMutex failed, reason:%s", pSched->label, strerror(errno));
pError("lock %s queueMutex failed(%s)", pSched->label, strerror(errno));
pSched->queue[pSched->emptySlot] = *pMsg;
pSched->emptySlot = (pSched->emptySlot + 1) % pSched->queueSize;
if (pthread_mutex_unlock(&pSched->queueMutex) != 0)
pError("unlock %s queueMutex failed, reason:%s", pSched->label, strerror(errno));
pError("unlock %s queueMutex failed(%s)", pSched->label, strerror(errno));
if (tsem_post(&pSched->fullSem) != 0) pError("post %s fullSem failed, reason:%s", pSched->label, strerror(errno));
if (tsem_post(&pSched->fullSem) != 0)
pError("post %s fullSem failed(%s)", pSched->label, strerror(errno));
return 0;
}
@ -183,10 +186,12 @@ void taosCleanUpScheduler(void *param) {
if (pSched == NULL) return;
for (int i = 0; i < pSched->numOfThreads; ++i) {
pthread_cancel(pSched->qthread[i]);
if (pSched->qthread[i])
pthread_cancel(pSched->qthread[i]);
}
for (int i = 0; i < pSched->numOfThreads; ++i) {
pthread_join(pSched->qthread[i], NULL);
if (pSched->qthread[i])
pthread_join(pSched->qthread[i], NULL);
}
tsem_destroy(&pSched->emptySem);
@ -197,8 +202,8 @@ void taosCleanUpScheduler(void *param) {
taosTmrStopA(&pSched->pTimer);
}
free(pSched->queue);
free(pSched->qthread);
if (pSched->queue) free(pSched->queue);
if (pSched->qthread) free(pSched->qthread);
free(pSched); // fix memory leak
}

18
src/vnode/CMakeLists.txt
View File

@ -1,6 +1,18 @@
CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
PROJECT(TDengine)
ADD_SUBDIRECTORY(wal)
ADD_SUBDIRECTORY(tsdb)
ADD_SUBDIRECTORY(main)
IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/tsdb/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/dnode/inc)
INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/inc)
INCLUDE_DIRECTORIES(inc)
AUX_SOURCE_DIRECTORY(src SRC)
ADD_LIBRARY(vnode ${SRC})
TARGET_LINK_LIBRARIES(vnode tsdb)
ENDIF ()

17
src/vnode/cache/inc/cache.h vendored
View File

@ -1,17 +0,0 @@
#if !defined(_TD_CACHE_H_)
#define _TD_CACHE_H_
#define TD_MIN_CACHE_BLOCK_SIZE 1024*1024 /* 1M */
#define TD_MAX_CACHE_BLOCK_SIZE 64*1024*1024 /* 64M */
typedef void cache_pool_t;
typedef struct SCacheBlock
{
int32_t blockId;
char data[];
} SCacheBlock;
#endif // _TD_CACHE_H_

0
src/vnode/main/inc/vnodeInt.h → src/vnode/inc/vnodeInt.h
View File

18
src/vnode/main/CMakeLists.txt
View File

@ -1,18 +0,0 @@
CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
PROJECT(TDengine)
IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/vnode/tsdb/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/dnode/inc)
INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/inc)
INCLUDE_DIRECTORIES(inc)
AUX_SOURCE_DIRECTORY(src SRC)
ADD_LIBRARY(vnode ${SRC})
TARGET_LINK_LIBRARIES(vnode tsdb)
ENDIF ()

6
src/vnode/main/src/vnodeMain.c → src/vnode/src/vnodeMain.c
View File

@ -160,7 +160,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
syncInfo.writeToCache = vnodeWriteToQueue;
syncInfo.confirmForward = dnodeSendRpcWriteRsp;
syncInfo.notifyRole = vnodeNotifyRole;
pVnode->sync = syncStart(&syncInfo);;
pVnode->sync = syncStart(&syncInfo);
pVnode->events = NULL;
pVnode->cq = NULL;
@ -224,10 +224,11 @@ void vnodeRelease(void *pVnodeRaw) {
// remove the whole directory
}
dTrace("pVnode:%p vgId:%d, vnode is released", pVnode, pVnode->vgId);
free(pVnode);
int32_t count = atomic_sub_fetch_32(&tsOpennedVnodes, 1);
dTrace("pVnode:%p vgId:%d, vnode is released, vnodes:%d", pVnode, vgId, count);
if (count <= 0) {
taosCleanUpIntHash(tsDnodeVnodesHash);
vnodeModuleInit = PTHREAD_ONCE_INIT;
@ -368,6 +369,7 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode) {
if (strcmp(option[0], "wals") != 0) return TSDB_CODE_INVALID_FILE_FORMAT;
if (wals == -1) return TSDB_CODE_INVALID_FILE_FORMAT;
pVnode->walCfg.wals = (int8_t)wals;
pVnode->walCfg.keep = 0;
int32_t arbitratorIp = -1;
num = fscanf(fp, "%s %u", option[0], &arbitratorIp);

0
src/vnode/main/src/vnodeRead.c → src/vnode/src/vnodeRead.c
View File

0
src/vnode/main/src/vnodeWrite.c → src/vnode/src/vnodeWrite.c
View File

0
src/vnode/wal/CMakeLists.txt → src/wal/CMakeLists.txt
View File

23
src/vnode/wal/src/walMain.c → src/wal/src/walMain.c
View File

@ -36,6 +36,7 @@
typedef struct {
int fd;
int keep;
int level;
int max; // maximum number of wal files
uint32_t id; // increase continuously
@ -61,6 +62,7 @@ void *walOpen(const char *path, const SWalCfg *pCfg) {
pWal->id = 0;
pWal->num = 0;
pWal->level = pCfg->commitLog;
pWal->keep = pCfg->keep;
strcpy(pWal->path, path);
pthread_mutex_init(&pWal->mutex, NULL);
@ -82,18 +84,21 @@ void *walOpen(const char *path, const SWalCfg *pCfg) {
void walClose(void *handle) {
if (handle == NULL) return;
SWal *pWal = handle;
SWal *pWal = handle;
close(pWal->fd);
// remove all files in the directory
for (int i=0; i<pWal->num; ++i) {
sprintf(pWal->name, "%s/%s%d", pWal->path, walPrefix, pWal->id-i);
if (remove(pWal->name) <0) {
wError("wal:%s, failed to remove", pWal->name);
} else {
wTrace("wal:%s, it is removed", pWal->name);
if (pWal->keep == 0) {
// remove all files in the directory
for (int i=0; i<pWal->num; ++i) {
sprintf(pWal->name, "%s/%s%d", pWal->path, walPrefix, pWal->id-i);
if (remove(pWal->name) <0) {
wError("wal:%s, failed to remove", pWal->name);
} else {
wTrace("wal:%s, it is removed", pWal->name);
}
}
} else {
wTrace("wal:%s, it is closed and kept", pWal->name);
}
pthread_mutex_destroy(&pWal->mutex);

Some files were not shown because too many files have changed in this diff Show More