From b9c4fc22654f6d7474348ed93f6a885a05b50459 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 11 Apr 2020 00:42:21 +0800 Subject: [PATCH 01/18] change go driver to git submodule. use following commands to retreive: git clone https://github.com/taosdata/TDengine git checkout develop git pull git submodule update --init --recursive --- .gitmodules | 3 + src/connector/go | 1 + src/connector/go/src/taosSql/connection.go | 368 ------------------ src/connector/go/src/taosSql/connector.go | 49 --- src/connector/go/src/taosSql/const.go | 51 --- src/connector/go/src/taosSql/driver.go | 43 --- src/connector/go/src/taosSql/dsn.go | 202 ---------- src/connector/go/src/taosSql/result.go | 200 ---------- src/connector/go/src/taosSql/rows.go | 296 --------------- src/connector/go/src/taosSql/statement.go | 158 -------- src/connector/go/src/taosSql/taosLog.go | 120 ------ src/connector/go/src/taosSql/taosSqlCgo.go | 84 ---- src/connector/go/src/taosSql/utils.go | 422 --------------------- 13 files changed, 4 insertions(+), 1993 deletions(-) create mode 100644 .gitmodules create mode 160000 src/connector/go delete mode 100755 src/connector/go/src/taosSql/connection.go delete mode 100755 src/connector/go/src/taosSql/connector.go delete mode 100755 src/connector/go/src/taosSql/const.go delete mode 100755 src/connector/go/src/taosSql/driver.go delete mode 100755 src/connector/go/src/taosSql/dsn.go delete mode 100755 src/connector/go/src/taosSql/result.go delete mode 100755 src/connector/go/src/taosSql/rows.go delete mode 100755 src/connector/go/src/taosSql/statement.go delete mode 100755 src/connector/go/src/taosSql/taosLog.go delete mode 100755 src/connector/go/src/taosSql/taosSqlCgo.go delete mode 100755 src/connector/go/src/taosSql/utils.go diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000000..35aea3658b --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "src/connector/go"] + path = src/connector/go + url = https://github.com/taosdata/driver-go diff --git a/src/connector/go b/src/connector/go new file mode 160000 index 0000000000..8c58c512b6 --- /dev/null +++ b/src/connector/go @@ -0,0 +1 @@ +Subproject commit 8c58c512b6acda8bcdfa48fdc7140227b5221766 diff --git a/src/connector/go/src/taosSql/connection.go b/src/connector/go/src/taosSql/connection.go deleted file mode 100755 index 2a3f5a9acd..0000000000 --- a/src/connector/go/src/taosSql/connection.go +++ /dev/null @@ -1,368 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package taosSql - -import "C" -import ( - "context" - "errors" - "database/sql/driver" - "unsafe" - "strconv" - "strings" - "time" -) - -type taosConn struct { - taos unsafe.Pointer - affectedRows int - insertId int - cfg *config - status statusFlag - parseTime bool - reset bool // set when the Go SQL package calls ResetSession -} - -type taosSqlResult struct { - affectedRows int64 - insertId int64 -} - -func (res *taosSqlResult) LastInsertId() (int64, error) { - return res.insertId, nil -} - -func (res *taosSqlResult) RowsAffected() (int64, error) { - return res.affectedRows, nil -} - -func (mc *taosConn) Begin() (driver.Tx, error) { - taosLog.Println("taosSql not support transaction") - return nil, errors.New("taosSql not support transaction") -} - -func (mc *taosConn) Close() (err error) { - if mc.taos == nil { - return errConnNoExist - } - mc.taos_close() - return nil -} - -func (mc *taosConn) Prepare(query string) (driver.Stmt, error) { - if mc.taos == nil { - return nil, errInvalidConn - } - - stmt := &taosSqlStmt{ - mc: mc, - pSql: query, - } - - // find ? count and save to stmt.paramCount - stmt.paramCount = strings.Count(query, "?") - - //fmt.Printf("prepare alloc stmt:%p, sql:%s\n", stmt, query) - taosLog.Printf("prepare alloc stmt:%p, sql:%s\n", stmt, query) - - return stmt, nil -} - -func (mc *taosConn) interpolateParams(query string, args []driver.Value) (string, error) { - // Number of ? should be same to len(args) - if strings.Count(query, "?") != len(args) { - return "", driver.ErrSkip - } - - buf := make([]byte, defaultBufSize) - buf = buf[:0] // clear buf - argPos := 0 - - for i := 0; i < len(query); i++ { - q := strings.IndexByte(query[i:], '?') - if q == -1 { - buf = append(buf, query[i:]...) - break - } - buf = append(buf, query[i:i+q]...) - i += q - - arg := args[argPos] - argPos++ - - if arg == nil { - buf = append(buf, "NULL"...) - continue - } - - switch v := arg.(type) { - case int64: - buf = strconv.AppendInt(buf, v, 10) - case uint64: - // Handle uint64 explicitly because our custom ConvertValue emits unsigned values - buf = strconv.AppendUint(buf, v, 10) - case float64: - buf = strconv.AppendFloat(buf, v, 'g', -1, 64) - case bool: - if v { - buf = append(buf, '1') - } else { - buf = append(buf, '0') - } - case time.Time: - if v.IsZero() { - buf = append(buf, "'0000-00-00'"...) - } else { - v := v.In(mc.cfg.loc) - v = v.Add(time.Nanosecond * 500) // To round under microsecond - year := v.Year() - year100 := year / 100 - year1 := year % 100 - month := v.Month() - day := v.Day() - hour := v.Hour() - minute := v.Minute() - second := v.Second() - micro := v.Nanosecond() / 1000 - - buf = append(buf, []byte{ - '\'', - digits10[year100], digits01[year100], - digits10[year1], digits01[year1], - '-', - digits10[month], digits01[month], - '-', - digits10[day], digits01[day], - ' ', - digits10[hour], digits01[hour], - ':', - digits10[minute], digits01[minute], - ':', - digits10[second], digits01[second], - }...) - - if micro != 0 { - micro10000 := micro / 10000 - micro100 := micro / 100 % 100 - micro1 := micro % 100 - buf = append(buf, []byte{ - '.', - digits10[micro10000], digits01[micro10000], - digits10[micro100], digits01[micro100], - digits10[micro1], digits01[micro1], - }...) - } - buf = append(buf, '\'') - } - case []byte: - if v == nil { - buf = append(buf, "NULL"...) - } else { - buf = append(buf, "_binary'"...) - if mc.status&statusNoBackslashEscapes == 0 { - buf = escapeBytesBackslash(buf, v) - } else { - buf = escapeBytesQuotes(buf, v) - } - buf = append(buf, '\'') - } - case string: - //buf = append(buf, '\'') - if mc.status&statusNoBackslashEscapes == 0 { - buf = escapeStringBackslash(buf, v) - } else { - buf = escapeStringQuotes(buf, v) - } - //buf = append(buf, '\'') - default: - return "", driver.ErrSkip - } - - //if len(buf)+4 > mc.maxAllowedPacket { - if len(buf)+4 > maxTaosSqlLen { - return "", driver.ErrSkip - } - } - if argPos != len(args) { - return "", driver.ErrSkip - } - return string(buf), nil -} - -func (mc *taosConn) Exec(query string, args []driver.Value) (driver.Result, error) { - if mc.taos == nil { - return nil, driver.ErrBadConn - } - if len(args) != 0 { - if !mc.cfg.interpolateParams { - return nil, driver.ErrSkip - } - // try to interpolate the parameters to save extra roundtrips for preparing and closing a statement - prepared, err := mc.interpolateParams(query, args) - if err != nil { - return nil, err - } - query = prepared - } - - mc.affectedRows = 0 - mc.insertId = 0 - _, err := mc.taosQuery(query) - if err == nil { - return &taosSqlResult{ - affectedRows: int64(mc.affectedRows), - insertId: int64(mc.insertId), - }, err - } - - return nil, err -} - -func (mc *taosConn) Query(query string, args []driver.Value) (driver.Rows, error) { - return mc.query(query, args) -} - -func (mc *taosConn) query(query string, args []driver.Value) (*textRows, error) { - if mc.taos == nil { - return nil, driver.ErrBadConn - } - if len(args) != 0 { - if !mc.cfg.interpolateParams { - return nil, driver.ErrSkip - } - // try client-side prepare to reduce roundtrip - prepared, err := mc.interpolateParams(query, args) - if err != nil { - return nil, err - } - query = prepared - } - - num_fields, err := mc.taosQuery(query) - if err == nil { - // Read Result - rows := new(textRows) - rows.mc = mc - - // Columns field - rows.rs.columns, err = mc.readColumns(num_fields) - return rows, err - } - return nil, err -} - -// Ping implements driver.Pinger interface -func (mc *taosConn) Ping(ctx context.Context) (err error) { - if mc.taos != nil { - return nil - } - return errInvalidConn -} - -// BeginTx implements driver.ConnBeginTx interface -func (mc *taosConn) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, error) { - taosLog.Println("taosSql not support transaction") - return nil, errors.New("taosSql not support transaction") -} - -func (mc *taosConn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) { - if mc.taos == nil { - return nil, errInvalidConn - } - - dargs, err := namedValueToValue(args) - if err != nil { - return nil, err - } - - rows, err := mc.query(query, dargs) - if err != nil { - return nil, err - } - - return rows, err -} - -func (mc *taosConn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) { - if mc.taos == nil { - return nil, errInvalidConn - } - - dargs, err := namedValueToValue(args) - if err != nil { - return nil, err - } - - return mc.Exec(query, dargs) -} - -func (mc *taosConn) PrepareContext(ctx context.Context, query string) (driver.Stmt, error) { - if mc.taos == nil { - return nil, errInvalidConn - } - - stmt, err := mc.Prepare(query) - if err != nil { - return nil, err - } - - return stmt, nil -} - -func (stmt *taosSqlStmt) QueryContext(ctx context.Context, args []driver.NamedValue) (driver.Rows, error) { - if stmt.mc == nil { - return nil, errInvalidConn - } - dargs, err := namedValueToValue(args) - - if err != nil { - return nil, err - } - - rows, err := stmt.query(dargs) - if err != nil { - return nil, err - } - return rows, err -} - -func (stmt *taosSqlStmt) ExecContext(ctx context.Context, args []driver.NamedValue) (driver.Result, error) { - if stmt.mc == nil { - return nil, errInvalidConn - } - - dargs, err := namedValueToValue(args) - if err != nil { - return nil, err - } - - return stmt.Exec(dargs) -} - -func (mc *taosConn) CheckNamedValue(nv *driver.NamedValue) (err error) { - nv.Value, err = converter{}.ConvertValue(nv.Value) - return -} - -// ResetSession implements driver.SessionResetter. -// (From Go 1.10) -func (mc *taosConn) ResetSession(ctx context.Context) error { - if mc.taos == nil { - return driver.ErrBadConn - } - mc.reset = true - return nil -} diff --git a/src/connector/go/src/taosSql/connector.go b/src/connector/go/src/taosSql/connector.go deleted file mode 100755 index 55715b949e..0000000000 --- a/src/connector/go/src/taosSql/connector.go +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package taosSql - -import ( - "context" - "database/sql/driver" -) - -type connector struct { - cfg *config -} - -// Connect implements driver.Connector interface. -// Connect returns a connection to the database. -func (c *connector) Connect(ctx context.Context) (driver.Conn, error) { - var err error - // New taosConn - mc := &taosConn{ - cfg: c.cfg, - parseTime: c.cfg.parseTime, - } - - // Connect to Server - mc.taos, err = mc.taosConnect(mc.cfg.addr, mc.cfg.user, mc.cfg.passwd, mc.cfg.dbName, mc.cfg.port) - if err != nil { - return nil, err - } - - return mc, nil -} - -// Driver implements driver.Connector interface. -// Driver returns &taosSQLDriver{}. -func (c *connector) Driver() driver.Driver { - return &taosSQLDriver{} -} diff --git a/src/connector/go/src/taosSql/const.go b/src/connector/go/src/taosSql/const.go deleted file mode 100755 index 89214a0d45..0000000000 --- a/src/connector/go/src/taosSql/const.go +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package taosSql - -const ( - timeFormat = "2006-01-02 15:04:05" - maxTaosSqlLen = 65380 - defaultBufSize = maxTaosSqlLen + 32 -) - -type fieldType byte - -type fieldFlag uint16 - -const ( - flagNotNULL fieldFlag = 1 << iota -) - -type statusFlag uint16 - -const ( - statusInTrans statusFlag = 1 << iota - statusInAutocommit - statusReserved // Not in documentation - statusMoreResultsExists - statusNoGoodIndexUsed - statusNoIndexUsed - statusCursorExists - statusLastRowSent - statusDbDropped - statusNoBackslashEscapes - statusMetadataChanged - statusQueryWasSlow - statusPsOutParams - statusInTransReadonly - statusSessionStateChanged -) - diff --git a/src/connector/go/src/taosSql/driver.go b/src/connector/go/src/taosSql/driver.go deleted file mode 100755 index 6839979c61..0000000000 --- a/src/connector/go/src/taosSql/driver.go +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package taosSql - -import ( - "context" - "database/sql" - "database/sql/driver" -) - -// taosSqlDriver is exported to make the driver directly accessible. -// In general the driver is used via the database/sql package. -type taosSQLDriver struct{} - -// Open new Connection. -// the DSN string is formatted -func (d taosSQLDriver) Open(dsn string) (driver.Conn, error) { - cfg, err := parseDSN(dsn) - if err != nil { - return nil, err - } - c := &connector{ - cfg: cfg, - } - return c.Connect(context.Background()) -} - -func init() { - sql.Register("taosSql", &taosSQLDriver{}) - taosLogInit() -} diff --git a/src/connector/go/src/taosSql/dsn.go b/src/connector/go/src/taosSql/dsn.go deleted file mode 100755 index c68ae1725c..0000000000 --- a/src/connector/go/src/taosSql/dsn.go +++ /dev/null @@ -1,202 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package taosSql - -import ( - "errors" - "net/url" - "strconv" - "strings" - "time" -) - -var ( - errInvalidDSNUnescaped = errors.New("invalid DSN: did you forget to escape a param value?") - errInvalidDSNAddr = errors.New("invalid DSN: network address not terminated (missing closing brace)") - errInvalidDSNPort = errors.New("invalid DSN: network port is not a valid number") - errInvalidDSNNoSlash = errors.New("invalid DSN: missing the slash separating the database name") -) - -// Config is a configuration parsed from a DSN string. -// If a new Config is created instead of being parsed from a DSN string, -// the NewConfig function should be used, which sets default values. -type config struct { - user string // Username - passwd string // Password (requires User) - net string // Network type - addr string // Network address (requires Net) - port int - dbName string // Database name - params map[string]string // Connection parameters - loc *time.Location // Location for time.Time values - columnsWithAlias bool // Prepend table alias to column names - interpolateParams bool // Interpolate placeholders into query string - parseTime bool // Parse time values to time.Time -} - -// NewConfig creates a new Config and sets default values. -func newConfig() *config { - return &config{ - loc: time.UTC, - interpolateParams: true, - parseTime: true, - } -} - -// ParseDSN parses the DSN string to a Config -func parseDSN(dsn string) (cfg *config, err error) { - taosLog.Println("input dsn:", dsn) - - // New config with some default values - cfg = newConfig() - - // [user[:password]@][net[(addr)]]/dbname[?param1=value1¶mN=valueN] - // Find the last '/' (since the password or the net addr might contain a '/') - foundSlash := false - for i := len(dsn) - 1; i >= 0; i-- { - if dsn[i] == '/' { - foundSlash = true - var j, k int - - // left part is empty if i <= 0 - if i > 0 { - // [username[:password]@][protocol[(address)]] - // Find the last '@' in dsn[:i] - for j = i; j >= 0; j-- { - if dsn[j] == '@' { - // username[:password] - // Find the first ':' in dsn[:j] - for k = 0; k < j; k++ { - if dsn[k] == ':' { - cfg.passwd = dsn[k+1 : j] - break - } - } - cfg.user = dsn[:k] - - break - } - } - - // [protocol[(address)]] - // Find the first '(' in dsn[j+1:i] - for k = j + 1; k < i; k++ { - if dsn[k] == '(' { - // dsn[i-1] must be == ')' if an address is specified - if dsn[i-1] != ')' { - if strings.ContainsRune(dsn[k+1:i], ')') { - return nil, errInvalidDSNUnescaped - } - return nil, errInvalidDSNAddr - } - strs := strings.Split(dsn[k+1:i-1], ":") - if len(strs) == 1 { - return nil, errInvalidDSNAddr - } - cfg.addr = strs[0] - cfg.port, err = strconv.Atoi(strs[1]) - if err != nil { - return nil, errInvalidDSNPort - } - break - } - } - cfg.net = dsn[j+1 : k] - } - - // dbname[?param1=value1&...¶mN=valueN] - // Find the first '?' in dsn[i+1:] - for j = i + 1; j < len(dsn); j++ { - if dsn[j] == '?' { - if err = parseDSNParams(cfg, dsn[j+1:]); err != nil { - return - } - break - } - } - cfg.dbName = dsn[i+1 : j] - - break - } - } - - if !foundSlash && len(dsn) > 0 { - return nil, errInvalidDSNNoSlash - } - - taosLog.Printf("cfg info: %+v", cfg) - - return -} - -// parseDSNParams parses the DSN "query string" -// Values must be url.QueryEscape'ed -func parseDSNParams(cfg *config, params string) (err error) { - for _, v := range strings.Split(params, "&") { - param := strings.SplitN(v, "=", 2) - if len(param) != 2 { - continue - } - - // cfg params - switch value := param[1]; param[0] { - case "columnsWithAlias": - var isBool bool - cfg.columnsWithAlias, isBool = readBool(value) - if !isBool { - return errors.New("invalid bool value: " + value) - } - - // Enable client side placeholder substitution - case "interpolateParams": - var isBool bool - cfg.interpolateParams, isBool = readBool(value) - if !isBool { - return errors.New("invalid bool value: " + value) - } - - // Time Location - case "loc": - if value, err = url.QueryUnescape(value); err != nil { - return - } - cfg.loc, err = time.LoadLocation(value) - if err != nil { - return - } - - // time.Time parsing - case "parseTime": - var isBool bool - cfg.parseTime, isBool = readBool(value) - if !isBool { - return errors.New("invalid bool value: " + value) - } - - default: - // lazy init - if cfg.params == nil { - cfg.params = make(map[string]string) - } - - if cfg.params[param[0]], err = url.QueryUnescape(value); err != nil { - return - } - } - } - - return -} diff --git a/src/connector/go/src/taosSql/result.go b/src/connector/go/src/taosSql/result.go deleted file mode 100755 index 2367560c31..0000000000 --- a/src/connector/go/src/taosSql/result.go +++ /dev/null @@ -1,200 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package taosSql - -/* -#cgo CFLAGS : -I/usr/include -#cgo LDFLAGS: -L/usr/lib -ltaos -#include -#include -#include -#include -*/ -import "C" - -import ( - "database/sql/driver" - "errors" - "fmt" - "io" - "strconv" - "time" - "unsafe" -) - -/****************************************************************************** -* Result * -******************************************************************************/ -// Read Packets as Field Packets until EOF-Packet or an Error appears -func (mc *taosConn) readColumns(count int) ([]taosSqlField, error) { - - columns := make([]taosSqlField, count) - var result unsafe.Pointer - result = C.taos_use_result(mc.taos) - if result == nil { - return nil, errors.New("invalid result") - } - - pFields := (*C.struct_taosField)(C.taos_fetch_fields(result)) - - // TODO: Optimized rewriting !!!! - fields := (*[1 << 30]C.struct_taosField)(unsafe.Pointer(pFields)) - - for i := 0; i < count; i++ { - //columns[i].tableName = ms.taos. - //fmt.Println(reflect.TypeOf(fields[i].name)) - var charray []byte - for j := range fields[i].name { - //fmt.Println("fields[i].name[j]: ", fields[i].name[j]) - if fields[i].name[j] != 0 { - charray = append(charray, byte(fields[i].name[j])) - } else { - break - } - } - columns[i].name = string(charray) - columns[i].length = (uint32)(fields[i].bytes) - columns[i].fieldType = fieldType(fields[i]._type) - columns[i].flags = 0 - // columns[i].decimals = 0 - //columns[i].charSet = 0 - } - return columns, nil -} - -func (rows *taosSqlRows) readRow(dest []driver.Value) error { - mc := rows.mc - - if rows.rs.done || mc == nil { - return io.EOF - } - - var result unsafe.Pointer - result = C.taos_use_result(mc.taos) - if result == nil { - return errors.New(C.GoString(C.taos_errstr(mc.taos))) - } - - //var row *unsafe.Pointer - row := C.taos_fetch_row(result) - if row == nil { - rows.rs.done = true - C.taos_free_result(result) - rows.mc = nil - return io.EOF - } - - // because sizeof(void*) == sizeof(int*) == 8 - // notes: sizeof(int) == 8 in go, but sizeof(int) == 4 in C. - for i := range dest { - currentRow := (unsafe.Pointer)(uintptr(*((*int)(unsafe.Pointer(uintptr(unsafe.Pointer(row)) + uintptr(i)*unsafe.Sizeof(int(0))))))) - - if currentRow == nil { - dest[i] = nil - continue - } - - switch rows.rs.columns[i].fieldType { - case C.TSDB_DATA_TYPE_BOOL: - if (*((*byte)(currentRow))) != 0 { - dest[i] = true - } else { - dest[i] = false - } - break - - case C.TSDB_DATA_TYPE_TINYINT: - dest[i] = (int)(*((*byte)(currentRow))) - break - - case C.TSDB_DATA_TYPE_SMALLINT: - dest[i] = (int16)(*((*int16)(currentRow))) - break - - case C.TSDB_DATA_TYPE_INT: - dest[i] = (int)(*((*int32)(currentRow))) // notes int32 of go <----> int of C - break - - case C.TSDB_DATA_TYPE_BIGINT: - dest[i] = (int64)(*((*int64)(currentRow))) - break - - case C.TSDB_DATA_TYPE_FLOAT: - dest[i] = (*((*float32)(currentRow))) - break - - case C.TSDB_DATA_TYPE_DOUBLE: - dest[i] = (*((*float64)(currentRow))) - break - - case C.TSDB_DATA_TYPE_BINARY, C.TSDB_DATA_TYPE_NCHAR: - charLen := rows.rs.columns[i].length - var index uint32 - binaryVal := make([]byte, charLen) - for index = 0; index < charLen; index++ { - binaryVal[index] = *((*byte)(unsafe.Pointer(uintptr(currentRow) + uintptr(index)))) - } - dest[i] = string(binaryVal[:]) - break - - case C.TSDB_DATA_TYPE_TIMESTAMP: - if mc.cfg.parseTime == true { - timestamp := (int64)(*((*int64)(currentRow))) - dest[i] = timestampConvertToString(timestamp, int(C.taos_result_precision(result))) - } else { - dest[i] = (int64)(*((*int64)(currentRow))) - } - break - - default: - fmt.Println("default fieldType: set dest[] to nil") - dest[i] = nil - break - } - } - - return nil -} - -// Read result as Field format until all rows or an Error appears -// call this func in conn mode -func (rows *textRows) readRow(dest []driver.Value) error { - return rows.taosSqlRows.readRow(dest) -} - -// call thsi func in stmt mode -func (rows *binaryRows) readRow(dest []driver.Value) error { - return rows.taosSqlRows.readRow(dest) -} - -func timestampConvertToString(timestamp int64, precision int) string { - var decimal, sVal, nsVal int64 - if precision == 0 { - decimal = timestamp % 1000 - sVal = timestamp / 1000 - nsVal = decimal * 1000 - } else { - decimal = timestamp % 1000000 - sVal = timestamp / 1000000 - nsVal = decimal * 1000000 - } - - date_time := time.Unix(sVal, nsVal) - - //const base_format = "2006-01-02 15:04:05" - str_time := date_time.Format(timeFormat) - - return (str_time + "." + strconv.Itoa(int(decimal))) -} diff --git a/src/connector/go/src/taosSql/rows.go b/src/connector/go/src/taosSql/rows.go deleted file mode 100755 index 5040dca06d..0000000000 --- a/src/connector/go/src/taosSql/rows.go +++ /dev/null @@ -1,296 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package taosSql -/* -#cgo CFLAGS : -I/usr/include -#cgo LDFLAGS: -L/usr/lib -ltaos -#include -#include -#include -#include -*/ -import "C" - -import ( - "database/sql" - "database/sql/driver" - "io" - "math" - "reflect" -) - -type taosSqlField struct { - tableName string - name string - length uint32 - flags fieldFlag // indicate whether this field can is null - fieldType fieldType - decimals byte - charSet uint8 -} - -type resultSet struct { - columns []taosSqlField - columnNames []string - done bool -} - -type taosSqlRows struct { - mc *taosConn - rs resultSet -} - -type binaryRows struct { - taosSqlRows -} - -type textRows struct { - taosSqlRows -} - -func (rows *taosSqlRows) Columns() []string { - if rows.rs.columnNames != nil { - return rows.rs.columnNames - } - - columns := make([]string, len(rows.rs.columns)) - if rows.mc != nil && rows.mc.cfg.columnsWithAlias { - for i := range columns { - if tableName := rows.rs.columns[i].tableName; len(tableName) > 0 { - columns[i] = tableName + "." + rows.rs.columns[i].name - } else { - columns[i] = rows.rs.columns[i].name - } - } - } else { - for i := range columns { - columns[i] = rows.rs.columns[i].name - } - } - - rows.rs.columnNames = columns - - return columns -} - -func (rows *taosSqlRows) ColumnTypeDatabaseTypeName(i int) string { - return rows.rs.columns[i].typeDatabaseName() -} - -func (rows *taosSqlRows) ColumnTypeLength(i int) (length int64, ok bool) { - return int64(rows.rs.columns[i].length), true -} - -func (rows *taosSqlRows) ColumnTypeNullable(i int) (nullable, ok bool) { - return rows.rs.columns[i].flags&flagNotNULL == 0, true -} - -func (rows *taosSqlRows) ColumnTypePrecisionScale(i int) (int64, int64, bool) { - column := rows.rs.columns[i] - decimals := int64(column.decimals) - - switch column.fieldType { - case C.TSDB_DATA_TYPE_FLOAT: - fallthrough - case C.TSDB_DATA_TYPE_DOUBLE: - if decimals == 0x1f { - return math.MaxInt64, math.MaxInt64, true - } - return math.MaxInt64, decimals, true - } - - return 0, 0, false -} - -func (rows *taosSqlRows) ColumnTypeScanType(i int) reflect.Type { - return rows.rs.columns[i].scanType() -} - -func (rows *taosSqlRows) Close() error { - if rows.mc != nil { - result := C.taos_use_result(rows.mc.taos) - if result != nil { - C.taos_free_result(result) - } - rows.mc = nil - } - return nil -} - -func (rows *taosSqlRows) HasNextResultSet() (b bool) { - if rows.mc == nil { - return false - } - return rows.mc.status&statusMoreResultsExists != 0 -} - -func (rows *taosSqlRows) nextResultSet() (int, error) { - if rows.mc == nil { - return 0, io.EOF - } - - // Remove unread packets from stream - if !rows.rs.done { - rows.rs.done = true - } - - if !rows.HasNextResultSet() { - rows.mc = nil - return 0, io.EOF - } - rows.rs = resultSet{} - return 0,nil -} - -func (rows *taosSqlRows) nextNotEmptyResultSet() (int, error) { - for { - resLen, err := rows.nextResultSet() - if err != nil { - return 0, err - } - - if resLen > 0 { - return resLen, nil - } - - rows.rs.done = true - } -} - -func (rows *binaryRows) NextResultSet() error { - resLen, err := rows.nextNotEmptyResultSet() - if err != nil { - return err - } - - rows.rs.columns, err = rows.mc.readColumns(resLen) - return err -} - -// stmt.Query return binary rows, and get row from this func -func (rows *binaryRows) Next(dest []driver.Value) error { - if mc := rows.mc; mc != nil { - // Fetch next row from stream - return rows.readRow(dest) - } - return io.EOF -} - -func (rows *textRows) NextResultSet() (err error) { - resLen, err := rows.nextNotEmptyResultSet() - if err != nil { - return err - } - - rows.rs.columns, err = rows.mc.readColumns(resLen) - return err -} - -// db.Query return text rows, and get row from this func -func (rows *textRows) Next(dest []driver.Value) error { - if mc := rows.mc; mc != nil { - // Fetch next row from stream - return rows.readRow(dest) - } - return io.EOF -} - -func (mf *taosSqlField) typeDatabaseName() string { - //fmt.Println("######## (mf *taosSqlField) typeDatabaseName() mf.fieldType:", mf.fieldType) - switch mf.fieldType { - case C.TSDB_DATA_TYPE_BOOL: - return "BOOL" - - case C.TSDB_DATA_TYPE_TINYINT: - return "TINYINT" - - case C.TSDB_DATA_TYPE_SMALLINT: - return "SMALLINT" - - case C.TSDB_DATA_TYPE_INT: - return "INT" - - case C.TSDB_DATA_TYPE_BIGINT: - return "BIGINT" - - case C.TSDB_DATA_TYPE_FLOAT: - return "FLOAT" - - case C.TSDB_DATA_TYPE_DOUBLE: - return "DOUBLE" - - case C.TSDB_DATA_TYPE_BINARY: - return "BINARY" - - case C.TSDB_DATA_TYPE_NCHAR: - return "NCHAR" - - case C.TSDB_DATA_TYPE_TIMESTAMP: - return "TIMESTAMP" - - default: - return "" - } -} - -var ( - scanTypeFloat32 = reflect.TypeOf(float32(0)) - scanTypeFloat64 = reflect.TypeOf(float64(0)) - scanTypeInt8 = reflect.TypeOf(int8(0)) - scanTypeInt16 = reflect.TypeOf(int16(0)) - scanTypeInt32 = reflect.TypeOf(int32(0)) - scanTypeInt64 = reflect.TypeOf(int64(0)) - scanTypeNullTime = reflect.TypeOf(NullTime{}) - scanTypeRawBytes = reflect.TypeOf(sql.RawBytes{}) - scanTypeUnknown = reflect.TypeOf(new(interface{})) -) - -func (mf *taosSqlField) scanType() reflect.Type { - //fmt.Println("######## (mf *taosSqlField) scanType() mf.fieldType:", mf.fieldType) - switch mf.fieldType { - case C.TSDB_DATA_TYPE_BOOL: - return scanTypeInt8 - - case C.TSDB_DATA_TYPE_TINYINT: - return scanTypeInt8 - - case C.TSDB_DATA_TYPE_SMALLINT: - return scanTypeInt16 - - case C.TSDB_DATA_TYPE_INT: - return scanTypeInt32 - - case C.TSDB_DATA_TYPE_BIGINT: - return scanTypeInt64 - - case C.TSDB_DATA_TYPE_FLOAT: - return scanTypeFloat32 - - case C.TSDB_DATA_TYPE_DOUBLE: - return scanTypeFloat64 - - case C.TSDB_DATA_TYPE_BINARY: - return scanTypeRawBytes - - case C.TSDB_DATA_TYPE_NCHAR: - return scanTypeRawBytes - - case C.TSDB_DATA_TYPE_TIMESTAMP: - return scanTypeNullTime - - default: - return scanTypeUnknown - } -} diff --git a/src/connector/go/src/taosSql/statement.go b/src/connector/go/src/taosSql/statement.go deleted file mode 100755 index 3aa3e01564..0000000000 --- a/src/connector/go/src/taosSql/statement.go +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package taosSql - -import ( - "database/sql/driver" - "fmt" - "reflect" -) - -type taosSqlStmt struct { - mc *taosConn - id uint32 - pSql string - paramCount int -} - -func (stmt *taosSqlStmt) Close() error { - return nil -} - -func (stmt *taosSqlStmt) NumInput() int { - return stmt.paramCount -} - -func (stmt *taosSqlStmt) Exec(args []driver.Value) (driver.Result, error) { - if stmt.mc == nil || stmt.mc.taos == nil { - return nil, errInvalidConn - } - return stmt.mc.Exec(stmt.pSql, args) -} - -func (stmt *taosSqlStmt) Query(args []driver.Value) (driver.Rows, error) { - if stmt.mc == nil || stmt.mc.taos == nil { - return nil, errInvalidConn - } - return stmt.query(args) -} - -func (stmt *taosSqlStmt) query(args []driver.Value) (*binaryRows, error) { - mc := stmt.mc - if mc == nil || mc.taos == nil { - return nil, errInvalidConn - } - - querySql := stmt.pSql - - if len(args) != 0 { - if !mc.cfg.interpolateParams { - return nil, driver.ErrSkip - } - // try client-side prepare to reduce roundtrip - prepared, err := mc.interpolateParams(stmt.pSql, args) - if err != nil { - return nil, err - } - querySql = prepared - } - - num_fields, err := mc.taosQuery(querySql) - if err == nil { - // Read Result - rows := new(binaryRows) - rows.mc = mc - // Columns field - rows.rs.columns, err = mc.readColumns(num_fields) - return rows, err - } - return nil, err -} - -type converter struct{} - -// ConvertValue mirrors the reference/default converter in database/sql/driver -// with _one_ exception. We support uint64 with their high bit and the default -// implementation does not. This function should be kept in sync with -// database/sql/driver defaultConverter.ConvertValue() except for that -// deliberate difference. -func (c converter) ConvertValue(v interface{}) (driver.Value, error) { - - if driver.IsValue(v) { - return v, nil - } - - if vr, ok := v.(driver.Valuer); ok { - sv, err := callValuerValue(vr) - if err != nil { - return nil, err - } - if !driver.IsValue(sv) { - return nil, fmt.Errorf("non-Value type %T returned from Value", sv) - } - - return sv, nil - } - - rv := reflect.ValueOf(v) - switch rv.Kind() { - case reflect.Ptr: - // indirect pointers - if rv.IsNil() { - return nil, nil - } else { - return c.ConvertValue(rv.Elem().Interface()) - } - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - return rv.Int(), nil - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: - return rv.Uint(), nil - case reflect.Float32, reflect.Float64: - return rv.Float(), nil - case reflect.Bool: - return rv.Bool(), nil - case reflect.Slice: - ek := rv.Type().Elem().Kind() - if ek == reflect.Uint8 { - return rv.Bytes(), nil - } - return nil, fmt.Errorf("unsupported type %T, a slice of %s", v, ek) - case reflect.String: - return rv.String(), nil - } - return nil, fmt.Errorf("unsupported type %T, a %s", v, rv.Kind()) -} - -var valuerReflectType = reflect.TypeOf((*driver.Valuer)(nil)).Elem() - -// callValuerValue returns vr.Value(), with one exception: -// If vr.Value is an auto-generated method on a pointer type and the -// pointer is nil, it would panic at runtime in the panicwrap -// method. Treat it like nil instead. -// -// This is so people can implement driver.Value on value types and -// still use nil pointers to those types to mean nil/NULL, just like -// string/*string. -// -// This is an exact copy of the same-named unexported function from the -// database/sql package. -func callValuerValue(vr driver.Valuer) (v driver.Value, err error) { - if rv := reflect.ValueOf(vr); rv.Kind() == reflect.Ptr && - rv.IsNil() && - rv.Type().Elem().Implements(valuerReflectType) { - return nil, nil - } - return vr.Value() -} diff --git a/src/connector/go/src/taosSql/taosLog.go b/src/connector/go/src/taosSql/taosLog.go deleted file mode 100755 index 63af5e34c1..0000000000 --- a/src/connector/go/src/taosSql/taosLog.go +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package taosSql - -import ( - "bufio" - "errors" - "fmt" - "io" - "log" - "os" - "strings" -) - -// Various errors the driver might return. -var ( - errInvalidConn = errors.New("invalid connection") - errConnNoExist = errors.New("no existent connection ") -) - -var taosLog *log.Logger - -// SetLogger is used to set the logger for critical errors. -// The initial logger -func taosLogInit() { - cfgName := "/etc/taos/taos.cfg" - logNameDefault := "/var/log/taos/taosgo.log" - var logName string - - // get log path from cfg file - cfgFile, err := os.OpenFile(cfgName, os.O_RDONLY, 0644) - defer cfgFile.Close() - if err != nil { - fmt.Println(err) - logName = logNameDefault - } else { - logName, err = getLogNameFromCfg(cfgFile) - if err != nil { - fmt.Println(err) - logName = logNameDefault - } - } - - logFile, err := os.OpenFile(logName, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - fmt.Println(err) - os.Exit(1) - } - taosLog = log.New(logFile, "", log.LstdFlags) - taosLog.SetPrefix("TAOS DRIVER ") - taosLog.SetFlags(log.LstdFlags|log.Lshortfile) -} - -func getLogNameFromCfg(f *os.File) (string, error) { - // Create file buf, *Reader - r := bufio.NewReader(f) - for { - //read one line, return to slice b - b, _, err := r.ReadLine() - if err != nil { - if err == io.EOF { - break - } - panic(err) - } - - // Remove space of left and right - s := strings.TrimSpace(string(b)) - if strings.Index(s, "#") == 0 { - // comment line - continue - } - - if len(s) == 0 { - continue - } - - var ns string - // If there is a comment on the right of the line, must be remove - index := strings.Index(s, "#") - if index > 0 { - // Gets the string to the left of the comment to determine whether it is empty - ns = s[:index] - if len(ns) == 0 { - continue - } - } else { - ns = s; - } - - ss := strings.Fields(ns) - if strings.Compare("logDir", ss[0]) != 0 { - continue - } - - if len(ss) < 2 { - break - } - - // Add a filename after the path - logName := ss[1] + "/taosgo.log" - return logName,nil - } - - return "", errors.New("no config log path, use default") -} - diff --git a/src/connector/go/src/taosSql/taosSqlCgo.go b/src/connector/go/src/taosSql/taosSqlCgo.go deleted file mode 100755 index cc3aaa1658..0000000000 --- a/src/connector/go/src/taosSql/taosSqlCgo.go +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package taosSql - -/* -#cgo CFLAGS : -I/usr/include -#cgo LDFLAGS: -L/usr/lib -ltaos -#include -#include -#include -#include -*/ -import "C" - -import ( - "errors" - "unsafe" -) - -func (mc *taosConn) taosConnect(ip, user, pass, db string, port int) (taos unsafe.Pointer, err error) { - cuser := C.CString(user) - cpass := C.CString(pass) - cip := C.CString(ip) - cdb := C.CString(db) - defer C.free(unsafe.Pointer(cip)) - defer C.free(unsafe.Pointer(cuser)) - defer C.free(unsafe.Pointer(cpass)) - defer C.free(unsafe.Pointer(cdb)) - - taosObj := C.taos_connect(cip, cuser, cpass, cdb, (C.ushort)(port)) - if taosObj == nil { - return nil, errors.New("taos_connect() fail!") - } - - return (unsafe.Pointer)(taosObj), nil -} - -func (mc *taosConn) taosQuery(sqlstr string) (int, error) { - //taosLog.Printf("taosQuery() input sql:%s\n", sqlstr) - - csqlstr := C.CString(sqlstr) - defer C.free(unsafe.Pointer(csqlstr)) - code := int(C.taos_query(mc.taos, csqlstr)) - - if 0 != code { - mc.taos_error() - errStr := C.GoString(C.taos_errstr(mc.taos)) - taosLog.Println("taos_query() failed:", errStr) - taosLog.Printf("taosQuery() input sql:%s\n", sqlstr) - return 0, errors.New(errStr) - } - - // read result and save into mc struct - num_fields := int(C.taos_field_count(mc.taos)) - if 0 == num_fields { // there are no select and show kinds of commands - mc.affectedRows = int(C.taos_affected_rows(mc.taos)) - mc.insertId = 0 - } - - return num_fields, nil -} - -func (mc *taosConn) taos_close() { - C.taos_close(mc.taos) -} - -func (mc *taosConn) taos_error() { - // free local resouce: allocated memory/metric-meta refcnt - //var pRes unsafe.Pointer - pRes := C.taos_use_result(mc.taos) - C.taos_free_result(pRes) -} diff --git a/src/connector/go/src/taosSql/utils.go b/src/connector/go/src/taosSql/utils.go deleted file mode 100755 index a104322fcc..0000000000 --- a/src/connector/go/src/taosSql/utils.go +++ /dev/null @@ -1,422 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package taosSql - -/* -#cgo CFLAGS : -I/usr/include -#include -#cgo LDFLAGS: -L/usr/lib -ltaos -void taosSetAllocMode(int mode, const char* path, _Bool autoDump); -void taosDumpMemoryLeak(); -*/ -import "C" - - -import ( - "database/sql/driver" - "errors" - "fmt" - "sync/atomic" - "time" - "unsafe" -) - -// Returns the bool value of the input. -// The 2nd return value indicates if the input was a valid bool value -func readBool(input string) (value bool, valid bool) { - switch input { - case "1", "true", "TRUE", "True": - return true, true - case "0", "false", "FALSE", "False": - return false, true - } - - // Not a valid bool value - return -} - -/****************************************************************************** -* Time related utils * -******************************************************************************/ - -// NullTime represents a time.Time that may be NULL. -// NullTime implements the Scanner interface so -// it can be used as a scan destination: -// -// var nt NullTime -// err := db.QueryRow("SELECT time FROM foo WHERE id=?", id).Scan(&nt) -// ... -// if nt.Valid { -// // use nt.Time -// } else { -// // NULL value -// } -// -// This NullTime implementation is not driver-specific -type NullTime struct { - Time time.Time - Valid bool // Valid is true if Time is not NULL -} - -// Scan implements the Scanner interface. -// The value type must be time.Time or string / []byte (formatted time-string), -// otherwise Scan fails. -func (nt *NullTime) Scan(value interface{}) (err error) { - if value == nil { - nt.Time, nt.Valid = time.Time{}, false - return - } - - switch v := value.(type) { - case time.Time: - nt.Time, nt.Valid = v, true - return - case []byte: - nt.Time, err = parseDateTime(string(v), time.UTC) - nt.Valid = (err == nil) - return - case string: - nt.Time, err = parseDateTime(v, time.UTC) - nt.Valid = (err == nil) - return - } - - nt.Valid = false - return fmt.Errorf("Can't convert %T to time.Time", value) -} - -// Value implements the driver Valuer interface. -func (nt NullTime) Value() (driver.Value, error) { - if !nt.Valid { - return nil, nil - } - return nt.Time, nil -} - -func parseDateTime(str string, loc *time.Location) (t time.Time, err error) { - base := "0000-00-00 00:00:00.0000000" - switch len(str) { - case 10, 19, 21, 22, 23, 24, 25, 26: // up to "YYYY-MM-DD HH:MM:SS.MMMMMM" - if str == base[:len(str)] { - return - } - t, err = time.Parse(timeFormat[:len(str)], str) - default: - err = fmt.Errorf("invalid time string: %s", str) - return - } - - // Adjust location - if err == nil && loc != time.UTC { - y, mo, d := t.Date() - h, mi, s := t.Clock() - t, err = time.Date(y, mo, d, h, mi, s, t.Nanosecond(), loc), nil - } - - return -} - -// zeroDateTime is used in formatBinaryDateTime to avoid an allocation -// if the DATE or DATETIME has the zero value. -// It must never be changed. -// The current behavior depends on database/sql copying the result. -var zeroDateTime = []byte("0000-00-00 00:00:00.000000") - -const digits01 = "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" -const digits10 = "0000000000111111111122222222223333333333444444444455555555556666666666777777777788888888889999999999" - -/****************************************************************************** -* Convert from and to bytes * -******************************************************************************/ - -func uint64ToBytes(n uint64) []byte { - return []byte{ - byte(n), - byte(n >> 8), - byte(n >> 16), - byte(n >> 24), - byte(n >> 32), - byte(n >> 40), - byte(n >> 48), - byte(n >> 56), - } -} - -func uint64ToString(n uint64) []byte { - var a [20]byte - i := 20 - - // U+0030 = 0 - // ... - // U+0039 = 9 - - var q uint64 - for n >= 10 { - i-- - q = n / 10 - a[i] = uint8(n-q*10) + 0x30 - n = q - } - - i-- - a[i] = uint8(n) + 0x30 - - return a[i:] -} - -// treats string value as unsigned integer representation -func stringToInt(b []byte) int { - val := 0 - for i := range b { - val *= 10 - val += int(b[i] - 0x30) - } - return val -} - -// reserveBuffer checks cap(buf) and expand buffer to len(buf) + appendSize. -// If cap(buf) is not enough, reallocate new buffer. -func reserveBuffer(buf []byte, appendSize int) []byte { - newSize := len(buf) + appendSize - if cap(buf) < newSize { - // Grow buffer exponentially - newBuf := make([]byte, len(buf)*2+appendSize) - copy(newBuf, buf) - buf = newBuf - } - return buf[:newSize] -} - -// escapeBytesBackslash escapes []byte with backslashes (\) -// This escapes the contents of a string (provided as []byte) by adding backslashes before special -// characters, and turning others into specific escape sequences, such as -// turning newlines into \n and null bytes into \0. -func escapeBytesBackslash(buf, v []byte) []byte { - pos := len(buf) - buf = reserveBuffer(buf, len(v)*2) - for _, c := range v { - switch c { - case '\x00': - buf[pos] = '\\' - buf[pos+1] = '0' - pos += 2 - case '\n': - buf[pos] = '\\' - buf[pos+1] = 'n' - pos += 2 - case '\r': - buf[pos] = '\\' - buf[pos+1] = 'r' - pos += 2 - case '\x1a': - buf[pos] = '\\' - buf[pos+1] = 'Z' - pos += 2 - case '\'': - buf[pos] = '\\' - buf[pos+1] = '\'' - pos += 2 - case '"': - buf[pos] = '\\' - buf[pos+1] = '"' - pos += 2 - case '\\': - buf[pos] = '\\' - buf[pos+1] = '\\' - pos += 2 - default: - buf[pos] = c - pos++ - } - } - return buf[:pos] -} - -// escapeStringBackslash is similar to escapeBytesBackslash but for string. -func escapeStringBackslash(buf []byte, v string) []byte { - pos := len(buf) - buf = reserveBuffer(buf, len(v)*2) - - for i := 0; i < len(v); i++ { - c := v[i] - switch c { - case '\x00': - buf[pos] = '\\' - buf[pos+1] = '0' - pos += 2 - case '\n': - buf[pos] = '\\' - buf[pos+1] = 'n' - pos += 2 - case '\r': - buf[pos] = '\\' - buf[pos+1] = 'r' - pos += 2 - case '\x1a': - buf[pos] = '\\' - buf[pos+1] = 'Z' - pos += 2 - //case '\'': - // buf[pos] = '\\' - // buf[pos+1] = '\'' - // pos += 2 - case '"': - buf[pos] = '\\' - buf[pos+1] = '"' - pos += 2 - case '\\': - buf[pos] = '\\' - buf[pos+1] = '\\' - pos += 2 - default: - buf[pos] = c - pos++ - } - } - - return buf[:pos] -} - -// escapeBytesQuotes escapes apostrophes in []byte by doubling them up. -// This escapes the contents of a string by doubling up any apostrophes that -// it contains. This is used when the NO_BACKSLASH_ESCAPES SQL_MODE is in -// effect on the server. -func escapeBytesQuotes(buf, v []byte) []byte { - pos := len(buf) - buf = reserveBuffer(buf, len(v)*2) - - for _, c := range v { - if c == '\'' { - buf[pos] = '\'' - buf[pos+1] = '\'' - pos += 2 - } else { - buf[pos] = c - pos++ - } - } - - return buf[:pos] -} - -// escapeStringQuotes is similar to escapeBytesQuotes but for string. -func escapeStringQuotes(buf []byte, v string) []byte { - pos := len(buf) - buf = reserveBuffer(buf, len(v)*2) - - for i := 0; i < len(v); i++ { - c := v[i] - if c == '\'' { - buf[pos] = '\'' - buf[pos+1] = '\'' - pos += 2 - } else { - buf[pos] = c - pos++ - } - } - - return buf[:pos] -} - -/****************************************************************************** -* Sync utils * -******************************************************************************/ - -// noCopy may be embedded into structs which must not be copied -// after the first use. -// -// See https://github.com/golang/go/issues/8005#issuecomment-190753527 -// for details. -type noCopy struct{} - -// Lock is a no-op used by -copylocks checker from `go vet`. -func (*noCopy) Lock() {} - -// atomicBool is a wrapper around uint32 for usage as a boolean value with -// atomic access. -type atomicBool struct { - _noCopy noCopy - value uint32 -} - -// IsSet returns whether the current boolean value is true -func (ab *atomicBool) IsSet() bool { - return atomic.LoadUint32(&ab.value) > 0 -} - -// Set sets the value of the bool regardless of the previous value -func (ab *atomicBool) Set(value bool) { - if value { - atomic.StoreUint32(&ab.value, 1) - } else { - atomic.StoreUint32(&ab.value, 0) - } -} - -// TrySet sets the value of the bool and returns whether the value changed -func (ab *atomicBool) TrySet(value bool) bool { - if value { - return atomic.SwapUint32(&ab.value, 1) == 0 - } - return atomic.SwapUint32(&ab.value, 0) > 0 -} - -// atomicError is a wrapper for atomically accessed error values -type atomicError struct { - _noCopy noCopy - value atomic.Value -} - -// Set sets the error value regardless of the previous value. -// The value must not be nil -func (ae *atomicError) Set(value error) { - ae.value.Store(value) -} - -// Value returns the current error value -func (ae *atomicError) Value() error { - if v := ae.value.Load(); v != nil { - // this will panic if the value doesn't implement the error interface - return v.(error) - } - return nil -} - -func namedValueToValue(named []driver.NamedValue) ([]driver.Value, error) { - dargs := make([]driver.Value, len(named)) - for n, param := range named { - if len(param.Name) > 0 { - // TODO: support the use of Named Parameters #561 - return nil, errors.New("taosSql: driver does not support the use of Named Parameters") - } - dargs[n] = param.Value - } - return dargs, nil -} - - -/****************************************************************************** -* Utils for C memory issues debugging * -******************************************************************************/ -func SetAllocMode(mode int32, path string) { - cpath := C.CString(path) - defer C.free(unsafe.Pointer(cpath)) - C.taosSetAllocMode(C.int(mode), cpath, false) -} - -func DumpMemoryLeak() { - C.taosDumpMemoryLeak() -} From 9d13fd243bcd9c7fa0ece51b2b7bcb4939697f19 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 11 Apr 2020 04:15:36 +0800 Subject: [PATCH 02/18] Upgrade Python to 3 instead of 2 for testcase development. [TD-139] --- tests/pytest/simpletest.sh | 2 +- tests/pytest/test.py | 1 - tests/pytest/util/log.py | 14 +++++++------- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/tests/pytest/simpletest.sh b/tests/pytest/simpletest.sh index aab36884f3..0a905beaa1 100755 --- a/tests/pytest/simpletest.sh +++ b/tests/pytest/simpletest.sh @@ -1 +1 @@ -sudo python2 ./test.py -f insert/basic.py +sudo python3 ./test.py -f insert/basic.py diff --git a/tests/pytest/test.py b/tests/pytest/test.py index b88e444665..0a524b44fa 100644 --- a/tests/pytest/test.py +++ b/tests/pytest/test.py @@ -73,7 +73,6 @@ if __name__ == "__main__": if fileName == "all": tdCases.runAllLinux(conn) else: - tdLog.info("CBD LN78: %s" % (fileName)) tdCases.runOneLinux(conn, fileName) conn.close() else: diff --git a/tests/pytest/util/log.py b/tests/pytest/util/log.py index 926e582448..e570af85f5 100644 --- a/tests/pytest/util/log.py +++ b/tests/pytest/util/log.py @@ -22,27 +22,27 @@ class TDLog: self.path = "" def info(self, info): - print "%s %s" % (datetime.datetime.now(), info) + print("%s %s" % (datetime.datetime.now(), info)) def sleep(self, sec): - print "%s sleep %d seconds" % (datetime.datetime.now(), sec) + print("%s sleep %d seconds" % (datetime.datetime.now(), sec)) time.sleep(sec) def debug(self, err): - print "\033[1;36m%s %s\033[0m" % (datetime.datetime.now(), err) + print("\033[1;36m%s %s\033[0m" % (datetime.datetime.now(), err)) def success(self, info): - print "\033[1;32m%s %s\033[0m" % (datetime.datetime.now(), info) + print("\033[1;32m%s %s\033[0m" % (datetime.datetime.now(), info)) def notice(self, err): - print "\033[1;33m%s %s\033[0m" % (datetime.datetime.now(), err) + print("\033[1;33m%s %s\033[0m" % (datetime.datetime.now(), err)) def exit(self, err): - print "\033[1;31m%s %s\033[0m" % (datetime.datetime.now(), err) + print("\033[1;31m%s %s\033[0m" % (datetime.datetime.now(), err)) sys.exit(1) def printNoPrefix(self, info): - print "\033[1;36m%s\033[0m" % (info) + print("\033[1;36m%s\033[0m" % (info)) tdLog = TDLog() From bd63cf2b4cc63c6ef386936722b4df7b5668ebe9 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sun, 12 Apr 2020 13:44:49 +0800 Subject: [PATCH 03/18] add general/user/basic1.sim --- tests/script/basicSuite.sim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/script/basicSuite.sim b/tests/script/basicSuite.sim index 440ec5592b..c282480bbe 100644 --- a/tests/script/basicSuite.sim +++ b/tests/script/basicSuite.sim @@ -10,4 +10,6 @@ run general/db/basic3.sim run general/db/basic4.sim run general/db/basic5.sim +run general/user/basic1.sim + ################################## From 50d7e6876d3a7e281be267dc6326f6bd0e84b32e Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sun, 12 Apr 2020 13:49:34 +0800 Subject: [PATCH 04/18] remove possible memory leaks --- src/rpc/src/rpcServer.c | 86 ++++++++++++++++++++++------------------- src/rpc/src/rpcUdp.c | 10 ++--- 2 files changed, 50 insertions(+), 46 deletions(-) diff --git a/src/rpc/src/rpcServer.c b/src/rpc/src/rpcServer.c index 538b3059e3..6fe5385bba 100644 --- a/src/rpc/src/rpcServer.c +++ b/src/rpc/src/rpcServer.c @@ -65,79 +65,85 @@ static void taosProcessTcpData(void *param); static void taosAcceptTcpConnection(void *arg); void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle) { - int i; SServerObj *pServerObj; - pthread_attr_t thattr; SThreadObj *pThreadObj; - pServerObj = (SServerObj *)malloc(sizeof(SServerObj)); + pServerObj = (SServerObj *)calloc(sizeof(SServerObj), 1); strcpy(pServerObj->ip, ip); pServerObj->port = port; strcpy(pServerObj->label, label); pServerObj->numOfThreads = numOfThreads; - pServerObj->pThreadObj = (SThreadObj *)malloc(sizeof(SThreadObj) * (size_t)numOfThreads); + pServerObj->pThreadObj = (SThreadObj *)calloc(sizeof(SThreadObj), numOfThreads); if (pServerObj->pThreadObj == NULL) { tError("TCP:%s no enough memory", label); + free(pServerObj); return NULL; } - memset(pServerObj->pThreadObj, 0, sizeof(SThreadObj) * (size_t)numOfThreads); - - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + int code = 0; pThreadObj = pServerObj->pThreadObj; - for (i = 0; i < numOfThreads; ++i) { + for (int i = 0; i < numOfThreads; ++i) { pThreadObj->processData = fp; strcpy(pThreadObj->label, label); pThreadObj->shandle = shandle; - if (pthread_mutex_init(&(pThreadObj->threadMutex), NULL) < 0) { - tError("%s failed to init TCP process data mutex, reason:%s", label, strerror(errno)); - return NULL; + code = pthread_mutex_init(&(pThreadObj->threadMutex), NULL); + if (code < 0) { + tError("%s failed to init TCP process data mutex(%s)", label, strerror(errno)); + break;; } - if (pthread_cond_init(&(pThreadObj->fdReady), NULL) != 0) { - tError("%s init TCP condition variable failed, reason:%s\n", label, strerror(errno)); - return NULL; + code = pthread_cond_init(&(pThreadObj->fdReady), NULL); + if (code != 0) { + tError("%s init TCP condition variable failed(%s)", label, strerror(errno)); + break; } pThreadObj->pollFd = epoll_create(10); // size does not matter if (pThreadObj->pollFd < 0) { tError("%s failed to create TCP epoll", label); - return NULL; + code = -1; + break; } - if (pthread_create(&(pThreadObj->thread), &thattr, (void *)taosProcessTcpData, (void *)(pThreadObj)) != 0) { - tError("%s failed to create TCP process data thread, reason:%s", label, strerror(errno)); - return NULL; + pthread_attr_t thattr; + pthread_attr_init(&thattr); + pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + code = pthread_create(&(pThreadObj->thread), &thattr, (void *)taosProcessTcpData, (void *)(pThreadObj)); + pthread_attr_destroy(&thattr); + if (code != 0) { + tError("%s failed to create TCP process data thread(%s)", label, strerror(errno)); + break; } pThreadObj->threadId = i; pThreadObj++; } - if (pthread_create(&(pServerObj->thread), &thattr, (void *)taosAcceptTcpConnection, (void *)(pServerObj)) != 0) { - tError("%s failed to create TCP accept thread, reason:%s", label, strerror(errno)); - return NULL; + if (code == 0) { + pthread_attr_t thattr; + pthread_attr_init(&thattr); + pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + code = pthread_create(&(pServerObj->thread), &thattr, (void *)taosAcceptTcpConnection, (void *)(pServerObj)); + pthread_attr_destroy(&thattr); + if (code != 0) { + tError("%s failed to create TCP accept thread(%s)", label, strerror(errno)); + } } - /* - if ( pthread_create(&(pServerObj->thread), &thattr, - (void*)taosAcceptUDConnection, (void *)(pServerObj)) != 0 ) { - tError("%s failed to create UD accept thread, reason:%s", label, - strerror(errno)); - return NULL; - } - */ - pthread_attr_destroy(&thattr); - tTrace("%s TCP server is initialized, ip:%s port:%hu numOfThreads:%d", label, ip, port, numOfThreads); + if (code != 0) { + free(pServerObj->pThreadObj); + free(pServerObj); + pServerObj = NULL; + } else { + tTrace("%s TCP server is initialized, ip:%s port:%hu numOfThreads:%d", label, ip, port, numOfThreads); + } return (void *)pServerObj; } void taosCleanUpTcpServer(void *handle) { - int i; SThreadObj *pThreadObj; SServerObj *pServerObj = (SServerObj *)handle; @@ -146,7 +152,7 @@ void taosCleanUpTcpServer(void *handle) { pthread_cancel(pServerObj->thread); pthread_join(pServerObj->thread, NULL); - for (i = 0; i < pServerObj->numOfThreads; ++i) { + for (int i = 0; i < pServerObj->numOfThreads; ++i) { pThreadObj = pServerObj->pThreadObj + i; while (pThreadObj->pHead) { @@ -161,9 +167,9 @@ void taosCleanUpTcpServer(void *handle) { pthread_mutex_destroy(&(pThreadObj->threadMutex)); } - tfree(pServerObj->pThreadObj); tTrace("TCP:%s, TCP server is cleaned up", pServerObj->label); + tfree(pServerObj->pThreadObj); tfree(pServerObj); } @@ -278,10 +284,10 @@ static void taosAcceptTcpConnection(void *arg) { sockFd = taosOpenTcpServerSocket(pServerObj->ip, pServerObj->port); if (sockFd < 0) { - tError("%s failed to open TCP socket, ip:%s, port:%hu", pServerObj->label, pServerObj->ip, pServerObj->port); + tError("%s failed to open TCP socket, ip:%s:%hu", pServerObj->label, pServerObj->ip, pServerObj->port); return; } else { - tTrace("%s TCP server is ready, ip:%s, port:%hu", pServerObj->label, pServerObj->ip, pServerObj->port); + tTrace("%s TCP server is ready, ip:%s:%hu", pServerObj->label, pServerObj->ip, pServerObj->port); } while (1) { @@ -289,11 +295,11 @@ static void taosAcceptTcpConnection(void *arg) { connFd = accept(sockFd, (struct sockaddr *)&clientAddr, &addrlen); if (connFd < 0) { - tError("%s TCP accept failure, errno:%d, reason:%s", pServerObj->label, errno, strerror(errno)); + tError("%s TCP accept failure(%s)", pServerObj->label, errno, strerror(errno)); continue; } - tTrace("%s TCP connection from ip:%s port:%hu", pServerObj->label, inet_ntoa(clientAddr.sin_addr), + tTrace("%s TCP connection from ip:%s:%hu", pServerObj->label, inet_ntoa(clientAddr.sin_addr), htons(clientAddr.sin_port)); taosKeepTcpAlive(connFd); @@ -318,7 +324,7 @@ static void taosAcceptTcpConnection(void *arg) { event.events = EPOLLIN | EPOLLPRI | EPOLLWAKEUP; event.data.ptr = pFdObj; if (epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_ADD, connFd, &event) < 0) { - tError("%s failed to add TCP FD for epoll, error:%s", pServerObj->label, strerror(errno)); + tError("%s failed to add TCP FD for epoll(%s)", pServerObj->label, strerror(errno)); tfree(pFdObj); close(connFd); continue; diff --git a/src/rpc/src/rpcUdp.c b/src/rpc/src/rpcUdp.c index 785288f5b9..e666187cf1 100644 --- a/src/rpc/src/rpcUdp.c +++ b/src/rpc/src/rpcUdp.c @@ -78,7 +78,6 @@ static SUdpBuf *taosCreateUdpBuf(SUdpConn *pConn, uint32_t ip, uint16_t port); static void taosProcessUdpBufTimer(void *param, void *tmrId); void *taosInitUdpConnection(char *ip, uint16_t port, char *label, int threads, void *fp, void *shandle) { - pthread_attr_t thAttr; SUdpConn * pConn; SUdpConnSet * pSet; @@ -106,9 +105,6 @@ void *taosInitUdpConnection(char *ip, uint16_t port, char *label, int threads, v } } - pthread_attr_init(&thAttr); - pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); - uint16_t ownPort; for (int i = 0; i < threads; ++i) { pConn = pSet->udpConn + i; @@ -146,19 +142,21 @@ void *taosInitUdpConnection(char *ip, uint16_t port, char *label, int threads, v pConn->tmrCtrl = pSet->tmrCtrl; } + pthread_attr_t thAttr; + pthread_attr_init(&thAttr); + pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); int code = pthread_create(&pConn->thread, &thAttr, taosRecvUdpData, pConn); + pthread_attr_destroy(&thAttr); if (code != 0) { tError("%s failed to create thread to process UDP data, reason:%s", label, strerror(errno)); taosCloseSocket(pConn->fd); taosCleanUpUdpConnection(pSet); - pthread_attr_destroy(&thAttr); return NULL; } ++pSet->threads; } - pthread_attr_destroy(&thAttr); tTrace("%s UDP connection is initialized, ip:%s port:%hu threads:%d", label, ip, port, threads); return pSet; From a5dabf7e9919851c861d3d9fa182de109733a884 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sun, 12 Apr 2020 17:05:06 +0800 Subject: [PATCH 05/18] remove the potential race condition on TCP FD --- src/rpc/src/rpcServer.c | 47 ++++++++++++++++++----------------------- 1 file changed, 21 insertions(+), 26 deletions(-) diff --git a/src/rpc/src/rpcServer.c b/src/rpc/src/rpcServer.c index 6fe5385bba..7430157bcf 100644 --- a/src/rpc/src/rpcServer.c +++ b/src/rpc/src/rpcServer.c @@ -145,7 +145,7 @@ void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, void taosCleanUpTcpServer(void *handle) { SThreadObj *pThreadObj; - SServerObj *pServerObj = (SServerObj *)handle; + SServerObj *pServerObj = handle; if (pServerObj == NULL) return; @@ -174,15 +174,15 @@ void taosCleanUpTcpServer(void *handle) { } void taosCloseTcpServerConnection(void *chandle) { - SFdObj *pFdObj = (SFdObj *)chandle; - + SFdObj *pFdObj = chandle; if (pFdObj == NULL) return; + pFdObj->thandle = NULL; taosCleanUpFdObj(pFdObj); } int taosSendTcpServerData(uint32_t ip, uint16_t port, void *data, int len, void *chandle) { - SFdObj *pFdObj = (SFdObj *)chandle; + SFdObj *pFdObj = chandle; if (chandle == NULL) return -1; @@ -354,14 +354,25 @@ static void taosCleanUpFdObj(SFdObj *pFdObj) { if (pFdObj == NULL) return; if (pFdObj->signature != pFdObj) return; + pFdObj->signature = NULL; pThreadObj = pFdObj->pThreadObj; - if (pThreadObj == NULL) { - tError("FdObj double clean up!!!"); - return; - } - epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_DEL, pFdObj->fd, NULL); + // notify the upper layer, so it will clean the associated context + if (pFdObj->thandle) { + SRecvInfo recvInfo; + recvInfo.msg = NULL; + recvInfo.msgLen = 0; + recvInfo.ip = 0; + recvInfo.port = 0; + recvInfo.shandle = pThreadObj->shandle; + recvInfo.thandle = pFdObj->thandle;; + recvInfo.chandle = NULL; + recvInfo.connType = RPC_CONN_TCP; + (*(pThreadObj->processData))(&recvInfo); + } + close(pFdObj->fd); + epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_DEL, pFdObj->fd, NULL); pthread_mutex_lock(&pThreadObj->threadMutex); @@ -384,24 +395,8 @@ static void taosCleanUpFdObj(SFdObj *pFdObj) { pthread_mutex_unlock(&pThreadObj->threadMutex); - // notify the upper layer, so it will clean the associated context - SRecvInfo recvInfo; - recvInfo.msg = NULL; - recvInfo.msgLen = 0; - recvInfo.ip = 0; - recvInfo.port = 0; - recvInfo.shandle = pThreadObj->shandle; - recvInfo.thandle = pFdObj->thandle;; - recvInfo.chandle = NULL; - recvInfo.connType = RPC_CONN_TCP; - - if (pFdObj->thandle) (*(pThreadObj->processData))(&recvInfo); - tTrace("%s TCP thread:%d, FD:%p is cleaned up, numOfFds:%d", pThreadObj->label, pThreadObj->threadId, - pFdObj, pThreadObj->numOfFds); - - memset(pFdObj, 0, sizeof(SFdObj)); - + pFdObj, pThreadObj->numOfFds); tfree(pFdObj); } From 8c22eed1bbfe2f0317c63be88ef0420087ba422e Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sun, 12 Apr 2020 18:10:06 +0800 Subject: [PATCH 06/18] remove a potential race condition --- src/rpc/src/rpcClient.c | 26 ++++++++++++-------------- src/rpc/test/rclient.c | 2 ++ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/rpc/src/rpcClient.c b/src/rpc/src/rpcClient.c index 264449bbb0..a3105ef516 100644 --- a/src/rpc/src/rpcClient.c +++ b/src/rpc/src/rpcClient.c @@ -188,10 +188,19 @@ static void taosCleanUpTcpFdObj(STcpFd *pFdObj) { if (pFdObj == NULL) return; if (pFdObj->signature != pFdObj) return; + pFdObj->signature = NULL; pTcp = pFdObj->pTcp; - if (pTcp == NULL) { - tError("double free TcpFdObj!!!!"); - return; + + if (pFdObj->thandle) { + recvInfo.msg = NULL; + recvInfo.msgLen = 0; + recvInfo.ip = 0; + recvInfo.port = 0; + recvInfo.shandle = pTcp->shandle; + recvInfo.thandle = pFdObj->thandle;; + recvInfo.chandle = NULL; + recvInfo.connType = RPC_CONN_TCP; + (*(pTcp->processData))(&recvInfo); } epoll_ctl(pTcp->pollFd, EPOLL_CTL_DEL, pFdObj->fd, NULL); @@ -216,19 +225,8 @@ static void taosCleanUpTcpFdObj(STcpFd *pFdObj) { pthread_mutex_unlock(&pTcp->mutex); - recvInfo.msg = NULL; - recvInfo.msgLen = 0; - recvInfo.ip = 0; - recvInfo.port = 0; - recvInfo.shandle = pTcp->shandle; - recvInfo.thandle = pFdObj->thandle;; - recvInfo.chandle = NULL; - recvInfo.connType = RPC_CONN_TCP; - - if (pFdObj->thandle) (*(pTcp->processData))(&recvInfo); tTrace("%s TCP is cleaned up, FD:%p numOfFds:%d", pTcp->label, pFdObj, pTcp->numOfFds); - memset(pFdObj, 0, sizeof(STcpFd)); tfree(pFdObj); } diff --git a/src/rpc/test/rclient.c b/src/rpc/test/rclient.c index f000ab91a2..732d7eb81c 100644 --- a/src/rpc/test/rclient.c +++ b/src/rpc/test/rclient.c @@ -204,6 +204,8 @@ int main(int argc, char *argv[]) { tPrint("it takes %.3f mseconds to send %d requests to server", usedTime, numOfReqs*appThreads); tPrint("Performance: %.3f requests per second, msgSize:%d bytes", 1000.0*numOfReqs*appThreads/usedTime, msgSize); + getchar(); + taosCloseLogger(); return 0; From f5a1ac4b48a3b45fcff16045a9285cab559b2009 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Sun, 12 Apr 2020 21:46:34 +0800 Subject: [PATCH 07/18] [td-98]fix bugs in descending order query for tables --- src/client/src/tscFunctionImpl.c | 115 ------ src/client/src/tscSQLParser.c | 1 + src/mnode/src/mgmtTable.c | 9 +- src/query/inc/queryExecutor.h | 4 +- src/query/inc/tsqlfunction.h | 16 - src/query/src/queryExecutor.c | 29 +- src/util/inc/tcompare.h | 24 ++ src/util/src/tcompare.c | 139 ++++++++ src/vnode/tsdb/src/tsdbRead.c | 590 ++++++++++++------------------- 9 files changed, 426 insertions(+), 501 deletions(-) diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 0fa9e01c2e..a5a0bf4a78 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -3578,121 +3578,6 @@ void spread_function_finalizer(SQLFunctionCtx *pCtx) { doFinalizer(pCtx); } -/* - * Compare two strings - * TSDB_MATCH: Match - * TSDB_NOMATCH: No match - * TSDB_NOWILDCARDMATCH: No match in spite of having * or % wildcards. - * Like matching rules: - * '%': Matches zero or more characters - * '_': Matches one character - * - */ -int patternMatch(const char *patterStr, const char *str, size_t size, const SPatternCompareInfo *pInfo) { - char c, c1; - - int32_t i = 0; - int32_t j = 0; - - while ((c = patterStr[i++]) != 0) { - if (c == pInfo->matchAll) { /* Match "*" */ - - while ((c = patterStr[i++]) == pInfo->matchAll || c == pInfo->matchOne) { - if (c == pInfo->matchOne && (j > size || str[j++] == 0)) { - // empty string, return not match - return TSDB_PATTERN_NOWILDCARDMATCH; - } - } - - if (c == 0) { - return TSDB_PATTERN_MATCH; /* "*" at the end of the pattern matches */ - } - - char next[3] = {toupper(c), tolower(c), 0}; - while (1) { - size_t n = strcspn(str, next); - str += n; - - if (str[0] == 0 || (n >= size - 1)) { - break; - } - - int32_t ret = patternMatch(&patterStr[i], ++str, size - n - 1, pInfo); - if (ret != TSDB_PATTERN_NOMATCH) { - return ret; - } - } - return TSDB_PATTERN_NOWILDCARDMATCH; - } - - c1 = str[j++]; - - if (j <= size) { - if (c == c1 || tolower(c) == tolower(c1) || (c == pInfo->matchOne && c1 != 0)) { - continue; - } - } - - return TSDB_PATTERN_NOMATCH; - } - - return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH; -} - -int WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t size, const SPatternCompareInfo *pInfo) { - wchar_t c, c1; - wchar_t matchOne = L'_'; // "_" - wchar_t matchAll = L'%'; // "%" - - int32_t i = 0; - int32_t j = 0; - - while ((c = patterStr[i++]) != 0) { - if (c == matchAll) { /* Match "%" */ - - while ((c = patterStr[i++]) == matchAll || c == matchOne) { - if (c == matchOne && (j > size || str[j++] == 0)) { - return TSDB_PATTERN_NOWILDCARDMATCH; - } - } - if (c == 0) { - return TSDB_PATTERN_MATCH; - } - - wchar_t accept[3] = {towupper(c), towlower(c), 0}; - while (1) { - size_t n = wcsspn(str, accept); - - str += n; - if (str[0] == 0 || (n >= size - 1)) { - break; - } - - str++; - - int32_t ret = WCSPatternMatch(&patterStr[i], str, wcslen(str), pInfo); - if (ret != TSDB_PATTERN_NOMATCH) { - return ret; - } - } - - return TSDB_PATTERN_NOWILDCARDMATCH; - } - - c1 = str[j++]; - - if (j <= size) { - if (c == c1 || towlower(c) == towlower(c1) || (c == matchOne && c1 != 0)) { - continue; - } - } - - return TSDB_PATTERN_NOMATCH; - } - - return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH; -} - static void getStatics_i8(int64_t *primaryKey, int32_t type, int8_t *data, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int32_t *numOfNull) { *min = INT64_MAX; diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 630e439397..5657b6b47d 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -30,6 +30,7 @@ #include "ttokendef.h" #include "name.h" +#include "tcompare.h" #define DEFAULT_PRIMARY_TIMESTAMP_COL_NAME "_c0" diff --git a/src/mnode/src/mgmtTable.c b/src/mnode/src/mgmtTable.c index b536bb5ac9..e6d597f5f5 100644 --- a/src/mnode/src/mgmtTable.c +++ b/src/mnode/src/mgmtTable.c @@ -15,19 +15,13 @@ #define _DEFAULT_SOURCE #include "os.h" + #include "taosmsg.h" -#include "tscompression.h" -#include "tskiplist.h" #include "ttime.h" #include "tutil.h" -#include "qast.h" -#include "qextbuffer.h" #include "taoserror.h" #include "taosmsg.h" #include "tscompression.h" -#include "tskiplist.h" -#include "tsqlfunction.h" -#include "ttime.h" #include "name.h" #include "taccount.h" #include "mgmtDClient.h" @@ -42,6 +36,7 @@ #include "mgmtTable.h" #include "mgmtUser.h" #include "mgmtVgroup.h" +#include "tcompare.h" void * tsChildTableSdb; void * tsSuperTableSdb; diff --git a/src/query/inc/queryExecutor.h b/src/query/inc/queryExecutor.h index 5aab3abf40..3932650e1f 100644 --- a/src/query/inc/queryExecutor.h +++ b/src/query/inc/queryExecutor.h @@ -160,7 +160,7 @@ typedef struct SQueryRuntimeEnv { SQueryCostSummary summary; bool stableQuery; // super table query or not void* pQueryHandle; - + void* pSubQueryHandle; // another thread for SDiskbasedResultBuf* pResultBuf; // query result buffer based on blocked-wised disk file } SQueryRuntimeEnv; @@ -172,6 +172,8 @@ typedef struct SQInfo { int32_t code; // error code to returned to client sem_t dataReady; SArray* pTableIdList; // table id list + void* tsdb; + SQueryRuntimeEnv runtimeEnv; int32_t subgroupIdx; int32_t offset; /* offset in group result set of subgroup */ diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/tsqlfunction.h index 7514f1e44c..90ae7df36a 100644 --- a/src/query/inc/tsqlfunction.h +++ b/src/query/inc/tsqlfunction.h @@ -79,17 +79,10 @@ extern "C" { #define TSDB_BASE_FUNC_SO TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_METRIC | TSDB_FUNCSTATE_OF #define TSDB_BASE_FUNC_MO TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_METRIC | TSDB_FUNCSTATE_OF -#define TSDB_PATTERN_MATCH 0 -#define TSDB_PATTERN_NOMATCH 1 -#define TSDB_PATTERN_NOWILDCARDMATCH 2 -#define TSDB_PATTERN_STRING_MAX_LEN 20 #define TSDB_FUNCTIONS_NAME_MAX_LENGTH 16 #define TSDB_AVG_FUNCTION_INTER_BUFFER_SIZE 50 -#define PATTERN_COMPARE_INFO_INITIALIZER \ - { '%', '_' } - #define DATA_SET_FLAG ',' // to denote the output area has data, not null value #define DATA_SET_FLAG_SIZE sizeof(DATA_SET_FLAG) @@ -222,20 +215,11 @@ typedef struct SQLAggFuncElem { int32_t (*dataReqFunc)(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId); } SQLAggFuncElem; -typedef struct SPatternCompareInfo { - char matchAll; // symbol for match all wildcard, default: '%' - char matchOne; // symbol for match one wildcard, default: '_' -} SPatternCompareInfo; - #define GET_RES_INFO(ctx) ((ctx)->resultInfo) int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionId, int32_t param, int16_t *type, int16_t *len, int16_t *interResBytes, int16_t extLength, bool isSuperTable); -int patternMatch(const char *zPattern, const char *zString, size_t size, const SPatternCompareInfo *pInfo); - -int WCSPatternMatch(const wchar_t *zPattern, const wchar_t *zString, size_t size, const SPatternCompareInfo *pInfo); - #define IS_STREAM_QUERY_VALID(x) (((x)&TSDB_FUNCSTATE_STREAM) != 0) #define IS_MULTIOUTPUT(x) (((x)&TSDB_FUNCSTATE_MO) != 0) #define IS_SINGLEOUTPUT(x) (((x)&TSDB_FUNCSTATE_SO) != 0) diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index 6e540f6ae1..0f74cf1beb 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -2565,7 +2565,7 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { dTrace("QInfo:%p query start, qrange:%" PRId64 "-%" PRId64 ", lastkey:%" PRId64 ", order:%d", GET_QINFO_ADDR(pRuntimeEnv), pQuery->window.skey, pQuery->window.ekey, pQuery->lastKey, pQuery->order.order); - tsdb_query_handle_t pQueryHandle = pRuntimeEnv->pQueryHandle; + tsdb_query_handle_t pQueryHandle = pRuntimeEnv->scanFlag == MASTER_SCAN? pRuntimeEnv->pQueryHandle:pRuntimeEnv->pSubQueryHandle; while (tsdbNextDataBlock(pQueryHandle)) { if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) { @@ -3520,8 +3520,9 @@ void scanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { setQueryStatus(pQuery, QUERY_NOT_COMPLETED); // store the start query position - void *pos = tsdbDataBlockTell(pRuntimeEnv->pQueryHandle); - +// void *pos = tsdbDataBlockTell(pRuntimeEnv->pQueryHandle); + SQInfo* pQInfo = (SQInfo*) GET_QINFO_ADDR(pRuntimeEnv); + int64_t skey = pQuery->lastKey; int32_t status = pQuery->status; int32_t activeSlot = pRuntimeEnv->windowResInfo.curIndex; @@ -3543,15 +3544,29 @@ void scanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { } // set the correct start position, and load the corresponding block in buffer for next round scan all data blocks. - /*int32_t ret =*/ tsdbDataBlockSeek(pRuntimeEnv->pQueryHandle, pos); - +// /*int32_t ret =*/ tsdbDataBlockSeek(pRuntimeEnv->pQueryHandle, pos); + + STsdbQueryCond cond = { + .twindow = {pQuery->window.skey, pQuery->lastKey}, + .order = pQuery->order.order, + .colList = pQuery->colList, + }; + + SArray *cols = taosArrayInit(pQuery->numOfCols, sizeof(pQuery->colList[0])); + for (int32_t i = 0; i < pQuery->numOfCols; ++i) { + taosArrayPush(cols, &pQuery->colList[i]); + } + + pRuntimeEnv->pSubQueryHandle = tsdbQueryByTableId(pQInfo->tsdb, &cond, pQInfo->pTableIdList, cols); + taosArrayDestroy(cols); + status = pQuery->status; pRuntimeEnv->windowResInfo.curIndex = activeSlot; setQueryStatus(pQuery, QUERY_NOT_COMPLETED); pRuntimeEnv->scanFlag = REPEAT_SCAN; - /* check if query is killed or not */ + // check if query is killed or not if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) { return; } @@ -4179,6 +4194,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, void *param, void* tsdb, bool isSTableQuery) pRuntimeEnv->pQueryHandle = tsdbQueryByTableId(tsdb, &cond, pQInfo->pTableIdList, cols); taosArrayDestroy(cols); + pQInfo->tsdb = tsdb; pRuntimeEnv->pQuery = pQuery; pRuntimeEnv->pTSBuf = param; @@ -4972,7 +4988,6 @@ static void tableIntervalProcessImpl(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; while (1) { -// initCtxOutputBuf(pRuntimeEnv); scanAllDataBlocks(pRuntimeEnv); if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) { diff --git a/src/util/inc/tcompare.h b/src/util/inc/tcompare.h index 3ac0d3d170..b8778b80c8 100644 --- a/src/util/inc/tcompare.h +++ b/src/util/inc/tcompare.h @@ -16,8 +16,24 @@ #ifndef TDENGINE_TCOMPARE_H #define TDENGINE_TCOMPARE_H +#ifdef __cplusplus +extern "C" { +#endif + #include "os.h" +#define TSDB_PATTERN_MATCH 0 +#define TSDB_PATTERN_NOMATCH 1 +#define TSDB_PATTERN_NOWILDCARDMATCH 2 +#define TSDB_PATTERN_STRING_MAX_LEN 20 + +#define PATTERN_COMPARE_INFO_INITIALIZER { '%', '_' } + +typedef struct SPatternCompareInfo { + char matchAll; // symbol for match all wildcard, default: '%' + char matchOne; // symbol for match one wildcard, default: '_' +} SPatternCompareInfo; + int32_t compareInt32Val(const void *pLeft, const void *pRight); int32_t compareInt64Val(const void *pLeft, const void *pRight); @@ -36,8 +52,16 @@ int32_t compareStrVal(const void *pLeft, const void *pRight); int32_t compareWStrVal(const void *pLeft, const void *pRight); +int patternMatch(const char *zPattern, const char *zString, size_t size, const SPatternCompareInfo *pInfo); + +int WCSPatternMatch(const wchar_t *zPattern, const wchar_t *zString, size_t size, const SPatternCompareInfo *pInfo); + __compar_fn_t getKeyComparFunc(int32_t keyType); __compar_fn_t getComparFunc(int32_t type, int32_t filterDataType); +#ifdef __cplusplus +} +#endif + #endif // TDENGINE_TCOMPARE_H diff --git a/src/util/src/tcompare.c b/src/util/src/tcompare.c index 401db9279a..bff10c3022 100644 --- a/src/util/src/tcompare.c +++ b/src/util/src/tcompare.c @@ -1,6 +1,8 @@ #include "taosdef.h" #include "tcompare.h" +#include "tutil.h" + int32_t compareInt32Val(const void *pLeft, const void *pRight) { int32_t ret = GET_INT32_VAL(pLeft) - GET_INT32_VAL(pRight); if (ret == 0) { @@ -102,6 +104,143 @@ int32_t compareWStrVal(const void *pLeft, const void *pRight) { return 0; } +/* + * Compare two strings + * TSDB_MATCH: Match + * TSDB_NOMATCH: No match + * TSDB_NOWILDCARDMATCH: No match in spite of having * or % wildcards. + * Like matching rules: + * '%': Matches zero or more characters + * '_': Matches one character + * + */ +int patternMatch(const char *patterStr, const char *str, size_t size, const SPatternCompareInfo *pInfo) { + char c, c1; + + int32_t i = 0; + int32_t j = 0; + + while ((c = patterStr[i++]) != 0) { + if (c == pInfo->matchAll) { /* Match "*" */ + + while ((c = patterStr[i++]) == pInfo->matchAll || c == pInfo->matchOne) { + if (c == pInfo->matchOne && (j > size || str[j++] == 0)) { + // empty string, return not match + return TSDB_PATTERN_NOWILDCARDMATCH; + } + } + + if (c == 0) { + return TSDB_PATTERN_MATCH; /* "*" at the end of the pattern matches */ + } + + char next[3] = {toupper(c), tolower(c), 0}; + while (1) { + size_t n = strcspn(str, next); + str += n; + + if (str[0] == 0 || (n >= size - 1)) { + break; + } + + int32_t ret = patternMatch(&patterStr[i], ++str, size - n - 1, pInfo); + if (ret != TSDB_PATTERN_NOMATCH) { + return ret; + } + } + return TSDB_PATTERN_NOWILDCARDMATCH; + } + + c1 = str[j++]; + + if (j <= size) { + if (c == c1 || tolower(c) == tolower(c1) || (c == pInfo->matchOne && c1 != 0)) { + continue; + } + } + + return TSDB_PATTERN_NOMATCH; + } + + return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH; +} + +int WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t size, const SPatternCompareInfo *pInfo) { + wchar_t c, c1; + wchar_t matchOne = L'_'; // "_" + wchar_t matchAll = L'%'; // "%" + + int32_t i = 0; + int32_t j = 0; + + while ((c = patterStr[i++]) != 0) { + if (c == matchAll) { /* Match "%" */ + + while ((c = patterStr[i++]) == matchAll || c == matchOne) { + if (c == matchOne && (j > size || str[j++] == 0)) { + return TSDB_PATTERN_NOWILDCARDMATCH; + } + } + if (c == 0) { + return TSDB_PATTERN_MATCH; + } + + wchar_t accept[3] = {towupper(c), towlower(c), 0}; + while (1) { + size_t n = wcsspn(str, accept); + + str += n; + if (str[0] == 0 || (n >= size - 1)) { + break; + } + + str++; + + int32_t ret = WCSPatternMatch(&patterStr[i], str, wcslen(str), pInfo); + if (ret != TSDB_PATTERN_NOMATCH) { + return ret; + } + } + + return TSDB_PATTERN_NOWILDCARDMATCH; + } + + c1 = str[j++]; + + if (j <= size) { + if (c == c1 || towlower(c) == towlower(c1) || (c == matchOne && c1 != 0)) { + continue; + } + } + + return TSDB_PATTERN_NOMATCH; + } + + return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH; +} + +static UNUSED_FUNC int32_t compareStrPatternComp(const void* pLeft, const void* pRight) { + SPatternCompareInfo pInfo = {'%', '_'}; + + const char* pattern = pRight; + const char* str = pLeft; + + int32_t ret = patternMatch(pattern, str, strlen(str), &pInfo); + + return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; +} + +static UNUSED_FUNC int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) { + SPatternCompareInfo pInfo = {'%', '_'}; + + const wchar_t* pattern = pRight; + const wchar_t* str = pLeft; + + int32_t ret = WCSPatternMatch(pattern, str, wcslen(str), &pInfo); + + return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; +} + __compar_fn_t getComparFunc(int32_t type, int32_t filterDataType) { __compar_fn_t comparFn = NULL; diff --git a/src/vnode/tsdb/src/tsdbRead.c b/src/vnode/tsdb/src/tsdbRead.c index 2fce73e547..966debda8d 100644 --- a/src/vnode/tsdb/src/tsdbRead.c +++ b/src/vnode/tsdb/src/tsdbRead.c @@ -87,6 +87,13 @@ typedef struct STableBlockInfo { int32_t groupIdx; /* number of group is less than the total number of tables */ } STableBlockInfo; +typedef struct SBlockOrderSupporter { + int32_t numOfTables; + STableBlockInfo** pDataBlockInfo; + int32_t* blockIndexArray; + int32_t* numOfBlocksPerMeter; +} SBlockOrderSupporter; + typedef struct STsdbQueryHandle { STsdbRepo* pTsdb; SQueryFilePos cur; // current position @@ -222,7 +229,7 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) { // todo dynamic get the daysperfile static int32_t getFileIdFromKey(TSKEY key) { - int64_t fid = (int64_t)(key / 10); // set the starting fileId + int64_t fid = (int64_t)(key / (10 * tsMsPerDay[0])); // set the starting fileId if (fid > INT32_MAX) { fid = INT32_MAX; } @@ -230,7 +237,32 @@ static int32_t getFileIdFromKey(TSKEY key) { return fid; } -static int32_t binarySearchForBlockImpl(SCompBlock* pBlock, int32_t numOfBlocks, TSKEY skey, int32_t order); +static int32_t binarySearchForBlockImpl(SCompBlock* pBlock, int32_t numOfBlocks, TSKEY skey, int32_t order) { + int32_t firstSlot = 0; + int32_t lastSlot = numOfBlocks - 1; + + int32_t midSlot = firstSlot; + + while (1) { + numOfBlocks = lastSlot - firstSlot + 1; + midSlot = (firstSlot + (numOfBlocks >> 1)); + + if (numOfBlocks == 1) break; + + if (skey > pBlock[midSlot].keyLast) { + if (numOfBlocks == 2) break; + if ((order == TSDB_ORDER_DESC) && (skey < pBlock[midSlot + 1].keyFirst)) break; + firstSlot = midSlot + 1; + } else if (skey < pBlock[midSlot].keyFirst) { + if ((order == TSDB_ORDER_ASC) && (skey > pBlock[midSlot - 1].keyLast)) break; + lastSlot = midSlot - 1; + } else { + break; // got the slot + } + } + + return midSlot; +} static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlocks, int32_t type) { // todo check open file failed @@ -299,33 +331,6 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo return TSDB_CODE_SUCCESS; } -int32_t binarySearchForBlockImpl(SCompBlock* pBlock, int32_t numOfBlocks, TSKEY skey, int32_t order) { - int32_t firstSlot = 0; - int32_t lastSlot = numOfBlocks - 1; - - int32_t midSlot = firstSlot; - - while (1) { - numOfBlocks = lastSlot - firstSlot + 1; - midSlot = (firstSlot + (numOfBlocks >> 1)); - - if (numOfBlocks == 1) break; - - if (skey > pBlock[midSlot].keyLast) { - if (numOfBlocks == 2) break; - if ((order == TSDB_ORDER_DESC) && (skey < pBlock[midSlot + 1].keyFirst)) break; - firstSlot = midSlot + 1; - } else if (skey < pBlock[midSlot].keyFirst) { - if ((order == TSDB_ORDER_ASC) && (skey > pBlock[midSlot - 1].keyLast)) break; - lastSlot = midSlot - 1; - } else { - break; // got the slot - } - } - - return midSlot; -} - static SDataBlockInfo getTrueDataBlockInfo(STableCheckInfo* pCheckInfo, SCompBlock* pBlock) { SDataBlockInfo info = { .window = {.skey = pBlock->keyFirst, .ekey = pBlock->keyLast}, @@ -338,7 +343,34 @@ static SDataBlockInfo getTrueDataBlockInfo(STableCheckInfo* pCheckInfo, SCompBlo return info; } -SArray* getDefaultLoadColumns(STsdbQueryHandle* pQueryHandle, bool loadTS); +static SArray* getColumnIdList(STsdbQueryHandle* pQueryHandle) { + size_t numOfCols = QH_GET_NUM_OF_COLS(pQueryHandle); + assert(numOfCols <= TSDB_MAX_COLUMNS); + + SArray* pIdList = taosArrayInit(numOfCols, sizeof(int16_t)); + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i); + taosArrayPush(pIdList, &pCol->info.colId); + } + + return pIdList; +} + +static SArray* getDefaultLoadColumns(STsdbQueryHandle* pQueryHandle, bool loadTS) { + SArray* pLocalIdList = getColumnIdList(pQueryHandle); + + // check if the primary time stamp column needs to load + int16_t colId = *(int16_t*)taosArrayGet(pLocalIdList, 0); + + // the primary timestamp column does not be included in the the specified load column list, add it + if (loadTS && colId != 0) { + int16_t columnId = 0; + taosArrayInsert(pLocalIdList, 0, &columnId); + } + + return pLocalIdList; +} + static void filterDataInDataBlock(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock, SArray* sa); static int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); @@ -388,7 +420,7 @@ static bool loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock if (QUERY_IS_ASC_QUERY(pQueryHandle->order)) { // query ended in current block - if (pQueryHandle->window.ekey < pBlock->keyLast) { + if (pQueryHandle->window.ekey < pBlock->keyLast || pCheckInfo->lastKey > pBlock->keyFirst) { if (!doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo)) { return false; } @@ -430,62 +462,62 @@ static bool loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock return pQueryHandle->realNumOfRows > 0; } -bool moveToNextBlock(STsdbQueryHandle* pQueryHandle, int32_t step) { - SQueryFilePos* cur = &pQueryHandle->cur; +//bool moveToNextBlock(STsdbQueryHandle* pQueryHandle, int32_t step) { +// SQueryFilePos* cur = &pQueryHandle->cur; +// +// if (pQueryHandle->cur.fid >= 0) { +// /* +// * 1. ascending order. The last data block of data file +// * 2. descending order. The first block of file +// */ +// STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex); +// int32_t tid = pCheckInfo->tableId.tid; +// +// if ((step == QUERY_ASC_FORWARD_STEP && +// (pQueryHandle->cur.slot == pQueryHandle->compIndex[tid].numOfSuperBlocks - 1)) || +// (step == QUERY_DESC_FORWARD_STEP && (pQueryHandle->cur.slot == 0))) { +// // temporarily keep the position value, in case of no data qualified when move forwards(backwards) +// // SQueryFilePos save = pQueryHandle->cur; +// pQueryHandle->pFileGroup = tsdbGetFileGroupNext(&pQueryHandle->fileIter); +// +// int32_t fid = -1; +// int32_t numOfBlocks = 0; +// +// if (pQueryHandle->pFileGroup != NULL) { +// if ((fid = getFileCompInfo(pQueryHandle, &numOfBlocks, 1)) < 0) { +// } else { +// cur->slot = (step == QUERY_ASC_FORWARD_STEP) ? 0 : pQueryHandle->numOfBlocks - 1; +// cur->pos = (step == QUERY_ASC_FORWARD_STEP) ? 0 : pQueryHandle->pBlock[cur->slot].numOfPoints - 1; +// +// SCompBlock* pBlock = &pCheckInfo->pCompInfo->blocks[cur->slot]; +// cur->fid = pQueryHandle->pFileGroup->fileId; +// assert(cur->pos >= 0 && cur->fid >= 0 && cur->slot >= 0); +// +// if (pBlock->keyFirst > pQueryHandle->window.ekey) { // done +// return false; +// } +// +// return loadFileDataBlock(pQueryHandle, pBlock, pCheckInfo); +// } +// } else { // check data in cache +// pQueryHandle->cur.fid = -1; +// return hasMoreDataInCache(pQueryHandle); +// } +// } else { // next block in the same file +// cur->slot += step; +// +// SCompBlock* pBlock = &pCheckInfo->pCompInfo->blocks[cur->slot]; +// cur->pos = (step == QUERY_ASC_FORWARD_STEP) ? 0 : pBlock->numOfPoints - 1; +// return loadFileDataBlock(pQueryHandle, pBlock, pCheckInfo); +// } +// } else { // data in cache +// return hasMoreDataInCache(pQueryHandle); +// } +// +// return false; +//} - if (pQueryHandle->cur.fid >= 0) { - /* - * 1. ascending order. The last data block of data file - * 2. descending order. The first block of file - */ - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex); - int32_t tid = pCheckInfo->tableId.tid; - - if ((step == QUERY_ASC_FORWARD_STEP && - (pQueryHandle->cur.slot == pQueryHandle->compIndex[tid].numOfSuperBlocks - 1)) || - (step == QUERY_DESC_FORWARD_STEP && (pQueryHandle->cur.slot == 0))) { - // temporarily keep the position value, in case of no data qualified when move forwards(backwards) - // SQueryFilePos save = pQueryHandle->cur; - pQueryHandle->pFileGroup = tsdbGetFileGroupNext(&pQueryHandle->fileIter); - - int32_t fid = -1; - int32_t numOfBlocks = 0; - - if (pQueryHandle->pFileGroup != NULL) { - if ((fid = getFileCompInfo(pQueryHandle, &numOfBlocks, 1)) < 0) { - } else { - cur->slot = (step == QUERY_ASC_FORWARD_STEP) ? 0 : pQueryHandle->numOfBlocks - 1; - cur->pos = (step == QUERY_ASC_FORWARD_STEP) ? 0 : pQueryHandle->pBlock[cur->slot].numOfPoints - 1; - - SCompBlock* pBlock = &pCheckInfo->pCompInfo->blocks[cur->slot]; - cur->fid = pQueryHandle->pFileGroup->fileId; - assert(cur->pos >= 0 && cur->fid >= 0 && cur->slot >= 0); - - if (pBlock->keyFirst > pQueryHandle->window.ekey) { // done - return false; - } - - return loadFileDataBlock(pQueryHandle, pBlock, pCheckInfo); - } - } else { // check data in cache - pQueryHandle->cur.fid = -1; - return hasMoreDataInCache(pQueryHandle); - } - } else { // next block in the same file - cur->slot += step; - - SCompBlock* pBlock = &pCheckInfo->pCompInfo->blocks[cur->slot]; - cur->pos = (step == QUERY_ASC_FORWARD_STEP) ? 0 : pBlock->numOfPoints - 1; - return loadFileDataBlock(pQueryHandle, pBlock, pCheckInfo); - } - } else { // data in cache - return hasMoreDataInCache(pQueryHandle); - } - - return false; -} - -int vnodeBinarySearchKey(char* pValue, int num, TSKEY key, int order) { +static int vnodeBinarySearchKey(char* pValue, int num, TSKEY key, int order) { int firstPos, lastPos, midPos = -1; int numOfPoints; TSKEY* keyList; @@ -572,7 +604,7 @@ static void filterDataInDataBlock(STsdbQueryHandle* pQueryHandle, STableCheckInf pQueryHandle->realNumOfRows = 0; return; } else { - pQueryHandle->realNumOfRows = endPos - cur->pos; + pQueryHandle->realNumOfRows = endPos - cur->pos + 1; } pCheckInfo->lastKey = ((int64_t*)(pCols->cols[0].pData))[endPos] + 1; @@ -581,10 +613,8 @@ static void filterDataInDataBlock(STsdbQueryHandle* pQueryHandle, STableCheckInf pQueryHandle->realNumOfRows = 0; return; } else { - pQueryHandle->realNumOfRows = cur->pos - endPos; + pQueryHandle->realNumOfRows = cur->pos - endPos + 1; } - - assert(0); } } @@ -614,34 +644,6 @@ static void filterDataInDataBlock(STsdbQueryHandle* pQueryHandle, STableCheckInf cur->pos = endPos; } -static SArray* getColumnIdList(STsdbQueryHandle* pQueryHandle) { - size_t numOfCols = QH_GET_NUM_OF_COLS(pQueryHandle); - assert(numOfCols <= TSDB_MAX_COLUMNS); - - SArray* pIdList = taosArrayInit(numOfCols, sizeof(int16_t)); - for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i); - taosArrayPush(pIdList, &pCol->info.colId); - } - - return pIdList; -} - -SArray* getDefaultLoadColumns(STsdbQueryHandle* pQueryHandle, bool loadTS) { - SArray* pLocalIdList = getColumnIdList(pQueryHandle); - - // check if the primary time stamp column needs to load - int16_t colId = *(int16_t*)taosArrayGet(pLocalIdList, 0); - - // the primary timestamp column does not be included in the the specified load column list, add it - if (loadTS && colId != 0) { - int16_t columnId = 0; - taosArrayInsert(pLocalIdList, 0, &columnId); - } - - return pLocalIdList; -} - int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) { int firstPos, lastPos, midPos = -1; int numOfPoints; @@ -702,79 +704,72 @@ int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) { return midPos; } -static bool getQualifiedDataBlock(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, int32_t type) { - STsdbFileH* pFileHandle = tsdbGetFile(pQueryHandle->pTsdb); - int32_t fid = getFileIdFromKey(pCheckInfo->lastKey); +//static bool getQualifiedDataBlock(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, int32_t type) { +// STsdbFileH* pFileHandle = tsdbGetFile(pQueryHandle->pTsdb); +// int32_t fid = getFileIdFromKey(pCheckInfo->lastKey); +// +// tsdbInitFileGroupIter(pFileHandle, &pQueryHandle->fileIter, TSDB_FGROUP_ITER_FORWARD); +// tsdbSeekFileGroupIter(&pQueryHandle->fileIter, fid); +// pQueryHandle->pFileGroup = tsdbGetFileGroupNext(&pQueryHandle->fileIter); +// +// SQueryFilePos* cur = &pQueryHandle->cur; +// +// int32_t tid = pCheckInfo->tableId.tid; +// int32_t numOfBlocks = 0; +// +// while (pQueryHandle->pFileGroup != NULL) { +// if (getFileCompInfo(pQueryHandle, &numOfBlocks, 1) != TSDB_CODE_SUCCESS) { +// break; +// } +// +// assert(pCheckInfo->numOfBlocks >= 0); +// +// // no data block in current file, try next +// if (pCheckInfo->numOfBlocks > 0) { +// cur->fid = pQueryHandle->pFileGroup->fileId; +// break; +// } +// +// dTrace("%p no data block in file, fid:%d, tid:%d, try next, %p", pQueryHandle, pQueryHandle->pFileGroup->fileId, +// tid, pQueryHandle->qinfo); +// +// pQueryHandle->pFileGroup = tsdbGetFileGroupNext(&pQueryHandle->fileIter); +// } +// +// if (pCheckInfo->numOfBlocks == 0) { +// return false; +// } +// +// cur->slot = 0; // always start from the first slot +// SCompBlock* pBlock = &pCheckInfo->pCompInfo->blocks[cur->slot]; +// return loadFileDataBlock(pQueryHandle, pBlock, pCheckInfo); +//} - tsdbInitFileGroupIter(pFileHandle, &pQueryHandle->fileIter, TSDB_FGROUP_ITER_FORWARD); - tsdbSeekFileGroupIter(&pQueryHandle->fileIter, fid); - pQueryHandle->pFileGroup = tsdbGetFileGroupNext(&pQueryHandle->fileIter); +//static UNUSED_FUNC bool hasMoreDataForSingleTable(STsdbQueryHandle* pHandle) { +// assert(pHandle->activeIndex == 0 && taosArrayGetSize(pHandle->pTableCheckInfo) == 1); +// +// STsdbFileH* pFileHandle = tsdbGetFile(pHandle->pTsdb); +// STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); +// +// if (!pCheckInfo->checkFirstFileBlock) { +// pCheckInfo->checkFirstFileBlock = true; +// +// if (pFileHandle != NULL) { +// bool found = getQualifiedDataBlock(pHandle, pCheckInfo, 1); +// if (found) { +// return true; +// } +// } +// +// // no data in file, try cache +// pHandle->cur.fid = -1; +// return hasMoreDataInCache(pHandle); +// } else { // move to next data block in file or in cache +// return moveToNextBlock(pHandle, 1); +// } +//} - SQueryFilePos* cur = &pQueryHandle->cur; - - int32_t tid = pCheckInfo->tableId.tid; - int32_t numOfBlocks = 0; - - while (pQueryHandle->pFileGroup != NULL) { - if (getFileCompInfo(pQueryHandle, &numOfBlocks, 1) != TSDB_CODE_SUCCESS) { - break; - } - - assert(pCheckInfo->numOfBlocks >= 0); - - // no data block in current file, try next - if (pCheckInfo->numOfBlocks > 0) { - cur->fid = pQueryHandle->pFileGroup->fileId; - break; - } - - dTrace("%p no data block in file, fid:%d, tid:%d, try next, %p", pQueryHandle, pQueryHandle->pFileGroup->fileId, - tid, pQueryHandle->qinfo); - - pQueryHandle->pFileGroup = tsdbGetFileGroupNext(&pQueryHandle->fileIter); - } - - if (pCheckInfo->numOfBlocks == 0) { - return false; - } - - cur->slot = 0; // always start from the first slot - SCompBlock* pBlock = &pCheckInfo->pCompInfo->blocks[cur->slot]; - return loadFileDataBlock(pQueryHandle, pBlock, pCheckInfo); -} - -static UNUSED_FUNC bool hasMoreDataForSingleTable(STsdbQueryHandle* pHandle) { - assert(pHandle->activeIndex == 0 && taosArrayGetSize(pHandle->pTableCheckInfo) == 1); - - STsdbFileH* pFileHandle = tsdbGetFile(pHandle->pTsdb); - STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); - - if (!pCheckInfo->checkFirstFileBlock) { - pCheckInfo->checkFirstFileBlock = true; - - if (pFileHandle != NULL) { - bool found = getQualifiedDataBlock(pHandle, pCheckInfo, 1); - if (found) { - return true; - } - } - - // no data in file, try cache - pHandle->cur.fid = -1; - return hasMoreDataInCache(pHandle); - } else { // move to next data block in file or in cache - return moveToNextBlock(pHandle, 1); - } -} - -typedef struct SBlockOrderSupporter { - int32_t numOfTables; - STableBlockInfo** pDataBlockInfo; - int32_t* blockIndexArray; - int32_t* numOfBlocksPerMeter; -} SBlockOrderSupporter; - -void cleanBlockOrderSupporter(SBlockOrderSupporter* pSupporter, int32_t numOfTables) { +static void cleanBlockOrderSupporter(SBlockOrderSupporter* pSupporter, int32_t numOfTables) { tfree(pSupporter->numOfBlocksPerMeter); tfree(pSupporter->blockIndexArray); @@ -815,7 +810,7 @@ static int32_t dataBlockOrderCompar(const void* pLeft, const void* pRight, void* return pLeftBlockInfoEx->pBlock.compBlock->offset > pRightBlockInfoEx->pBlock.compBlock->offset ? 1 : -1; } -int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numOfBlocks, int32_t* numOfAllocBlocks) { +static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numOfBlocks, int32_t* numOfAllocBlocks) { char* tmp = realloc(pQueryHandle->pDataBlockInfo, sizeof(STableBlockInfo) * numOfBlocks); if (tmp == NULL) { return TSDB_CODE_SERV_OUT_OF_MEMORY; @@ -912,6 +907,53 @@ int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numOfBlocks return TSDB_CODE_SUCCESS; } +// todo opt for only one table case +static bool getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle) { + pQueryHandle->numOfBlocks = 0; + SQueryFilePos* cur = &pQueryHandle->cur; + + int32_t numOfBlocks = 0; + int32_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + + while ((pQueryHandle->pFileGroup = tsdbGetFileGroupNext(&pQueryHandle->fileIter)) != NULL) { + int32_t type = QUERY_IS_ASC_QUERY(pQueryHandle->order)? QUERY_RANGE_GREATER_EQUAL:QUERY_RANGE_LESS_EQUAL; + if (getFileCompInfo(pQueryHandle, &numOfBlocks, type) != TSDB_CODE_SUCCESS) { + break; + } + + assert(numOfBlocks >= 0); + dTrace("%p %d blocks found in file for %d table(s), fid:%d", pQueryHandle, numOfBlocks, + numOfTables, pQueryHandle->pFileGroup->fileId); + + // todo return error code to query engine + if (createDataBlocksInfo(pQueryHandle, numOfBlocks, &pQueryHandle->numOfBlocks) != TSDB_CODE_SUCCESS) { + break; + } + + assert(numOfBlocks >= pQueryHandle->numOfBlocks); + if (pQueryHandle->numOfBlocks > 0) { + break; + } + } + + // no data in file anymore + if (pQueryHandle->numOfBlocks <= 0) { + assert(pQueryHandle->pFileGroup == NULL); + cur->fid = -1; + + return false; + } + + cur->slot = QUERY_IS_ASC_QUERY(pQueryHandle->order)? 0:pQueryHandle->numOfBlocks-1; + cur->fid = pQueryHandle->pFileGroup->fileId; + + STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot]; + STableCheckInfo* pCheckInfo = pBlockInfo->pTableCheckInfo; + SCompBlock* pBlock = pBlockInfo->pBlock.compBlock; + + return loadFileDataBlock(pQueryHandle, pBlock, pCheckInfo); +} + static bool getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle) { STsdbFileH* pFileHandle = tsdbGetFile(pQueryHandle->pTsdb); SQueryFilePos* cur = &pQueryHandle->cur; @@ -925,92 +967,12 @@ static bool getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle) { tsdbInitFileGroupIter(pFileHandle, &pQueryHandle->fileIter, pQueryHandle->order); tsdbSeekFileGroupIter(&pQueryHandle->fileIter, fid); - int32_t numOfBlocks = -1; - - // todo opt for only one table case - pQueryHandle->numOfBlocks = 0; - int32_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); - - while ((pQueryHandle->pFileGroup = tsdbGetFileGroupNext(&pQueryHandle->fileIter)) != NULL) { - int32_t type = QUERY_IS_ASC_QUERY(pQueryHandle->order)? QUERY_RANGE_GREATER_EQUAL:QUERY_RANGE_LESS_EQUAL; - if (getFileCompInfo(pQueryHandle, &numOfBlocks, type) != TSDB_CODE_SUCCESS) { - break; - } - - assert(numOfBlocks >= 0); - dTrace("%p %d blocks found in file for %d table(s), fid:%d", pQueryHandle, numOfBlocks, - numOfTables, pQueryHandle->pFileGroup->fileId); - - // todo return error code to query engine - if (createDataBlocksInfo(pQueryHandle, numOfBlocks, &pQueryHandle->numOfBlocks) != TSDB_CODE_SUCCESS) { - break; - } - - assert(numOfBlocks >= pQueryHandle->numOfBlocks); - if (pQueryHandle->numOfBlocks > 0) { - break; - } - } - - // no data in file anymore - if (pQueryHandle->numOfBlocks <= 0) { - assert(pQueryHandle->pFileGroup == NULL); - cur->fid = -1; - - return false; - } - - cur->slot = QUERY_IS_ASC_QUERY(pQueryHandle->order)? 0:pQueryHandle->numOfBlocks-1; - cur->fid = pQueryHandle->pFileGroup->fileId; - - STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot]; - STableCheckInfo* pCheckInfo = pBlockInfo->pTableCheckInfo; - SCompBlock* pBlock = pBlockInfo->pBlock.compBlock; - - return loadFileDataBlock(pQueryHandle, pBlock, pCheckInfo); + return getDataBlocksInFilesImpl(pQueryHandle); } else { if ((cur->slot == pQueryHandle->numOfBlocks - 1 && QUERY_IS_ASC_QUERY(pQueryHandle->order)) || (cur->slot == 0 && !QUERY_IS_ASC_QUERY(pQueryHandle->order))) { // all blocks - int32_t numOfBlocks = -1; - int32_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); - pQueryHandle->numOfBlocks = 0; - - while ((pQueryHandle->pFileGroup = tsdbGetFileGroupNext(&pQueryHandle->fileIter)) != NULL) { - if (getFileCompInfo(pQueryHandle, &numOfBlocks, 1) != TSDB_CODE_SUCCESS) { - break; - } - - assert(numOfBlocks >= 0); - dTrace("%p %d blocks found in file for %d table(s), fid:%d", pQueryHandle, numOfBlocks, numOfTables, - pQueryHandle->pFileGroup->fileId); - - // todo return error code to query engine - if (createDataBlocksInfo(pQueryHandle, numOfBlocks, &pQueryHandle->numOfBlocks) != TSDB_CODE_SUCCESS) { - break; - } - - assert(numOfBlocks >= pQueryHandle->numOfBlocks); - if (pQueryHandle->numOfBlocks > 0) { - break; - } - } - - // no data in file anymore - if (pQueryHandle->numOfBlocks <= 0) { - assert(pQueryHandle->pFileGroup == NULL); - cur->fid = -1; - return false; - } - - cur->slot = QUERY_IS_ASC_QUERY(pQueryHandle->order)? 0:pQueryHandle->numOfBlocks-1; - cur->fid = pQueryHandle->pFileGroup->fileId; - - STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot]; - STableCheckInfo* pCheckInfo = pBlockInfo->pTableCheckInfo; - SCompBlock* pBlock = pBlockInfo->pBlock.compBlock; - - return loadFileDataBlock(pQueryHandle, pBlock, pCheckInfo); + return getDataBlocksInFilesImpl(pQueryHandle); } else { // next block of the same file int32_t step = QUERY_IS_ASC_QUERY(pQueryHandle->order)? 1:-1; cur->slot += step; @@ -1027,11 +989,9 @@ static bool getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle) { } } - static bool doHasDataInBuffer(STsdbQueryHandle* pQueryHandle) { size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); // todo add assert, the value of numOfTables should be less than the maximum value for each vnode capacity - assert(numOfTables > 0); while (pQueryHandle->activeIndex < numOfTables) { if (hasMoreDataInCache(pQueryHandle)) { @@ -1259,7 +1219,7 @@ static void convertQueryResult(SArray* pRes, SArray* pTableList) { } } -void destroyHelper(void* param) { +static void destroyHelper(void* param) { if (param == NULL) { return; } @@ -1269,86 +1229,6 @@ void destroyHelper(void* param) { free(param); } -static UNUSED_FUNC int32_t compareStrPatternComp(const void* pLeft, const void* pRight) { - SPatternCompareInfo pInfo = {'%', '_'}; - - const char* pattern = pRight; - const char* str = pLeft; - - int32_t ret = patternMatch(pattern, str, strlen(str), &pInfo); - - return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; -} - -static UNUSED_FUNC int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) { - SPatternCompareInfo pInfo = {'%', '_'}; - - const wchar_t* pattern = pRight; - const wchar_t* str = pLeft; - - int32_t ret = WCSPatternMatch(pattern, str, wcslen(str), &pInfo); - - return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; -} - -// static __compar_fn_t getFilterComparator(int32_t type, int32_t filterType, int32_t optr) { -// __compar_fn_t comparator = NULL; -// -// switch (type) { -// case TSDB_DATA_TYPE_TINYINT: -// case TSDB_DATA_TYPE_SMALLINT: -// case TSDB_DATA_TYPE_INT: -// case TSDB_DATA_TYPE_BIGINT: -// case TSDB_DATA_TYPE_BOOL: { -// if (filterType >= TSDB_DATA_TYPE_BOOL && filterType <= TSDB_DATA_TYPE_BIGINT) { -// comparator = compareIntVal; -// } else if (filterType >= TSDB_DATA_TYPE_FLOAT && filterType <= TSDB_DATA_TYPE_DOUBLE) { -// comparator = compareIntDoubleVal; -// } -// break; -// } -// -// case TSDB_DATA_TYPE_FLOAT: -// case TSDB_DATA_TYPE_DOUBLE: { -// if (filterType >= TSDB_DATA_TYPE_BOOL && filterType <= TSDB_DATA_TYPE_BIGINT) { -// comparator = compareDoubleIntVal; -// } else if (filterType >= TSDB_DATA_TYPE_FLOAT && filterType <= TSDB_DATA_TYPE_DOUBLE) { -// comparator = compareDoubleVal; -// } -// break; -// } -// -// case TSDB_DATA_TYPE_BINARY: { -// assert(filterType == TSDB_DATA_TYPE_BINARY); -// -// if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */ -// comparator = compareStrPatternComp; -// } else { /* normal relational comparator */ -// comparator = compareStrVal; -// } -// -// break; -// } -// -// case TSDB_DATA_TYPE_NCHAR: { -// assert(filterType == TSDB_DATA_TYPE_NCHAR); -// -// if (optr == TSDB_RELATION_LIKE) { -// comparator = compareWStrPatternComp; -// } else { -// comparator = compareWStrVal; -// } -// -// break; -// } -// default: -// comparator = compareIntVal; -// break; -// } -// -// return comparator; -//} - static void getTagColumnInfo(SExprTreeSupporter* pSupporter, SSchema* pSchema, int32_t* index, int32_t* offset) { *index = 0; *offset = 0; From 5e58bd5c5f5702d82ce07507b01e5d3f8a646381 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sun, 12 Apr 2020 22:38:19 +0800 Subject: [PATCH 08/18] remove potential race condition on FdObj --- src/rpc/src/rpcClient.c | 54 +++++++++++++++++--------------- src/rpc/src/rpcMain.c | 2 +- src/rpc/src/rpcServer.c | 68 ++++++++++++++++++++++------------------- 3 files changed, 67 insertions(+), 57 deletions(-) diff --git a/src/rpc/src/rpcClient.c b/src/rpc/src/rpcClient.c index a3105ef516..7ca24f6229 100644 --- a/src/rpc/src/rpcClient.c +++ b/src/rpc/src/rpcClient.c @@ -65,18 +65,18 @@ void *taosInitTcpClient(char *ip, uint16_t port, char *label, int num, void *fp, pTcp->shandle = shandle; if (pthread_mutex_init(&(pTcp->mutex), NULL) < 0) { - tError("%s failed to init TCP mutex, reason:%s", label, strerror(errno)); + tError("%s failed to init TCP client mutex(%s)", label, strerror(errno)); return NULL; } if (pthread_cond_init(&(pTcp->fdReady), NULL) != 0) { - tError("%s init TCP condition variable failed, reason:%s\n", label, strerror(errno)); + tError("%s init TCP condition variable failed(%s)", label, strerror(errno)); return NULL; } pTcp->pollFd = epoll_create(10); // size does not matter if (pTcp->pollFd < 0) { - tError("%s failed to create TCP epoll", label); + tError("%s failed to create TCP client epoll", label); return NULL; } @@ -87,11 +87,11 @@ void *taosInitTcpClient(char *ip, uint16_t port, char *label, int num, void *fp, int code = pthread_create(&(pTcp->thread), &thattr, taosReadTcpData, (void *)(pTcp)); pthread_attr_destroy(&thattr); if (code != 0) { - tError("%s failed to create TCP read data thread, reason:%s", label, strerror(errno)); + tError("%s failed to create TCP read data thread(%s)", label, strerror(errno)); return NULL; } - tTrace("%s TCP client is initialized, ip:%s port:%hu", label, ip, port); + tTrace("%s TCP client is initialized, ip:%s:%hu", label, ip, port); return pTcp; } @@ -181,15 +181,9 @@ int taosSendTcpClientData(uint32_t ip, uint16_t port, void *data, int len, void return (int)send(pFdObj->fd, data, (size_t)len, 0); } -static void taosCleanUpTcpFdObj(STcpFd *pFdObj) { - STcpClient *pTcp; +static void taosReportBrokenLink(STcpFd *pFdObj) { SRecvInfo recvInfo; - - if (pFdObj == NULL) return; - if (pFdObj->signature != pFdObj) return; - - pFdObj->signature = NULL; - pTcp = pFdObj->pTcp; + STcpClient *pTcp = pFdObj->pTcp; if (pFdObj->thandle) { recvInfo.msg = NULL; @@ -202,6 +196,15 @@ static void taosCleanUpTcpFdObj(STcpFd *pFdObj) { recvInfo.connType = RPC_CONN_TCP; (*(pTcp->processData))(&recvInfo); } +} + +static void taosCleanUpTcpFdObj(STcpFd *pFdObj) { + + if (pFdObj == NULL) return; + if (pFdObj->signature != pFdObj) return; + + pFdObj->signature = NULL; + STcpClient *pTcp = pFdObj->pTcp; epoll_ctl(pTcp->pollFd, EPOLL_CTL_DEL, pFdObj->fd, NULL); close(pFdObj->fd); @@ -211,7 +214,7 @@ static void taosCleanUpTcpFdObj(STcpFd *pFdObj) { pTcp->numOfFds--; if (pTcp->numOfFds < 0) - tError("%s number of TCP FDs shall never be negative, FD:%p", pTcp->label, pFdObj); + tError("%s %p, number of FDs is negative!!!, FD:%p", pTcp->label, pFdObj->thandle, pFdObj); if (pFdObj->prev) { (pFdObj->prev)->next = pFdObj->next; @@ -225,7 +228,7 @@ static void taosCleanUpTcpFdObj(STcpFd *pFdObj) { pthread_mutex_unlock(&pTcp->mutex); - tTrace("%s TCP is cleaned up, FD:%p numOfFds:%d", pTcp->label, pFdObj, pTcp->numOfFds); + tTrace("%s %p, FD:%p is cleaned, numOfFds:%d", pTcp->label, pFdObj->thandle, pFdObj, pTcp->numOfFds); tfree(pFdObj); } @@ -250,29 +253,29 @@ static void *taosReadTcpData(void *param) { pFdObj = events[i].data.ptr; if (events[i].events & EPOLLERR) { - tTrace("%s TCP error happened on FD\n", pTcp->label); - taosCleanUpTcpFdObj(pFdObj); + tTrace("%s %p, TCP error happened on FD", pTcp->label, pFdObj->thandle); + taosReportBrokenLink(pFdObj); continue; } if (events[i].events & EPOLLHUP) { - tTrace("%s TCP FD hang up\n", pTcp->label); - taosCleanUpTcpFdObj(pFdObj); + tTrace("%s %p, TCP FD hang up", pTcp->label, pFdObj->thandle); + taosReportBrokenLink(pFdObj); continue; } int headLen = taosReadMsg(pFdObj->fd, &rpcHead, sizeof(SRpcHead)); if (headLen != sizeof(SRpcHead)) { - tError("%s read error, headLen:%d", pTcp->label, headLen); - taosCleanUpTcpFdObj(pFdObj); + tError("%s %p, read error, headLen:%d", pTcp->label, pFdObj->thandle, headLen); + taosReportBrokenLink(pFdObj); continue; } int32_t msgLen = (int32_t)htonl((uint32_t)rpcHead.msgLen); char *buffer = (char *)malloc((size_t)msgLen + tsRpcOverhead); if (NULL == buffer) { - tTrace("%s TCP malloc(size:%d) fail\n", pTcp->label, msgLen); - taosCleanUpTcpFdObj(pFdObj); + tTrace("%s %p, TCP malloc(size:%d) fail", pTcp->label, pFdObj->thandle, msgLen); + taosReportBrokenLink(pFdObj); continue; } @@ -281,9 +284,10 @@ static void *taosReadTcpData(void *param) { int32_t retLen = taosReadMsg(pFdObj->fd, msg + headLen, leftLen); if (leftLen != retLen) { - tError("%s read error, leftLen:%d retLen:%d", pTcp->label, leftLen, retLen); + tError("%s %p, read error, leftLen:%d retLen:%d", + pTcp->label, pFdObj->thandle, leftLen, retLen); tfree(buffer); - taosCleanUpTcpFdObj(pFdObj); + taosReportBrokenLink(pFdObj); continue; } diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index bec9621e3b..8280264764 100755 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -817,7 +817,7 @@ static void rpcProcessBrokenLink(SRpcConn *pConn) { SRpcInfo *pRpc = pConn->pRpc; tTrace("%s %p, link is broken", pRpc->label, pConn); - pConn->chandle = NULL; + // pConn->chandle = NULL; if (pConn->outType) { SRpcReqContext *pContext = pConn->pContext; diff --git a/src/rpc/src/rpcServer.c b/src/rpc/src/rpcServer.c index 7430157bcf..37576fa0f6 100644 --- a/src/rpc/src/rpcServer.c +++ b/src/rpc/src/rpcServer.c @@ -177,7 +177,6 @@ void taosCloseTcpServerConnection(void *chandle) { SFdObj *pFdObj = chandle; if (pFdObj == NULL) return; - pFdObj->thandle = NULL; taosCleanUpFdObj(pFdObj); } @@ -191,6 +190,25 @@ int taosSendTcpServerData(uint32_t ip, uint16_t port, void *data, int len, void #define maxEvents 10 +static void taosReportBrokenLink(SFdObj *pFdObj) { + + SThreadObj *pThreadObj = pFdObj->pThreadObj; + + // notify the upper layer, so it will clean the associated context + if (pFdObj->thandle) { + SRecvInfo recvInfo; + recvInfo.msg = NULL; + recvInfo.msgLen = 0; + recvInfo.ip = 0; + recvInfo.port = 0; + recvInfo.shandle = pThreadObj->shandle; + recvInfo.thandle = pFdObj->thandle;; + recvInfo.chandle = NULL; + recvInfo.connType = RPC_CONN_TCP; + (*(pThreadObj->processData))(&recvInfo); + } +} + static void taosProcessTcpData(void *param) { SThreadObj * pThreadObj; int i, fdNum; @@ -214,29 +232,29 @@ static void taosProcessTcpData(void *param) { pFdObj = events[i].data.ptr; if (events[i].events & EPOLLERR) { - tTrace("%s TCP thread:%d, error happened on FD", pThreadObj->label, pThreadObj->threadId); - taosCleanUpFdObj(pFdObj); + tTrace("%s %p, error happened on FD", pThreadObj->label, pFdObj->thandle); + taosReportBrokenLink(pFdObj); continue; } if (events[i].events & EPOLLHUP) { - tTrace("%s TCP thread:%d, FD hang up", pThreadObj->label, pThreadObj->threadId); - taosCleanUpFdObj(pFdObj); + tTrace("%s %p, FD hang up", pThreadObj->label, pFdObj->thandle); + taosReportBrokenLink(pFdObj); continue; } int32_t headLen = taosReadMsg(pFdObj->fd, &rpcHead, sizeof(SRpcHead)); if (headLen != sizeof(SRpcHead)) { - tError("%s read error, headLen:%d, errno:%d", pThreadObj->label, headLen, errno); - taosCleanUpFdObj(pFdObj); + tError("%s %p, read error, headLen:%d", pThreadObj->label, pFdObj->thandle, headLen); + taosReportBrokenLink(pFdObj); continue; } int32_t msgLen = (int32_t)htonl((uint32_t)rpcHead.msgLen); char *buffer = malloc(msgLen + tsRpcOverhead); if ( NULL == buffer) { - tError("%s TCP malloc(size:%d) fail\n", pThreadObj->label, msgLen); - taosCleanUpFdObj(pFdObj); + tError("%s %p, TCP malloc(size:%d) fail", pThreadObj->label, pFdObj->thandle, msgLen); + taosReportBrokenLink(pFdObj); continue; } @@ -245,8 +263,9 @@ static void taosProcessTcpData(void *param) { int32_t retLen = taosReadMsg(pFdObj->fd, msg + headLen, leftLen); if (leftLen != retLen) { - tError("%s read error, leftLen:%d retLen:%d", pThreadObj->label, leftLen, retLen); - taosCleanUpFdObj(pFdObj); + tError("%s %p, read error, leftLen:%d retLen:%d", + pThreadObj->label, pFdObj->thandle, leftLen, retLen); + taosReportBrokenLink(pFdObj); tfree(buffer); continue; } @@ -339,7 +358,7 @@ static void taosAcceptTcpConnection(void *arg) { pthread_cond_signal(&pThreadObj->fdReady); pthread_mutex_unlock(&(pThreadObj->threadMutex)); - tTrace("%s TCP thread:%d, a new connection from %s:%hu, FD:%p, numOfFds:%d", pServerObj->label, + tTrace("%s TCP thread:%d, new connection from %s:%hu, FD:%p, numOfFds:%d", pServerObj->label, pThreadObj->threadId, pFdObj->ipstr, pFdObj->port, pFdObj, pThreadObj->numOfFds); // pick up next thread for next connection @@ -349,27 +368,12 @@ static void taosAcceptTcpConnection(void *arg) { } static void taosCleanUpFdObj(SFdObj *pFdObj) { - SThreadObj *pThreadObj; if (pFdObj == NULL) return; if (pFdObj->signature != pFdObj) return; pFdObj->signature = NULL; - pThreadObj = pFdObj->pThreadObj; - - // notify the upper layer, so it will clean the associated context - if (pFdObj->thandle) { - SRecvInfo recvInfo; - recvInfo.msg = NULL; - recvInfo.msgLen = 0; - recvInfo.ip = 0; - recvInfo.port = 0; - recvInfo.shandle = pThreadObj->shandle; - recvInfo.thandle = pFdObj->thandle;; - recvInfo.chandle = NULL; - recvInfo.connType = RPC_CONN_TCP; - (*(pThreadObj->processData))(&recvInfo); - } + SThreadObj *pThreadObj = pFdObj->pThreadObj; close(pFdObj->fd); epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_DEL, pFdObj->fd, NULL); @@ -379,7 +383,8 @@ static void taosCleanUpFdObj(SFdObj *pFdObj) { pThreadObj->numOfFds--; if (pThreadObj->numOfFds < 0) - tError("%s TCP thread:%d, number of FDs shall never be negative", pThreadObj->label, pThreadObj->threadId); + tError("%s %p, TCP thread:%d, number of FDs is negative!!!", + pThreadObj->label, pFdObj->thandle, pThreadObj->threadId); // remove from the FdObject list @@ -395,8 +400,9 @@ static void taosCleanUpFdObj(SFdObj *pFdObj) { pthread_mutex_unlock(&pThreadObj->threadMutex); - tTrace("%s TCP thread:%d, FD:%p is cleaned up, numOfFds:%d", pThreadObj->label, pThreadObj->threadId, - pFdObj, pThreadObj->numOfFds); + tTrace("%s %p, FD:%p is cleaned, numOfFds:%d", + pThreadObj->label, pFdObj->thandle, pFdObj, pThreadObj->numOfFds); + tfree(pFdObj); } From d2e0f7de02140879668527edb9b62af4adea6cc4 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Sun, 12 Apr 2020 22:53:40 +0800 Subject: [PATCH 09/18] fix valgrind warning --- src/vnode/tsdb/src/tsdbMain.c | 10 +++++++++- src/vnode/tsdb/src/tsdbMeta.c | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c index 6873c69c2a..ee9413fc12 100644 --- a/src/vnode/tsdb/src/tsdbMain.c +++ b/src/vnode/tsdb/src/tsdbMain.c @@ -813,6 +813,13 @@ static SSkipListIterator **tsdbCreateTableIters(STsdbMeta *pMeta, int maxTables) return iters; } +static void tsdbFreeMemTable(SMemTable *pMemTable) { + if (pMemTable) { + tSkipListDestroy(pMemTable->pData); + free(pMemTable); + } +} + // Commit to file static void *tsdbCommitData(void *arg) { // TODO @@ -859,7 +866,8 @@ static void *tsdbCommitData(void *arg) { // TODO: free the skiplist for (int i = 0; i < pCfg->maxTables; i++) { STable *pTable = pMeta->tables[i]; - if (pTable && pTable->imem) { // Here has memory leak + if (pTable && pTable->imem) { + tsdbFreeMemTable(pTable->imem); pTable->imem = NULL; } } diff --git a/src/vnode/tsdb/src/tsdbMeta.c b/src/vnode/tsdb/src/tsdbMeta.c index 0eb6dde1d0..0ab276c120 100644 --- a/src/vnode/tsdb/src/tsdbMeta.c +++ b/src/vnode/tsdb/src/tsdbMeta.c @@ -34,7 +34,7 @@ void *tsdbEncodeTable(STable *pTable, int *contLen) { *contLen = tsdbEstimateTableEncodeSize(pTable); if (*contLen < 0) return NULL; - void *ret = malloc(*contLen); + void *ret = calloc(1, *contLen); if (ret == NULL) return NULL; void *ptr = ret; From 593e7bb79ac360f61d1486471df339fb77095185 Mon Sep 17 00:00:00 2001 From: lihui Date: Mon, 13 Apr 2020 09:27:53 +0800 Subject: [PATCH 10/18] [modify webdoc] --- documentation/webdocs/markdowndocs/administrator-ch.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/webdocs/markdowndocs/administrator-ch.md b/documentation/webdocs/markdowndocs/administrator-ch.md index cbccc03cbb..289f2dbf91 100644 --- a/documentation/webdocs/markdowndocs/administrator-ch.md +++ b/documentation/webdocs/markdowndocs/administrator-ch.md @@ -105,7 +105,7 @@ RESTful服务使用的端口号，所有的HTTP请求（TCP）都需要向该接 **shellActivityTimer** - 默认值：3 -系统在服务端保持结果集的最长时间，范围[1-120]。 +系统在服务端保持结果集的最长时间，单位：秒，范围[1-120]。 **maxUsers** - 默认值：10,000 @@ -138,7 +138,7 @@ RESTful服务使用的端口号，所有的HTTP请求（TCP）都需要向该接 系统（服务端和客户端）运行日志开关： - 131 仅输出错误和警告信息 -- 135 输入错误（ERROR）、警告（WARN）、信息（Info） +- 135 输出错误（ERROR）、警告（WARN）、信息（Info） 不同应用场景的数据往往具有不同的数据特征，比如保留天数、副本数、采集频次、记录大小、采集点的数量、压缩等都可完全不同。为获得在存储上的最高效率，TDengine提供如下存储相关的系统配置参数： From 65f0ff6d89e47bb6d5dcddfbd382a3d8dacfdd2e Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 13 Apr 2020 10:18:01 +0800 Subject: [PATCH 11/18] change tscPrint to tscTrace to avoid print debug info to shell console. --- src/client/src/tscServer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index ebac676b77..1d6f7d67e7 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -191,7 +191,7 @@ int tscSendMsgToServer(SSqlObj *pSql) { } if (pSql->cmd.command < TSDB_SQL_MGMT) { - tscPrint("%p msg:%s is sent to server %d", pSql, taosMsg[pSql->cmd.msgType], pSql->ipList.port); + tscTrace("%p msg:%s is sent to server %d", pSql, taosMsg[pSql->cmd.msgType], pSql->ipList.port); memcpy(pMsg, pSql->cmd.payload + tsRpcHeadSize, pSql->cmd.payloadLen); SRpcMsg rpcMsg = { From 2589fc9efb826b9e93a1828a39be82f83488b03e Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Mon, 13 Apr 2020 16:09:31 +0800 Subject: [PATCH 12/18] [td-98] fix time range query bug when data kept in buffer --- src/query/inc/queryExecutor.h | 2 +- src/query/src/queryExecutor.c | 9 +++++-- src/vnode/tsdb/src/tsdbRead.c | 47 +++++++++++++++++++++++++++-------- 3 files changed, 44 insertions(+), 14 deletions(-) diff --git a/src/query/inc/queryExecutor.h b/src/query/inc/queryExecutor.h index 3932650e1f..0302732995 100644 --- a/src/query/inc/queryExecutor.h +++ b/src/query/inc/queryExecutor.h @@ -160,7 +160,7 @@ typedef struct SQueryRuntimeEnv { SQueryCostSummary summary; bool stableQuery; // super table query or not void* pQueryHandle; - void* pSubQueryHandle; // another thread for + void* pSecQueryHandle; // another thread for SDiskbasedResultBuf* pResultBuf; // query result buffer based on blocked-wised disk file } SQueryRuntimeEnv; diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index 0f74cf1beb..8015360919 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -1552,6 +1552,8 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { destroyResultBuf(pRuntimeEnv->pResultBuf); tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle); + tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle); + pRuntimeEnv->pTSBuf = tsBufDestory(pRuntimeEnv->pTSBuf); } @@ -2565,7 +2567,7 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { dTrace("QInfo:%p query start, qrange:%" PRId64 "-%" PRId64 ", lastkey:%" PRId64 ", order:%d", GET_QINFO_ADDR(pRuntimeEnv), pQuery->window.skey, pQuery->window.ekey, pQuery->lastKey, pQuery->order.order); - tsdb_query_handle_t pQueryHandle = pRuntimeEnv->scanFlag == MASTER_SCAN? pRuntimeEnv->pQueryHandle:pRuntimeEnv->pSubQueryHandle; + tsdb_query_handle_t pQueryHandle = pRuntimeEnv->scanFlag == MASTER_SCAN? pRuntimeEnv->pQueryHandle:pRuntimeEnv->pSecQueryHandle; while (tsdbNextDataBlock(pQueryHandle)) { if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) { @@ -3557,7 +3559,10 @@ void scanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { taosArrayPush(cols, &pQuery->colList[i]); } - pRuntimeEnv->pSubQueryHandle = tsdbQueryByTableId(pQInfo->tsdb, &cond, pQInfo->pTableIdList, cols); + if (pRuntimeEnv->pSecQueryHandle != NULL) { + pRuntimeEnv->pSecQueryHandle = tsdbQueryByTableId(pQInfo->tsdb, &cond, pQInfo->pTableIdList, cols); + } + taosArrayDestroy(cols); status = pQuery->status; diff --git a/src/vnode/tsdb/src/tsdbRead.c b/src/vnode/tsdb/src/tsdbRead.c index 966debda8d..ee95199e5b 100644 --- a/src/vnode/tsdb/src/tsdbRead.c +++ b/src/vnode/tsdb/src/tsdbRead.c @@ -218,9 +218,31 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) { if (pTable->mem == NULL && pTable->imem == NULL) { return false; } + + STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); + pTable = pCheckInfo->pTableObj; + + if (pCheckInfo->iter == NULL) { + pCheckInfo->iter = tSkipListCreateIter(pTable->mem->pData); + if (pCheckInfo->iter == NULL) { + return false; + } + } + + if (!tSkipListIterNext(pCheckInfo->iter)) { // buffer is empty + return false; + } + SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter); + if (node == NULL) { + return false; + } + + SDataRow row = SL_GET_NODE_DATA(node); + pCheckInfo->lastKey = dataRowKey(row); // first timestamp in buffer + // all data in mem are checked already. - if (pTableCheckInfo->lastKey > pTable->mem->keyLast) { + if (pTableCheckInfo->lastKey > pHandle->window.ekey) { return false; } @@ -522,13 +544,15 @@ static int vnodeBinarySearchKey(char* pValue, int num, TSKEY key, int order) { int numOfPoints; TSKEY* keyList; + assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC); + if (num <= 0) return -1; keyList = (TSKEY*)pValue; firstPos = 0; lastPos = num - 1; - if (order == 0) { + if (order == TSDB_ORDER_DESC) { // find the first position which is smaller than the key while (1) { if (key >= keyList[lastPos]) return lastPos; @@ -596,8 +620,8 @@ static void filterDataInDataBlock(STsdbQueryHandle* pQueryHandle, STableCheckInf pQueryHandle->realNumOfRows = cur->pos + 1; pCheckInfo->lastKey = blockInfo.window.ekey - 1; } else { - endPos = - vnodeBinarySearchKey(pCols->cols[0].pData, pCols->numOfPoints, pQueryHandle->window.ekey, pQueryHandle->order); + int32_t order = (pQueryHandle->order == TSDB_ORDER_ASC)? TSDB_ORDER_DESC:TSDB_ORDER_ASC; + endPos = vnodeBinarySearchKey(pCols->cols[0].pData, pCols->numOfPoints, pQueryHandle->window.ekey, order); if (QUERY_IS_ASC_QUERY(pQueryHandle->order)) { if (endPos < cur->pos) { @@ -1042,7 +1066,7 @@ static int tsdbReadRowsFromCache(SSkipListIterator* pIter, TSKEY maxKey, int max int32_t numOfCols = taosArrayGetSize(pQueryHandle->pColumns); *skey = INT64_MIN; - while (tSkipListIterNext(pIter)) { + do/* (1) */{ SSkipListNode* node = tSkipListIterGet(pIter); if (node == NULL) break; @@ -1063,8 +1087,11 @@ static int tsdbReadRowsFromCache(SSkipListIterator* pIter, TSKEY maxKey, int max } numOfRows++; - if (numOfRows >= maxRowsToRead) break; - }; + if (numOfRows >= maxRowsToRead) { + break; + } + + } while(tSkipListIterNext(pIter)); return numOfRows; } @@ -1107,10 +1134,8 @@ SDataBlockInfo tsdbRetrieveDataBlockInfo(tsdb_query_handle_t* pQueryHandle) { if (pTable->mem != NULL) { // create mem table iterator if it is not created yet - if (pCheckInfo->iter == NULL) { - pCheckInfo->iter = tSkipListCreateIter(pTable->mem->pData); - } - rows = tsdbReadRowsFromCache(pCheckInfo->iter, INT64_MAX, 2, &skey, &ekey, pHandle); + assert(pCheckInfo->iter != NULL); + rows = tsdbReadRowsFromCache(pCheckInfo->iter, pHandle->window.ekey, 2, &skey, &ekey, pHandle); // update the last key value pCheckInfo->lastKey = ekey + 1; From 704d8da049519cfdc6e1da01e251e36e4254ddaf Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Mon, 13 Apr 2020 18:03:47 +0800 Subject: [PATCH 13/18] re-organize TCP connection code --- src/rpc/inc/rpcServer.h | 34 --- src/rpc/inc/{rpcClient.h => rpcTcp.h} | 12 +- src/rpc/src/rpcClient.c | 315 ------------------- src/rpc/src/rpcMain.c | 15 +- src/rpc/src/{rpcServer.c => rpcTcp.c} | 424 ++++++++++++-------------- 5 files changed, 212 insertions(+), 588 deletions(-) delete mode 100644 src/rpc/inc/rpcServer.h rename src/rpc/inc/{rpcClient.h => rpcTcp.h} (74%) delete mode 100644 src/rpc/src/rpcClient.c mode change 100755 => 100644 src/rpc/src/rpcMain.c rename src/rpc/src/{rpcServer.c => rpcTcp.c} (55%) diff --git a/src/rpc/inc/rpcServer.h b/src/rpc/inc/rpcServer.h deleted file mode 100644 index 6b238733a4..0000000000 --- a/src/rpc/inc/rpcServer.h +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _rpc_server_header_ -#define _rpc_server_header_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include "taosdef.h" - -void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle); -void taosCleanUpTcpServer(void *param); -void taosCloseTcpServerConnection(void *param); -int taosSendTcpServerData(uint32_t ip, uint16_t port, void *data, int len, void *chandle); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/rpc/inc/rpcClient.h b/src/rpc/inc/rpcTcp.h similarity index 74% rename from src/rpc/inc/rpcClient.h rename to src/rpc/inc/rpcTcp.h index c87ae79312..16972dbc7e 100644 --- a/src/rpc/inc/rpcClient.h +++ b/src/rpc/inc/rpcTcp.h @@ -13,20 +13,22 @@ * along with this program. If not, see . */ -#ifndef _rpc_client_header_ -#define _rpc_client_header_ +#ifndef _rpc_tcp_header_ +#define _rpc_tcp_header_ #ifdef __cplusplus extern "C" { #endif -#include "taosdef.h" +void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle); +void taosCleanUpTcpServer(void *param); void *taosInitTcpClient(char *ip, uint16_t port, char *label, int num, void *fp, void *shandle); void taosCleanUpTcpClient(void *chandle); void *taosOpenTcpClientConnection(void *shandle, void *thandle, char *ip, uint16_t port); -void taosCloseTcpClientConnection(void *chandle); -int taosSendTcpClientData(uint32_t ip, uint16_t port, void *data, int len, void *chandle); + +void taosCloseTcpConnection(void *chandle); +int taosSendTcpData(uint32_t ip, uint16_t port, void *data, int len, void *chandle); #ifdef __cplusplus } diff --git a/src/rpc/src/rpcClient.c b/src/rpc/src/rpcClient.c deleted file mode 100644 index 7ca24f6229..0000000000 --- a/src/rpc/src/rpcClient.c +++ /dev/null @@ -1,315 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "os.h" -#include "taosmsg.h" -#include "tlog.h" -#include "tsocket.h" -#include "tutil.h" -#include "rpcClient.h" -#include "rpcHead.h" - -#ifndef EPOLLWAKEUP -#define EPOLLWAKEUP (1u << 29) -#endif - -typedef struct _tcp_fd { - void *signature; - int fd; // TCP socket FD - void * thandle; - uint32_t ip; - char ipstr[20]; - uint16_t port; - struct _tcp_client *pTcp; - struct _tcp_fd * prev, *next; -} STcpFd; - -typedef struct _tcp_client { - pthread_t thread; - STcpFd * pHead; - pthread_mutex_t mutex; - pthread_cond_t fdReady; - int pollFd; - int numOfFds; - char label[12]; - char ipstr[20]; - void *shandle; // handle passed by upper layer during server initialization - void *(*processData)(SRecvInfo *pRecv); -} STcpClient; - -#define maxTcpEvents 100 - -static void taosCleanUpTcpFdObj(STcpFd *pFdObj); -static void *taosReadTcpData(void *param); - -void *taosInitTcpClient(char *ip, uint16_t port, char *label, int num, void *fp, void *shandle) { - STcpClient *pTcp; - pthread_attr_t thattr; - - pTcp = (STcpClient *)malloc(sizeof(STcpClient)); - memset(pTcp, 0, sizeof(STcpClient)); - strcpy(pTcp->label, label); - strcpy(pTcp->ipstr, ip); - pTcp->shandle = shandle; - - if (pthread_mutex_init(&(pTcp->mutex), NULL) < 0) { - tError("%s failed to init TCP client mutex(%s)", label, strerror(errno)); - return NULL; - } - - if (pthread_cond_init(&(pTcp->fdReady), NULL) != 0) { - tError("%s init TCP condition variable failed(%s)", label, strerror(errno)); - return NULL; - } - - pTcp->pollFd = epoll_create(10); // size does not matter - if (pTcp->pollFd < 0) { - tError("%s failed to create TCP client epoll", label); - return NULL; - } - - pTcp->processData = fp; - - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); - int code = pthread_create(&(pTcp->thread), &thattr, taosReadTcpData, (void *)(pTcp)); - pthread_attr_destroy(&thattr); - if (code != 0) { - tError("%s failed to create TCP read data thread(%s)", label, strerror(errno)); - return NULL; - } - - tTrace("%s TCP client is initialized, ip:%s:%hu", label, ip, port); - - return pTcp; -} - -void taosCleanUpTcpClient(void *chandle) { - STcpClient *pTcp = (STcpClient *)chandle; - if (pTcp == NULL) return; - - while (pTcp->pHead) { - taosCleanUpTcpFdObj(pTcp->pHead); - pTcp->pHead = pTcp->pHead->next; - } - - close(pTcp->pollFd); - - pthread_cancel(pTcp->thread); - pthread_join(pTcp->thread, NULL); - - // tTrace (":%s, all connections are cleaned up", pTcp->label); - - tfree(pTcp); -} - -void *taosOpenTcpClientConnection(void *shandle, void *thandle, char *ip, uint16_t port) { - STcpClient * pTcp = (STcpClient *)shandle; - STcpFd * pFdObj; - struct epoll_event event; - struct in_addr destIp; - int fd; - - fd = taosOpenTcpClientSocket(ip, port, pTcp->ipstr); - if (fd <= 0) return NULL; - - pFdObj = (STcpFd *)malloc(sizeof(STcpFd)); - if (pFdObj == NULL) { - tError("%s no enough resource to allocate TCP FD IDs", pTcp->label); - tclose(fd); - return NULL; - } - - memset(pFdObj, 0, sizeof(STcpFd)); - pFdObj->fd = fd; - strcpy(pFdObj->ipstr, ip); - inet_aton(ip, &destIp); - pFdObj->ip = destIp.s_addr; - pFdObj->port = port; - pFdObj->pTcp = pTcp; - pFdObj->thandle = thandle; - pFdObj->signature = pFdObj; - - event.events = EPOLLIN | EPOLLPRI | EPOLLWAKEUP; - event.data.ptr = pFdObj; - if (epoll_ctl(pTcp->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) { - tError("%s failed to add TCP FD for epoll, error:%s", pTcp->label, strerror(errno)); - tfree(pFdObj); - tclose(fd); - return NULL; - } - - // notify the data process, add into the FdObj list - pthread_mutex_lock(&(pTcp->mutex)); - pFdObj->next = pTcp->pHead; - if (pTcp->pHead) (pTcp->pHead)->prev = pFdObj; - pTcp->pHead = pFdObj; - pTcp->numOfFds++; - pthread_cond_signal(&pTcp->fdReady); - pthread_mutex_unlock(&(pTcp->mutex)); - - tTrace("%s TCP connection to %s:%hu is created, FD:%p numOfFds:%d", pTcp->label, ip, port, pFdObj, pTcp->numOfFds); - - return pFdObj; -} - -void taosCloseTcpClientConnection(void *chandle) { - STcpFd *pFdObj = (STcpFd *)chandle; - - if (pFdObj == NULL) return; - - taosCleanUpTcpFdObj(pFdObj); -} - -int taosSendTcpClientData(uint32_t ip, uint16_t port, void *data, int len, void *chandle) { - STcpFd *pFdObj = (STcpFd *)chandle; - - if (chandle == NULL) return -1; - - return (int)send(pFdObj->fd, data, (size_t)len, 0); -} - -static void taosReportBrokenLink(STcpFd *pFdObj) { - SRecvInfo recvInfo; - STcpClient *pTcp = pFdObj->pTcp; - - if (pFdObj->thandle) { - recvInfo.msg = NULL; - recvInfo.msgLen = 0; - recvInfo.ip = 0; - recvInfo.port = 0; - recvInfo.shandle = pTcp->shandle; - recvInfo.thandle = pFdObj->thandle;; - recvInfo.chandle = NULL; - recvInfo.connType = RPC_CONN_TCP; - (*(pTcp->processData))(&recvInfo); - } -} - -static void taosCleanUpTcpFdObj(STcpFd *pFdObj) { - - if (pFdObj == NULL) return; - if (pFdObj->signature != pFdObj) return; - - pFdObj->signature = NULL; - STcpClient *pTcp = pFdObj->pTcp; - - epoll_ctl(pTcp->pollFd, EPOLL_CTL_DEL, pFdObj->fd, NULL); - close(pFdObj->fd); - - pthread_mutex_lock(&pTcp->mutex); - - pTcp->numOfFds--; - - if (pTcp->numOfFds < 0) - tError("%s %p, number of FDs is negative!!!, FD:%p", pTcp->label, pFdObj->thandle, pFdObj); - - if (pFdObj->prev) { - (pFdObj->prev)->next = pFdObj->next; - } else { - pTcp->pHead = pFdObj->next; - } - - if (pFdObj->next) { - (pFdObj->next)->prev = pFdObj->prev; - } - - pthread_mutex_unlock(&pTcp->mutex); - - tTrace("%s %p, FD:%p is cleaned, numOfFds:%d", pTcp->label, pFdObj->thandle, pFdObj, pTcp->numOfFds); - - tfree(pFdObj); -} - -static void *taosReadTcpData(void *param) { - STcpClient *pTcp = (STcpClient *)param; - int i, fdNum; - STcpFd *pFdObj; - struct epoll_event events[maxTcpEvents]; - SRecvInfo recvInfo; - SRpcHead rpcHead; - - while (1) { - pthread_mutex_lock(&pTcp->mutex); - if (pTcp->numOfFds < 1) pthread_cond_wait(&pTcp->fdReady, &pTcp->mutex); - pthread_mutex_unlock(&pTcp->mutex); - - fdNum = epoll_wait(pTcp->pollFd, events, maxTcpEvents, -1); - if (fdNum < 0) continue; - - for (i = 0; i < fdNum; ++i) { - pFdObj = events[i].data.ptr; - - if (events[i].events & EPOLLERR) { - tTrace("%s %p, TCP error happened on FD", pTcp->label, pFdObj->thandle); - taosReportBrokenLink(pFdObj); - continue; - } - - if (events[i].events & EPOLLHUP) { - tTrace("%s %p, TCP FD hang up", pTcp->label, pFdObj->thandle); - taosReportBrokenLink(pFdObj); - continue; - } - - int headLen = taosReadMsg(pFdObj->fd, &rpcHead, sizeof(SRpcHead)); - if (headLen != sizeof(SRpcHead)) { - tError("%s %p, read error, headLen:%d", pTcp->label, pFdObj->thandle, headLen); - taosReportBrokenLink(pFdObj); - continue; - } - - int32_t msgLen = (int32_t)htonl((uint32_t)rpcHead.msgLen); - char *buffer = (char *)malloc((size_t)msgLen + tsRpcOverhead); - if (NULL == buffer) { - tTrace("%s %p, TCP malloc(size:%d) fail", pTcp->label, pFdObj->thandle, msgLen); - taosReportBrokenLink(pFdObj); - continue; - } - - char *msg = buffer + tsRpcOverhead; - int32_t leftLen = msgLen - headLen; - int32_t retLen = taosReadMsg(pFdObj->fd, msg + headLen, leftLen); - - if (leftLen != retLen) { - tError("%s %p, read error, leftLen:%d retLen:%d", - pTcp->label, pFdObj->thandle, leftLen, retLen); - tfree(buffer); - taosReportBrokenLink(pFdObj); - continue; - } - - // tTrace("%s TCP data is received, ip:%s:%u len:%d", pTcp->label, pFdObj->ipstr, pFdObj->port, msgLen); - - memcpy(msg, &rpcHead, sizeof(SRpcHead)); - recvInfo.msg = msg; - recvInfo.msgLen = msgLen; - recvInfo.ip = pFdObj->ip; - recvInfo.port = pFdObj->port; - recvInfo.shandle = pTcp->shandle; - recvInfo.thandle = pFdObj->thandle;; - recvInfo.chandle = pFdObj; - recvInfo.connType = RPC_CONN_TCP; - - pFdObj->thandle = (*(pTcp->processData))(&recvInfo); - - if (pFdObj->thandle == NULL) taosCleanUpTcpFdObj(pFdObj); - } - } - - return NULL; -} - - diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c old mode 100755 new mode 100644 index 8280264764..5048d5db14 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -27,8 +27,7 @@ #include "taosmsg.h" #include "rpcUdp.h" #include "rpcCache.h" -#include "rpcClient.h" -#include "rpcServer.h" +#include "rpcTcp.h" #include "rpcHead.h" #include "trpc.h" #include "hash.h" @@ -67,7 +66,7 @@ typedef struct { void *udphandle;// returned handle from UDP initialization void *pCache; // connection cache pthread_mutex_t mutex; - struct _RpcConn *connList; // connection list + struct SRpcConn *connList; // connection list } SRpcInfo; typedef struct { @@ -88,7 +87,7 @@ typedef struct { char msg[0]; // RpcHead starts from here } SRpcReqContext; -typedef struct _RpcConn { +typedef struct SRpcConn { int sid; // session ID uint32_t ownId; // own link ID uint32_t peerId; // peer link ID @@ -156,8 +155,8 @@ void (*taosCleanUpConn[])(void *thandle) = { int (*taosSendData[])(uint32_t ip, uint16_t port, void *data, int len, void *chandle) = { taosSendUdpData, taosSendUdpData, - taosSendTcpServerData, - taosSendTcpClientData + taosSendTcpData, + taosSendTcpData }; void *(*taosOpenConn[])(void *shandle, void *thandle, char *ip, uint16_t port) = { @@ -170,8 +169,8 @@ void *(*taosOpenConn[])(void *shandle, void *thandle, char *ip, uint16_t port) = void (*taosCloseConn[])(void *chandle) = { NULL, NULL, - taosCloseTcpServerConnection, - taosCloseTcpClientConnection + taosCloseTcpConnection, + taosCloseTcpConnection }; static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerIpStr, uint16_t peerPort, int8_t connType); diff --git a/src/rpc/src/rpcServer.c b/src/rpc/src/rpcTcp.c similarity index 55% rename from src/rpc/src/rpcServer.c rename to src/rpc/src/rpcTcp.c index 37576fa0f6..27b81deda5 100644 --- a/src/rpc/src/rpcServer.c +++ b/src/rpc/src/rpcTcp.c @@ -18,30 +18,30 @@ #include "tlog.h" #include "tsocket.h" #include "tutil.h" -#include "rpcServer.h" #include "rpcHead.h" +#include "rpcTcp.h" -#define TAOS_IPv4ADDR_LEN 16 #ifndef EPOLLWAKEUP #define EPOLLWAKEUP (1u << 29) #endif -typedef struct _fd_obj { - void *signature; - int fd; // TCP socket FD - void * thandle; // handle from upper layer, like TAOS - char ipstr[TAOS_IPv4ADDR_LEN]; - unsigned int ip; - uint16_t port; - struct _thread_obj *pThreadObj; - struct _fd_obj * prev, *next; +typedef struct SFdObj { + void *signature; + int fd; // TCP socket FD + void *thandle; // handle from upper layer, like TAOS + uint32_t ip; + uint16_t port; + struct SThreadObj *pThreadObj; + struct SFdObj *prev; + struct SFdObj *next; } SFdObj; -typedef struct _thread_obj { +typedef struct SThreadObj { pthread_t thread; SFdObj * pHead; - pthread_mutex_t threadMutex; + pthread_mutex_t mutex; pthread_cond_t fdReady; + char ipstr[TSDB_IPv4ADDR_LEN]; int pollFd; int numOfFds; int threadId; @@ -51,7 +51,7 @@ typedef struct _thread_obj { } SThreadObj; typedef struct { - char ip[40]; + char ip[TSDB_IPv4ADDR_LEN]; uint16_t port; char label[12]; int numOfThreads; @@ -60,13 +60,15 @@ typedef struct { pthread_t thread; } SServerObj; -static void taosCleanUpFdObj(SFdObj *pFdObj); -static void taosProcessTcpData(void *param); -static void taosAcceptTcpConnection(void *arg); +static void *taosProcessTcpData(void *param); +static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, int fd); +static void taosFreeFdObj(SFdObj *pFdObj); +static void taosReportBrokenLink(SFdObj *pFdObj); +static void taosAcceptTcpConnection(void *arg); void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle) { - SServerObj *pServerObj; - SThreadObj *pThreadObj; + SServerObj *pServerObj; + SThreadObj *pThreadObj; pServerObj = (SServerObj *)calloc(sizeof(SServerObj), 1); strcpy(pServerObj->ip, ip); @@ -88,7 +90,7 @@ void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, strcpy(pThreadObj->label, label); pThreadObj->shandle = shandle; - code = pthread_mutex_init(&(pThreadObj->threadMutex), NULL); + code = pthread_mutex_init(&(pThreadObj->mutex), NULL); if (code < 0) { tError("%s failed to init TCP process data mutex(%s)", label, strerror(errno)); break;; @@ -110,7 +112,7 @@ void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, pthread_attr_t thattr; pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); - code = pthread_create(&(pThreadObj->thread), &thattr, (void *)taosProcessTcpData, (void *)(pThreadObj)); + code = pthread_create(&(pThreadObj->thread), &thattr, taosProcessTcpData, (void *)(pThreadObj)); pthread_attr_destroy(&thattr); if (code != 0) { tError("%s failed to create TCP process data thread(%s)", label, strerror(errno)); @@ -144,8 +146,8 @@ void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, } void taosCleanUpTcpServer(void *handle) { - SThreadObj *pThreadObj; SServerObj *pServerObj = handle; + SThreadObj *pThreadObj; if (pServerObj == NULL) return; @@ -156,7 +158,7 @@ void taosCleanUpTcpServer(void *handle) { pThreadObj = pServerObj->pThreadObj + i; while (pThreadObj->pHead) { - taosCleanUpFdObj(pThreadObj->pHead); + taosFreeFdObj(pThreadObj->pHead); pThreadObj->pHead = pThreadObj->pHead; } @@ -164,7 +166,7 @@ void taosCleanUpTcpServer(void *handle) { pthread_cancel(pThreadObj->thread); pthread_join(pThreadObj->thread, NULL); pthread_cond_destroy(&(pThreadObj->fdReady)); - pthread_mutex_destroy(&(pThreadObj->threadMutex)); + pthread_mutex_destroy(&(pThreadObj->mutex)); } tTrace("TCP:%s, TCP server is cleaned up", pServerObj->label); @@ -173,14 +175,146 @@ void taosCleanUpTcpServer(void *handle) { tfree(pServerObj); } -void taosCloseTcpServerConnection(void *chandle) { +static void taosAcceptTcpConnection(void *arg) { + int connFd = -1; + struct sockaddr_in caddr; + int sockFd; + int threadId = 0; + SThreadObj *pThreadObj; + SServerObj *pServerObj; + + pServerObj = (SServerObj *)arg; + + sockFd = taosOpenTcpServerSocket(pServerObj->ip, pServerObj->port); + if (sockFd < 0) return; + + tTrace("%s TCP server is ready, ip:%s:%hu", pServerObj->label, pServerObj->ip, pServerObj->port); + + while (1) { + socklen_t addrlen = sizeof(caddr); + connFd = accept(sockFd, (struct sockaddr *)&caddr, &addrlen); + + if (connFd < 0) { + tError("%s TCP accept failure(%s)", pServerObj->label, errno, strerror(errno)); + continue; + } + + tTrace("%s TCP connection from ip:%s:%hu", pServerObj->label, inet_ntoa(caddr.sin_addr), caddr.sin_port); + taosKeepTcpAlive(connFd); + + // pick up the thread to handle this connection + pThreadObj = pServerObj->pThreadObj + threadId; + + SFdObj *pFdObj = taosMallocFdObj(pThreadObj, connFd); + if (pFdObj) { + pFdObj->ip = caddr.sin_addr.s_addr; + pFdObj->port = caddr.sin_port; + tTrace("%s new connection from %s:%hu, FD:%p, numOfFds:%d", pServerObj->label, + inet_ntoa(caddr.sin_addr), pFdObj->port, pFdObj, pThreadObj->numOfFds); + } else { + close(connFd); + tError("%s failed to malloc FdObj(%s)", pServerObj->label, strerror(errno)); + } + + // pick up next thread for next connection + threadId++; + threadId = threadId % pServerObj->numOfThreads; + } +} + +void *taosInitTcpClient(char *ip, uint16_t port, char *label, int num, void *fp, void *shandle) { + SThreadObj *pThreadObj; + pthread_attr_t thattr; + + pThreadObj = (SThreadObj *)malloc(sizeof(SThreadObj)); + memset(pThreadObj, 0, sizeof(SThreadObj)); + strcpy(pThreadObj->label, label); + strcpy(pThreadObj->ipstr, ip); + pThreadObj->shandle = shandle; + + if (pthread_mutex_init(&(pThreadObj->mutex), NULL) < 0) { + tError("%s failed to init TCP client mutex(%s)", label, strerror(errno)); + return NULL; + } + + if (pthread_cond_init(&(pThreadObj->fdReady), NULL) != 0) { + tError("%s init TCP condition variable failed(%s)", label, strerror(errno)); + return NULL; + } + + pThreadObj->pollFd = epoll_create(10); // size does not matter + if (pThreadObj->pollFd < 0) { + tError("%s failed to create TCP client epoll", label); + return NULL; + } + + pThreadObj->processData = fp; + + pthread_attr_init(&thattr); + pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + int code = pthread_create(&(pThreadObj->thread), &thattr, taosProcessTcpData, (void *)(pThreadObj)); + pthread_attr_destroy(&thattr); + if (code != 0) { + tError("%s failed to create TCP read data thread(%s)", label, strerror(errno)); + return NULL; + } + + tTrace("%s TCP client is initialized, ip:%s:%hu", label, ip, port); + + return pThreadObj; +} + +void taosCleanUpTcpClient(void *chandle) { + SThreadObj *pThreadObj = chandle; + if (pThreadObj == NULL) return; + + while (pThreadObj->pHead) { + taosFreeFdObj(pThreadObj->pHead); + pThreadObj->pHead = pThreadObj->pHead->next; + } + + close(pThreadObj->pollFd); + + pthread_cancel(pThreadObj->thread); + pthread_join(pThreadObj->thread, NULL); + + tTrace (":%s, all connections are cleaned up", pThreadObj->label); + + tfree(pThreadObj); +} + +void *taosOpenTcpClientConnection(void *shandle, void *thandle, char *ip, uint16_t port) { + SThreadObj * pThreadObj = shandle; + struct in_addr destIp; + + int fd = taosOpenTcpClientSocket(ip, port, pThreadObj->ipstr); + if (fd <= 0) return NULL; + + inet_aton(ip, &destIp); + SFdObj *pFdObj = taosMallocFdObj(pThreadObj, fd); + + if (pFdObj) { + pFdObj->thandle = thandle; + pFdObj->port = port; + pFdObj->ip = destIp.s_addr; + tTrace("%s %p, TCP connection to %s:%hu is created, FD:%p numOfFds:%d", + pThreadObj->label, thandle, ip, port, pFdObj, pThreadObj->numOfFds); + } else { + close(fd); + tError("%s failed to malloc client FdObj(%s)", pThreadObj->label, strerror(errno)); + } + + return pFdObj; +} + +void taosCloseTcpConnection(void *chandle) { SFdObj *pFdObj = chandle; if (pFdObj == NULL) return; - taosCleanUpFdObj(pFdObj); + taosFreeFdObj(pFdObj); } -int taosSendTcpServerData(uint32_t ip, uint16_t port, void *data, int len, void *chandle) { +int taosSendTcpData(uint32_t ip, uint16_t port, void *data, int len, void *chandle) { SFdObj *pFdObj = chandle; if (chandle == NULL) return -1; @@ -188,8 +322,6 @@ int taosSendTcpServerData(uint32_t ip, uint16_t port, void *data, int len, void return (int)send(pFdObj->fd, data, (size_t)len, 0); } -#define maxEvents 10 - static void taosReportBrokenLink(SFdObj *pFdObj) { SThreadObj *pThreadObj = pFdObj->pThreadObj; @@ -209,26 +341,26 @@ static void taosReportBrokenLink(SFdObj *pFdObj) { } } -static void taosProcessTcpData(void *param) { - SThreadObj * pThreadObj; - int i, fdNum; - SFdObj * pFdObj; +#define maxEvents 10 + +static void *taosProcessTcpData(void *param) { + SThreadObj *pThreadObj = param; + SFdObj *pFdObj; struct epoll_event events[maxEvents]; SRecvInfo recvInfo; - pThreadObj = (SThreadObj *)param; SRpcHead rpcHead; while (1) { - pthread_mutex_lock(&pThreadObj->threadMutex); + pthread_mutex_lock(&pThreadObj->mutex); if (pThreadObj->numOfFds < 1) { - pthread_cond_wait(&pThreadObj->fdReady, &pThreadObj->threadMutex); + pthread_cond_wait(&pThreadObj->fdReady, &pThreadObj->mutex); } - pthread_mutex_unlock(&pThreadObj->threadMutex); + pthread_mutex_unlock(&pThreadObj->mutex); - fdNum = epoll_wait(pThreadObj->pollFd, events, maxEvents, -1); + int fdNum = epoll_wait(pThreadObj->pollFd, events, maxEvents, -1); if (fdNum < 0) continue; - for (i = 0; i < fdNum; ++i) { + for (int i = 0; i < fdNum; ++i) { pFdObj = events[i].data.ptr; if (events[i].events & EPOLLERR) { @@ -270,7 +402,7 @@ static void taosProcessTcpData(void *param) { continue; } - // tTrace("%s TCP data is received, ip:%s:%u len:%d", pTcp->label, pFdObj->ipstr, pFdObj->port, msgLen); + // tTrace("%s TCP data is received, ip:%s:%u len:%d", pThreadObj->label, pFdObj->ipstr, pFdObj->port, msgLen); memcpy(msg, &rpcHead, sizeof(SRpcHead)); recvInfo.msg = msg; @@ -283,91 +415,43 @@ static void taosProcessTcpData(void *param) { recvInfo.connType = RPC_CONN_TCP; pFdObj->thandle = (*(pThreadObj->processData))(&recvInfo); - if (pFdObj->thandle == NULL) taosCleanUpFdObj(pFdObj); + if (pFdObj->thandle == NULL) taosFreeFdObj(pFdObj); } } + + return NULL; } -static void taosAcceptTcpConnection(void *arg) { - int connFd = -1; - struct sockaddr_in clientAddr; - int sockFd; - int threadId = 0; - SThreadObj * pThreadObj; - SServerObj * pServerObj; - SFdObj * pFdObj; +static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, int fd) { struct epoll_event event; - pServerObj = (SServerObj *)arg; + SFdObj *pFdObj = (SFdObj *)calloc(sizeof(SFdObj), 1); + if (pFdObj == NULL) return NULL; - sockFd = taosOpenTcpServerSocket(pServerObj->ip, pServerObj->port); + pFdObj->fd = fd; + pFdObj->pThreadObj = pThreadObj; + pFdObj->signature = pFdObj; - if (sockFd < 0) { - tError("%s failed to open TCP socket, ip:%s:%hu", pServerObj->label, pServerObj->ip, pServerObj->port); - return; - } else { - tTrace("%s TCP server is ready, ip:%s:%hu", pServerObj->label, pServerObj->ip, pServerObj->port); + event.events = EPOLLIN | EPOLLPRI | EPOLLWAKEUP; + event.data.ptr = pFdObj; + if (epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) { + tfree(pFdObj); + return NULL; } - while (1) { - socklen_t addrlen = sizeof(clientAddr); - connFd = accept(sockFd, (struct sockaddr *)&clientAddr, &addrlen); + // notify the data process, add into the FdObj list + pthread_mutex_lock(&(pThreadObj->mutex)); + pFdObj->next = pThreadObj->pHead; + if (pThreadObj->pHead) (pThreadObj->pHead)->prev = pFdObj; + pThreadObj->pHead = pFdObj; + pThreadObj->numOfFds++; + pthread_cond_signal(&pThreadObj->fdReady); + pthread_mutex_unlock(&(pThreadObj->mutex)); - if (connFd < 0) { - tError("%s TCP accept failure(%s)", pServerObj->label, errno, strerror(errno)); - continue; - } - - tTrace("%s TCP connection from ip:%s:%hu", pServerObj->label, inet_ntoa(clientAddr.sin_addr), - htons(clientAddr.sin_port)); - taosKeepTcpAlive(connFd); - - // pick up the thread to handle this connection - pThreadObj = pServerObj->pThreadObj + threadId; - - pFdObj = (SFdObj *)malloc(sizeof(SFdObj)); - if (pFdObj == NULL) { - tError("%s no enough resource to allocate TCP FD IDs", pServerObj->label); - close(connFd); - continue; - } - - memset(pFdObj, 0, sizeof(SFdObj)); - pFdObj->fd = connFd; - strcpy(pFdObj->ipstr, inet_ntoa(clientAddr.sin_addr)); - pFdObj->ip = clientAddr.sin_addr.s_addr; - pFdObj->port = htons(clientAddr.sin_port); - pFdObj->pThreadObj = pThreadObj; - pFdObj->signature = pFdObj; - - event.events = EPOLLIN | EPOLLPRI | EPOLLWAKEUP; - event.data.ptr = pFdObj; - if (epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_ADD, connFd, &event) < 0) { - tError("%s failed to add TCP FD for epoll(%s)", pServerObj->label, strerror(errno)); - tfree(pFdObj); - close(connFd); - continue; - } - - // notify the data process, add into the FdObj list - pthread_mutex_lock(&(pThreadObj->threadMutex)); - pFdObj->next = pThreadObj->pHead; - if (pThreadObj->pHead) (pThreadObj->pHead)->prev = pFdObj; - pThreadObj->pHead = pFdObj; - pThreadObj->numOfFds++; - pthread_cond_signal(&pThreadObj->fdReady); - pthread_mutex_unlock(&(pThreadObj->threadMutex)); - - tTrace("%s TCP thread:%d, new connection from %s:%hu, FD:%p, numOfFds:%d", pServerObj->label, - pThreadObj->threadId, pFdObj->ipstr, pFdObj->port, pFdObj, pThreadObj->numOfFds); - - // pick up next thread for next connection - threadId++; - threadId = threadId % pServerObj->numOfThreads; - } + return pFdObj; } -static void taosCleanUpFdObj(SFdObj *pFdObj) { +static void taosFreeFdObj(SFdObj *pFdObj) { if (pFdObj == NULL) return; if (pFdObj->signature != pFdObj) return; @@ -378,7 +462,7 @@ static void taosCleanUpFdObj(SFdObj *pFdObj) { close(pFdObj->fd); epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_DEL, pFdObj->fd, NULL); - pthread_mutex_lock(&pThreadObj->threadMutex); + pthread_mutex_lock(&pThreadObj->mutex); pThreadObj->numOfFds--; @@ -398,7 +482,7 @@ static void taosCleanUpFdObj(SFdObj *pFdObj) { (pFdObj->next)->prev = pFdObj->prev; } - pthread_mutex_unlock(&pThreadObj->threadMutex); + pthread_mutex_unlock(&pThreadObj->mutex); tTrace("%s %p, FD:%p is cleaned, numOfFds:%d", pThreadObj->label, pFdObj->thandle, pFdObj, pThreadObj->numOfFds); @@ -406,116 +490,4 @@ static void taosCleanUpFdObj(SFdObj *pFdObj) { tfree(pFdObj); } -#if 0 -static void taosAcceptUDConnection(void *arg) { - int connFd = -1; - int sockFd; - int threadId = 0; - SThreadObj * pThreadObj; - SServerObj * pServerObj; - SFdObj * pFdObj; - struct epoll_event event; - - pServerObj = (SServerObj *)arg; - sockFd = taosOpenUDServerSocket(pServerObj->ip, pServerObj->port); - - if (sockFd < 0) { - tError("%s failed to open UD socket, ip:%s, port:%hu", pServerObj->label, pServerObj->ip, pServerObj->port); - return; - } else { - tTrace("%s UD server is ready, ip:%s, port:%hu", pServerObj->label, pServerObj->ip, pServerObj->port); - } - - while (1) { - connFd = accept(sockFd, NULL, NULL); - - if (connFd < 0) { - tError("%s UD accept failure, errno:%d, reason:%s", pServerObj->label, errno, strerror(errno)); - continue; - } - - // pick up the thread to handle this connection - pThreadObj = pServerObj->pThreadObj + threadId; - - pFdObj = (SFdObj *)malloc(sizeof(SFdObj)); - if (pFdObj == NULL) { - tError("%s no enough resource to allocate TCP FD IDs", pServerObj->label); - close(connFd); - continue; - } - - memset(pFdObj, 0, sizeof(SFdObj)); - pFdObj->fd = connFd; - pFdObj->pThreadObj = pThreadObj; - - event.events = EPOLLIN | EPOLLPRI | EPOLLWAKEUP; - event.data.ptr = pFdObj; - if (epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_ADD, connFd, &event) < 0) { - tError("%s failed to add UD FD for epoll, error:%s", pServerObj->label, strerror(errno)); - tfree(pFdObj); - close(connFd); - continue; - } - - // notify the data process, add into the FdObj list - pthread_mutex_lock(&(pThreadObj->threadMutex)); - pFdObj->next = pThreadObj->pHead; - if (pThreadObj->pHead) (pThreadObj->pHead)->prev = pFdObj; - pThreadObj->pHead = pFdObj; - pThreadObj->numOfFds++; - pthread_cond_signal(&pThreadObj->fdReady); - pthread_mutex_unlock(&(pThreadObj->threadMutex)); - - tTrace("%s UD thread:%d, a new connection, numOfFds:%d", pServerObj->label, pThreadObj->threadId, - pThreadObj->numOfFds); - - // pick up next thread for next connection - threadId++; - threadId = threadId % pServerObj->numOfThreads; - } -} -#endif - -#if 0 -void taosListTcpConnection(void *handle, char *buffer) { - SServerObj *pServerObj; - SThreadObj *pThreadObj; - SFdObj * pFdObj; - int i, numOfFds, numOfConns; - char * msg; - - pServerObj = (SServerObj *)handle; - buffer[0] = 0; - msg = buffer; - numOfConns = 0; - - pThreadObj = pServerObj->pThreadObj; - - for (i = 0; i < pServerObj->numOfThreads; ++i) { - numOfFds = 0; - sprintf(msg, "TCP:%s Thread:%d number of connections:%d\n", pServerObj->label, pThreadObj->threadId, - pThreadObj->numOfFds); - msg = msg + strlen(msg); - pFdObj = pThreadObj->pHead; - while (pFdObj) { - sprintf(msg, " ip:%s port:%hu\n", pFdObj->ipstr, pFdObj->port); - msg = msg + strlen(msg); - numOfFds++; - numOfConns++; - pFdObj = pFdObj->next; - } - - if (numOfFds != pThreadObj->numOfFds) - tError("TCP:%s thread:%d BIG error, numOfFds:%d actual numOfFds:%d", pServerObj->label, pThreadObj->threadId, - pThreadObj->numOfFds, numOfFds); - - pThreadObj++; - } - - sprintf(msg, "TCP:%s total connections:%d\n", pServerObj->label, numOfConns); - - return; -} -#endif - From e01081f2755596a1822bd4ea1d2cda189bb40468 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Mon, 13 Apr 2020 18:50:30 +0800 Subject: [PATCH 14/18] rename some structures --- src/rpc/src/rpcCache.c | 10 +++++----- src/rpc/src/rpcHaship.c | 14 +++++++------- src/rpc/src/rpcUdp.c | 24 ++++++++++++------------ 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/src/rpc/src/rpcCache.c b/src/rpc/src/rpcCache.c index a4863ef61d..3370b31277 100644 --- a/src/rpc/src/rpcCache.c +++ b/src/rpc/src/rpcCache.c @@ -22,18 +22,18 @@ #include "tutil.h" #include "rpcCache.h" -typedef struct _c_hash_t { +typedef struct SConnHash { uint32_t ip; uint16_t port; char connType; - struct _c_hash_t *prev; - struct _c_hash_t *next; - void * data; + struct SConnHash *prev; + struct SConnHash *next; + void *data; uint64_t time; } SConnHash; typedef struct { - SConnHash ** connHashList; + SConnHash **connHashList; mpool_h connHashMemPool; int maxSessions; int total; diff --git a/src/rpc/src/rpcHaship.c b/src/rpc/src/rpcHaship.c index c904dba4a1..43f2d138de 100644 --- a/src/rpc/src/rpcHaship.c +++ b/src/rpc/src/rpcHaship.c @@ -17,13 +17,13 @@ #include "tlog.h" #include "tmempool.h" -typedef struct _ip_hash_t { +typedef struct SIpHash { uint32_t ip; uint16_t port; int hash; - struct _ip_hash_t *prev; - struct _ip_hash_t *next; - void * data; + struct SIpHash *prev; + struct SIpHash *next; + void *data; } SIpHash; typedef struct { @@ -47,7 +47,7 @@ int rpcHashIp(void *handle, uint32_t ip, uint16_t port) { void *rpcAddIpHash(void *handle, void *data, uint32_t ip, uint16_t port) { int hash; - SIpHash * pNode; + SIpHash *pNode; SHashObj *pObj; pObj = (SHashObj *)handle; @@ -70,7 +70,7 @@ void *rpcAddIpHash(void *handle, void *data, uint32_t ip, uint16_t port) { void rpcDeleteIpHash(void *handle, uint32_t ip, uint16_t port) { int hash; - SIpHash * pNode; + SIpHash *pNode; SHashObj *pObj; pObj = (SHashObj *)handle; @@ -102,7 +102,7 @@ void rpcDeleteIpHash(void *handle, uint32_t ip, uint16_t port) { void *rpcGetIpHash(void *handle, uint32_t ip, uint16_t port) { int hash; - SIpHash * pNode; + SIpHash *pNode; SHashObj *pObj; pObj = (SHashObj *)handle; diff --git a/src/rpc/src/rpcUdp.c b/src/rpc/src/rpcUdp.c index e666187cf1..1e5ac3c6b5 100644 --- a/src/rpc/src/rpcUdp.c +++ b/src/rpc/src/rpcUdp.c @@ -32,7 +32,7 @@ int tsUdpDelay = 0; typedef struct { - void * signature; + void *signature; int index; int fd; uint16_t port; // peer port @@ -53,23 +53,23 @@ typedef struct { int server; char ip[16]; // local IP uint16_t port; // local Port - void * shandle; // handle passed by upper layer during server initialization + void *shandle; // handle passed by upper layer during server initialization int threads; char label[12]; - void * tmrCtrl; + void *tmrCtrl; void *(*fp)(SRecvInfo *pPacket); SUdpConn udpConn[]; } SUdpConnSet; typedef struct { - void * signature; + void *signature; uint32_t ip; // dest IP uint16_t port; // dest Port - SUdpConn * pConn; + SUdpConn *pConn; struct sockaddr_in destAdd; - void * msgHdr; + void *msgHdr; int totalLen; - void * timer; + void *timer; int emptyNum; } SUdpBuf; @@ -78,8 +78,8 @@ static SUdpBuf *taosCreateUdpBuf(SUdpConn *pConn, uint32_t ip, uint16_t port); static void taosProcessUdpBufTimer(void *param, void *tmrId); void *taosInitUdpConnection(char *ip, uint16_t port, char *label, int threads, void *fp, void *shandle) { - SUdpConn * pConn; - SUdpConnSet * pSet; + SUdpConn *pConn; + SUdpConnSet *pSet; int size = (int)sizeof(SUdpConnSet) + threads * (int)sizeof(SUdpConn); pSet = (SUdpConnSet *)malloc((size_t)size); @@ -164,7 +164,7 @@ void *taosInitUdpConnection(char *ip, uint16_t port, char *label, int threads, v void taosCleanUpUdpConnection(void *handle) { SUdpConnSet *pSet = (SUdpConnSet *)handle; - SUdpConn * pConn; + SUdpConn *pConn; if (pSet == NULL) return; @@ -205,10 +205,10 @@ void *taosOpenUdpConnection(void *shandle, void *thandle, char *ip, uint16_t por } static void *taosRecvUdpData(void *param) { + SUdpConn *pConn = param; struct sockaddr_in sourceAdd; int dataLen; unsigned int addLen; - SUdpConn * pConn = (SUdpConn *)param; uint16_t port; int minSize = sizeof(SRpcHead); SRecvInfo recvInfo; @@ -274,7 +274,7 @@ static void *taosRecvUdpData(void *param) { int taosSendUdpData(uint32_t ip, uint16_t port, void *data, int dataLen, void *chandle) { SUdpConn *pConn = (SUdpConn *)chandle; - SUdpBuf * pBuf; + SUdpBuf *pBuf; if (pConn == NULL || pConn->signature != pConn) return -1; From f2b41206ce816b3e751b89fca931058aaa271bb4 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Mon, 13 Apr 2020 19:25:08 +0800 Subject: [PATCH 15/18] rename a structure --- src/dnode/src/dnodeWrite.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dnode/src/dnodeWrite.c b/src/dnode/src/dnodeWrite.c index b56e0d8ad7..7eb4add831 100644 --- a/src/dnode/src/dnodeWrite.c +++ b/src/dnode/src/dnodeWrite.c @@ -41,7 +41,7 @@ typedef struct { SRpcMsg rpcMsg; } SWriteMsg; -typedef struct _wworker_pool { +typedef struct { int32_t max; // max number of workers int32_t nextId; // from 0 to max-1, cyclic SWriteWorker *writeWorker; From ea6898a26de3c92e7c98d5d122af9434d345b4db Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Mon, 13 Apr 2020 22:59:40 +0800 Subject: [PATCH 16/18] [td-98] fix time range query bug when data kept in buffer --- src/query/src/queryExecutor.c | 27 +++++---- src/util/src/tcompare.c | 7 ++- src/util/src/tskiplist.c | 52 ++--------------- src/vnode/tsdb/src/tsdbRead.c | 103 ++++++++++++++++++++++++---------- 4 files changed, 98 insertions(+), 91 deletions(-) diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index 8015360919..9607157bda 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -362,10 +362,10 @@ bool isTSCompQuery(SQuery *pQuery) { return pQuery->pSelectExpr[0].pBase.functio bool doRevisedResultsByLimit(SQInfo *pQInfo) { SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - if ((pQuery->limit.limit > 0) && (pQuery->rec.rows + pQuery->rec.rows > pQuery->limit.limit)) { - pQuery->rec.rows = pQuery->limit.limit - pQuery->rec.rows; - - // query completed + if ((pQuery->limit.limit > 0) && (pQuery->rec.total + pQuery->rec.rows > pQuery->limit.limit)) { + pQuery->rec.rows = pQuery->limit.limit - pQuery->rec.total; + assert(pQuery->rec.rows > 0); + setQueryStatus(pQuery, QUERY_COMPLETED); return true; } @@ -2503,17 +2503,20 @@ SArray *loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBl } int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order) { - int firstPos, lastPos, midPos = -1; - int numOfPoints; - TSKEY *keyList; + int32_t midPos = -1; + int32_t numOfPoints; - if (num <= 0) return -1; + if (num <= 0) { + return -1; + } - keyList = (TSKEY *)pValue; - firstPos = 0; - lastPos = num - 1; + assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC); + + TSKEY* keyList = (TSKEY *)pValue; + int32_t firstPos = 0; + int32_t lastPos = num - 1; - if (order == 0) { + if (order == TSDB_ORDER_DESC) { // find the first position which is smaller than the key while (1) { if (key >= keyList[lastPos]) return lastPos; diff --git a/src/util/src/tcompare.c b/src/util/src/tcompare.c index bff10c3022..848eeee573 100644 --- a/src/util/src/tcompare.c +++ b/src/util/src/tcompare.c @@ -13,7 +13,7 @@ int32_t compareInt32Val(const void *pLeft, const void *pRight) { } int32_t compareInt64Val(const void *pLeft, const void *pRight) { - int32_t ret = GET_INT64_VAL(pLeft) - GET_INT64_VAL(pRight); + int64_t ret = GET_INT64_VAL(pLeft) - GET_INT64_VAL(pRight); if (ret == 0) { return 0; } else { @@ -248,8 +248,9 @@ __compar_fn_t getComparFunc(int32_t type, int32_t filterDataType) { case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_SMALLINT: case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_BIGINT: { - if (filterDataType == TSDB_DATA_TYPE_BIGINT) { + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: { + if (filterDataType == TSDB_DATA_TYPE_BIGINT || filterDataType == TSDB_DATA_TYPE_TIMESTAMP) { comparFn = compareInt64Val; break; } diff --git a/src/util/src/tskiplist.c b/src/util/src/tskiplist.c index 138c75c197..beb831ea67 100644 --- a/src/util/src/tskiplist.c +++ b/src/util/src/tskiplist.c @@ -75,53 +75,6 @@ static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **forward, SSk static SSkipListNode* tSkipListDoAppend(SSkipList *pSkipList, SSkipListNode *pNode); static SSkipListIterator* doCreateSkipListIterator(SSkipList *pSkipList, int32_t order); -//static __compar_fn_t getComparFunc(SSkipList *pSkipList, int32_t filterDataType) { -// __compar_fn_t comparFn = NULL; -// -// switch (pSkipList->keyInfo.type) { -// case TSDB_DATA_TYPE_TINYINT: -// case TSDB_DATA_TYPE_SMALLINT: -// case TSDB_DATA_TYPE_INT: -// case TSDB_DATA_TYPE_BIGINT: { -// if (filterDataType == TSDB_DATA_TYPE_BIGINT) { -// comparFn = compareInt64Val; -// break; -// } -// } -// case TSDB_DATA_TYPE_BOOL: { -// if (filterDataType >= TSDB_DATA_TYPE_BOOL && filterDataType <= TSDB_DATA_TYPE_BIGINT) { -// comparFn = compareInt32Val; -// } else if (filterDataType >= TSDB_DATA_TYPE_FLOAT && filterDataType <= TSDB_DATA_TYPE_DOUBLE) { -// comparFn = compareIntDoubleVal; -// } -// break; -// } -// case TSDB_DATA_TYPE_FLOAT: -// case TSDB_DATA_TYPE_DOUBLE: { -//// if (filterDataType >= TSDB_DATA_TYPE_BOOL && filterDataType <= TSDB_DATA_TYPE_BIGINT) { -//// comparFn = compareDoubleIntVal; -//// } else if (filterDataType >= TSDB_DATA_TYPE_FLOAT && filterDataType <= TSDB_DATA_TYPE_DOUBLE) { -//// comparFn = compareDoubleVal; -//// } -// if (filterDataType == TSDB_DATA_TYPE_DOUBLE) { -// comparFn = compareDoubleVal; -// } -// break; -// } -// case TSDB_DATA_TYPE_BINARY: -// comparFn = compareStrVal; -// break; -// case TSDB_DATA_TYPE_NCHAR: -// comparFn = compareWStrVal; -// break; -// default: -// comparFn = compareInt32Val; -// break; -// } -// -// return comparFn; -//} - static bool initForwardBackwardPtr(SSkipList* pSkipList) { uint32_t maxLevel = pSkipList->maxLevel; @@ -445,6 +398,11 @@ SSkipListIterator *tSkipListCreateIterFromVal(SSkipList* pSkipList, const char* iter->cur = forward[0]; // greater equals than the value } else { iter->cur = SL_GET_FORWARD_POINTER(forward[0], 0); + + if (ret == 0) { + assert(iter->cur != pSkipList->pTail); + iter->cur = SL_GET_FORWARD_POINTER(iter->cur, 0); + } } return iter; diff --git a/src/vnode/tsdb/src/tsdbRead.c b/src/vnode/tsdb/src/tsdbRead.c index ee95199e5b..bf45c2d0af 100644 --- a/src/vnode/tsdb/src/tsdbRead.c +++ b/src/vnode/tsdb/src/tsdbRead.c @@ -26,7 +26,7 @@ #define EXTRA_BYTES 2 #define PRIMARY_TSCOL_REQUIRED(c) (((SColumnInfoData*)taosArrayGet(c, 0))->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) -#define QUERY_IS_ASC_QUERY(o) (o == TSDB_ORDER_ASC) +#define ASCENDING_ORDER_TRAVERSE(o) (o == TSDB_ORDER_ASC) #define QH_GET_NUM_OF_COLS(handle) ((size_t)(taosArrayGetSize((handle)->pColumns))) enum { @@ -154,6 +154,7 @@ tsdb_query_handle_t* tsdbQueryByTableId(tsdb_repo_t* tsdb, STsdbQueryCond* pCond pQueryHandle->loadDataAfterSeek = false; pQueryHandle->isFirstSlot = true; + pQueryHandle->cur.fid = -1; size_t size = taosArrayGetSize(idList); assert(size >= 1); @@ -183,7 +184,7 @@ tsdb_query_handle_t* tsdbQueryByTableId(tsdb_repo_t* tsdb, STsdbQueryCond* pCond * For ascending timestamp order query, query starts from data files. In contrast, buffer will be checked in the first place * in case of descending timestamp order query. */ - pQueryHandle->checkFiles = QUERY_IS_ASC_QUERY(pQueryHandle->order); + pQueryHandle->checkFiles = ASCENDING_ORDER_TRAVERSE(pQueryHandle->order); pQueryHandle->activeIndex = 0; // allocate buffer in order to load data blocks from file @@ -208,10 +209,11 @@ tsdb_query_handle_t* tsdbQueryByTableId(tsdb_repo_t* tsdb, STsdbQueryCond* pCond static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) { assert(pHandle->activeIndex == 0 && taosArrayGetSize(pHandle->pTableCheckInfo) == 1); + pHandle->cur.fid = -1; + + STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); - STableCheckInfo* pTableCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); - - STable* pTable = pTableCheckInfo->pTableObj; + STable* pTable = pCheckInfo->pTableObj; assert(pTable != NULL); // no data in cache, abort @@ -219,11 +221,10 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) { return false; } - STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); - pTable = pCheckInfo->pTableObj; - if (pCheckInfo->iter == NULL) { - pCheckInfo->iter = tSkipListCreateIter(pTable->mem->pData); + pCheckInfo->iter = tSkipListCreateIterFromVal(pTable->mem->pData, (const char*) &pCheckInfo->lastKey, + TSDB_DATA_TYPE_TIMESTAMP, pHandle->order); + if (pCheckInfo->iter == NULL) { return false; } @@ -240,9 +241,12 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) { SDataRow row = SL_GET_NODE_DATA(node); pCheckInfo->lastKey = dataRowKey(row); // first timestamp in buffer + dTrace("%p uid:%" PRId64", tid:%d check data in buffer from skey:%" PRId64 ", order:%d", pHandle, + pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, pCheckInfo->lastKey, pHandle->order); // all data in mem are checked already. - if (pTableCheckInfo->lastKey > pHandle->window.ekey) { + if ((pCheckInfo->lastKey > pHandle->window.ekey && ASCENDING_ORDER_TRAVERSE(pHandle->order)) || + (pCheckInfo->lastKey < pHandle->window.ekey && !ASCENDING_ORDER_TRAVERSE(pHandle->order))) { return false; } @@ -440,7 +444,7 @@ static bool loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock SArray* sa = getDefaultLoadColumns(pQueryHandle, true); SQueryFilePos* cur = &pQueryHandle->cur; - if (QUERY_IS_ASC_QUERY(pQueryHandle->order)) { + if (ASCENDING_ORDER_TRAVERSE(pQueryHandle->order)) { // query ended in current block if (pQueryHandle->window.ekey < pBlock->keyLast || pCheckInfo->lastKey > pBlock->keyFirst) { if (!doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo)) { @@ -611,11 +615,11 @@ static void filterDataInDataBlock(STsdbQueryHandle* pQueryHandle, STableCheckInf SDataCols* pCols = pCheckInfo->pDataCols; int32_t endPos = cur->pos; - if (QUERY_IS_ASC_QUERY(pQueryHandle->order) && pQueryHandle->window.ekey > blockInfo.window.ekey) { + if (ASCENDING_ORDER_TRAVERSE(pQueryHandle->order) && pQueryHandle->window.ekey > blockInfo.window.ekey) { endPos = blockInfo.rows - 1; pQueryHandle->realNumOfRows = endPos - cur->pos + 1; pCheckInfo->lastKey = blockInfo.window.ekey + 1; - } else if (!QUERY_IS_ASC_QUERY(pQueryHandle->order) && pQueryHandle->window.ekey < blockInfo.window.skey) { + } else if (!ASCENDING_ORDER_TRAVERSE(pQueryHandle->order) && pQueryHandle->window.ekey < blockInfo.window.skey) { endPos = 0; pQueryHandle->realNumOfRows = cur->pos + 1; pCheckInfo->lastKey = blockInfo.window.ekey - 1; @@ -623,7 +627,7 @@ static void filterDataInDataBlock(STsdbQueryHandle* pQueryHandle, STableCheckInf int32_t order = (pQueryHandle->order == TSDB_ORDER_ASC)? TSDB_ORDER_DESC:TSDB_ORDER_ASC; endPos = vnodeBinarySearchKey(pCols->cols[0].pData, pCols->numOfPoints, pQueryHandle->window.ekey, order); - if (QUERY_IS_ASC_QUERY(pQueryHandle->order)) { + if (ASCENDING_ORDER_TRAVERSE(pQueryHandle->order)) { if (endPos < cur->pos) { pQueryHandle->realNumOfRows = 0; return; @@ -940,7 +944,7 @@ static bool getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle) { int32_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); while ((pQueryHandle->pFileGroup = tsdbGetFileGroupNext(&pQueryHandle->fileIter)) != NULL) { - int32_t type = QUERY_IS_ASC_QUERY(pQueryHandle->order)? QUERY_RANGE_GREATER_EQUAL:QUERY_RANGE_LESS_EQUAL; + int32_t type = ASCENDING_ORDER_TRAVERSE(pQueryHandle->order)? QUERY_RANGE_GREATER_EQUAL:QUERY_RANGE_LESS_EQUAL; if (getFileCompInfo(pQueryHandle, &numOfBlocks, type) != TSDB_CODE_SUCCESS) { break; } @@ -968,7 +972,7 @@ static bool getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle) { return false; } - cur->slot = QUERY_IS_ASC_QUERY(pQueryHandle->order)? 0:pQueryHandle->numOfBlocks-1; + cur->slot = ASCENDING_ORDER_TRAVERSE(pQueryHandle->order)? 0:pQueryHandle->numOfBlocks-1; cur->fid = pQueryHandle->pFileGroup->fileId; STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot]; @@ -993,16 +997,16 @@ static bool getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle) { return getDataBlocksInFilesImpl(pQueryHandle); } else { - if ((cur->slot == pQueryHandle->numOfBlocks - 1 && QUERY_IS_ASC_QUERY(pQueryHandle->order)) || - (cur->slot == 0 && !QUERY_IS_ASC_QUERY(pQueryHandle->order))) { // all blocks + if ((cur->slot == pQueryHandle->numOfBlocks - 1 && ASCENDING_ORDER_TRAVERSE(pQueryHandle->order)) || + (cur->slot == 0 && !ASCENDING_ORDER_TRAVERSE(pQueryHandle->order))) { // all blocks return getDataBlocksInFilesImpl(pQueryHandle); } else { // next block of the same file - int32_t step = QUERY_IS_ASC_QUERY(pQueryHandle->order)? 1:-1; + int32_t step = ASCENDING_ORDER_TRAVERSE(pQueryHandle->order)? 1:-1; cur->slot += step; STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot]; - if (QUERY_IS_ASC_QUERY(pQueryHandle->order)) { + if (ASCENDING_ORDER_TRAVERSE(pQueryHandle->order)) { cur->pos = 0; } else { cur->pos = pBlockInfo->pBlock.compBlock->numOfPoints - 1; @@ -1035,7 +1039,7 @@ bool tsdbNextDataBlock(tsdb_query_handle_t* pqHandle) { size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); assert(numOfTables > 0); - if (QUERY_IS_ASC_QUERY(pQueryHandle->order)) { + if (ASCENDING_ORDER_TRAVERSE(pQueryHandle->order)) { if (pQueryHandle->checkFiles) { if (getDataBlocksInFiles(pQueryHandle)) { return true; @@ -1066,12 +1070,23 @@ static int tsdbReadRowsFromCache(SSkipListIterator* pIter, TSKEY maxKey, int max int32_t numOfCols = taosArrayGetSize(pQueryHandle->pColumns); *skey = INT64_MIN; - do/* (1) */{ + do { SSkipListNode* node = tSkipListIterGet(pIter); - if (node == NULL) break; + if (node == NULL) { + break; + } SDataRow row = SL_GET_NODE_DATA(node); - if (dataRowKey(row) > maxKey) break; + TSKEY key = dataRowKey(row); + + if ((key > maxKey && ASCENDING_ORDER_TRAVERSE(pQueryHandle->order)) || + (key < maxKey && !ASCENDING_ORDER_TRAVERSE(pQueryHandle->order))) { + + dTrace("%p key:%"PRIu64" beyond qrange:%"PRId64" - %"PRId64", no more data in buffer", pQueryHandle, key, pQueryHandle->window.skey, + pQueryHandle->window.ekey); + + break; + } if (*skey == INT64_MIN) { *skey = dataRowKey(row); @@ -1080,9 +1095,18 @@ static int tsdbReadRowsFromCache(SSkipListIterator* pIter, TSKEY maxKey, int max *ekey = dataRowKey(row); int32_t offset = 0; + char* pData = NULL; + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); - memcpy(pColInfo->pData + numOfRows * pColInfo->info.bytes, dataRowTuple(row) + offset, pColInfo->info.bytes); + + if (ASCENDING_ORDER_TRAVERSE(pQueryHandle->order)) { + pData = pColInfo->pData + numOfRows * pColInfo->info.bytes; + } else { + pData = pColInfo->pData + (maxRowsToRead - numOfRows - 1) * pColInfo->info.bytes; + } + + memcpy(pData, dataRowTuple(row) + offset, pColInfo->info.bytes); offset += pColInfo->info.bytes; } @@ -1093,6 +1117,18 @@ static int tsdbReadRowsFromCache(SSkipListIterator* pIter, TSKEY maxKey, int max } while(tSkipListIterNext(pIter)); + assert(numOfRows <= maxRowsToRead); + + // if the buffer is not full in case of descending order query, move the data in the front of the buffer + if (!ASCENDING_ORDER_TRAVERSE(pQueryHandle->order) && numOfRows < maxRowsToRead) { + int32_t emptySize = maxRowsToRead - numOfRows; + + for(int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); + memmove(pColInfo->pData, pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes); + } + } + return numOfRows; } @@ -1105,6 +1141,8 @@ SDataBlockInfo tsdbRetrieveDataBlockInfo(tsdb_query_handle_t* pQueryHandle) { TSKEY skey = 0, ekey = 0; int32_t rows = 0; + int32_t step = ASCENDING_ORDER_TRAVERSE(pHandle->order)? 1:-1; + // data in file if (pHandle->cur.fid >= 0) { STableBlockInfo* pBlockInfo = &pHandle->pDataBlockInfo[pHandle->cur.slot]; @@ -1126,7 +1164,7 @@ SDataBlockInfo tsdbRetrieveDataBlockInfo(tsdb_query_handle_t* pQueryHandle) { ekey = *(TSKEY*)((char*)pColInfoEx->pData + TSDB_KEYSIZE * (rows - 1)); // update the last key value - pBlockInfo->pTableCheckInfo->lastKey = ekey + 1; + pBlockInfo->pTableCheckInfo->lastKey = ekey + step; } } else { STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); @@ -1138,12 +1176,16 @@ SDataBlockInfo tsdbRetrieveDataBlockInfo(tsdb_query_handle_t* pQueryHandle) { rows = tsdbReadRowsFromCache(pCheckInfo->iter, pHandle->window.ekey, 2, &skey, &ekey, pHandle); // update the last key value - pCheckInfo->lastKey = ekey + 1; + pCheckInfo->lastKey = ekey + step; } } SDataBlockInfo blockInfo = { - .uid = pTable->tableId.uid, .sid = pTable->tableId.tid, .rows = rows, .window = {.skey = skey, .ekey = ekey}}; + .uid = pTable->tableId.uid, + .sid = pTable->tableId.tid, + .rows = rows, + .window = {.skey = MIN(skey, ekey), .ekey = MAX(skey, ekey)} + }; return blockInfo; } @@ -1399,7 +1441,10 @@ int32_t tsdbQueryTags(tsdb_repo_t* tsdb, int64_t uid, const char* pTagCond, size void tsdbCleanupQueryHandle(tsdb_query_handle_t queryHandle) { STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*)queryHandle; - + if (pQueryHandle == NULL) { + return; + } + size_t size = taosArrayGetSize(pQueryHandle->pTableCheckInfo); for (int32_t i = 0; i < size; ++i) { STableCheckInfo* pTableCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); From 8863766f7322372f2b72534797ae8d80ede44399 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 14 Apr 2020 11:02:01 +0800 Subject: [PATCH 17/18] make print output be python2 and python3 compatible. --- tests/pytest/util/log.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/tests/pytest/util/log.py b/tests/pytest/util/log.py index e570af85f5..c7032df3c4 100644 --- a/tests/pytest/util/log.py +++ b/tests/pytest/util/log.py @@ -15,6 +15,7 @@ import sys import os import time import datetime +from distutils.log import warn as printf class TDLog: @@ -22,27 +23,27 @@ class TDLog: self.path = "" def info(self, info): - print("%s %s" % (datetime.datetime.now(), info)) + printf("%s %s" % (datetime.datetime.now(), info)) def sleep(self, sec): - print("%s sleep %d seconds" % (datetime.datetime.now(), sec)) + printf("%s sleep %d seconds" % (datetime.datetime.now(), sec)) time.sleep(sec) def debug(self, err): - print("\033[1;36m%s %s\033[0m" % (datetime.datetime.now(), err)) + printf("\033[1;36m%s %s\033[0m" % (datetime.datetime.now(), err)) def success(self, info): - print("\033[1;32m%s %s\033[0m" % (datetime.datetime.now(), info)) + printf("\033[1;32m%s %s\033[0m" % (datetime.datetime.now(), info)) def notice(self, err): - print("\033[1;33m%s %s\033[0m" % (datetime.datetime.now(), err)) + printf("\033[1;33m%s %s\033[0m" % (datetime.datetime.now(), err)) def exit(self, err): - print("\033[1;31m%s %s\033[0m" % (datetime.datetime.now(), err)) + printf("\033[1;31m%s %s\033[0m" % (datetime.datetime.now(), err)) sys.exit(1) - def printNoPrefix(self, info): - print("\033[1;36m%s\033[0m" % (info)) + def printfNoPrefix(self, info): + printf("\033[1;36m%s\033[0m" % (info)) tdLog = TDLog() From 09189682e68edddf24099230f046d6368bbcd0ea Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Tue, 14 Apr 2020 15:38:12 +0800 Subject: [PATCH 18/18] [td-98] fix unit test compile failure bug. --- src/query/tests/patternMatchTest.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/query/tests/patternMatchTest.cpp b/src/query/tests/patternMatchTest.cpp index bca27ff42f..2e70f26269 100644 --- a/src/query/tests/patternMatchTest.cpp +++ b/src/query/tests/patternMatchTest.cpp @@ -4,6 +4,7 @@ #include #include "tsqlfunction.h" +#include "tcompare.h" TEST(testCase, patternMatchTest) { SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER;