xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch 'develop' into coverity_scan

This commit is contained in:
Shuduo Sang 2020-07-17 15:32:11 +08:00
parent 4edf364dac 99137f5a09
commit 2a36506229
223 changed files with 8758 additions and 4023 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

246
importSampleData/app/main.go
View File

@ -7,7 +7,6 @@ import (
"encoding/json" "encoding/json"
"flag" "flag"
"fmt" "fmt"
"github.com/taosdata/TDengine/importSampleData/import"
"hash/crc32" "hash/crc32"
"io" "io"
"log" "log"
@ -18,88 +17,89 @@ import (
"sync" "sync"
"time" "time"
dataimport "github.com/taosdata/TDengine/importSampleData/import"
_ "github.com/taosdata/driver-go/taosSql" _ "github.com/taosdata/driver-go/taosSql"
) )
const ( const (
TIMESTAMP = "timestamp" TIMESTAMP = "timestamp"
DATETIME = "datetime" DATETIME = "datetime"
MILLISECOND = "millisecond" MILLISECOND = "millisecond"
DEFAULT_STARTTIME int64 = -1 DEFAULT_STARTTIME int64 = -1
DEFAULT_INTERVAL int64 = 1*1000 DEFAULT_INTERVAL int64 = 1 * 1000
DEFAULT_DELAY int64 = -1 DEFAULT_DELAY int64 = -1
DEFAULT_STATISTIC_TABLE = "statistic" DEFAULT_STATISTIC_TABLE = "statistic"
JSON_FORMAT = "json" JSON_FORMAT = "json"
CSV_FORMAT = "csv" CSV_FORMAT = "csv"
SUPERTABLE_PREFIX = "s_" SUPERTABLE_PREFIX = "s_"
SUBTABLE_PREFIX = "t_" SUBTABLE_PREFIX = "t_"
DRIVER_NAME = "taosSql" DRIVER_NAME = "taosSql"
STARTTIME_LAYOUT = "2006-01-02 15:04:05.000" STARTTIME_LAYOUT = "2006-01-02 15:04:05.000"
INSERT_PREFIX = "insert into " INSERT_PREFIX = "insert into "
) )
var ( var (
cfg string
cfg string cases string
cases string hnum int
hnum int vnum int
vnum int thread int
thread int batch int
batch int auto int
auto int starttimestr string
starttimestr string interval int64
interval int64 host string
host string port int
port int user string
user string password string
password string dropdb int
dropdb int db string
db string dbparam string
dbparam string
dataSourceName string dataSourceName string
startTime int64 startTime int64
superTableConfigMap = make(map[string]*superTableConfig) superTableConfigMap = make(map[string]*superTableConfig)
subTableMap = make(map[string]*dataRows) subTableMap = make(map[string]*dataRows)
scaleTableNames []string scaleTableNames []string
scaleTableMap = make(map[string]*scaleTableInfo) scaleTableMap = make(map[string]*scaleTableInfo)
successRows []int64 successRows []int64
lastStaticTime time.Time lastStaticTime time.Time
lastTotalRows int64 lastTotalRows int64
timeTicker *time.Ticker timeTicker *time.Ticker
delay int64 // default 10 milliseconds delay int64 // default 10 milliseconds
tick int64 tick int64
save int save int
saveTable string saveTable string
) )
type superTableConfig struct { type superTableConfig struct {
startTime int64 startTime int64
endTime int64 endTime int64
cycleTime int64 cycleTime int64
avgInterval int64 avgInterval int64
config dataimport.CaseConfig config dataimport.CaseConfig
} }
type scaleTableInfo struct { type scaleTableInfo struct {
scaleTableName string scaleTableName string
subTableName string subTableName string
insertRows int64 insertRows int64
} }
type tableRows struct { type tableRows struct {
tableName string // tableName tableName string // tableName
value string // values(...) value string // values(...)
} }
type dataRows struct { type dataRows struct {
rows []map[string]interface{} rows []map[string]interface{}
config dataimport.CaseConfig config dataimport.CaseConfig
} }
func (rows dataRows) Len() int { func (rows dataRows) Len() int {
@ -127,10 +127,10 @@ func init() {
if db == "" { if db == "" {
//db = "go" //db = "go"
db = fmt.Sprintf("test_%s",time.Now().Format("20060102")) db = fmt.Sprintf("test_%s", time.Now().Format("20060102"))
} }
if auto == 1 && len(starttimestr) == 0 { if auto == 1 && len(starttimestr) == 0 {
log.Fatalf("startTime must be set when auto is 1, the format is \"yyyy-MM-dd HH:mm:ss.SSS\" ") log.Fatalf("startTime must be set when auto is 1, the format is \"yyyy-MM-dd HH:mm:ss.SSS\" ")
} }
@ -141,7 +141,7 @@ func init() {
} }
startTime = t.UnixNano() / 1e6 // as millisecond startTime = t.UnixNano() / 1e6 // as millisecond
}else{ } else {
startTime = DEFAULT_STARTTIME startTime = DEFAULT_STARTTIME
} }
@ -179,7 +179,7 @@ func main() {
_, exists := superTableConfigMap[caseConfig.Stname] _, exists := superTableConfigMap[caseConfig.Stname]
if !exists { if !exists {
superTableConfigMap[caseConfig.Stname] = &superTableConfig{config:caseConfig} superTableConfigMap[caseConfig.Stname] = &superTableConfig{config: caseConfig}
} else { } else {
log.Fatalf("the stname of case %s already exist.\n", caseConfig.Stname) log.Fatalf("the stname of case %s already exist.\n", caseConfig.Stname)
} }
@ -201,9 +201,9 @@ func main() {
if DEFAULT_DELAY == delay { if DEFAULT_DELAY == delay {
// default delay // default delay
delay = caseMinumInterval / 2 delay = caseMinumInterval / 2
if delay < 1 { if delay < 1 {
delay = 1 delay = 1
} }
log.Printf("actual delay is %d ms.", delay) log.Printf("actual delay is %d ms.", delay)
} }
@ -232,7 +232,7 @@ func main() {
filePerThread := subTableNum / thread filePerThread := subTableNum / thread
leftFileNum := subTableNum % thread leftFileNum := subTableNum % thread
var wg sync.WaitGroup var wg sync.WaitGroup
start = time.Now() start = time.Now()
@ -255,31 +255,31 @@ func main() {
go staticSpeed() go staticSpeed()
wg.Wait() wg.Wait()
usedTime := time.Since(start) usedTime := time.Since(start)
total := getTotalRows(successRows) total := getTotalRows(successRows)
log.Printf("finished insert %d rows, used %d ms, speed %d rows/s", total, usedTime/1e6, total * 1e9 / int64(usedTime)) log.Printf("finished insert %d rows, used %d ms, speed %d rows/s", total, usedTime/1e6, total*1e3/usedTime.Milliseconds())
if vnum == 0 { if vnum == 0 {
// continue waiting for insert data // continue waiting for insert data
wait := make(chan string) wait := make(chan string)
v := <- wait v := <-wait
log.Printf("program receive %s, exited.\n", v) log.Printf("program receive %s, exited.\n", v)
}else{ } else {
timeTicker.Stop() timeTicker.Stop()
} }
} }
func staticSpeed(){ func staticSpeed() {
connection := getConnection() connection := getConnection()
defer connection.Close() defer connection.Close()
if save == 1 { if save == 1 {
connection.Exec("use " + db) connection.Exec("use " + db)
_, err := connection.Exec("create table if not exists " + saveTable +"(ts timestamp, speed int)") _, err := connection.Exec("create table if not exists " + saveTable + "(ts timestamp, speed int)")
if err != nil { if err != nil {
log.Fatalf("create %s Table error: %s\n", saveTable, err) log.Fatalf("create %s Table error: %s\n", saveTable, err)
} }
@ -287,13 +287,13 @@ func staticSpeed(){
for { for {
<-timeTicker.C <-timeTicker.C
currentTime := time.Now() currentTime := time.Now()
usedTime := currentTime.UnixNano() - lastStaticTime.UnixNano() usedTime := currentTime.UnixNano() - lastStaticTime.UnixNano()
total := getTotalRows(successRows) total := getTotalRows(successRows)
currentSuccessRows := total - lastTotalRows currentSuccessRows := total - lastTotalRows
speed := currentSuccessRows * 1e9 / int64(usedTime) speed := currentSuccessRows * 1e9 / int64(usedTime)
log.Printf("insert %d rows, used %d ms, speed %d rows/s", currentSuccessRows, usedTime/1e6, speed) log.Printf("insert %d rows, used %d ms, speed %d rows/s", currentSuccessRows, usedTime/1e6, speed)
@ -301,14 +301,14 @@ func staticSpeed(){
insertSql := fmt.Sprintf("insert into %s values(%d, %d)", saveTable, currentTime.UnixNano()/1e6, speed) insertSql := fmt.Sprintf("insert into %s values(%d, %d)", saveTable, currentTime.UnixNano()/1e6, speed)
connection.Exec(insertSql) connection.Exec(insertSql)
} }
lastStaticTime = currentTime lastStaticTime = currentTime
lastTotalRows = total lastTotalRows = total
} }
} }
func getTotalRows(successRows []int64) int64{ func getTotalRows(successRows []int64) int64 {
var total int64 = 0 var total int64 = 0
for j := 0; j < len(successRows); j++ { for j := 0; j < len(successRows); j++ {
total += successRows[j] total += successRows[j]
@ -316,18 +316,18 @@ func getTotalRows(successRows []int64) int64{
return total return total
} }
func getSuperTableTimeConfig(fileRows dataRows) (start, cycleTime, avgInterval int64){ func getSuperTableTimeConfig(fileRows dataRows) (start, cycleTime, avgInterval int64) {
if auto == 1 { if auto == 1 {
// use auto generate data time // use auto generate data time
start = startTime start = startTime
avgInterval = interval avgInterval = interval
maxTableRows := normalizationDataWithSameInterval(fileRows, avgInterval) maxTableRows := normalizationDataWithSameInterval(fileRows, avgInterval)
cycleTime = maxTableRows * avgInterval + avgInterval cycleTime = maxTableRows*avgInterval + avgInterval
} else { } else {
// use the sample data primary timestamp // use the sample data primary timestamp
sort.Sort(fileRows)// sort the file data by the primarykey sort.Sort(fileRows) // sort the file data by the primarykey
minTime := getPrimaryKey(fileRows.rows[0][fileRows.config.Timestamp]) minTime := getPrimaryKey(fileRows.rows[0][fileRows.config.Timestamp])
maxTime := getPrimaryKey(fileRows.rows[len(fileRows.rows)-1][fileRows.config.Timestamp]) maxTime := getPrimaryKey(fileRows.rows[len(fileRows.rows)-1][fileRows.config.Timestamp])
@ -340,21 +340,21 @@ func getSuperTableTimeConfig(fileRows dataRows) (start, cycleTime, avgInterval i
if minTime == maxTime { if minTime == maxTime {
avgInterval = interval avgInterval = interval
cycleTime = tableNum * avgInterval + avgInterval cycleTime = tableNum*avgInterval + avgInterval
}else{ } else {
avgInterval = (maxTime - minTime) / int64(len(fileRows.rows)) * tableNum avgInterval = (maxTime - minTime) / int64(len(fileRows.rows)) * tableNum
cycleTime = maxTime - minTime + avgInterval cycleTime = maxTime - minTime + avgInterval
} }
} }
return return
} }
func createStatisticTable(){ func createStatisticTable() {
connection := getConnection() connection := getConnection()
defer connection.Close() defer connection.Close()
_, err := connection.Exec("create table if not exist " + db + "."+ saveTable +"(ts timestamp, speed int)") _, err := connection.Exec("create table if not exist " + db + "." + saveTable + "(ts timestamp, speed int)")
if err != nil { if err != nil {
log.Fatalf("createStatisticTable error: %s\n", err) log.Fatalf("createStatisticTable error: %s\n", err)
} }
@ -379,8 +379,8 @@ func createSubTable(subTableMaps map[string]*dataRows) {
tableName := getScaleSubTableName(subTableName, i) tableName := getScaleSubTableName(subTableName, i)
scaleTableMap[tableName] = &scaleTableInfo{ scaleTableMap[tableName] = &scaleTableInfo{
subTableName: subTableName, subTableName: subTableName,
insertRows: 0, insertRows: 0,
} }
scaleTableNames = append(scaleTableNames, tableName) scaleTableNames = append(scaleTableNames, tableName)
@ -389,12 +389,12 @@ func createSubTable(subTableMaps map[string]*dataRows) {
buffers.WriteString(" using ") buffers.WriteString(" using ")
buffers.WriteString(superTableName) buffers.WriteString(superTableName)
buffers.WriteString(" tags(") buffers.WriteString(" tags(")
for _, tag := range subTableMaps[subTableName].config.Tags{ for _, tag := range subTableMaps[subTableName].config.Tags {
tagValue := fmt.Sprintf("%v", tagValues[strings.ToLower(tag.Name)]) tagValue := fmt.Sprintf("%v", tagValues[strings.ToLower(tag.Name)])
buffers.WriteString("'" + tagValue + "'") buffers.WriteString("'" + tagValue + "'")
buffers.WriteString(",") buffers.WriteString(",")
} }
buffers.Truncate(buffers.Len()-1) buffers.Truncate(buffers.Len() - 1)
buffers.WriteString(")") buffers.WriteString(")")
createTableSql := buffers.String() createTableSql := buffers.String()
@ -451,14 +451,14 @@ func createSuperTable(superTableConfigMap map[string]*superTableConfig) {
buffer.WriteString(field.Name + " " + field.Type + ",") buffer.WriteString(field.Name + " " + field.Type + ",")
} }
buffer.Truncate(buffer.Len()-1) buffer.Truncate(buffer.Len() - 1)
buffer.WriteString(") tags( ") buffer.WriteString(") tags( ")
for _, tag := range superTableConf.config.Tags { for _, tag := range superTableConf.config.Tags {
buffer.WriteString(tag.Name + " " + tag.Type + ",") buffer.WriteString(tag.Name + " " + tag.Type + ",")
} }
buffer.Truncate(buffer.Len()-1) buffer.Truncate(buffer.Len() - 1)
buffer.WriteString(")") buffer.WriteString(")")
createSql := buffer.String() createSql := buffer.String()
@ -475,16 +475,15 @@ func createSuperTable(superTableConfigMap map[string]*superTableConfig) {
func getScaleSubTableName(subTableName string, hnum int) string { func getScaleSubTableName(subTableName string, hnum int) string {
if hnum == 0 { if hnum == 0 {
return subTableName return subTableName
} }
return fmt.Sprintf( "%s_%d", subTableName, hnum) return fmt.Sprintf("%s_%d", subTableName, hnum)
} }
func getSuperTableName(stname string) string { func getSuperTableName(stname string) string {
return SUPERTABLE_PREFIX + stname return SUPERTABLE_PREFIX + stname
} }
/** /**
* normalizationData , and return the num of subTables * normalizationData , and return the num of subTables
*/ */
@ -505,12 +504,12 @@ func normalizationData(fileRows dataRows, minTime int64) int64 {
value, ok := subTableMap[subTableName] value, ok := subTableMap[subTableName]
if !ok { if !ok {
subTableMap[subTableName] = &dataRows{ subTableMap[subTableName] = &dataRows{
rows: []map[string]interface{}{row}, rows: []map[string]interface{}{row},
config: fileRows.config, config: fileRows.config,
} }
tableNum++ tableNum++
}else{ } else {
value.rows = append(value.rows, row) value.rows = append(value.rows, row)
} }
} }
@ -518,9 +517,9 @@ func normalizationData(fileRows dataRows, minTime int64) int64 {
} }
// return the maximum table rows // return the maximum table rows
func normalizationDataWithSameInterval(fileRows dataRows, avgInterval int64) int64{ func normalizationDataWithSameInterval(fileRows dataRows, avgInterval int64) int64 {
// subTableMap // subTableMap
currSubTableMap := make(map[string]*dataRows) currSubTableMap := make(map[string]*dataRows)
for _, row := range fileRows.rows { for _, row := range fileRows.rows {
// get subTableName // get subTableName
tableValue := getSubTableNameValue(row[fileRows.config.SubTableName]) tableValue := getSubTableNameValue(row[fileRows.config.SubTableName])
@ -534,10 +533,10 @@ func normalizationDataWithSameInterval(fileRows dataRows, avgInterval int64) int
if !ok { if !ok {
row[fileRows.config.Timestamp] = 0 row[fileRows.config.Timestamp] = 0
currSubTableMap[subTableName] = &dataRows{ currSubTableMap[subTableName] = &dataRows{
rows: []map[string]interface{}{row}, rows: []map[string]interface{}{row},
config: fileRows.config, config: fileRows.config,
} }
}else{ } else {
row[fileRows.config.Timestamp] = int64(len(value.rows)) * avgInterval row[fileRows.config.Timestamp] = int64(len(value.rows)) * avgInterval
value.rows = append(value.rows, row) value.rows = append(value.rows, row)
} }
@ -545,7 +544,7 @@ func normalizationDataWithSameInterval(fileRows dataRows, avgInterval int64) int
} }
var maxRows, tableRows int = 0, 0 var maxRows, tableRows int = 0, 0
for tableName := range currSubTableMap{ for tableName := range currSubTableMap {
tableRows = len(currSubTableMap[tableName].rows) tableRows = len(currSubTableMap[tableName].rows)
subTableMap[tableName] = currSubTableMap[tableName] // add to global subTableMap subTableMap[tableName] = currSubTableMap[tableName] // add to global subTableMap
if tableRows > maxRows { if tableRows > maxRows {
@ -556,13 +555,11 @@ func normalizationDataWithSameInterval(fileRows dataRows, avgInterval int64) int
return int64(maxRows) return int64(maxRows)
} }
func getSubTableName(subTableValue string, superTableName string) string {
func getSubTableName(subTableValue string, superTableName string) string {
return SUBTABLE_PREFIX + subTableValue + "_" + superTableName return SUBTABLE_PREFIX + subTableValue + "_" + superTableName
} }
func insertData(threadIndex, start, end int, wg *sync.WaitGroup, successRows []int64) {
func insertData(threadIndex, start, end int, wg *sync.WaitGroup, successRows []int64) {
connection := getConnection() connection := getConnection()
defer connection.Close() defer connection.Close()
defer wg.Done() defer wg.Done()
@ -591,9 +588,9 @@ func insertData(threadIndex, start, end int, wg *sync.WaitGroup, successRows []
var tableEndTime int64 var tableEndTime int64
if vnum == 0 { if vnum == 0 {
// need continue generate data // need continue generate data
tableEndTime = time.Now().UnixNano()/1e6 tableEndTime = time.Now().UnixNano() / 1e6
}else { } else {
tableEndTime = tableStartTime + superTableConf.cycleTime * int64(vnum) - superTableConf.avgInterval tableEndTime = tableStartTime + superTableConf.cycleTime*int64(vnum) - superTableConf.avgInterval
} }
insertRows := scaleTableMap[tableName].insertRows insertRows := scaleTableMap[tableName].insertRows
@ -603,10 +600,10 @@ func insertData(threadIndex, start, end int, wg *sync.WaitGroup, successRows []
rowIndex := insertRows % subTableRows rowIndex := insertRows % subTableRows
currentRow := subTableInfo.rows[rowIndex] currentRow := subTableInfo.rows[rowIndex]
currentTime := getPrimaryKey(currentRow[subTableInfo.config.Timestamp]) + loopNum * superTableConf.cycleTime + tableStartTime currentTime := getPrimaryKey(currentRow[subTableInfo.config.Timestamp]) + loopNum*superTableConf.cycleTime + tableStartTime
if currentTime <= tableEndTime { if currentTime <= tableEndTime {
// append // append
if lastTableName != tableName { if lastTableName != tableName {
buffers.WriteString(tableName) buffers.WriteString(tableName)
buffers.WriteString(" values") buffers.WriteString(" values")
@ -616,22 +613,22 @@ func insertData(threadIndex, start, end int, wg *sync.WaitGroup, successRows []
buffers.WriteString("(") buffers.WriteString("(")
buffers.WriteString(fmt.Sprintf("%v", currentTime)) buffers.WriteString(fmt.Sprintf("%v", currentTime))
buffers.WriteString(",") buffers.WriteString(",")
// fieldNum := len(subTableInfo.config.Fields) // fieldNum := len(subTableInfo.config.Fields)
for _,field := range subTableInfo.config.Fields { for _, field := range subTableInfo.config.Fields {
buffers.WriteString(getFieldValue(currentRow[strings.ToLower(field.Name)])) buffers.WriteString(getFieldValue(currentRow[strings.ToLower(field.Name)]))
buffers.WriteString(",") buffers.WriteString(",")
// if( i != fieldNum -1){ // if( i != fieldNum -1){
// } // }
} }
buffers.Truncate(buffers.Len()-1) buffers.Truncate(buffers.Len() - 1)
buffers.WriteString(") ") buffers.WriteString(") ")
appendRows++ appendRows++
insertRows++ insertRows++
if appendRows == batch { if appendRows == batch {
// executebatch // executebatch
insertSql := buffers.String() insertSql := buffers.String()
connection.Exec("use " + db) connection.Exec("use " + db)
@ -645,7 +642,7 @@ func insertData(threadIndex, start, end int, wg *sync.WaitGroup, successRows []
lastTableName = "" lastTableName = ""
appendRows = 0 appendRows = 0
} }
}else { } else {
// finished insert current table // finished insert current table
break break
} }
@ -654,14 +651,14 @@ func insertData(threadIndex, start, end int, wg *sync.WaitGroup, successRows []
scaleTableMap[tableName].insertRows = insertRows scaleTableMap[tableName].insertRows = insertRows
} }
// left := len(rows) // left := len(rows)
if appendRows > 0 { if appendRows > 0 {
// executebatch // executebatch
insertSql := buffers.String() insertSql := buffers.String()
connection.Exec("use " + db) connection.Exec("use " + db)
affectedRows := executeBatchInsert(insertSql, connection) affectedRows := executeBatchInsert(insertSql, connection)
successRows[threadIndex] += affectedRows successRows[threadIndex] += affectedRows
currSuccessRows += affectedRows currSuccessRows += affectedRows
@ -676,7 +673,7 @@ func insertData(threadIndex, start, end int, wg *sync.WaitGroup, successRows []
break break
} }
if(num == 0){ if num == 0 {
wg.Done() //finished insert history data wg.Done() //finished insert history data
num++ num++
} }
@ -691,7 +688,7 @@ func insertData(threadIndex, start, end int, wg *sync.WaitGroup, successRows []
} }
func buildSql(rows []tableRows) string{ func buildSql(rows []tableRows) string {
var lastTableName string var lastTableName string
@ -709,7 +706,7 @@ func buildSql(rows []tableRows) string{
if lastTableName == row.tableName { if lastTableName == row.tableName {
buffers.WriteString(row.value) buffers.WriteString(row.value)
}else { } else {
buffers.WriteString(" ") buffers.WriteString(" ")
buffers.WriteString(row.tableName) buffers.WriteString(row.tableName)
buffers.WriteString(" values") buffers.WriteString(" values")
@ -722,7 +719,7 @@ func buildSql(rows []tableRows) string{
return inserSql return inserSql
} }
func buildRow(tableName string, currentTime int64, subTableInfo *dataRows, currentRow map[string]interface{}) tableRows{ func buildRow(tableName string, currentTime int64, subTableInfo *dataRows, currentRow map[string]interface{}) tableRows {
tableRows := tableRows{tableName: tableName} tableRows := tableRows{tableName: tableName}
@ -732,12 +729,12 @@ func buildRow(tableName string, currentTime int64, subTableInfo *dataRows, curre
buffers.WriteString(fmt.Sprintf("%v", currentTime)) buffers.WriteString(fmt.Sprintf("%v", currentTime))
buffers.WriteString(",") buffers.WriteString(",")
for _,field := range subTableInfo.config.Fields { for _, field := range subTableInfo.config.Fields {
buffers.WriteString(getFieldValue(currentRow[strings.ToLower(field.Name)])) buffers.WriteString(getFieldValue(currentRow[strings.ToLower(field.Name)]))
buffers.WriteString(",") buffers.WriteString(",")
} }
buffers.Truncate(buffers.Len()-1) buffers.Truncate(buffers.Len() - 1)
buffers.WriteString(")") buffers.WriteString(")")
insertSql := buffers.String() insertSql := buffers.String()
@ -764,7 +761,7 @@ func getFieldValue(fieldValue interface{}) string {
return fmt.Sprintf("'%v'", fieldValue) return fmt.Sprintf("'%v'", fieldValue)
} }
func getConnection() *sql.DB{ func getConnection() *sql.DB {
db, err := sql.Open(DRIVER_NAME, dataSourceName) db, err := sql.Open(DRIVER_NAME, dataSourceName)
if err != nil { if err != nil {
panic(err) panic(err)
@ -772,7 +769,6 @@ func getConnection() *sql.DB{
return db return db
} }
func getSubTableNameValue(suffix interface{}) string { func getSubTableNameValue(suffix interface{}) string {
return fmt.Sprintf("%v", suffix) return fmt.Sprintf("%v", suffix)
} }
@ -950,7 +946,7 @@ func parseMillisecond(str interface{}, layout string) int64 {
log.Println(err) log.Println(err)
return -1 return -1
} }
return t.UnixNano()/1e6 return t.UnixNano() / 1e6
} }
// lowerMapKey transfer all the map key to lowercase // lowerMapKey transfer all the map key to lowercase
@ -1009,7 +1005,7 @@ func checkUserCaseConfig(caseName string, caseConfig *dataimport.CaseConfig) {
if i < len(caseConfig.Fields)-1 { if i < len(caseConfig.Fields)-1 {
// delete middle item, a = a[:i+copy(a[i:], a[i+1:])] // delete middle item, a = a[:i+copy(a[i:], a[i+1:])]
caseConfig.Fields = caseConfig.Fields[:i+copy(caseConfig.Fields[i:], caseConfig.Fields[i+1:])] caseConfig.Fields = caseConfig.Fields[:i+copy(caseConfig.Fields[i:], caseConfig.Fields[i+1:])]
}else { } else {
// delete the last item // delete the last item
caseConfig.Fields = caseConfig.Fields[:len(caseConfig.Fields)-1] caseConfig.Fields = caseConfig.Fields[:len(caseConfig.Fields)-1]
} }
@ -1057,7 +1053,7 @@ func parseArg() {
flag.Parse() flag.Parse()
} }
func printArg() { func printArg() {
fmt.Println("used param: ") fmt.Println("used param: ")
fmt.Println("-cfg: ", cfg) fmt.Println("-cfg: ", cfg)
fmt.Println("-cases:", cases) fmt.Println("-cases:", cases)

BIN
importSampleData/bin/taosimport
View File

Binary file not shown.

13
packaging/cfg/taos.cfg
View File

@ -51,18 +51,15 @@
# number of threads per CPU core # number of threads per CPU core
# numOfThreadsPerCore 1.0 # numOfThreadsPerCore 1.0
# number of vnodes per core in DNode # number of vgroups per db
# numOfVnodesPerCore 8 # maxVgroupsPerDb 0
# max number of tables per vnode
# maxTablesPerVnode 1000000
# the ratio of threads responsible for querying in the total thread # the ratio of threads responsible for querying in the total thread
# ratioOfQueryThreads 0.5 # ratioOfQueryThreads 0.5
# number of total vnodes in DNode
# numOfTotalVnodes 0
# max number of tables per vnode
# maxtablesPerVnode 1000
# interval of check load balance when the management node is in normal operation # interval of check load balance when the management node is in normal operation
# balanceInterval 300 # balanceInterval 300

10
src/client/inc/tscLog.h
View File

@ -27,11 +27,11 @@ extern int32_t tscEmbedded;
#define tscFatal(...) { if (cDebugFlag & DEBUG_FATAL) { taosPrintLog("TSC FATAL ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }} #define tscFatal(...) { if (cDebugFlag & DEBUG_FATAL) { taosPrintLog("TSC FATAL ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }}
#define tscError(...) { if (cDebugFlag & DEBUG_ERROR) { taosPrintLog("TSC ERROR ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }} #define tscError(...) { if (cDebugFlag & DEBUG_ERROR) { taosPrintLog("TSC ERROR ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }}
#define tscWarn(...) { if (cDebugFlag & DEBUG_WARN) { taosPrintLog("TSC WARN ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }} #define tscWarn(...) { if (cDebugFlag & DEBUG_WARN) { taosPrintLog("TSC WARN ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }}
#define tscInfo(...) { if (cDebugFlag & DEBUG_INFO) { taosPrintLog("TSC INFO ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }} #define tscInfo(...) { if (cDebugFlag & DEBUG_INFO) { taosPrintLog("TSC ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }}
#define tscDebug(...) { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSC DEBUG ", cDebugFlag, __VA_ARGS__); }} #define tscDebug(...) { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }}
#define tscTrace(...) { if (cDebugFlag & DEBUG_TRACE) { taosPrintLog("TSC TRACE ", cDebugFlag, __VA_ARGS__); }} #define tscTrace(...) { if (cDebugFlag & DEBUG_TRACE) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }}
#define tscDebugL(...){ if (cDebugFlag & DEBUG_DEBUG) { taosPrintLongString("TSC DEBUG ", cDebugFlag, __VA_ARGS__); }} #define tscDebugL(...){ if (cDebugFlag & DEBUG_DEBUG) { taosPrintLongString("TSC ", cDebugFlag, __VA_ARGS__); }}
#ifdef __cplusplus #ifdef __cplusplus
} }

4
src/client/inc/tscUtil.h
View File

@ -160,7 +160,9 @@ void tscFieldInfoUpdateOffsetForInterResult(SQueryInfo* pQueryInfo);
int16_t tscFieldInfoGetOffset(SQueryInfo* pQueryInfo, int32_t index); int16_t tscFieldInfoGetOffset(SQueryInfo* pQueryInfo, int32_t index);
void tscFieldInfoClear(SFieldInfo* pFieldInfo); void tscFieldInfoClear(SFieldInfo* pFieldInfo);
int32_t tscNumOfFields(SQueryInfo* pQueryInfo);
static FORCE_INLINE int32_t tscNumOfFields(SQueryInfo* pQueryInfo) { return pQueryInfo->fieldsInfo.numOfOutput; }
int32_t tscFieldInfoCompare(const SFieldInfo* pFieldInfo1, const SFieldInfo* pFieldInfo2); int32_t tscFieldInfoCompare(const SFieldInfo* pFieldInfo1, const SFieldInfo* pFieldInfo2);
void addExprParams(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes, int16_t tableIndex); void addExprParams(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes, int16_t tableIndex);

39
src/client/inc/tsclient.h
View File

@ -412,7 +412,44 @@ char *tscGetErrorMsgPayload(SSqlCmd *pCmd);
int32_t tscInvalidSQLErrMsg(char *msg, const char *additionalInfo, const char *sql); int32_t tscInvalidSQLErrMsg(char *msg, const char *additionalInfo, const char *sql);
int32_t tscToSQLCmd(SSqlObj *pSql, struct SSqlInfo *pInfo); int32_t tscToSQLCmd(SSqlObj *pSql, struct SSqlInfo *pInfo);
void tscGetResultColumnChr(SSqlRes *pRes, SFieldInfo* pFieldInfo, int32_t column); //void tscGetResultColumnChr(SSqlRes *pRes, SFieldInfo* pFieldInfo, int32_t column);
static FORCE_INLINE void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pFieldInfo, int32_t columnIndex) {
SFieldSupInfo* pInfo = (SFieldSupInfo*) TARRAY_GET_ELEM(pFieldInfo->pSupportInfo, columnIndex);
assert(pInfo->pSqlExpr != NULL);
int32_t type = pInfo->pSqlExpr->resType;
int32_t bytes = pInfo->pSqlExpr->resBytes;
char* pData = pRes->data + pInfo->pSqlExpr->offset * pRes->numOfRows + bytes * pRes->row;
if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BINARY) {
int32_t realLen = varDataLen(pData);
assert(realLen <= bytes - VARSTR_HEADER_SIZE);
if (isNull(pData, type)) {
pRes->tsrow[columnIndex] = NULL;
} else {
pRes->tsrow[columnIndex] = ((tstr*)pData)->data;
}
if (realLen < pInfo->pSqlExpr->resBytes - VARSTR_HEADER_SIZE) { // todo refactor
*(pData + realLen + VARSTR_HEADER_SIZE) = 0;
}
pRes->length[columnIndex] = realLen;
} else {
assert(bytes == tDataTypeDesc[type].nSize);
if (isNull(pData, type)) {
pRes->tsrow[columnIndex] = NULL;
} else {
pRes->tsrow[columnIndex] = pData;
}
pRes->length[columnIndex] = bytes;
}
}
extern void * tscCacheHandle; extern void * tscCacheHandle;
extern void * tscTmr; extern void * tscTmr;

8
src/client/src/TSDBJNIConnector.c
View File

@ -24,10 +24,10 @@
#define jniFatal(...) { if (jniDebugFlag & DEBUG_FATAL) { taosPrintLog("JNI FATAL ", tscEmbedded ? 255 : jniDebugFlag, __VA_ARGS__); }} #define jniFatal(...) { if (jniDebugFlag & DEBUG_FATAL) { taosPrintLog("JNI FATAL ", tscEmbedded ? 255 : jniDebugFlag, __VA_ARGS__); }}
#define jniError(...) { if (jniDebugFlag & DEBUG_ERROR) { taosPrintLog("JNI ERROR ", tscEmbedded ? 255 : jniDebugFlag, __VA_ARGS__); }} #define jniError(...) { if (jniDebugFlag & DEBUG_ERROR) { taosPrintLog("JNI ERROR ", tscEmbedded ? 255 : jniDebugFlag, __VA_ARGS__); }}
#define jniWarn(...) { if (jniDebugFlag & DEBUG_WARN) { taosPrintLog("JNI WARN ", tscEmbedded ? 255 : jniDebugFlag, __VA_ARGS__); }} #define jniWarn(...) { if (jniDebugFlag & DEBUG_WARN) { taosPrintLog("JNI WARN ", tscEmbedded ? 255 : jniDebugFlag, __VA_ARGS__); }}
#define jniInfo(...) { if (jniDebugFlag & DEBUG_INFO) { taosPrintLog("JNI INFO ", tscEmbedded ? 255 : jniDebugFlag, __VA_ARGS__); }} #define jniInfo(...) { if (jniDebugFlag & DEBUG_INFO) { taosPrintLog("JNI ", tscEmbedded ? 255 : jniDebugFlag, __VA_ARGS__); }}
#define jniDebug(...) { if (jniDebugFlag & DEBUG_DEBUG) { taosPrintLog("JNI DEBUG ", jniDebugFlag, __VA_ARGS__); }} #define jniDebug(...) { if (jniDebugFlag & DEBUG_DEBUG) { taosPrintLog("JNI ", jniDebugFlag, __VA_ARGS__); }}
#define jniTrace(...) { if (jniDebugFlag & DEBUG_TRACE) { taosPrintLog("JNI TRACE ", jniDebugFlag, __VA_ARGS__); }} #define jniTrace(...) { if (jniDebugFlag & DEBUG_TRACE) { taosPrintLog("JNI ", jniDebugFlag, __VA_ARGS__); }}
int __init = 0; int __init = 0;

110
src/client/src/tscAsync.c
View File

@ -43,8 +43,9 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const
pSql->signature = pSql; pSql->signature = pSql;
pSql->param = param; pSql->param = param;
pSql->pTscObj = pObj; pSql->pTscObj = pObj;
pSql->maxRetry = TSDB_MAX_REPLICA_NUM; pSql->maxRetry = TSDB_MAX_REPLICA;
pSql->fp = fp; pSql->fp = fp;
pSql->fetchFp = fp;
pSql->sqlstr = calloc(1, sqlLen + 1); pSql->sqlstr = calloc(1, sqlLen + 1);
if (pSql->sqlstr == NULL) { if (pSql->sqlstr == NULL) {
@ -53,7 +54,7 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const
return; return;
} }
strtolower(pSql->sqlstr, sqlstr); strntolower(pSql->sqlstr, sqlstr, sqlLen);
tscDebugL("%p SQL: %s", pSql, pSql->sqlstr); tscDebugL("%p SQL: %s", pSql, pSql->sqlstr);
pSql->cmd.curSql = pSql->sqlstr; pSql->cmd.curSql = pSql->sqlstr;
@ -159,7 +160,7 @@ static void tscProcessAsyncRetrieveImpl(void *param, TAOS_RES *tres, int numOfRo
pRes->code = numOfRows; pRes->code = numOfRows;
} }
tscQueueAsyncError(pSql->fetchFp, param, pRes->code); tscQueueAsyncRes(pSql);
return; return;
} }
@ -167,6 +168,7 @@ static void tscProcessAsyncRetrieveImpl(void *param, TAOS_RES *tres, int numOfRo
if (pCmd->command != TSDB_SQL_RETRIEVE_LOCALMERGE && pCmd->command < TSDB_SQL_LOCAL) { if (pCmd->command != TSDB_SQL_RETRIEVE_LOCALMERGE && pCmd->command < TSDB_SQL_LOCAL) {
pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH;
} }
tscProcessSql(pSql); tscProcessSql(pSql);
} }
@ -196,6 +198,10 @@ void taos_fetch_rows_a(TAOS_RES *taosa, void (*fp)(void *, TAOS_RES *, int), voi
SSqlRes *pRes = &pSql->res; SSqlRes *pRes = &pSql->res;
SSqlCmd *pCmd = &pSql->cmd; SSqlCmd *pCmd = &pSql->cmd;
// user-defined callback function is stored in fetchFp
pSql->fetchFp = fp;
pSql->fp = tscAsyncFetchRowsProxy;
if (pRes->qhandle == 0) { if (pRes->qhandle == 0) {
tscError("qhandle is NULL"); tscError("qhandle is NULL");
pRes->code = TSDB_CODE_TSC_INVALID_QHANDLE; pRes->code = TSDB_CODE_TSC_INVALID_QHANDLE;
@ -203,10 +209,6 @@ void taos_fetch_rows_a(TAOS_RES *taosa, void (*fp)(void *, TAOS_RES *, int), voi
return; return;
} }
// user-defined callback function is stored in fetchFp
pSql->fetchFp = fp;
pSql->fp = tscAsyncFetchRowsProxy;
pSql->param = param; pSql->param = param;
tscResetForNextRetrieve(pRes); tscResetForNextRetrieve(pRes);
@ -346,31 +348,32 @@ void tscProcessFetchRow(SSchedMsg *pMsg) {
void tscProcessAsyncRes(SSchedMsg *pMsg) { void tscProcessAsyncRes(SSchedMsg *pMsg) {
SSqlObj *pSql = (SSqlObj *)pMsg->ahandle; SSqlObj *pSql = (SSqlObj *)pMsg->ahandle;
SSqlCmd *pCmd = &pSql->cmd; // SSqlCmd *pCmd = &pSql->cmd;
SSqlRes *pRes = &pSql->res; SSqlRes *pRes = &pSql->res;
void *taosres = pSql; // void *taosres = pSql;
// pCmd may be released, so cache pCmd->command // pCmd may be released, so cache pCmd->command
int cmd = pCmd->command; // int cmd = pCmd->command;
int code = pRes->code; // int code = pRes->code;
// in case of async insert, restore the user specified callback function // in case of async insert, restore the user specified callback function
bool shouldFree = tscShouldBeFreed(pSql); // bool shouldFree = tscShouldBeFreed(pSql);
if (cmd == TSDB_SQL_INSERT) { // if (pCmd->command == TSDB_SQL_INSERT) {
assert(pSql->fp != NULL); // assert(pSql->fp != NULL);
pSql->fp = pSql->fetchFp; assert(pSql->fp != NULL && pSql->fetchFp != NULL);
} // }
if (pSql->fp) { // if (pSql->fp) {
(*pSql->fp)(pSql->param, taosres, code); pSql->fp = pSql->fetchFp;
} (*pSql->fp)(pSql->param, pSql, pRes->code);
// }
if (shouldFree) { // if (shouldFree) {
tscDebug("%p sqlObj is automatically freed in async res", pSql); // tscDebug("%p sqlObj is automatically freed in async res", pSql);
tscFreeSqlObj(pSql); // tscFreeSqlObj(pSql);
} // }
} }
static void tscProcessAsyncError(SSchedMsg *pMsg) { static void tscProcessAsyncError(SSchedMsg *pMsg) {
@ -420,15 +423,15 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
SSqlCmd *pCmd = &pSql->cmd; SSqlCmd *pCmd = &pSql->cmd;
SSqlRes *pRes = &pSql->res; SSqlRes *pRes = &pSql->res;
pRes->code = code;
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
pRes->code = code; tscError("%p ge tableMeta failed, code:%s", pSql, tstrerror(code));
tscQueueAsyncRes(pSql); goto _error;
return; } else {
tscDebug("%p get tableMeta successfully", pSql);
} }
tscDebug("%p get tableMeta successfully", pSql);
if (pSql->pStream == NULL) { if (pSql->pStream == NULL) {
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
@ -453,11 +456,9 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
assert(pParObj->signature == pParObj && trs->subqueryIndex == pTableMetaInfo->vgroupIndex && assert(pParObj->signature == pParObj && trs->subqueryIndex == pTableMetaInfo->vgroupIndex &&
pTableMetaInfo->vgroupIndex >= 0 && pTableMetaInfo->vgroupList != NULL); pTableMetaInfo->vgroupIndex >= 0 && pTableMetaInfo->vgroupList != NULL);
if ((code = tscProcessSql(pSql)) == TSDB_CODE_SUCCESS) { // tscProcessSql can add error into async res
return; tscProcessSql(pSql);
} return;
goto _error;
} else { // continue to process normal async query } else { // continue to process normal async query
if (pCmd->parseFinished) { if (pCmd->parseFinished) {
tscDebug("%p update table meta in local cache, continue to process sql and send corresponding query", pSql); tscDebug("%p update table meta in local cache, continue to process sql and send corresponding query", pSql);
@ -477,30 +478,43 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
tscDebug("%p redo parse sql string to build submit block", pSql); tscDebug("%p redo parse sql string to build submit block", pSql);
pCmd->parseFinished = false; pCmd->parseFinished = false;
tscResetSqlCmdObj(pCmd);
code = tsParseSql(pSql, true); code = tsParseSql(pSql, true);
if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
return; return;
} else if (code != TSDB_CODE_SUCCESS) {
goto _error;
} }
if (code == TSDB_CODE_SUCCESS) { /*
/* * Discard previous built submit blocks, and then parse the sql string again and build up all submit blocks,
* Discard previous built submit blocks, and then parse the sql string again and build up all submit blocks, * and send the required submit block according to index value in supporter to server.
* and send the required submit block according to index value in supporter to server. */
*/ pSql->fp = pSql->fetchFp; // restore the fp
pSql->fp = pSql->fetchFp; // restore the fp tscHandleInsertRetry(pSql);
if ((code = tscHandleInsertRetry(pSql)) == TSDB_CODE_SUCCESS) { } else if (pCmd->command == TSDB_SQL_SELECT) { // in case of other query type, continue
return; tscDebug("%p redo parse sql string and proceed", pSql);
} //tscDebug("before %p fp:%p, fetchFp:%p", pSql, pSql->fp, pSql->fetchFp);
} pCmd->parseFinished = false;
tscResetSqlCmdObj(pCmd);
} else {// in case of other query type, continue //tscDebug("after %p fp:%p, fetchFp:%p", pSql, pSql->fp, pSql->fetchFp);
if ((code = tscProcessSql(pSql)) == TSDB_CODE_SUCCESS) { code = tsParseSql(pSql, true);
if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
return; return;
} else if (code != TSDB_CODE_SUCCESS) {
goto _error;
} }
tscProcessSql(pSql);
} else { // in all other cases, simple retry
tscProcessSql(pSql);
} }
goto _error; return;
} else { } else {
tscDebug("%p continue parse sql after get table meta", pSql); tscDebug("%p continue parse sql after get table meta", pSql);
@ -538,7 +552,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
goto _error; goto _error;
} }
if (code == TSDB_CODE_SUCCESS && UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
code = tscGetSTableVgroupInfo(pSql, pCmd->clauseIndex); code = tscGetSTableVgroupInfo(pSql, pCmd->clauseIndex);
if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
return; return;
@ -550,8 +564,8 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
tscDebug("%p stream:%p meta is updated, start new query, command:%d", pSql, pSql->pStream, pSql->cmd.command); tscDebug("%p stream:%p meta is updated, start new query, command:%d", pSql, pSql->pStream, pSql->cmd.command);
if (!pSql->cmd.parseFinished) { if (!pSql->cmd.parseFinished) {
tsParseSql(pSql, false); tsParseSql(pSql, false);
sem_post(&pSql->rspSem);
} }
(*pSql->fp)(pSql->param, pSql, code);
return; return;
} }

257
src/client/src/tscFunctionImpl.c
View File

@ -330,10 +330,6 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
bool stableQueryFunctChanged(int32_t funcId) {
return (aAggs[funcId].stableFuncId != funcId);
}
/** /**
* the numOfRes should be kept, since it may be used later * the numOfRes should be kept, since it may be used later
* and allow the ResultInfo to be re initialized * and allow the ResultInfo to be re initialized
@ -361,7 +357,6 @@ static bool function_setup(SQLFunctionCtx *pCtx) {
} }
memset(pCtx->aOutputBuf, 0, (size_t)pCtx->outputBytes); memset(pCtx->aOutputBuf, 0, (size_t)pCtx->outputBytes);
initResultInfo(pResInfo); initResultInfo(pResInfo);
return true; return true;
} }
@ -675,16 +670,16 @@ static void sum_func_second_merge(SQLFunctionCtx *pCtx) {
} }
} }
static int32_t precal_req_load_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { static int32_t statisRequired(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) {
return BLK_DATA_STATIS_NEEDED; return BLK_DATA_STATIS_NEEDED;
} }
static int32_t data_req_load_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { static int32_t dataBlockRequired(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) {
return BLK_DATA_ALL_NEEDED; return BLK_DATA_ALL_NEEDED;
} }
// todo: if column in current data block are null, opt for this case // todo: if column in current data block are null, opt for this case
static int32_t first_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { static int32_t firstFuncRequired(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) {
if (pCtx->order == TSDB_ORDER_DESC) { if (pCtx->order == TSDB_ORDER_DESC) {
return BLK_DATA_NO_NEEDED; return BLK_DATA_NO_NEEDED;
} }
@ -697,7 +692,7 @@ static int32_t first_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end,
} }
} }
static int32_t last_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { static int32_t lastFuncRequired(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) {
if (pCtx->order != pCtx->param[0].i64Key) { if (pCtx->order != pCtx->param[0].i64Key) {
return BLK_DATA_NO_NEEDED; return BLK_DATA_NO_NEEDED;
} }
@ -709,34 +704,40 @@ static int32_t last_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end,
} }
} }
static int32_t first_dist_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { static int32_t firstDistFuncRequired(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) {
if (pCtx->order == TSDB_ORDER_DESC) { if (pCtx->order == TSDB_ORDER_DESC) {
return BLK_DATA_NO_NEEDED; return BLK_DATA_NO_NEEDED;
} }
// not initialized yet, it is the first block, load it.
if (pCtx->aOutputBuf == NULL) {
return BLK_DATA_ALL_NEEDED;
}
// result buffer has not been set yet. SFirstLastInfo *pInfo = (SFirstLastInfo*) (pCtx->aOutputBuf + pCtx->inputBytes);
return BLK_DATA_ALL_NEEDED; if (pInfo->hasResult != DATA_SET_FLAG) {
//todo optimize the filter info return BLK_DATA_ALL_NEEDED;
// SFirstLastInfo *pInfo = (SFirstLastInfo*) (pCtx->aOutputBuf + pCtx->inputBytes); } else { // data in current block is not earlier than current result
// if (pInfo->hasResult != DATA_SET_FLAG) { return (pInfo->ts <= start) ? BLK_DATA_NO_NEEDED : BLK_DATA_ALL_NEEDED;
// return BLK_DATA_ALL_NEEDED; }
// } else { // data in current block is not earlier than current result
// return (pInfo->ts <= start) ? BLK_DATA_NO_NEEDED : BLK_DATA_ALL_NEEDED;
// }
} }
static int32_t last_dist_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { static int32_t lastDistFuncRequired(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) {
if (pCtx->order != pCtx->param[0].i64Key) { if (pCtx->order != pCtx->param[0].i64Key) {
return BLK_DATA_NO_NEEDED; return BLK_DATA_NO_NEEDED;
} }
return BLK_DATA_ALL_NEEDED; // not initialized yet, it is the first block, load it.
// SFirstLastInfo *pInfo = (SFirstLastInfo*) (pCtx->aOutputBuf + pCtx->inputBytes); if (pCtx->aOutputBuf == NULL) {
// if (pInfo->hasResult != DATA_SET_FLAG) { return BLK_DATA_ALL_NEEDED;
// return BLK_DATA_ALL_NEEDED; }
// } else {
// return (pInfo->ts > end) ? BLK_DATA_NO_NEEDED : BLK_DATA_ALL_NEEDED; SFirstLastInfo *pInfo = (SFirstLastInfo*) (pCtx->aOutputBuf + pCtx->inputBytes);
// } if (pInfo->hasResult != DATA_SET_FLAG) {
return BLK_DATA_ALL_NEEDED;
} else {
return (pInfo->ts > end) ? BLK_DATA_NO_NEEDED : BLK_DATA_ALL_NEEDED;
}
} }
////////////////////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////////////////////
@ -1549,6 +1550,8 @@ static void first_data_assign_impl(SQLFunctionCtx *pCtx, char *pData, int32_t in
* to decide if the value is earlier than current intermediate result * to decide if the value is earlier than current intermediate result
*/ */
static void first_dist_function(SQLFunctionCtx *pCtx) { static void first_dist_function(SQLFunctionCtx *pCtx) {
assert(pCtx->size > 0);
if (pCtx->size == 0) { if (pCtx->size == 0) {
return; return;
} }
@ -1563,7 +1566,12 @@ static void first_dist_function(SQLFunctionCtx *pCtx) {
} }
int32_t notNullElems = 0; int32_t notNullElems = 0;
// data block is discard, not loaded, do not need to check it
if (!pCtx->preAggVals.dataBlockLoaded) {
return;
}
// find the first not null value // find the first not null value
for (int32_t i = 0; i < pCtx->size; ++i) { for (int32_t i = 0; i < pCtx->size; ++i) {
char *data = GET_INPUT_CHAR_INDEX(pCtx, i); char *data = GET_INPUT_CHAR_INDEX(pCtx, i);
@ -1584,10 +1592,6 @@ static void first_dist_function(SQLFunctionCtx *pCtx) {
} }
static void first_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) { static void first_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) {
if (pCtx->size == 0) {
return;
}
char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); char *pData = GET_INPUT_CHAR_INDEX(pCtx, index);
if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { if (pCtx->hasNull && isNull(pData, pCtx->inputType)) {
return; return;
@ -1715,10 +1719,6 @@ static void last_data_assign_impl(SQLFunctionCtx *pCtx, char *pData, int32_t ind
} }
static void last_dist_function(SQLFunctionCtx *pCtx) { static void last_dist_function(SQLFunctionCtx *pCtx) {
if (pCtx->size == 0) {
return;
}
/* /*
* 1. for scan data in asc order, no need to check data * 1. for scan data in asc order, no need to check data
* 2. for data blocks that are not loaded, no need to check data * 2. for data blocks that are not loaded, no need to check data
@ -1726,7 +1726,12 @@ static void last_dist_function(SQLFunctionCtx *pCtx) {
if (pCtx->order != pCtx->param[0].i64Key) { if (pCtx->order != pCtx->param[0].i64Key) {
return; return;
} }
// data block is discard, not loaded, do not need to check it
if (!pCtx->preAggVals.dataBlockLoaded) {
return;
}
int32_t notNullElems = 0; int32_t notNullElems = 0;
for (int32_t i = pCtx->size - 1; i >= 0; --i) { for (int32_t i = pCtx->size - 1; i >= 0; --i) {
@ -2123,55 +2128,6 @@ static void copyTopBotRes(SQLFunctionCtx *pCtx, int32_t type) {
tfree(pData); tfree(pData);
} }
bool top_bot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, char *minval, char *maxval) {
STopBotInfo *pTopBotInfo = (STopBotInfo *)GET_RES_INFO(pCtx)->interResultBuf;
int32_t numOfExistsRes = pTopBotInfo->num;
// required number of results are not reached, continue load data block
if (numOfExistsRes < pCtx->param[0].i64Key) {
return true;
}
tValuePair *pRes = (tValuePair*) pTopBotInfo->res;
if (functionId == TSDB_FUNC_TOP) {
switch (pCtx->inputType) {
case TSDB_DATA_TYPE_TINYINT:
return GET_INT8_VAL(maxval) > pRes[0].v.i64Key;
case TSDB_DATA_TYPE_SMALLINT:
return GET_INT16_VAL(maxval) > pRes[0].v.i64Key;
case TSDB_DATA_TYPE_INT:
return GET_INT32_VAL(maxval) > pRes[0].v.i64Key;
case TSDB_DATA_TYPE_BIGINT:
return GET_INT64_VAL(maxval) > pRes[0].v.i64Key;
case TSDB_DATA_TYPE_FLOAT:
return GET_FLOAT_VAL(maxval) > pRes[0].v.dKey;
case TSDB_DATA_TYPE_DOUBLE:
return GET_DOUBLE_VAL(maxval) > pRes[0].v.dKey;
default:
return true;
}
} else {
switch (pCtx->inputType) {
case TSDB_DATA_TYPE_TINYINT:
return GET_INT8_VAL(minval) < pRes[0].v.i64Key;
case TSDB_DATA_TYPE_SMALLINT:
return GET_INT16_VAL(minval) < pRes[0].v.i64Key;
case TSDB_DATA_TYPE_INT:
return GET_INT32_VAL(minval) < pRes[0].v.i64Key;
case TSDB_DATA_TYPE_BIGINT:
return GET_INT64_VAL(minval) < pRes[0].v.i64Key;
case TSDB_DATA_TYPE_FLOAT:
return GET_FLOAT_VAL(minval) < pRes[0].v.dKey;
case TSDB_DATA_TYPE_DOUBLE:
return GET_DOUBLE_VAL(minval) < pRes[0].v.dKey;
default:
return true;
}
}
}
/* /*
* Parameters values: * Parameters values:
* 1. param[0]: maximum allowable results * 1. param[0]: maximum allowable results
@ -2182,15 +2138,62 @@ bool top_bot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, char *mi
*/ */
static STopBotInfo *getTopBotOutputInfo(SQLFunctionCtx *pCtx) { static STopBotInfo *getTopBotOutputInfo(SQLFunctionCtx *pCtx) {
SResultInfo *pResInfo = GET_RES_INFO(pCtx); SResultInfo *pResInfo = GET_RES_INFO(pCtx);
// only the first_stage_merge is directly written data into final output buffer // only the first_stage_merge is directly written data into final output buffer
if (pResInfo->superTableQ && pCtx->currentStage != SECONDARY_STAGE_MERGE) { if (pResInfo->superTableQ && pCtx->currentStage != SECONDARY_STAGE_MERGE) {
return (STopBotInfo*) pCtx->aOutputBuf; return (STopBotInfo*) pCtx->aOutputBuf;
} else { // during normal table query and super table at the secondary_stage, result is written to intermediate buffer } else { // during normal table query and super table at the secondary_stage, result is written to intermediate buffer
return pResInfo->interResultBuf; return pResInfo->interResultBuf;
} }
} }
bool topbot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, const char *minval, const char *maxval) {
STopBotInfo *pTopBotInfo = getTopBotOutputInfo(pCtx);
// required number of results are not reached, continue load data block
if (pTopBotInfo->num < pCtx->param[0].i64Key) {
return true;
}
tValuePair **pRes = (tValuePair**) pTopBotInfo->res;
if (functionId == TSDB_FUNC_TOP) {
switch (pCtx->inputType) {
case TSDB_DATA_TYPE_TINYINT:
return GET_INT8_VAL(maxval) > pRes[0]->v.i64Key;
case TSDB_DATA_TYPE_SMALLINT:
return GET_INT16_VAL(maxval) > pRes[0]->v.i64Key;
case TSDB_DATA_TYPE_INT:
return GET_INT32_VAL(maxval) > pRes[0]->v.i64Key;
case TSDB_DATA_TYPE_BIGINT:
return GET_INT64_VAL(maxval) > pRes[0]->v.i64Key;
case TSDB_DATA_TYPE_FLOAT:
return GET_FLOAT_VAL(maxval) > pRes[0]->v.dKey;
case TSDB_DATA_TYPE_DOUBLE:
return GET_DOUBLE_VAL(maxval) > pRes[0]->v.dKey;
default:
return true;
}
} else {
switch (pCtx->inputType) {
case TSDB_DATA_TYPE_TINYINT:
return GET_INT8_VAL(minval) < pRes[0]->v.i64Key;
case TSDB_DATA_TYPE_SMALLINT:
return GET_INT16_VAL(minval) < pRes[0]->v.i64Key;
case TSDB_DATA_TYPE_INT:
return GET_INT32_VAL(minval) < pRes[0]->v.i64Key;
case TSDB_DATA_TYPE_BIGINT:
return GET_INT64_VAL(minval) < pRes[0]->v.i64Key;
case TSDB_DATA_TYPE_FLOAT:
return GET_FLOAT_VAL(minval) < pRes[0]->v.dKey;
case TSDB_DATA_TYPE_DOUBLE:
return GET_DOUBLE_VAL(minval) < pRes[0]->v.dKey;
default:
return true;
}
}
}
/* /*
* keep the intermediate results during scan data blocks in the format of: * keep the intermediate results during scan data blocks in the format of:
* +-----------------------------------+-------------one value pair-----------+------------next value pair-----------+ * +-----------------------------------+-------------one value pair-----------+------------next value pair-----------+
@ -2952,10 +2955,14 @@ static void tag_project_function(SQLFunctionCtx *pCtx) {
INC_INIT_VAL(pCtx, pCtx->size); INC_INIT_VAL(pCtx, pCtx->size);
assert(pCtx->inputBytes == pCtx->outputBytes); assert(pCtx->inputBytes == pCtx->outputBytes);
for (int32_t i = 0; i < pCtx->size; ++i) { tVariantDump(&pCtx->tag, pCtx->aOutputBuf, pCtx->outputType, true);
tVariantDump(&pCtx->tag, pCtx->aOutputBuf, pCtx->outputType, true); char* data = pCtx->aOutputBuf;
pCtx->aOutputBuf += pCtx->outputBytes;
// directly copy from the first one
for (int32_t i = 1; i < pCtx->size; ++i) {
memmove(pCtx->aOutputBuf, data, pCtx->outputBytes);
pCtx->aOutputBuf += pCtx->outputBytes; pCtx->aOutputBuf += pCtx->outputBytes;
} }
} }
@ -3372,7 +3379,7 @@ static void spread_function(SQLFunctionCtx *pCtx) {
SResultInfo *pResInfo = GET_RES_INFO(pCtx); SResultInfo *pResInfo = GET_RES_INFO(pCtx);
SSpreadInfo *pInfo = pResInfo->interResultBuf; SSpreadInfo *pInfo = pResInfo->interResultBuf;
int32_t numOfElems = pCtx->size; int32_t numOfElems = 0;
// todo : opt with pre-calculated result // todo : opt with pre-calculated result
// column missing cause the hasNull to be true // column missing cause the hasNull to be true
@ -3941,7 +3948,7 @@ static void ts_comp_finalize(SQLFunctionCtx *pCtx) {
tsBufFlush(pTSbuf); tsBufFlush(pTSbuf);
strcpy(pCtx->aOutputBuf, pTSbuf->path); strcpy(pCtx->aOutputBuf, pTSbuf->path);
tsBufDestory(pTSbuf); tsBufDestroy(pTSbuf);
doFinalizer(pCtx); doFinalizer(pCtx);
} }
@ -4408,7 +4415,7 @@ static void sumrate_finalizer(SQLFunctionCtx *pCtx) {
* e.g., count/sum/avg/min/max/stddev/percentile/apercentile/first/last... * e.g., count/sum/avg/min/max/stddev/percentile/apercentile/first/last...
* *
*/ */
int32_t funcCompatDefList[] = { int32_t functionCompatList[] = {
// count, sum, avg, min, max, stddev, percentile, apercentile, first, last // count, sum, avg, min, max, stddev, percentile, apercentile, first, last
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
// last_row, top, bottom, spread, twa, leastsqr, ts, ts_dummy, tag_dummy, ts_z // last_row, top, bottom, spread, twa, leastsqr, ts, ts_dummy, tag_dummy, ts_z
@ -4447,7 +4454,7 @@ SQLAggFuncElem aAggs[] = {{
function_finalizer, function_finalizer,
sum_func_merge, sum_func_merge,
sum_func_second_merge, sum_func_second_merge,
precal_req_load_info, statisRequired,
}, },
{ {
// 2 // 2
@ -4462,7 +4469,7 @@ SQLAggFuncElem aAggs[] = {{
avg_finalizer, avg_finalizer,
avg_func_merge, avg_func_merge,
avg_func_second_merge, avg_func_second_merge,
precal_req_load_info, statisRequired,
}, },
{ {
// 3 // 3
@ -4477,7 +4484,7 @@ SQLAggFuncElem aAggs[] = {{
function_finalizer, function_finalizer,
min_func_merge, min_func_merge,
min_func_second_merge, min_func_second_merge,
precal_req_load_info, statisRequired,
}, },
{ {
// 4 // 4
@ -4492,7 +4499,7 @@ SQLAggFuncElem aAggs[] = {{
function_finalizer, function_finalizer,
max_func_merge, max_func_merge,
max_func_second_merge, max_func_second_merge,
precal_req_load_info, statisRequired,
}, },
{ {
// 5 // 5
@ -4507,7 +4514,7 @@ SQLAggFuncElem aAggs[] = {{
stddev_finalizer, stddev_finalizer,
noop1, noop1,
noop1, noop1,
data_req_load_info, dataBlockRequired,
}, },
{ {
// 6 // 6
@ -4522,7 +4529,7 @@ SQLAggFuncElem aAggs[] = {{
percentile_finalizer, percentile_finalizer,
noop1, noop1,
noop1, noop1,
data_req_load_info, dataBlockRequired,
}, },
{ {
// 7 // 7
@ -4537,7 +4544,7 @@ SQLAggFuncElem aAggs[] = {{
apercentile_finalizer, apercentile_finalizer,
apercentile_func_merge, apercentile_func_merge,
apercentile_func_second_merge, apercentile_func_second_merge,
data_req_load_info, dataBlockRequired,
}, },
{ {
// 8 // 8
@ -4552,7 +4559,7 @@ SQLAggFuncElem aAggs[] = {{
function_finalizer, function_finalizer,
noop1, noop1,
noop1, noop1,
first_data_req_info, firstFuncRequired,
}, },
{ {
// 9 // 9
@ -4567,7 +4574,7 @@ SQLAggFuncElem aAggs[] = {{
function_finalizer, function_finalizer,
noop1, noop1,
noop1, noop1,
last_data_req_info, lastFuncRequired,
}, },
{ {
// 10 // 10
@ -4583,7 +4590,7 @@ SQLAggFuncElem aAggs[] = {{
last_row_finalizer, last_row_finalizer,
noop1, noop1,
last_dist_func_second_merge, last_dist_func_second_merge,
data_req_load_info, dataBlockRequired,
}, },
{ {
// 11 // 11
@ -4599,7 +4606,7 @@ SQLAggFuncElem aAggs[] = {{
top_bottom_func_finalizer, top_bottom_func_finalizer,
top_func_merge, top_func_merge,
top_func_second_merge, top_func_second_merge,
data_req_load_info, dataBlockRequired,
}, },
{ {
// 12 // 12
@ -4615,7 +4622,7 @@ SQLAggFuncElem aAggs[] = {{
top_bottom_func_finalizer, top_bottom_func_finalizer,
bottom_func_merge, bottom_func_merge,
bottom_func_second_merge, bottom_func_second_merge,
data_req_load_info, dataBlockRequired,
}, },
{ {
// 13 // 13
@ -4645,7 +4652,7 @@ SQLAggFuncElem aAggs[] = {{
twa_function_finalizer, twa_function_finalizer,
twa_func_merge, twa_func_merge,
twa_function_copy, twa_function_copy,
data_req_load_info, dataBlockRequired,
}, },
{ {
// 15 // 15
@ -4660,7 +4667,7 @@ SQLAggFuncElem aAggs[] = {{
leastsquares_finalizer, leastsquares_finalizer,
noop1, noop1,
noop1, noop1,
data_req_load_info, dataBlockRequired,
}, },
{ {
// 16 // 16
@ -4690,7 +4697,7 @@ SQLAggFuncElem aAggs[] = {{
doFinalizer, doFinalizer,
copy_function, copy_function,
copy_function, copy_function,
data_req_load_info, dataBlockRequired,
}, },
{ {
// 18 // 18
@ -4720,7 +4727,7 @@ SQLAggFuncElem aAggs[] = {{
ts_comp_finalize, ts_comp_finalize,
copy_function, copy_function,
copy_function, copy_function,
data_req_load_info, dataBlockRequired,
}, },
{ {
// 20 // 20
@ -4750,7 +4757,7 @@ SQLAggFuncElem aAggs[] = {{
doFinalizer, doFinalizer,
copy_function, copy_function,
copy_function, copy_function,
data_req_load_info, dataBlockRequired,
}, },
{ {
// 22, multi-output, tag function has only one result // 22, multi-output, tag function has only one result
@ -4780,7 +4787,7 @@ SQLAggFuncElem aAggs[] = {{
doFinalizer, doFinalizer,
copy_function, copy_function,
copy_function, copy_function,
data_req_load_info, dataBlockRequired,
}, },
{ {
// 24 // 24
@ -4795,7 +4802,7 @@ SQLAggFuncElem aAggs[] = {{
doFinalizer, doFinalizer,
noop1, noop1,
noop1, noop1,
data_req_load_info, dataBlockRequired,
}, },
// distributed version used in two-stage aggregation processes // distributed version used in two-stage aggregation processes
{ {
@ -4811,7 +4818,7 @@ SQLAggFuncElem aAggs[] = {{
function_finalizer, function_finalizer,
first_dist_func_merge, first_dist_func_merge,
first_dist_func_second_merge, first_dist_func_second_merge,
first_dist_data_req_info, firstDistFuncRequired,
}, },
{ {
// 26 // 26
@ -4826,7 +4833,7 @@ SQLAggFuncElem aAggs[] = {{
function_finalizer, function_finalizer,
last_dist_func_merge, last_dist_func_merge,
last_dist_func_second_merge, last_dist_func_second_merge,
last_dist_data_req_info, lastDistFuncRequired,
}, },
{ {
// 27 // 27
@ -4841,7 +4848,7 @@ SQLAggFuncElem aAggs[] = {{
doFinalizer, doFinalizer,
noop1, noop1,
copy_function, copy_function,
data_req_load_info, dataBlockRequired,
}, },
{ {
// 28 // 28
@ -4856,7 +4863,7 @@ SQLAggFuncElem aAggs[] = {{
rate_finalizer, rate_finalizer,
rate_func_merge, rate_func_merge,
rate_func_copy, rate_func_copy,
data_req_load_info, dataBlockRequired,
}, },
{ {
// 29 // 29
@ -4871,7 +4878,7 @@ SQLAggFuncElem aAggs[] = {{
rate_finalizer, rate_finalizer,
rate_func_merge, rate_func_merge,
rate_func_copy, rate_func_copy,
data_req_load_info, dataBlockRequired,
}, },
{ {
// 30 // 30
@ -4886,7 +4893,7 @@ SQLAggFuncElem aAggs[] = {{
sumrate_finalizer, sumrate_finalizer,
sumrate_func_merge, sumrate_func_merge,
sumrate_func_second_merge, sumrate_func_second_merge,
data_req_load_info, dataBlockRequired,
}, },
{ {
// 31 // 31
@ -4901,7 +4908,7 @@ SQLAggFuncElem aAggs[] = {{
sumrate_finalizer, sumrate_finalizer,
sumrate_func_merge, sumrate_func_merge,
sumrate_func_second_merge, sumrate_func_second_merge,
data_req_load_info, dataBlockRequired,
}, },
{ {
// 32 // 32
@ -4916,7 +4923,7 @@ SQLAggFuncElem aAggs[] = {{
sumrate_finalizer, sumrate_finalizer,
sumrate_func_merge, sumrate_func_merge,
sumrate_func_second_merge, sumrate_func_second_merge,
data_req_load_info, dataBlockRequired,
}, },
{ {
// 33 // 33
@ -4931,7 +4938,7 @@ SQLAggFuncElem aAggs[] = {{
sumrate_finalizer, sumrate_finalizer,
sumrate_func_merge, sumrate_func_merge,
sumrate_func_second_merge, sumrate_func_second_merge,
data_req_load_info, dataBlockRequired,
}, },
{ {
// 34 // 34
@ -4946,5 +4953,5 @@ SQLAggFuncElem aAggs[] = {{
noop1, noop1,
noop1, noop1,
noop1, noop1,
data_req_load_info, dataBlockRequired,
}}; }};

2
src/client/src/tscPrepare.c
View File

@ -497,7 +497,7 @@ TAOS_STMT* taos_stmt_init(TAOS* taos) {
tsem_init(&pSql->rspSem, 0, 0); tsem_init(&pSql->rspSem, 0, 0);
pSql->signature = pSql; pSql->signature = pSql;
pSql->pTscObj = pObj; pSql->pTscObj = pObj;
pSql->maxRetry = TSDB_MAX_REPLICA_NUM; pSql->maxRetry = TSDB_MAX_REPLICA;
pStmt->pSql = pSql; pStmt->pSql = pSql;
return pStmt; return pStmt;

46
src/client/src/tscSQLParser.c
View File

@ -175,7 +175,6 @@ static int32_t handlePassword(SSqlCmd* pCmd, SSQLToken* pPwd) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
// todo handle memory leak in error handle function
int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) {
if (pInfo == NULL || pSql == NULL || pSql->signature != pSql) { if (pInfo == NULL || pSql == NULL || pSql->signature != pSql) {
return TSDB_CODE_TSC_APP_ERROR; return TSDB_CODE_TSC_APP_ERROR;
@ -2472,7 +2471,7 @@ static bool functionCompatibleCheck(SQueryInfo* pQueryInfo) {
startIdx++; startIdx++;
} }
int32_t factor = funcCompatDefList[tscSqlExprGet(pQueryInfo, startIdx)->functionId]; int32_t factor = functionCompatList[tscSqlExprGet(pQueryInfo, startIdx)->functionId];
// diff function cannot be executed with other function // diff function cannot be executed with other function
// arithmetic function can be executed with other arithmetic functions // arithmetic function can be executed with other arithmetic functions
@ -2490,7 +2489,7 @@ static bool functionCompatibleCheck(SQueryInfo* pQueryInfo) {
continue; continue;
} }
if (funcCompatDefList[functionId] != factor) { if (functionCompatList[functionId] != factor) {
return false; return false;
} }
} }
@ -4643,21 +4642,24 @@ typedef struct SDNodeDynConfOption {
} SDNodeDynConfOption; } SDNodeDynConfOption;
int32_t validateEp(char* ep) { int32_t validateEp(char* ep) {
char buf[TSDB_EP_LEN + 1] = {0}; char buf[TSDB_EP_LEN + 1] = {0};
tstrncpy(buf, ep, TSDB_EP_LEN); tstrncpy(buf, ep, TSDB_EP_LEN);
char *pos = strchr(buf, ':'); char* pos = strchr(buf, ':');
if (NULL == pos) { if (NULL == pos) {
return TSDB_CODE_TSC_INVALID_SQL; int32_t val = strtol(ep, NULL, 10);
if (val <= 0 || val > 65536) {
return TSDB_CODE_TSC_INVALID_SQL;
}
} else {
uint16_t port = atoi(pos + 1);
if (0 == port) {
return TSDB_CODE_TSC_INVALID_SQL;
}
} }
uint16_t port = atoi(pos+1);
if (0 == port) {
return TSDB_CODE_TSC_INVALID_SQL;
}
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t validateDNodeConfig(tDCLSQL* pOptions) { int32_t validateDNodeConfig(tDCLSQL* pOptions) {
@ -4665,13 +4667,13 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) {
return TSDB_CODE_TSC_INVALID_SQL; return TSDB_CODE_TSC_INVALID_SQL;
} }
const int DNODE_DYNAMIC_CFG_OPTIONS_SIZE = 17; const int DNODE_DYNAMIC_CFG_OPTIONS_SIZE = 19;
const SDNodeDynConfOption DNODE_DYNAMIC_CFG_OPTIONS[] = { const SDNodeDynConfOption DNODE_DYNAMIC_CFG_OPTIONS[] = {
{"resetLog", 8}, {"resetQueryCache", 15}, {"debugFlag", 9}, {"mDebugFlag", 10}, {"resetLog", 8}, {"resetQueryCache", 15}, {"debugFlag", 9}, {"mDebugFlag", 10},
{"dDebugFlag", 10}, {"sdbDebugFlag", 12}, {"vDebugFlag", 10}, {"cDebugFlag", 10}, {"dDebugFlag", 10}, {"sdbDebugFlag", 12}, {"vDebugFlag", 10}, {"cDebugFlag", 10},
{"httpDebugFlag", 13}, {"monitorDebugFlag", 16}, {"rpcDebugFlag", 12}, {"uDebugFlag", 10}, {"httpDebugFlag", 13}, {"monitorDebugFlag", 16}, {"rpcDebugFlag", 12}, {"uDebugFlag", 10},
{"tmrDebugFlag", 12}, {"qDebugflag", 10}, {"sDebugflag", 10}, {"tsdbDebugFlag", 13}, {"tmrDebugFlag", 12}, {"qDebugflag", 10}, {"sDebugflag", 10}, {"tsdbDebugFlag", 13},
{"monitor", 7}}; {"mqttDebugFlag", 13}, {"wDebugFlag", 10}, {"monitor", 7}};
SSQLToken* pOptionToken = &pOptions->a[1]; SSQLToken* pOptionToken = &pOptions->a[1];
@ -4695,7 +4697,7 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) {
SSQLToken* pValToken = &pOptions->a[2]; SSQLToken* pValToken = &pOptions->a[2];
int32_t val = strtol(pValToken->z, NULL, 10); int32_t val = strtol(pValToken->z, NULL, 10);
if (val < 131 || val > 199) { if (val < 0 || val > 256) {
/* options value is out of valid range */ /* options value is out of valid range */
return TSDB_CODE_TSC_INVALID_SQL; return TSDB_CODE_TSC_INVALID_SQL;
} }
@ -4963,6 +4965,7 @@ static void setCreateDBOption(SCMCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) {
pMsg->commitTime = htonl(pCreateDb->commitTime); pMsg->commitTime = htonl(pCreateDb->commitTime);
pMsg->minRowsPerFileBlock = htonl(pCreateDb->minRowsPerBlock); pMsg->minRowsPerFileBlock = htonl(pCreateDb->minRowsPerBlock);
pMsg->maxRowsPerFileBlock = htonl(pCreateDb->maxRowsPerBlock); pMsg->maxRowsPerFileBlock = htonl(pCreateDb->maxRowsPerBlock);
pMsg->fsyncPeriod = htonl(pCreateDb->fsyncPeriod);
pMsg->compression = pCreateDb->compressionLevel; pMsg->compression = pCreateDb->compressionLevel;
pMsg->walLevel = (char)pCreateDb->walLevel; pMsg->walLevel = (char)pCreateDb->walLevel;
pMsg->replications = pCreateDb->replica; pMsg->replications = pCreateDb->replica;
@ -5490,9 +5493,9 @@ int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCMCreateDbMsg* pCreate) {
} }
if (pCreate->replications != -1 && if (pCreate->replications != -1 &&
(pCreate->replications < TSDB_MIN_REPLICA_NUM || pCreate->replications > TSDB_MAX_REPLICA_NUM)) { (pCreate->replications < TSDB_MIN_DB_REPLICA_OPTION || pCreate->replications > TSDB_MAX_DB_REPLICA_OPTION)) {
snprintf(msg, tListLen(msg), "invalid db option replications: %d valid range: [%d, %d]", pCreate->replications, snprintf(msg, tListLen(msg), "invalid db option replications: %d valid range: [%d, %d]", pCreate->replications,
TSDB_MIN_REPLICA_NUM, TSDB_MAX_REPLICA_NUM); TSDB_MIN_DB_REPLICA_OPTION, TSDB_MAX_DB_REPLICA_OPTION);
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg);
} }
@ -5530,6 +5533,13 @@ int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCMCreateDbMsg* pCreate) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg);
} }
val = htonl(pCreate->fsyncPeriod);
if (val != -1 && (val < TSDB_MIN_FSYNC_PERIOD || val > TSDB_MAX_FSYNC_PERIOD)) {
snprintf(msg, tListLen(msg), "invalid db option fsyncPeriod: %d valid range: [%d, %d]", val,
TSDB_MIN_FSYNC_PERIOD, TSDB_MAX_FSYNC_PERIOD);
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg);
}
if (pCreate->compression != -1 && if (pCreate->compression != -1 &&
(pCreate->compression < TSDB_MIN_COMP_LEVEL || pCreate->compression > TSDB_MAX_COMP_LEVEL)) { (pCreate->compression < TSDB_MIN_COMP_LEVEL || pCreate->compression > TSDB_MAX_COMP_LEVEL)) {
snprintf(msg, tListLen(msg), "invalid db option compression: %d valid range: [%d, %d]", pCreate->compression, snprintf(msg, tListLen(msg), "invalid db option compression: %d valid range: [%d, %d]", pCreate->compression,

79
src/client/src/tscServer.c
View File

@ -246,43 +246,52 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet) {
rpcMsg->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; rpcMsg->code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
} else { } else {
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
if (rpcMsg->code == TSDB_CODE_TDB_INVALID_TABLE_ID || rpcMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID || // if (rpcMsg->code != TSDB_CODE_RPC_NETWORK_UNAVAIL) {
rpcMsg->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || rpcMsg->code == TSDB_CODE_TDB_TABLE_RECONFIGURE) { // if (pCmd->command == TSDB_SQL_CONNECT) {
if (pCmd->command == TSDB_SQL_CONNECT) { // rpcMsg->code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
rpcMsg->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; // rpcFreeCont(rpcMsg->pCont);
rpcFreeCont(rpcMsg->pCont); // return;
return; // }
} else if (pCmd->command == TSDB_SQL_HB) {
rpcMsg->code = TSDB_CODE_RPC_NOT_READY; // if (pCmd->command == TSDB_SQL_HB) {
rpcFreeCont(rpcMsg->pCont); // rpcMsg->code = TSDB_CODE_RPC_NOT_READY;
return; // rpcFreeCont(rpcMsg->pCont);
} else if (pCmd->command == TSDB_SQL_META) { // return;
// get table meta query will not retry, do nothing // }
// if (pCmd->command == TSDB_SQL_META || pCmd->command == TSDB_SQL_DESCRIBE_TABLE ||
// pCmd->command == TSDB_SQL_STABLEVGROUP || pCmd->command == TSDB_SQL_SHOW ||
// pCmd->command == TSDB_SQL_RETRIEVE) {
// // get table meta/vgroup query will not retry, do nothing
// }
// }
if ((pCmd->command == TSDB_SQL_SELECT || pCmd->command == TSDB_SQL_FETCH || pCmd->command == TSDB_SQL_INSERT ||
pCmd->command == TSDB_SQL_UPDATE_TAGS_VAL) &&
(rpcMsg->code == TSDB_CODE_TDB_INVALID_TABLE_ID || rpcMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID ||
rpcMsg->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || rpcMsg->code == TSDB_CODE_TDB_TABLE_RECONFIGURE)) {
tscWarn("%p it shall renew table meta, code:%s, retry:%d", pSql, tstrerror(rpcMsg->code), ++pSql->retry);
// set the flag to denote that sql string needs to be re-parsed and build submit block with table schema
if (rpcMsg->code == TSDB_CODE_TDB_TABLE_RECONFIGURE) {
pSql->cmd.submitSchema = 1;
}
pSql->res.code = rpcMsg->code; // keep the previous error code
if (pSql->retry > pSql->maxRetry) {
tscError("%p max retry %d reached, give up", pSql, pSql->maxRetry);
} else { } else {
tscWarn("%p it shall renew table meta, code:%s, retry:%d", pSql, tstrerror(rpcMsg->code), ++pSql->retry); rpcMsg->code = tscRenewTableMeta(pSql, pTableMetaInfo->name);
// set the flag to denote that sql string needs to be re-parsed and build submit block with table schema // if there is an error occurring, proceed to the following error handling procedure.
if (rpcMsg->code == TSDB_CODE_TDB_TABLE_RECONFIGURE) { // todo add test cases
pSql->cmd.submitSchema = 1; if (rpcMsg->code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
} rpcFreeCont(rpcMsg->pCont);
return;
pSql->res.code = rpcMsg->code; // keep the previous error code
if (pSql->retry > pSql->maxRetry) {
tscError("%p max retry %d reached, give up", pSql, pSql->maxRetry);
} else {
rpcMsg->code = tscRenewTableMeta(pSql, pTableMetaInfo->name);
// if there is an error occurring, proceed to the following error handling procedure.
// todo add test cases
if (rpcMsg->code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
rpcFreeCont(rpcMsg->pCont);
return;
}
} }
} }
} }
} }
pRes->rspLen = 0; pRes->rspLen = 0;
if (pRes->code != TSDB_CODE_TSC_QUERY_CANCELLED) { if (pRes->code != TSDB_CODE_TSC_QUERY_CANCELLED) {
@ -339,7 +348,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet) {
} }
if (rpcMsg->code != TSDB_CODE_TSC_ACTION_IN_PROGRESS) { if (rpcMsg->code != TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
rpcMsg->code = (pRes->code == TSDB_CODE_SUCCESS) ? pRes->numOfRows: pRes->code; rpcMsg->code = (pRes->code == TSDB_CODE_SUCCESS)? pRes->numOfRows: pRes->code;
bool shouldFree = tscShouldBeFreed(pSql); bool shouldFree = tscShouldBeFreed(pSql);
(*pSql->fp)(pSql->param, pSql, rpcMsg->code); (*pSql->fp)(pSql->param, pSql, rpcMsg->code);
@ -412,7 +421,7 @@ int tscProcessSql(SSqlObj *pSql) {
return pSql->res.code; return pSql->res.code;
} }
} else if (pCmd->command < TSDB_SQL_LOCAL) { } else if (pCmd->command < TSDB_SQL_LOCAL) {
pSql->ipList = tscMgmtIpSet; //? pSql->ipList = tscMgmtIpSet;
} else { // local handler } else { // local handler
return (*tscProcessMsgRsp[pCmd->command])(pSql); return (*tscProcessMsgRsp[pCmd->command])(pSql);
} }
@ -476,6 +485,8 @@ int tscBuildFetchMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
int32_t vgIndex = pTableMetaInfo->vgroupIndex; int32_t vgIndex = pTableMetaInfo->vgroupIndex;
SVgroupsInfo* pVgroupInfo = pTableMetaInfo->vgroupList; SVgroupsInfo* pVgroupInfo = pTableMetaInfo->vgroupList;
assert(pVgroupInfo->vgroups[vgIndex].vgId > 0 && vgIndex < pTableMetaInfo->vgroupList->numOfVgroups);
pRetrieveMsg->header.vgId = htonl(pVgroupInfo->vgroups[vgIndex].vgId); pRetrieveMsg->header.vgId = htonl(pVgroupInfo->vgroups[vgIndex].vgId);
} else { } else {
STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; STableMeta* pTableMeta = pTableMetaInfo->pTableMeta;
@ -549,6 +560,7 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char
assert(index >= 0); assert(index >= 0);
if (pTableMetaInfo->vgroupList->numOfVgroups > 0) { if (pTableMetaInfo->vgroupList->numOfVgroups > 0) {
assert(index < pTableMetaInfo->vgroupList->numOfVgroups);
pVgroupInfo = &pTableMetaInfo->vgroupList->vgroups[index]; pVgroupInfo = &pTableMetaInfo->vgroupList->vgroups[index];
} }
tscDebug("%p query on stable, vgIndex:%d, numOfVgroups:%d", pSql, index, pTableMetaInfo->vgroupList->numOfVgroups); tscDebug("%p query on stable, vgIndex:%d, numOfVgroups:%d", pSql, index, pTableMetaInfo->vgroupList->numOfVgroups);
@ -1372,7 +1384,6 @@ static int tscLocalResultCommonBuilder(SSqlObj *pSql, int32_t numOfRes) {
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
pRes->code = TSDB_CODE_SUCCESS; pRes->code = TSDB_CODE_SUCCESS;
if (pRes->rspType == 0) { if (pRes->rspType == 0) {
pRes->numOfRows = numOfRes; pRes->numOfRows = numOfRes;
pRes->row = 0; pRes->row = 0;

62
src/client/src/tscSql.c
View File

@ -113,7 +113,7 @@ SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con
pSql->pTscObj = pObj; pSql->pTscObj = pObj;
pSql->signature = pSql; pSql->signature = pSql;
pSql->maxRetry = TSDB_MAX_REPLICA_NUM; pSql->maxRetry = TSDB_MAX_REPLICA;
tsem_init(&pSql->rspSem, 0, 0); tsem_init(&pSql->rspSem, 0, 0);
pObj->pDnodeConn = pDnodeConn; pObj->pDnodeConn = pDnodeConn;
@ -181,6 +181,19 @@ TAOS *taos_connect(const char *ip, const char *user, const char *pass, const cha
return NULL; return NULL;
} }
TAOS *taos_connect_c(const char *ip, uint8_t ipLen, const char *user, uint8_t userLen,
const char *pass, uint8_t passLen, const char *db, uint8_t dbLen, uint16_t port) {
char ipBuf[TSDB_EP_LEN] = {0};
char userBuf[TSDB_USER_LEN] = {0};
char passBuf[TSDB_PASSWORD_LEN] = {0};
char dbBuf[TSDB_DB_NAME_LEN] = {0};
strncpy(ipBuf, ip, MIN(TSDB_EP_LEN - 1, ipLen));
strncpy(userBuf, user, MIN(TSDB_USER_LEN - 1, userLen));
strncpy(passBuf, pass, MIN(TSDB_PASSWORD_LEN - 1,passLen));
strncpy(dbBuf, db, MIN(TSDB_DB_NAME_LEN - 1, dbLen));
return taos_connect(ipBuf, userBuf, passBuf, dbBuf, port);
}
TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int), TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int),
void *param, void **taos) { void *param, void **taos) {
@ -249,7 +262,31 @@ TAOS_RES* taos_query(TAOS *taos, const char *sqlstr) {
tsem_wait(&pSql->rspSem); tsem_wait(&pSql->rspSem);
return pSql; return pSql;
} }
TAOS_RES* taos_query_c(TAOS *taos, const char *sqlstr, uint32_t sqlLen) {
STscObj *pObj = (STscObj *)taos;
if (pObj == NULL || pObj->signature != pObj) {
terrno = TSDB_CODE_TSC_DISCONNECTED;
return NULL;
}
if (sqlLen > tsMaxSQLStringLen) {
tscError("sql string exceeds max length:%d", tsMaxSQLStringLen);
terrno = TSDB_CODE_TSC_INVALID_SQL;
return NULL;
}
SSqlObj* pSql = calloc(1, sizeof(SSqlObj));
if (pSql == NULL) {
tscError("failed to malloc sqlObj");
terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
return NULL;
}
doAsyncQuery(pObj, pSql, waitForQueryRsp, taos, sqlstr, sqlLen);
tsem_wait(&pSql->rspSem);
return pSql;
}
int taos_result_precision(TAOS_RES *res) { int taos_result_precision(TAOS_RES *res) {
SSqlObj *pSql = (SSqlObj *)res; SSqlObj *pSql = (SSqlObj *)res;
if (pSql == NULL || pSql->signature != pSql) return 0; if (pSql == NULL || pSql->signature != pSql) return 0;
@ -403,7 +440,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) {
taos_fetch_rows_a(res, waitForRetrieveRsp, pSql->pTscObj); taos_fetch_rows_a(res, waitForRetrieveRsp, pSql->pTscObj);
sem_wait(&pSql->rspSem); sem_wait(&pSql->rspSem);
} }
return doSetResultRowData(pSql, true); return doSetResultRowData(pSql, true);
} }
@ -481,25 +518,14 @@ static bool tscFreeQhandleInVnode(SSqlObj* pSql) {
if (pRes->code == TSDB_CODE_SUCCESS && pRes->completed == false && !tscIsTwoStageSTableQuery(pQueryInfo, 0) && if (pRes->code == TSDB_CODE_SUCCESS && pRes->completed == false && !tscIsTwoStageSTableQuery(pQueryInfo, 0) &&
(pCmd->command == TSDB_SQL_SELECT || (pCmd->command == TSDB_SQL_SELECT ||
pCmd->command == TSDB_SQL_SHOW || pCmd->command == TSDB_SQL_SHOW ||
pCmd->command == TSDB_SQL_RETRIEVE || pCmd->command == TSDB_SQL_RETRIEVE ||
pCmd->command == TSDB_SQL_FETCH) && pCmd->command == TSDB_SQL_FETCH) &&
(pCmd->command == TSDB_SQL_SELECT && pSql->pStream == NULL && pTableMetaInfo->pTableMeta != NULL)) { (pSql->pStream == NULL && pTableMetaInfo->pTableMeta != NULL)) {
pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH;
tscDebug("%p send msg to dnode to free qhandle ASAP, command:%s", pSql, sqlCmd[pCmd->command]); tscDebug("%p send msg to dnode to free qhandle ASAP, command:%s, ", pSql, sqlCmd[pCmd->command]);
tscProcessSql(pSql); tscProcessSql(pSql);
// in case of sync model query, waits for response and then goes on
// if (pSql->fp == waitForQueryRsp || pSql->fp == waitForRetrieveRsp) {
// sem_wait(&pSql->rspSem);
// tscFreeSqlObj(pSql);
// tscDebug("%p sqlObj is freed by app", pSql);
// } else {
tscDebug("%p sqlObj will be freed while rsp received", pSql);
// }
return true; return true;
} }

85
src/client/src/tscStream.c
View File

@ -70,6 +70,7 @@ static void tscProcessStreamLaunchQuery(SSchedMsg *pMsg) {
SSqlObj * pSql = pStream->pSql; SSqlObj * pSql = pStream->pSql;
pSql->fp = tscProcessStreamQueryCallback; pSql->fp = tscProcessStreamQueryCallback;
pSql->fetchFp = tscProcessStreamQueryCallback;
pSql->param = pStream; pSql->param = pStream;
pSql->res.completed = false; pSql->res.completed = false;
@ -471,6 +472,41 @@ static void setErrorInfo(SSqlObj* pSql, int32_t code, char* info) {
} }
} }
static void tscCreateStream(void *param, TAOS_RES *res, int code) {
SSqlStream* pStream = (SSqlStream*)param;
SSqlObj* pSql = pStream->pSql;
SSqlCmd* pCmd = &pSql->cmd;
if (code != TSDB_CODE_SUCCESS) {
setErrorInfo(pSql, code, pCmd->payload);
tscError("%p open stream failed, sql:%s, reason:%s, code:0x%08x", pSql, pSql->sqlstr, pCmd->payload, code);
pStream->fp(pStream->param, NULL, NULL);
return;
}
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);
pStream->isProject = isProjectStream(pQueryInfo);
pStream->precision = tinfo.precision;
pStream->ctime = taosGetTimestamp(pStream->precision);
pStream->etime = pQueryInfo->window.ekey;
tscAddIntoStreamList(pStream);
tscSetSlidingWindowInfo(pSql, pStream);
pStream->stime = tscGetStreamStartTimestamp(pSql, pStream, pStream->stime);
int64_t starttime = tscGetLaunchTimestamp(pStream);
pCmd->command = TSDB_SQL_SELECT;
taosTmrReset(tscProcessStreamTimer, starttime, pStream, tscTmr, &pStream->pTimer);
tscDebug("%p stream:%p is opened, query on:%s, interval:%" PRId64 ", sliding:%" PRId64 ", first launched in:%" PRId64 ", sql:%s", pSql,
pStream, pTableMetaInfo->name, pStream->interval, pStream->slidingTime, starttime, pSql->sqlstr);
}
TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row), TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row),
int64_t stime, void *param, void (*callback)(void *)) { int64_t stime, void *param, void (*callback)(void *)) {
STscObj *pObj = (STscObj *)taos; STscObj *pObj = (STscObj *)taos;
@ -482,7 +518,6 @@ TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *p
} }
pSql->signature = pSql; pSql->signature = pSql;
pSql->param = pSql;
pSql->pTscObj = pObj; pSql->pTscObj = pObj;
SSqlCmd *pCmd = &pSql->cmd; SSqlCmd *pCmd = &pSql->cmd;
@ -494,7 +529,14 @@ TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *p
tscFreeSqlObj(pSql); tscFreeSqlObj(pSql);
return NULL; return NULL;
} }
pStream->stime = stime;
pStream->fp = fp;
pStream->callback = callback;
pStream->param = param;
pStream->pSql = pSql;
pSql->pStream = pStream; pSql->pStream = pStream;
pSql->param = pStream;
pSql->sqlstr = calloc(1, strlen(sqlstr) + 1); pSql->sqlstr = calloc(1, strlen(sqlstr) + 1);
if (pSql->sqlstr == NULL) { if (pSql->sqlstr == NULL) {
@ -507,45 +549,18 @@ TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *p
tscDebugL("%p SQL: %s", pSql, pSql->sqlstr); tscDebugL("%p SQL: %s", pSql, pSql->sqlstr);
tsem_init(&pSql->rspSem, 0, 0); tsem_init(&pSql->rspSem, 0, 0);
pSql->fp = tscCreateStream;
pSql->fetchFp = tscCreateStream;
int32_t code = tsParseSql(pSql, true); int32_t code = tsParseSql(pSql, true);
if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { if (code == TSDB_CODE_SUCCESS) {
sem_wait(&pSql->rspSem); tscCreateStream(pStream, pSql, code);
} } else if (code != TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
tscError("%p open stream failed, sql:%s, code:%s", pSql, sqlstr, tstrerror(pRes->code));
if (pRes->code != TSDB_CODE_SUCCESS) {
setErrorInfo(pSql, pRes->code, pCmd->payload);
tscError("%p open stream failed, sql:%s, reason:%s, code:0x%08x", pSql, sqlstr, pCmd->payload, pRes->code);
tscFreeSqlObj(pSql); tscFreeSqlObj(pSql);
free(pStream);
return NULL; return NULL;
} }
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);
pStream->isProject = isProjectStream(pQueryInfo);
pStream->fp = fp;
pStream->callback = callback;
pStream->param = param;
pStream->pSql = pSql;
pStream->precision = tinfo.precision;
pStream->ctime = taosGetTimestamp(pStream->precision);
pStream->etime = pQueryInfo->window.ekey;
tscAddIntoStreamList(pStream);
tscSetSlidingWindowInfo(pSql, pStream);
pStream->stime = tscGetStreamStartTimestamp(pSql, pStream, stime);
int64_t starttime = tscGetLaunchTimestamp(pStream);
pCmd->command = TSDB_SQL_SELECT;
taosTmrReset(tscProcessStreamTimer, starttime, pStream, tscTmr, &pStream->pTimer);
tscDebug("%p stream:%p is opened, query on:%s, interval:%" PRId64 ", sliding:%" PRId64 ", first launched in:%" PRId64 ", sql:%s", pSql,
pStream, pTableMetaInfo->name, pStream->interval, pStream->slidingTime, starttime, sqlstr);
return pStream; return pStream;
} }

2
src/client/src/tscSub.c
View File

@ -107,7 +107,7 @@ static SSub* tscCreateSubscription(STscObj* pObj, const char* topic, const char*
pSql->signature = pSql; pSql->signature = pSql;
pSql->param = pSql; pSql->param = pSql;
pSql->pTscObj = pObj; pSql->pTscObj = pObj;
pSql->maxRetry = TSDB_MAX_REPLICA_NUM; pSql->maxRetry = TSDB_MAX_REPLICA;
pSql->fp = asyncCallback; pSql->fp = asyncCallback;
int code = tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE); int code = tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE);

65
src/client/src/tscSubquery.c
View File

@ -152,8 +152,8 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJ
tsBufFlush(output1); tsBufFlush(output1);
tsBufFlush(output2); tsBufFlush(output2);
tsBufDestory(pSupporter1->pTSBuf); tsBufDestroy(pSupporter1->pTSBuf);
tsBufDestory(pSupporter2->pTSBuf); tsBufDestroy(pSupporter2->pTSBuf);
tscDebug("%p input1:%" PRId64 ", input2:%" PRId64 ", final:%" PRId64 " for secondary query after ts blocks " tscDebug("%p input1:%" PRId64 ", input2:%" PRId64 ", final:%" PRId64 " for secondary query after ts blocks "
"intersecting, skey:%" PRId64 ", ekey:%" PRId64, pSql, numOfInput1, numOfInput2, output1->numOfTotal, "intersecting, skey:%" PRId64 ", ekey:%" PRId64, pSql, numOfInput1, numOfInput2, output1->numOfTotal,
@ -550,7 +550,7 @@ static bool checkForDuplicateTagVal(SQueryInfo* pQueryInfo, SJoinSupporter* p1,
return true; return true;
} }
static void getIntersectionOfTableTuple(SQueryInfo* pQueryInfo, SSqlObj* pParentSql, SArray** s1, SArray** s2) { static int32_t getIntersectionOfTableTuple(SQueryInfo* pQueryInfo, SSqlObj* pParentSql, SArray** s1, SArray** s2) {
tscDebug("%p all subqueries retrieve <tid, tags> complete, do tags match", pParentSql); tscDebug("%p all subqueries retrieve <tid, tags> complete, do tags match", pParentSql);
SJoinSupporter* p1 = pParentSql->pSubs[0]->param; SJoinSupporter* p1 = pParentSql->pSubs[0]->param;
@ -568,10 +568,7 @@ static void getIntersectionOfTableTuple(SQueryInfo* pQueryInfo, SSqlObj* pParent
*s2 = taosArrayInit(p2->num, p2->tagSize); *s2 = taosArrayInit(p2->num, p2->tagSize);
if (!(checkForDuplicateTagVal(pQueryInfo, p1, pParentSql) && checkForDuplicateTagVal(pQueryInfo, p2, pParentSql))) { if (!(checkForDuplicateTagVal(pQueryInfo, p1, pParentSql) && checkForDuplicateTagVal(pQueryInfo, p2, pParentSql))) {
freeJoinSubqueryObj(pParentSql); return TSDB_CODE_QRY_DUP_JOIN_KEY;
pParentSql->res.code = TSDB_CODE_QRY_DUP_JOIN_KEY;
tscQueueAsyncRes(pParentSql);
return;
} }
int32_t i = 0, j = 0; int32_t i = 0, j = 0;
@ -594,6 +591,8 @@ static void getIntersectionOfTableTuple(SQueryInfo* pQueryInfo, SSqlObj* pParent
i++; i++;
} }
} }
return TSDB_CODE_SUCCESS;
} }
static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRows) { static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRows) {
@ -680,7 +679,14 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow
} }
SArray *s1 = NULL, *s2 = NULL; SArray *s1 = NULL, *s2 = NULL;
getIntersectionOfTableTuple(pQueryInfo, pParentSql, &s1, &s2); int32_t code = getIntersectionOfTableTuple(pQueryInfo, pParentSql, &s1, &s2);
if (code != TSDB_CODE_SUCCESS) {
freeJoinSubqueryObj(pParentSql);
pParentSql->res.code = code;
tscQueueAsyncRes(pParentSql);
return;
}
if (taosArrayGetSize(s1) == 0 || taosArrayGetSize(s2) == 0) { // no results,return. if (taosArrayGetSize(s1) == 0 || taosArrayGetSize(s2) == 0) { // no results,return.
tscDebug("%p tag intersect does not generated qualified tables for join, free all sub SqlObj and quit", pParentSql); tscDebug("%p tag intersect does not generated qualified tables for join, free all sub SqlObj and quit", pParentSql);
freeJoinSubqueryObj(pParentSql); freeJoinSubqueryObj(pParentSql);
@ -762,7 +768,7 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
tsBufMerge(pSupporter->pTSBuf, pBuf, pTableMetaInfo->vgroupIndex); tsBufMerge(pSupporter->pTSBuf, pBuf, pTableMetaInfo->vgroupIndex);
tsBufDestory(pBuf); tsBufDestroy(pBuf);
} }
// continue to retrieve ts-comp data from vnode // continue to retrieve ts-comp data from vnode
@ -1778,7 +1784,6 @@ static SSqlObj *tscCreateSqlObjForSubquery(SSqlObj *pSql, SRetrieveSupport *trsu
pSql->pSubs[trsupport->subqueryIndex] = pNew; pSql->pSubs[trsupport->subqueryIndex] = pNew;
} }
printf("------------alloc:%p\n", pNew);
return pNew; return pNew;
} }
@ -1890,9 +1895,11 @@ int32_t tscHandleInsertRetry(SSqlObj* pSql) {
assert(pSupporter->index < pSupporter->pState->numOfTotal); assert(pSupporter->index < pSupporter->pState->numOfTotal);
STableDataBlocks* pTableDataBlock = taosArrayGetP(pCmd->pDataBlocks, pSupporter->index); STableDataBlocks* pTableDataBlock = taosArrayGetP(pCmd->pDataBlocks, pSupporter->index);
pRes->code = tscCopyDataBlockToPayload(pSql, pTableDataBlock); int32_t code = tscCopyDataBlockToPayload(pSql, pTableDataBlock);
if (pRes->code != TSDB_CODE_SUCCESS) {
return pRes->code; if ((pRes->code = code)!= TSDB_CODE_SUCCESS) {
tscQueueAsyncRes(pSql);
return code; // here the pSql may have been released already.
} }
return tscProcessSql(pSql); return tscProcessSql(pSql);
@ -2107,9 +2114,9 @@ static char *getArithemicInputSrc(void *param, const char *name, int32_t colId)
void **doSetResultRowData(SSqlObj *pSql, bool finalResult) { void **doSetResultRowData(SSqlObj *pSql, bool finalResult) {
SSqlCmd *pCmd = &pSql->cmd; SSqlCmd *pCmd = &pSql->cmd;
SSqlRes *pRes = &pSql->res; SSqlRes *pRes = &pSql->res;
assert(pRes->row >= 0 && pRes->row <= pRes->numOfRows); assert(pRes->row >= 0 && pRes->row <= pRes->numOfRows);
if(pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE) { if(pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE) {
if (pRes->completed) { if (pRes->completed) {
tfree(pRes->tsrow); tfree(pRes->tsrow);
@ -2117,29 +2124,31 @@ void **doSetResultRowData(SSqlObj *pSql, bool finalResult) {
return pRes->tsrow; return pRes->tsrow;
} }
if (pRes->row >= pRes->numOfRows) { // all the results has returned to invoker if (pRes->row >= pRes->numOfRows) { // all the results has returned to invoker
tfree(pRes->tsrow); tfree(pRes->tsrow);
return pRes->tsrow; return pRes->tsrow;
} }
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
size_t size = tscNumOfFields(pQueryInfo); size_t size = tscNumOfFields(pQueryInfo);
for (int i = 0; i < size; ++i) { for (int i = 0; i < size; ++i) {
SFieldSupInfo* pSup = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, i); SFieldSupInfo* pSup = TARRAY_GET_ELEM(pQueryInfo->fieldsInfo.pSupportInfo, i);
if (pSup->pSqlExpr != NULL) { if (pSup->pSqlExpr != NULL) {
tscGetResultColumnChr(pRes, &pQueryInfo->fieldsInfo, i); tscGetResultColumnChr(pRes, &pQueryInfo->fieldsInfo, i);
} }
// primary key column cannot be null in interval query, no need to check // primary key column cannot be null in interval query, no need to check
if (i == 0 && pQueryInfo->intervalTime > 0) { if (i == 0 && pQueryInfo->intervalTime > 0) {
continue; continue;
} }
TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); TAOS_FIELD *pField = TARRAY_GET_ELEM(pQueryInfo->fieldsInfo.pFields, i);
transferNcharData(pSql, i, pField); if (pRes->tsrow[i] != NULL && pField->type == TSDB_DATA_TYPE_NCHAR) {
transferNcharData(pSql, i, pField);
}
// calculate the result from several other columns // calculate the result from several other columns
if (pSup->pArithExprInfo != NULL) { if (pSup->pArithExprInfo != NULL) {
if (pRes->pArithSup == NULL) { if (pRes->pArithSup == NULL) {
@ -2149,10 +2158,10 @@ void **doSetResultRowData(SSqlObj *pSql, bool finalResult) {
sas->numOfCols = tscSqlExprNumOfExprs(pQueryInfo); sas->numOfCols = tscSqlExprNumOfExprs(pQueryInfo);
sas->exprList = pQueryInfo->exprList; sas->exprList = pQueryInfo->exprList;
sas->data = calloc(sas->numOfCols, POINTER_BYTES); sas->data = calloc(sas->numOfCols, POINTER_BYTES);
pRes->pArithSup = sas; pRes->pArithSup = sas;
} }
if (pRes->buffer[i] == NULL) { if (pRes->buffer[i] == NULL) {
TAOS_FIELD* field = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); TAOS_FIELD* field = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i);
pRes->buffer[i] = malloc(field->bytes); pRes->buffer[i] = malloc(field->bytes);
@ -2162,13 +2171,13 @@ void **doSetResultRowData(SSqlObj *pSql, bool finalResult) {
SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, k); SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, k);
pRes->pArithSup->data[k] = (pRes->data + pRes->numOfRows* pExpr->offset) + pRes->row*pExpr->resBytes; pRes->pArithSup->data[k] = (pRes->data + pRes->numOfRows* pExpr->offset) + pRes->row*pExpr->resBytes;
} }
tExprTreeCalcTraverse(pRes->pArithSup->pArithExpr->pExpr, 1, pRes->buffer[i], pRes->pArithSup, tExprTreeCalcTraverse(pRes->pArithSup->pArithExpr->pExpr, 1, pRes->buffer[i], pRes->pArithSup,
TSDB_ORDER_ASC, getArithemicInputSrc); TSDB_ORDER_ASC, getArithemicInputSrc);
pRes->tsrow[i] = pRes->buffer[i]; pRes->tsrow[i] = pRes->buffer[i];
} }
} }
pRes->row++; // index increase one-step pRes->row++; // index increase one-step
return pRes->tsrow; return pRes->tsrow;
} }

97
src/client/src/tscUtil.c
View File

@ -794,7 +794,7 @@ SFieldSupInfo* tscFieldInfoAppend(SFieldInfo* pFieldInfo, TAOS_FIELD* pField) {
} }
SFieldSupInfo* tscFieldInfoGetSupp(SFieldInfo* pFieldInfo, int32_t index) { SFieldSupInfo* tscFieldInfoGetSupp(SFieldInfo* pFieldInfo, int32_t index) {
return taosArrayGet(pFieldInfo->pSupportInfo, index); return TARRAY_GET_ELEM(pFieldInfo->pSupportInfo, index);
} }
SFieldSupInfo* tscFieldInfoInsert(SFieldInfo* pFieldInfo, int32_t index, TAOS_FIELD* field) { SFieldSupInfo* tscFieldInfoInsert(SFieldInfo* pFieldInfo, int32_t index, TAOS_FIELD* field) {
@ -858,11 +858,9 @@ void tscFieldInfoCopy(SFieldInfo* dst, const SFieldInfo* src) {
} }
TAOS_FIELD* tscFieldInfoGetField(SFieldInfo* pFieldInfo, int32_t index) { TAOS_FIELD* tscFieldInfoGetField(SFieldInfo* pFieldInfo, int32_t index) {
return taosArrayGet(pFieldInfo->pFields, index); return TARRAY_GET_ELEM(pFieldInfo->pFields, index);
} }
int32_t tscNumOfFields(SQueryInfo* pQueryInfo) { return pQueryInfo->fieldsInfo.numOfOutput; }
int16_t tscFieldInfoGetOffset(SQueryInfo* pQueryInfo, int32_t index) { int16_t tscFieldInfoGetOffset(SQueryInfo* pQueryInfo, int32_t index) {
SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, index); SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, index);
assert(pInfo != NULL); assert(pInfo != NULL);
@ -1546,7 +1544,7 @@ static void freeQueryInfoImpl(SQueryInfo* pQueryInfo) {
pQueryInfo->groupbyExpr.columnInfo = NULL; pQueryInfo->groupbyExpr.columnInfo = NULL;
} }
pQueryInfo->tsBuf = tsBufDestory(pQueryInfo->tsBuf); pQueryInfo->tsBuf = tsBufDestroy(pQueryInfo->tsBuf);
tfree(pQueryInfo->fillVal); tfree(pQueryInfo->fillVal);
} }
@ -1650,8 +1648,9 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cm
} }
pNew->fp = fp; pNew->fp = fp;
pNew->fetchFp = fp;
pNew->param = param; pNew->param = param;
pNew->maxRetry = TSDB_MAX_REPLICA_NUM; pNew->maxRetry = TSDB_MAX_REPLICA;
pNew->sqlstr = strdup(pSql->sqlstr); pNew->sqlstr = strdup(pSql->sqlstr);
if (pNew->sqlstr == NULL) { if (pNew->sqlstr == NULL) {
@ -1805,8 +1804,10 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
} }
pNew->fp = fp; pNew->fp = fp;
pNew->fetchFp = fp;
pNew->param = param; pNew->param = param;
pNew->maxRetry = TSDB_MAX_REPLICA_NUM; pNew->maxRetry = TSDB_MAX_REPLICA;
char* name = pTableMetaInfo->name; char* name = pTableMetaInfo->name;
STableMetaInfo* pFinalInfo = NULL; STableMetaInfo* pFinalInfo = NULL;
@ -2007,7 +2008,7 @@ void tscTryQueryNextVnode(SSqlObj* pSql, __async_cb_func_t fp) {
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
int32_t totalVgroups = pTableMetaInfo->vgroupList->numOfVgroups; int32_t totalVgroups = pTableMetaInfo->vgroupList->numOfVgroups;
while (++pTableMetaInfo->vgroupIndex < totalVgroups) { if (++pTableMetaInfo->vgroupIndex < totalVgroups) {
tscDebug("%p results from vgroup index:%d completed, try next:%d. total vgroups:%d. current numOfRes:%" PRId64, pSql, tscDebug("%p results from vgroup index:%d completed, try next:%d. total vgroups:%d. current numOfRes:%" PRId64, pSql,
pTableMetaInfo->vgroupIndex - 1, pTableMetaInfo->vgroupIndex, totalVgroups, pRes->numOfClauseTotal); pTableMetaInfo->vgroupIndex - 1, pTableMetaInfo->vgroupIndex, totalVgroups, pRes->numOfClauseTotal);
@ -2043,11 +2044,9 @@ void tscTryQueryNextVnode(SSqlObj* pSql, __async_cb_func_t fp) {
// set the callback function // set the callback function
pSql->fp = fp; pSql->fp = fp;
int32_t ret = tscProcessSql(pSql); tscProcessSql(pSql);
if (ret == TSDB_CODE_SUCCESS) { } else {
return; tscDebug("%p try all %d vnodes, query complete. current numOfRes:%" PRId64, pSql, totalVgroups, pRes->numOfClauseTotal);
} else {// todo check for failure
}
} }
} }
@ -2085,42 +2084,42 @@ void tscTryQueryNextClause(SSqlObj* pSql, void (*queryFp)()) {
} }
} }
void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pFieldInfo, int32_t columnIndex) { //void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pFieldInfo, int32_t columnIndex) {
SFieldSupInfo* pInfo = taosArrayGet(pFieldInfo->pSupportInfo, columnIndex); // SFieldSupInfo* pInfo = TARRAY_GET_ELEM(pFieldInfo->pSupportInfo, columnIndex);
assert(pInfo->pSqlExpr != NULL); // assert(pInfo->pSqlExpr != NULL);
//
int32_t type = pInfo->pSqlExpr->resType; // int32_t type = pInfo->pSqlExpr->resType;
int32_t bytes = pInfo->pSqlExpr->resBytes; // int32_t bytes = pInfo->pSqlExpr->resBytes;
//
char* pData = pRes->data + pInfo->pSqlExpr->offset * pRes->numOfRows + bytes * pRes->row; // char* pData = pRes->data + pInfo->pSqlExpr->offset * pRes->numOfRows + bytes * pRes->row;
//
if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BINARY) { // if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BINARY) {
int32_t realLen = varDataLen(pData); // int32_t realLen = varDataLen(pData);
assert(realLen <= bytes - VARSTR_HEADER_SIZE); // assert(realLen <= bytes - VARSTR_HEADER_SIZE);
//
if (isNull(pData, type)) { // if (isNull(pData, type)) {
pRes->tsrow[columnIndex] = NULL; // pRes->tsrow[columnIndex] = NULL;
} else { // } else {
pRes->tsrow[columnIndex] = ((tstr*)pData)->data; // pRes->tsrow[columnIndex] = ((tstr*)pData)->data;
} // }
//
if (realLen < pInfo->pSqlExpr->resBytes - VARSTR_HEADER_SIZE) { // todo refactor // if (realLen < pInfo->pSqlExpr->resBytes - VARSTR_HEADER_SIZE) { // todo refactor
*(pData + realLen + VARSTR_HEADER_SIZE) = 0; // *(pData + realLen + VARSTR_HEADER_SIZE) = 0;
} // }
//
pRes->length[columnIndex] = realLen; // pRes->length[columnIndex] = realLen;
} else { // } else {
assert(bytes == tDataTypeDesc[type].nSize); // assert(bytes == tDataTypeDesc[type].nSize);
//
if (isNull(pData, type)) { // if (isNull(pData, type)) {
pRes->tsrow[columnIndex] = NULL; // pRes->tsrow[columnIndex] = NULL;
} else { // } else {
pRes->tsrow[columnIndex] = pData; // pRes->tsrow[columnIndex] = pData;
} // }
//
pRes->length[columnIndex] = bytes; // pRes->length[columnIndex] = bytes;
} // }
} //}
void* malloc_throw(size_t size) { void* malloc_throw(size_t size) {
void* p = malloc(size); void* p = malloc(size);

4
src/common/inc/tglobal.h
View File

@ -68,7 +68,10 @@ extern int64_t tsMaxRetentWindow;
// db parameters in client // db parameters in client
extern int32_t tsCacheBlockSize; extern int32_t tsCacheBlockSize;
extern int32_t tsBlocksPerVnode; extern int32_t tsBlocksPerVnode;
extern int32_t tsMinTablePerVnode;
extern int32_t tsMaxTablePerVnode; extern int32_t tsMaxTablePerVnode;
extern int32_t tsTableIncStepPerVnode;
extern int32_t tsMaxVgroupsPerDb;
extern int16_t tsDaysPerFile; extern int16_t tsDaysPerFile;
extern int32_t tsDaysToKeep; extern int32_t tsDaysToKeep;
extern int32_t tsMinRowsInFileBlock; extern int32_t tsMinRowsInFileBlock;
@ -77,6 +80,7 @@ extern int16_t tsCommitTime; // seconds
extern int32_t tsTimePrecision; extern int32_t tsTimePrecision;
extern int16_t tsCompression; extern int16_t tsCompression;
extern int16_t tsWAL; extern int16_t tsWAL;
extern int32_t tsFsyncPeriod;
extern int32_t tsReplications; extern int32_t tsReplications;
// balance // balance

12
src/common/inc/tulog.h
View File

@ -25,15 +25,15 @@ extern "C" {
extern int32_t uDebugFlag; extern int32_t uDebugFlag;
extern int32_t tscEmbedded; extern int32_t tscEmbedded;
#define uFatal(...) { if (uDebugFlag & DEBUG_FATAL) { taosPrintLog("UTL FATAL ", tscEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }} #define uFatal(...) { if (uDebugFlag & DEBUG_FATAL) { taosPrintLog("UTL FATAL", tscEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }}
#define uError(...) { if (uDebugFlag & DEBUG_ERROR) { taosPrintLog("UTL ERROR ", tscEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }} #define uError(...) { if (uDebugFlag & DEBUG_ERROR) { taosPrintLog("UTL ERROR ", tscEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }}
#define uWarn(...) { if (uDebugFlag & DEBUG_WARN) { taosPrintLog("UTL WARN ", tscEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }} #define uWarn(...) { if (uDebugFlag & DEBUG_WARN) { taosPrintLog("UTL WARN ", tscEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }}
#define uInfo(...) { if (uDebugFlag & DEBUG_INFO) { taosPrintLog("UTL INFO ", tscEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }} #define uInfo(...) { if (uDebugFlag & DEBUG_INFO) { taosPrintLog("UTL ", tscEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }}
#define uDebug(...) { if (uDebugFlag & DEBUG_DEBUG) { taosPrintLog("UTL DEBUG ", uDebugFlag, __VA_ARGS__); }} #define uDebug(...) { if (uDebugFlag & DEBUG_DEBUG) { taosPrintLog("UTL ", uDebugFlag, __VA_ARGS__); }}
#define uTrace(...) { if (uDebugFlag & DEBUG_TRACE) { taosPrintLog("UTL TRACE ", uDebugFlag, __VA_ARGS__); }} #define uTrace(...) { if (uDebugFlag & DEBUG_TRACE) { taosPrintLog("UTL ", uDebugFlag, __VA_ARGS__); }}
#define pError(...) { taosPrintLog("APP ERROR ", 255, __VA_ARGS__); } #define pError(...) { taosPrintLog("APP ERROR ", 255, __VA_ARGS__); }
#define pPrint(...) { taosPrintLog("APP INFO ", 255, __VA_ARGS__); } #define pPrint(...) { taosPrintLog("APP ", 255, __VA_ARGS__); }
#ifdef __cplusplus #ifdef __cplusplus
} }

10
src/common/src/tdataformat.c
View File

@ -318,7 +318,7 @@ SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows) {
pCols->maxPoints = maxRows; pCols->maxPoints = maxRows;
pCols->bufSize = maxRowSize * maxRows; pCols->bufSize = maxRowSize * maxRows;
pCols->buf = malloc(pCols->bufSize); pCols->buf = calloc(1, pCols->bufSize);
if (pCols->buf == NULL) { if (pCols->buf == NULL) {
free(pCols); free(pCols);
return NULL; return NULL;
@ -384,9 +384,11 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
} }
void tdResetDataCols(SDataCols *pCols) { void tdResetDataCols(SDataCols *pCols) {
pCols->numOfRows = 0; if (pCols != NULL) {
for (int i = 0; i < pCols->maxCols; i++) { pCols->numOfRows = 0;
dataColReset(pCols->cols + i); for (int i = 0; i < pCols->maxCols; i++) {
dataColReset(pCols->cols + i);
}
} }
} }

79
src/common/src/tglobal.c
View File

@ -38,7 +38,7 @@ uint16_t tsDnodeShellPort = 6030; // udp[6035-6039] tcp[6035]
uint16_t tsDnodeDnodePort = 6035; // udp/tcp uint16_t tsDnodeDnodePort = 6035; // udp/tcp
uint16_t tsSyncPort = 6040; uint16_t tsSyncPort = 6040;
int32_t tsStatusInterval = 1; // second int32_t tsStatusInterval = 1; // second
int16_t tsNumOfVnodesPerCore = 8; int16_t tsNumOfVnodesPerCore = 32;
int16_t tsNumOfTotalVnodes = TSDB_INVALID_VNODE_NUM; int16_t tsNumOfTotalVnodes = TSDB_INVALID_VNODE_NUM;
int32_t tsNumOfMnodes = 3; int32_t tsNumOfMnodes = 3;
int32_t tsEnableVnodeBak = 1; int32_t tsEnableVnodeBak = 1;
@ -110,13 +110,12 @@ int16_t tsCommitTime = TSDB_DEFAULT_COMMIT_TIME; // seconds
int32_t tsTimePrecision = TSDB_DEFAULT_PRECISION; int32_t tsTimePrecision = TSDB_DEFAULT_PRECISION;
int16_t tsCompression = TSDB_DEFAULT_COMP_LEVEL; int16_t tsCompression = TSDB_DEFAULT_COMP_LEVEL;
int16_t tsWAL = TSDB_DEFAULT_WAL_LEVEL; int16_t tsWAL = TSDB_DEFAULT_WAL_LEVEL;
int32_t tsReplications = TSDB_DEFAULT_REPLICA_NUM; int32_t tsFsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD;
int32_t tsReplications = TSDB_DEFAULT_DB_REPLICA_OPTION;
#ifdef _TD_ARM_32_ int32_t tsMaxVgroupsPerDb = 0;
int32_t tsMaxTablePerVnode = 100; int32_t tsMinTablePerVnode = 100;
#else int32_t tsMaxTablePerVnode = TSDB_DEFAULT_TABLES;
int32_t tsMaxTablePerVnode = TSDB_DEFAULT_TABLES; int32_t tsTableIncStepPerVnode = TSDB_TABLES_STEP;
#endif
// balance // balance
int32_t tsEnableBalance = 1; int32_t tsEnableBalance = 1;
@ -195,7 +194,7 @@ int32_t monitorDebugFlag = 131;
int32_t qDebugFlag = 131; int32_t qDebugFlag = 131;
int32_t rpcDebugFlag = 131; int32_t rpcDebugFlag = 131;
int32_t uDebugFlag = 131; int32_t uDebugFlag = 131;
int32_t debugFlag = 131; int32_t debugFlag = 0;
int32_t sDebugFlag = 135; int32_t sDebugFlag = 135;
int32_t wDebugFlag = 135; int32_t wDebugFlag = 135;
int32_t tsdbDebugFlag = 131; int32_t tsdbDebugFlag = 131;
@ -203,7 +202,7 @@ int32_t tsdbDebugFlag = 131;
static pthread_once_t tsInitGlobalCfgOnce = PTHREAD_ONCE_INIT; static pthread_once_t tsInitGlobalCfgOnce = PTHREAD_ONCE_INIT;
void taosSetAllDebugFlag() { void taosSetAllDebugFlag() {
for (int32_t i = 0; i < tsGlobalConfigNum; ++i) { if (debugFlag != 0) {
mDebugFlag = debugFlag; mDebugFlag = debugFlag;
sdbDebugFlag = debugFlag; sdbDebugFlag = debugFlag;
dDebugFlag = debugFlag; dDebugFlag = debugFlag;
@ -220,8 +219,8 @@ void taosSetAllDebugFlag() {
wDebugFlag = debugFlag; wDebugFlag = debugFlag;
tsdbDebugFlag = debugFlag; tsdbDebugFlag = debugFlag;
qDebugFlag = debugFlag; qDebugFlag = debugFlag;
uInfo("all debug flag are set to %d", debugFlag);
} }
uInfo("all debug flag are set to %d", debugFlag);
} }
bool taosCfgDynamicOptions(char *msg) { bool taosCfgDynamicOptions(char *msg) {
@ -394,16 +393,6 @@ static void doInitGlobalConfig() {
cfg.unitType = TAOS_CFG_UTYPE_NONE; cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg); taosInitConfigOption(cfg);
cfg.option = "numOfVnodesPerCore";
cfg.ptr = &tsNumOfVnodesPerCore;
cfg.valType = TAOS_CFG_VTYPE_INT16;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 1;
cfg.maxValue = 64;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "numOfTotalVnodes"; cfg.option = "numOfTotalVnodes";
cfg.ptr = &tsNumOfTotalVnodes; cfg.ptr = &tsNumOfTotalVnodes;
cfg.valType = TAOS_CFG_VTYPE_INT16; cfg.valType = TAOS_CFG_VTYPE_INT16;
@ -560,7 +549,7 @@ static void doInitGlobalConfig() {
cfg.ptr = &tsMinIntervalTime; cfg.ptr = &tsMinIntervalTime;
cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 10; cfg.minValue = 1;
cfg.maxValue = 1000000; cfg.maxValue = 1000000;
cfg.ptrLength = 0; cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_MS; cfg.unitType = TAOS_CFG_UTYPE_MS;
@ -606,8 +595,18 @@ static void doInitGlobalConfig() {
cfg.unitType = TAOS_CFG_UTYPE_NONE; cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg); taosInitConfigOption(cfg);
cfg.option = "maxVgroupsPerDb";
cfg.ptr = &tsMaxVgroupsPerDb;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 0;
cfg.maxValue = 8192;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
// database configs // database configs
cfg.option = "maxtablesPerVnode"; cfg.option = "maxTablesPerVnode";
cfg.ptr = &tsMaxTablePerVnode; cfg.ptr = &tsMaxTablePerVnode;
cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
@ -617,6 +616,26 @@ static void doInitGlobalConfig() {
cfg.unitType = TAOS_CFG_UTYPE_NONE; cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg); taosInitConfigOption(cfg);
cfg.option = "minTablesPerVnode";
cfg.ptr = &tsMinTablePerVnode;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = TSDB_MIN_TABLES;
cfg.maxValue = TSDB_MAX_TABLES;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "tableIncStepPerVnode";
cfg.ptr = &tsTableIncStepPerVnode;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = TSDB_MIN_TABLES;
cfg.maxValue = TSDB_MAX_TABLES;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "cache"; cfg.option = "cache";
cfg.ptr = &tsCacheBlockSize; cfg.ptr = &tsCacheBlockSize;
cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.valType = TAOS_CFG_VTYPE_INT32;
@ -697,12 +716,22 @@ static void doInitGlobalConfig() {
cfg.unitType = TAOS_CFG_UTYPE_NONE; cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg); taosInitConfigOption(cfg);
cfg.option = "fsync";
cfg.ptr = &tsFsyncPeriod;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = TSDB_MIN_FSYNC_PERIOD;
cfg.maxValue = TSDB_MAX_FSYNC_PERIOD;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "replica"; cfg.option = "replica";
cfg.ptr = &tsReplications; cfg.ptr = &tsReplications;
cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = TSDB_MIN_REPLICA_NUM; cfg.minValue = TSDB_MIN_DB_REPLICA_OPTION;
cfg.maxValue = TSDB_MAX_REPLICA_NUM; cfg.maxValue = TSDB_MAX_DB_REPLICA_OPTION;
cfg.ptrLength = 0; cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE; cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg); taosInitConfigOption(cfg);

2
src/common/src/tname.c
View File

@ -82,7 +82,7 @@ int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, in
} }
int64_t start = ((startTime - intervalTime) / slidingTime + 1) * slidingTime; int64_t start = ((startTime - intervalTime) / slidingTime + 1) * slidingTime;
if (!(timeUnit == 'a' || timeUnit == 'm' || timeUnit == 's' || timeUnit == 'h')) { if (!(timeUnit == 'u' || timeUnit == 'a' || timeUnit == 'm' || timeUnit == 's' || timeUnit == 'h')) {
/* /*
* here we revised the start time of day according to the local time zone, * here we revised the start time of day according to the local time zone,
* but in case of DST, the start time of one day need to be dynamically decided. * but in case of DST, the start time of one day need to be dynamically decided.

50
src/common/src/ttypes.c
View File

@ -367,31 +367,31 @@ bool isValidDataType(int32_t type) {
return type >= TSDB_DATA_TYPE_NULL && type <= TSDB_DATA_TYPE_NCHAR; return type >= TSDB_DATA_TYPE_NULL && type <= TSDB_DATA_TYPE_NCHAR;
} }
bool isNull(const char *val, int32_t type) { //bool isNull(const char *val, int32_t type) {
switch (type) { // switch (type) {
case TSDB_DATA_TYPE_BOOL: // case TSDB_DATA_TYPE_BOOL:
return *(uint8_t *)val == TSDB_DATA_BOOL_NULL; // return *(uint8_t *)val == TSDB_DATA_BOOL_NULL;
case TSDB_DATA_TYPE_TINYINT: // case TSDB_DATA_TYPE_TINYINT:
return *(uint8_t *)val == TSDB_DATA_TINYINT_NULL; // return *(uint8_t *)val == TSDB_DATA_TINYINT_NULL;
case TSDB_DATA_TYPE_SMALLINT: // case TSDB_DATA_TYPE_SMALLINT:
return *(uint16_t *)val == TSDB_DATA_SMALLINT_NULL; // return *(uint16_t *)val == TSDB_DATA_SMALLINT_NULL;
case TSDB_DATA_TYPE_INT: // case TSDB_DATA_TYPE_INT:
return *(uint32_t *)val == TSDB_DATA_INT_NULL; // return *(uint32_t *)val == TSDB_DATA_INT_NULL;
case TSDB_DATA_TYPE_BIGINT: // case TSDB_DATA_TYPE_BIGINT:
case TSDB_DATA_TYPE_TIMESTAMP: // case TSDB_DATA_TYPE_TIMESTAMP:
return *(uint64_t *)val == TSDB_DATA_BIGINT_NULL; // return *(uint64_t *)val == TSDB_DATA_BIGINT_NULL;
case TSDB_DATA_TYPE_FLOAT: // case TSDB_DATA_TYPE_FLOAT:
return *(uint32_t *)val == TSDB_DATA_FLOAT_NULL; // return *(uint32_t *)val == TSDB_DATA_FLOAT_NULL;
case TSDB_DATA_TYPE_DOUBLE: // case TSDB_DATA_TYPE_DOUBLE:
return *(uint64_t *)val == TSDB_DATA_DOUBLE_NULL; // return *(uint64_t *)val == TSDB_DATA_DOUBLE_NULL;
case TSDB_DATA_TYPE_NCHAR: // case TSDB_DATA_TYPE_NCHAR:
return *(uint32_t*) varDataVal(val) == TSDB_DATA_NCHAR_NULL; // return *(uint32_t*) varDataVal(val) == TSDB_DATA_NCHAR_NULL;
case TSDB_DATA_TYPE_BINARY: // case TSDB_DATA_TYPE_BINARY:
return *(uint8_t *) varDataVal(val) == TSDB_DATA_BINARY_NULL; // return *(uint8_t *) varDataVal(val) == TSDB_DATA_BINARY_NULL;
default: // default:
return false; // return false;
}; // };
} //}
void setVardataNull(char* val, int32_t type) { void setVardataNull(char* val, int32_t type) {
if (type == TSDB_DATA_TYPE_BINARY) { if (type == TSDB_DATA_TYPE_BINARY) {

4
src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNode.java
View File

@ -52,7 +52,7 @@ public class TDNode {
public void start() { public void start() {
String selfPath = System.getProperty("user.dir"); String selfPath = System.getProperty("user.dir");
String binPath = ""; String binPath = "";
String projDir = selfPath + "/../../../../"; String projDir = selfPath + "/../../../";
try { try {
ArrayList<String> taosdPath = new ArrayList<>(); ArrayList<String> taosdPath = new ArrayList<>();
@ -68,7 +68,7 @@ public class TDNode {
return; return;
} else { } else {
for(String p : taosdPath) { for(String p : taosdPath) {
if(!p.contains("packing")) { if(!p.contains("packaging")) {
binPath = p; binPath = p;
} }
} }

123
src/connector/jdbc/src/test/java/com/taosdata/jdbc/BatchInsertTest.java Normal file
View File

@ -0,0 +1,123 @@
package com.taosdata.jdbc;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.sql.*;
import java.util.Properties;
import java.util.Random;
import static org.junit.Assert.assertEquals;
import java.util.Properties;
import java.util.concurrent.Executors;
import java.util.concurrent.*;
import static org.junit.Assert.assertTrue;
public class BatchInsertTest extends BaseTest {
static Connection connection = null;
static Statement statement = null;
static String dbName = "test";
static String stbName = "meters";
static String host = "localhost";
static int numOfTables = 30;
final static int numOfRecordsPerTable = 1000;
static long ts = 1496732686000l;
final static String tablePrefix = "t";
@Before
public void createDatabase() throws SQLException {
try {
Class.forName("com.taosdata.jdbc.TSDBDriver");
} catch (ClassNotFoundException e) {
return;
}
Properties properties = new Properties();
properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host);
properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");
connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + "?user=root&password=taosdata"
, properties);
statement = connection.createStatement();
statement.executeUpdate("drop database if exists " + dbName);
statement.executeUpdate("create database if not exists " + dbName);
statement.executeUpdate("use " + dbName);
String createTableSql = "create table " + stbName + "(ts timestamp, f1 int, f2 int, f3 int) tags(areaid int, loc binary(20))";
statement.executeUpdate(createTableSql);
for(int i = 0; i < numOfTables; i++) {
String loc = i % 2 == 0 ? "beijing" : "shanghai";
String createSubTalbesSql = "create table " + tablePrefix + i + " using " + stbName + " tags(" + i + ", '" + loc + "')";
statement.executeUpdate(createSubTalbesSql);
}
}
@Test
public void testBatchInsert() throws SQLException{
ExecutorService executorService = Executors.newFixedThreadPool(numOfTables);
for (int i = 0; i < numOfTables; i++) {
final int index = i;
executorService.execute(new Runnable() {
@Override
public void run() {
try {
long startTime = System.currentTimeMillis();
Statement statement = connection.createStatement(); // get statement
StringBuilder sb = new StringBuilder();
sb.append("INSERT INTO " + tablePrefix + index + " VALUES");
Random rand = new Random();
for (int j = 1; j <= numOfRecordsPerTable; j++) {
sb.append("(" + (ts + j) + ", ");
sb.append(rand.nextInt(100) + ", ");
sb.append(rand.nextInt(100) + ", ");
sb.append(rand.nextInt(100) + ")");
}
statement.addBatch(sb.toString());
statement.executeBatch();
long endTime = System.currentTimeMillis();
System.out.println("Thread " + index + " takes " + (endTime - startTime) + " microseconds");
connection.commit();
statement.close();
} catch (Exception e) {
e.printStackTrace();
}
}
});
}
executorService.shutdown();
try {
executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS);
} catch (InterruptedException e) {
e.printStackTrace();
}
Statement statement = connection.createStatement();
ResultSet rs = statement.executeQuery("select * from meters");
int num = 0;
while (rs.next()) {
num++;
}
assertEquals(num, numOfTables * numOfRecordsPerTable);
rs.close();
}
@After
public void close() throws Exception {
statement.close();
connection.close();
Thread.sleep(10);
}
}

42
src/cq/src/cqMain.c
View File

@ -23,6 +23,7 @@
#include "taos.h" #include "taos.h"
#include "taosdef.h" #include "taosdef.h"
#include "taosmsg.h" #include "taosmsg.h"
#include "ttimer.h"
#include "tcq.h" #include "tcq.h"
#include "tdataformat.h" #include "tdataformat.h"
#include "tglobal.h" #include "tglobal.h"
@ -45,10 +46,12 @@ typedef struct {
struct SCqObj *pHead; struct SCqObj *pHead;
void *dbConn; void *dbConn;
int master; int master;
void *tmrCtrl;
pthread_mutex_t mutex; pthread_mutex_t mutex;
} SCqContext; } SCqContext;
typedef struct SCqObj { typedef struct SCqObj {
tmr_h tmrId;
uint64_t uid; uint64_t uid;
int32_t tid; // table ID int32_t tid; // table ID
int rowSize; // bytes of a row int rowSize; // bytes of a row
@ -66,13 +69,14 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row);
static void cqCreateStream(SCqContext *pContext, SCqObj *pObj); static void cqCreateStream(SCqContext *pContext, SCqObj *pObj);
void *cqOpen(void *ahandle, const SCqCfg *pCfg) { void *cqOpen(void *ahandle, const SCqCfg *pCfg) {
SCqContext *pContext = calloc(sizeof(SCqContext), 1); SCqContext *pContext = calloc(sizeof(SCqContext), 1);
if (pContext == NULL) { if (pContext == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
return NULL; return NULL;
} }
pContext->tmrCtrl = taosTmrInit(0, 0, 0, "CQ");
tstrncpy(pContext->user, pCfg->user, sizeof(pContext->user)); tstrncpy(pContext->user, pCfg->user, sizeof(pContext->user));
tstrncpy(pContext->pass, pCfg->pass, sizeof(pContext->pass)); tstrncpy(pContext->pass, pCfg->pass, sizeof(pContext->pass));
const char* db = pCfg->db; const char* db = pCfg->db;
@ -99,6 +103,9 @@ void cqClose(void *handle) {
SCqContext *pContext = handle; SCqContext *pContext = handle;
if (handle == NULL) return; if (handle == NULL) return;
taosTmrCleanUp(pContext->tmrCtrl);
pContext->tmrCtrl = NULL;
// stop all CQs // stop all CQs
cqStop(pContext); cqStop(pContext);
@ -154,8 +161,10 @@ void cqStop(void *handle) {
taos_close_stream(pObj->pStream); taos_close_stream(pObj->pStream);
pObj->pStream = NULL; pObj->pStream = NULL;
cTrace("vgId:%d, id:%d CQ:%s is closed", pContext->vgId, pObj->tid, pObj->sqlStr); cTrace("vgId:%d, id:%d CQ:%s is closed", pContext->vgId, pObj->tid, pObj->sqlStr);
} else {
taosTmrStop(pObj->tmrId);
pObj->tmrId = 0;
} }
pObj = pObj->next; pObj = pObj->next;
} }
@ -211,8 +220,13 @@ void cqDrop(void *handle) {
} }
// free the resources associated // free the resources associated
if (pObj->pStream) taos_close_stream(pObj->pStream); if (pObj->pStream) {
pObj->pStream = NULL; taos_close_stream(pObj->pStream);
pObj->pStream = NULL;
} else {
taosTmrStop(pObj->tmrId);
pObj->tmrId = 0;
}
cTrace("vgId:%d, id:%d CQ:%s is dropped", pContext->vgId, pObj->tid, pObj->sqlStr); cTrace("vgId:%d, id:%d CQ:%s is dropped", pContext->vgId, pObj->tid, pObj->sqlStr);
tdFreeSchema(pObj->pSchema); tdFreeSchema(pObj->pSchema);
@ -222,18 +236,30 @@ void cqDrop(void *handle) {
pthread_mutex_unlock(&pContext->mutex); pthread_mutex_unlock(&pContext->mutex);
} }
static void cqCreateStream(SCqContext *pContext, SCqObj *pObj) { static void cqProcessCreateTimer(void *param, void *tmrId) {
SCqObj* pObj = (SCqObj*)param;
SCqContext* pContext = pObj->pContext;
if (pContext->dbConn == NULL) { if (pContext->dbConn == NULL) {
pContext->dbConn = taos_connect("localhost", pContext->user, pContext->pass, pContext->db, 0); pContext->dbConn = taos_connect("localhost", pContext->user, pContext->pass, pContext->db, 0);
if (pContext->dbConn == NULL) { if (pContext->dbConn == NULL) {
cError("vgId:%d, failed to connect to TDengine(%s)", pContext->vgId, tstrerror(terrno)); cError("vgId:%d, failed to connect to TDengine(%s)", pContext->vgId, tstrerror(terrno));
return;
} }
} }
cqCreateStream(pContext, pObj);
}
int64_t lastKey = 0; static void cqCreateStream(SCqContext *pContext, SCqObj *pObj) {
pObj->pContext = pContext; pObj->pContext = pContext;
pObj->pStream = taos_open_stream(pContext->dbConn, pObj->sqlStr, cqProcessStreamRes, lastKey, pObj, NULL);
if (pContext->dbConn == NULL) {
pObj->tmrId = taosTmrStart(cqProcessCreateTimer, 1000, pObj, pContext->tmrCtrl);
return;
}
pObj->tmrId = 0;
pObj->pStream = taos_open_stream(pContext->dbConn, pObj->sqlStr, cqProcessStreamRes, 0, pObj, NULL);
if (pObj->pStream) { if (pObj->pStream) {
pContext->num++; pContext->num++;
cTrace("vgId:%d, id:%d CQ:%s is openned", pContext->vgId, pObj->tid, pObj->sqlStr); cTrace("vgId:%d, id:%d CQ:%s is openned", pContext->vgId, pObj->tid, pObj->sqlStr);

8
src/dnode/inc/dnodeInt.h
View File

@ -26,10 +26,10 @@ extern int32_t dDebugFlag;
#define dFatal(...) { if (dDebugFlag & DEBUG_FATAL) { taosPrintLog("DND FATAL ", 255, __VA_ARGS__); }} #define dFatal(...) { if (dDebugFlag & DEBUG_FATAL) { taosPrintLog("DND FATAL ", 255, __VA_ARGS__); }}
#define dError(...) { if (dDebugFlag & DEBUG_ERROR) { taosPrintLog("DND ERROR ", 255, __VA_ARGS__); }} #define dError(...) { if (dDebugFlag & DEBUG_ERROR) { taosPrintLog("DND ERROR ", 255, __VA_ARGS__); }}
#define dWarn(...) { if (dDebugFlag & DEBUG_WARN) { taosPrintLog("DND WARN ", 255, __VA_ARGS__); }} #define dWarn(...) { if (dDebugFlag & DEBUG_WARN) { taosPrintLog("DND WARN ", 255, __VA_ARGS__); }}
#define dInfo(...) { if (dDebugFlag & DEBUG_INFO) { taosPrintLog("DND INFO ", 255, __VA_ARGS__); }} #define dInfo(...) { if (dDebugFlag & DEBUG_INFO) { taosPrintLog("DND ", 255, __VA_ARGS__); }}
#define dDebug(...) { if (dDebugFlag & DEBUG_DEBUG) { taosPrintLog("DND DEBUG ", dDebugFlag, __VA_ARGS__); }} #define dDebug(...) { if (dDebugFlag & DEBUG_DEBUG) { taosPrintLog("DND ", dDebugFlag, __VA_ARGS__); }}
#define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("DND TRACE ", dDebugFlag, __VA_ARGS__); }} #define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("DND ", dDebugFlag, __VA_ARGS__); }}
#ifdef __cplusplus #ifdef __cplusplus
} }

1
src/dnode/inc/dnodeModule.h
View File

@ -22,7 +22,6 @@ extern "C" {
int32_t dnodeInitModules(); int32_t dnodeInitModules();
void dnodeStartModules(); void dnodeStartModules();
void dnodeStartStream();
void dnodeCleanupModules(); void dnodeCleanupModules();
void dnodeProcessModuleStatus(uint32_t moduleStatus); void dnodeProcessModuleStatus(uint32_t moduleStatus);

1
src/dnode/src/dnodeMain.c
View File

@ -123,7 +123,6 @@ int32_t dnodeInitSystem() {
dnodeStartModules(); dnodeStartModules();
dnodeSetRunStatus(TSDB_DNODE_RUN_STATUS_RUNING); dnodeSetRunStatus(TSDB_DNODE_RUN_STATUS_RUNING);
dnodeStartStream();
dInfo("TDengine is initialized successfully"); dInfo("TDengine is initialized successfully");

62
src/dnode/src/dnodeMgmt.c
View File

@ -72,6 +72,7 @@ static void *dnodeProcessMgmtQueue(void *param);
static int32_t dnodeOpenVnodes(); static int32_t dnodeOpenVnodes();
static void dnodeCloseVnodes(); static void dnodeCloseVnodes();
static int32_t dnodeProcessCreateVnodeMsg(SRpcMsg *pMsg); static int32_t dnodeProcessCreateVnodeMsg(SRpcMsg *pMsg);
static int32_t dnodeProcessAlterVnodeMsg(SRpcMsg *pMsg);
static int32_t dnodeProcessDropVnodeMsg(SRpcMsg *pMsg); static int32_t dnodeProcessDropVnodeMsg(SRpcMsg *pMsg);
static int32_t dnodeProcessAlterStreamMsg(SRpcMsg *pMsg); static int32_t dnodeProcessAlterStreamMsg(SRpcMsg *pMsg);
static int32_t dnodeProcessConfigDnodeMsg(SRpcMsg *pMsg); static int32_t dnodeProcessConfigDnodeMsg(SRpcMsg *pMsg);
@ -79,6 +80,7 @@ static int32_t (*dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *pMsg);
int32_t dnodeInitMgmt() { int32_t dnodeInitMgmt() {
dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE] = dnodeProcessCreateVnodeMsg; dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE] = dnodeProcessCreateVnodeMsg;
dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE] = dnodeProcessAlterVnodeMsg;
dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_DROP_VNODE] = dnodeProcessDropVnodeMsg; dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_DROP_VNODE] = dnodeProcessDropVnodeMsg;
dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM] = dnodeProcessAlterStreamMsg; dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM] = dnodeProcessAlterStreamMsg;
dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE] = dnodeProcessConfigDnodeMsg; dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE] = dnodeProcessConfigDnodeMsg;
@ -352,23 +354,6 @@ static int32_t dnodeOpenVnodes() {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
void dnodeStartStream() {
int32_t vnodeList[TSDB_MAX_VNODES] = {0};
int32_t numOfVnodes = 0;
int32_t status = vnodeGetVnodeList(vnodeList, &numOfVnodes);
if (status != TSDB_CODE_SUCCESS) {
dInfo("get dnode list failed");
return;
}
for (int32_t i = 0; i < numOfVnodes; ++i) {
vnodeStartStream(vnodeList[i]);
}
dInfo("streams started");
}
static void dnodeCloseVnodes() { static void dnodeCloseVnodes() {
int32_t vnodeList[TSDB_MAX_VNODES]= {0}; int32_t vnodeList[TSDB_MAX_VNODES]= {0};
int32_t numOfVnodes = 0; int32_t numOfVnodes = 0;
@ -388,7 +373,7 @@ static void dnodeCloseVnodes() {
dInfo("total vnodes:%d are all closed", numOfVnodes); dInfo("total vnodes:%d are all closed", numOfVnodes);
} }
static int32_t dnodeProcessCreateVnodeMsg(SRpcMsg *rpcMsg) { static void* dnodeParseVnodeMsg(SRpcMsg *rpcMsg) {
SMDCreateVnodeMsg *pCreate = rpcMsg->pCont; SMDCreateVnodeMsg *pCreate = rpcMsg->pCont;
pCreate->cfg.vgId = htonl(pCreate->cfg.vgId); pCreate->cfg.vgId = htonl(pCreate->cfg.vgId);
pCreate->cfg.cfgVersion = htonl(pCreate->cfg.cfgVersion); pCreate->cfg.cfgVersion = htonl(pCreate->cfg.cfgVersion);
@ -401,20 +386,42 @@ static int32_t dnodeProcessCreateVnodeMsg(SRpcMsg *rpcMsg) {
pCreate->cfg.daysToKeep = htonl(pCreate->cfg.daysToKeep); pCreate->cfg.daysToKeep = htonl(pCreate->cfg.daysToKeep);
pCreate->cfg.minRowsPerFileBlock = htonl(pCreate->cfg.minRowsPerFileBlock); pCreate->cfg.minRowsPerFileBlock = htonl(pCreate->cfg.minRowsPerFileBlock);
pCreate->cfg.maxRowsPerFileBlock = htonl(pCreate->cfg.maxRowsPerFileBlock); pCreate->cfg.maxRowsPerFileBlock = htonl(pCreate->cfg.maxRowsPerFileBlock);
pCreate->cfg.fsyncPeriod = htonl(pCreate->cfg.fsyncPeriod);
pCreate->cfg.commitTime = htonl(pCreate->cfg.commitTime); pCreate->cfg.commitTime = htonl(pCreate->cfg.commitTime);
for (int32_t j = 0; j < pCreate->cfg.replications; ++j) { for (int32_t j = 0; j < pCreate->cfg.replications; ++j) {
pCreate->nodes[j].nodeId = htonl(pCreate->nodes[j].nodeId); pCreate->nodes[j].nodeId = htonl(pCreate->nodes[j].nodeId);
} }
void *pVnode = vnodeAcquireVnode(pCreate->cfg.vgId); return pCreate;
}
static int32_t dnodeProcessCreateVnodeMsg(SRpcMsg *rpcMsg) {
SMDCreateVnodeMsg *pCreate = dnodeParseVnodeMsg(rpcMsg);
void *pVnode = vnodeAcquire(pCreate->cfg.vgId);
if (pVnode != NULL) { if (pVnode != NULL) {
dDebug("vgId:%d, already exist, processed as alter msg", pCreate->cfg.vgId); dDebug("vgId:%d, already exist, return success", pCreate->cfg.vgId);
int32_t code = vnodeAlter(pVnode, pCreate); vnodeRelease(pVnode);
return TSDB_CODE_SUCCESS;
} else {
dDebug("vgId:%d, create vnode msg is received", pCreate->cfg.vgId);
return vnodeCreate(pCreate);
}
}
static int32_t dnodeProcessAlterVnodeMsg(SRpcMsg *rpcMsg) {
SMDAlterVnodeMsg *pAlter = dnodeParseVnodeMsg(rpcMsg);
void *pVnode = vnodeAcquire(pAlter->cfg.vgId);
if (pVnode != NULL) {
dDebug("vgId:%d, alter vnode msg is received", pAlter->cfg.vgId);
int32_t code = vnodeAlter(pVnode, pAlter);
vnodeRelease(pVnode); vnodeRelease(pVnode);
return code; return code;
} else { } else {
return vnodeCreate(pCreate); dError("vgId:%d, vnode not exist, can't alter it", pAlter->cfg.vgId);
return TSDB_CODE_VND_INVALID_VGROUP_ID;
} }
} }
@ -698,10 +705,13 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) {
pStatus->alternativeRole = (uint8_t) tsAlternativeRole; pStatus->alternativeRole = (uint8_t) tsAlternativeRole;
// fill cluster cfg parameters // fill cluster cfg parameters
pStatus->clusterCfg.numOfMnodes = tsNumOfMnodes; pStatus->clusterCfg.numOfMnodes = htonl(tsNumOfMnodes);
pStatus->clusterCfg.mnodeEqualVnodeNum = tsMnodeEqualVnodeNum; pStatus->clusterCfg.enableBalance = htonl(tsEnableBalance);
pStatus->clusterCfg.offlineThreshold = tsOfflineThreshold; pStatus->clusterCfg.mnodeEqualVnodeNum = htonl(tsMnodeEqualVnodeNum);
pStatus->clusterCfg.statusInterval = tsStatusInterval; pStatus->clusterCfg.offlineThreshold = htonl(tsOfflineThreshold);
pStatus->clusterCfg.statusInterval = htonl(tsStatusInterval);
pStatus->clusterCfg.maxtablesPerVnode = htonl(tsMaxTablePerVnode);
pStatus->clusterCfg.maxVgroupsPerDb = htonl(tsMaxVgroupsPerDb);
strcpy(pStatus->clusterCfg.arbitrator, tsArbitrator); strcpy(pStatus->clusterCfg.arbitrator, tsArbitrator);
strcpy(pStatus->clusterCfg.timezone, tsTimezone); strcpy(pStatus->clusterCfg.timezone, tsTimezone);
strcpy(pStatus->clusterCfg.locale, tsLocale); strcpy(pStatus->clusterCfg.locale, tsLocale);

5
src/dnode/src/dnodePeer.c
View File

@ -44,6 +44,7 @@ int32_t dnodeInitServer() {
dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_DROP_STABLE] = dnodeDispatchToVnodeWriteQueue; dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_DROP_STABLE] = dnodeDispatchToVnodeWriteQueue;
dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE] = dnodeDispatchToMgmtQueue; dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE] = dnodeDispatchToMgmtQueue;
dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE] = dnodeDispatchToMgmtQueue;
dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_DROP_VNODE] = dnodeDispatchToMgmtQueue; dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_DROP_VNODE] = dnodeDispatchToMgmtQueue;
dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM] = dnodeDispatchToMgmtQueue; dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM] = dnodeDispatchToMgmtQueue;
dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE] = dnodeDispatchToMgmtQueue; dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE] = dnodeDispatchToMgmtQueue;
@ -60,7 +61,7 @@ int32_t dnodeInitServer() {
rpcInit.label = "DND-S"; rpcInit.label = "DND-S";
rpcInit.numOfThreads = 1; rpcInit.numOfThreads = 1;
rpcInit.cfp = dnodeProcessReqMsgFromDnode; rpcInit.cfp = dnodeProcessReqMsgFromDnode;
rpcInit.sessions = 100; rpcInit.sessions = TSDB_MAX_VNODES;
rpcInit.connType = TAOS_CONN_SERVER; rpcInit.connType = TAOS_CONN_SERVER;
rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.idleTime = tsShellActivityTimer * 1000;
@ -122,7 +123,7 @@ int32_t dnodeInitClient() {
rpcInit.label = "DND-C"; rpcInit.label = "DND-C";
rpcInit.numOfThreads = 1; rpcInit.numOfThreads = 1;
rpcInit.cfp = dnodeProcessRspFromDnode; rpcInit.cfp = dnodeProcessRspFromDnode;
rpcInit.sessions = 100; rpcInit.sessions = TSDB_MAX_VNODES;
rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.connType = TAOS_CONN_CLIENT;
rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.idleTime = tsShellActivityTimer * 1000;
rpcInit.user = "t"; rpcInit.user = "t";

10
src/dnode/src/dnodeSystem.c
View File

@ -70,7 +70,13 @@ int32_t main(int32_t argc, char *argv[]) {
} }
#endif #endif
#ifdef TAOS_RANDOM_FILE_FAIL #ifdef TAOS_RANDOM_FILE_FAIL
else if (strcmp(argv[i], "--random-file-fail-factor") == 0) { else if (strcmp(argv[i], "--random-file-fail-output") == 0) {
if ((i < argc - 1) && (argv[i + 1][0] != '-')) {
taosSetRandomFileFailOutput(argv[++i]);
} else {
taosSetRandomFileFailOutput(NULL);
}
} else if (strcmp(argv[i], "--random-file-fail-factor") == 0) {
if ( (i+1) < argc ) { if ( (i+1) < argc ) {
int factor = atoi(argv[i+1]); int factor = atoi(argv[i+1]);
printf("The factor of random failure is %d\n", factor); printf("The factor of random failure is %d\n", factor);
@ -129,7 +135,7 @@ int32_t main(int32_t argc, char *argv[]) {
static void signal_handler(int32_t signum, siginfo_t *sigInfo, void *context) { static void signal_handler(int32_t signum, siginfo_t *sigInfo, void *context) {
if (signum == SIGUSR1) { if (signum == SIGUSR1) {
taosCfgDynamicOptions("debugFlag 151"); taosCfgDynamicOptions("debugFlag 143");
return; return;
} }
if (signum == SIGUSR2) { if (signum == SIGUSR2) {

18
src/dnode/src/dnodeVRead.c
View File

@ -91,23 +91,21 @@ void dnodeDispatchToVnodeReadQueue(SRpcMsg *pMsg) {
int32_t queuedMsgNum = 0; int32_t queuedMsgNum = 0;
int32_t leftLen = pMsg->contLen; int32_t leftLen = pMsg->contLen;
char *pCont = (char *) pMsg->pCont; char *pCont = (char *) pMsg->pCont;
void *pVnode;
while (leftLen > 0) { while (leftLen > 0) {
SMsgHead *pHead = (SMsgHead *) pCont; SMsgHead *pHead = (SMsgHead *) pCont;
pHead->vgId = htonl(pHead->vgId); pHead->vgId = htonl(pHead->vgId);
pHead->contLen = htonl(pHead->contLen); pHead->contLen = htonl(pHead->contLen);
pVnode = vnodeAcquireVnode(pHead->vgId); taos_queue queue = vnodeAcquireRqueue(pHead->vgId);
if (pVnode == NULL) { if (queue == NULL) {
leftLen -= pHead->contLen; leftLen -= pHead->contLen;
pCont -= pHead->contLen; pCont -= pHead->contLen;
continue; continue;
} }
// put message into queue // put message into queue
taos_queue queue = vnodeGetRqueue(pVnode);
SReadMsg *pRead = (SReadMsg *)taosAllocateQitem(sizeof(SReadMsg)); SReadMsg *pRead = (SReadMsg *)taosAllocateQitem(sizeof(SReadMsg));
pRead->rpcMsg = *pMsg; pRead->rpcMsg = *pMsg;
pRead->pCont = pCont; pRead->pCont = pCont;
@ -175,18 +173,6 @@ void dnodeFreeVnodeRqueue(void *rqueue) {
// dynamically adjust the number of threads // dynamically adjust the number of threads
} }
void dnodePutItemIntoReadQueue(void *pVnode, void *qhandle) {
SReadMsg *pRead = (SReadMsg *)taosAllocateQitem(sizeof(SReadMsg));
pRead->rpcMsg.msgType = TSDB_MSG_TYPE_QUERY;
pRead->pCont = qhandle;
pRead->contLen = 0;
assert(pVnode != NULL);
taos_queue queue = vnodeAcquireRqueue(pVnode);
taosWriteQitem(queue, TAOS_QTYPE_QUERY, pRead);
}
void dnodeSendRpcReadRsp(void *pVnode, SReadMsg *pRead, int32_t code) { void dnodeSendRpcReadRsp(void *pVnode, SReadMsg *pRead, int32_t code) {
SRpcMsg rpcRsp = { SRpcMsg rpcRsp = {
.handle = pRead->rpcMsg.handle, .handle = pRead->rpcMsg.handle,

5
src/dnode/src/dnodeVWrite.c
View File

@ -104,7 +104,7 @@ void dnodeDispatchToVnodeWriteQueue(SRpcMsg *pMsg) {
pHead->vgId = htonl(pHead->vgId); pHead->vgId = htonl(pHead->vgId);
pHead->contLen = htonl(pHead->contLen); pHead->contLen = htonl(pHead->contLen);
taos_queue queue = vnodeGetWqueue(pHead->vgId); taos_queue queue = vnodeAcquireWqueue(pHead->vgId);
if (queue) { if (queue) {
// put message into queue // put message into queue
SWriteMsg *pWrite = (SWriteMsg *)taosAllocateQitem(sizeof(SWriteMsg)); SWriteMsg *pWrite = (SWriteMsg *)taosAllocateQitem(sizeof(SWriteMsg));
@ -232,9 +232,10 @@ static void *dnodeProcessWriteQueue(void *param) {
pHead->msgType = pWrite->rpcMsg.msgType; pHead->msgType = pWrite->rpcMsg.msgType;
pHead->version = 0; pHead->version = 0;
pHead->len = pWrite->contLen; pHead->len = pWrite->contLen;
dDebug("%p, msg:%s will be processed in vwrite queue", pWrite->rpcMsg.ahandle, taosMsg[pWrite->rpcMsg.msgType]); dDebug("%p, rpc msg:%s will be processed in vwrite queue", pWrite->rpcMsg.ahandle, taosMsg[pWrite->rpcMsg.msgType]);
} else { } else {
pHead = (SWalHead *)item; pHead = (SWalHead *)item;
dTrace("%p, wal msg:%s will be processed in vwrite queue, version:%" PRIu64, pHead, taosMsg[pHead->msgType], pHead->version);
} }
int32_t code = vnodeProcessWrite(pVnode, type, pHead, pRspRet); int32_t code = vnodeProcessWrite(pVnode, type, pHead, pRspRet);

1
src/inc/dnode.h
View File

@ -53,7 +53,6 @@ void *dnodeAllocateVnodeWqueue(void *pVnode);
void dnodeFreeVnodeWqueue(void *queue); void dnodeFreeVnodeWqueue(void *queue);
void *dnodeAllocateVnodeRqueue(void *pVnode); void *dnodeAllocateVnodeRqueue(void *pVnode);
void dnodeFreeVnodeRqueue(void *rqueue); void dnodeFreeVnodeRqueue(void *rqueue);
void dnodePutItemIntoReadQueue(void *pVnode, void *qhandle);
void dnodeSendRpcVnodeWriteRsp(void *pVnode, void *param, int32_t code); void dnodeSendRpcVnodeWriteRsp(void *pVnode, void *param, int32_t code);
int32_t dnodeAllocateMnodePqueue(); int32_t dnodeAllocateMnodePqueue();

16
src/inc/query.h
View File

@ -20,7 +20,6 @@ extern "C" {
#endif #endif
typedef void* qinfo_t; typedef void* qinfo_t;
typedef void (*_qinfo_free_fn_t)(void*);
/** /**
* create the qinfo object according to QueryTableMsg * create the qinfo object according to QueryTableMsg
@ -29,13 +28,8 @@ typedef void (*_qinfo_free_fn_t)(void*);
* @param qinfo * @param qinfo
* @return * @return
*/ */
int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryTableMsg, void* param, _qinfo_free_fn_t fn, qinfo_t* qinfo); int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryTableMsg, void* param, qinfo_t* qinfo);
/**
* Destroy QInfo object
* @param qinfo qhandle
*/
void qDestroyQueryInfo(qinfo_t qinfo);
/** /**
* the main query execution function, including query on both table and multitables, * the main query execution function, including query on both table and multitables,
@ -84,8 +78,14 @@ bool qHasMoreResultsToRetrieve(qinfo_t qinfo);
*/ */
int32_t qKillQuery(qinfo_t qinfo); int32_t qKillQuery(qinfo_t qinfo);
/**
* destroy query info structure
* @param qHandle
*/
void qDestroyQueryInfo(qinfo_t qHandle);
void* qOpenQueryMgmt(int32_t vgId); void* qOpenQueryMgmt(int32_t vgId);
void qSetQueryMgmtClosed(void* pExecutor); void qQueryMgmtNotifyClosed(void* pExecutor);
void qCleanupQueryMgmt(void* pExecutor); void qCleanupQueryMgmt(void* pExecutor);
void** qRegisterQInfo(void* pMgmt, uint64_t qInfo); void** qRegisterQInfo(void* pMgmt, uint64_t qInfo);
void** qAcquireQInfo(void* pMgmt, uint64_t key); void** qAcquireQInfo(void* pMgmt, uint64_t key);

3
src/inc/taos.h
View File

@ -67,6 +67,8 @@ DLL_EXPORT void taos_init();
DLL_EXPORT void taos_cleanup(); DLL_EXPORT void taos_cleanup();
DLL_EXPORT int taos_options(TSDB_OPTION option, const void *arg, ...); DLL_EXPORT int taos_options(TSDB_OPTION option, const void *arg, ...);
DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port); DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port);
DLL_EXPORT TAOS *taos_connect_c(const char *ip, uint8_t ipLen, const char *user, uint8_t userLen,
const char *pass, uint8_t passLen, const char *db, uint8_t dbLen, uint16_t port);
DLL_EXPORT void taos_close(TAOS *taos); DLL_EXPORT void taos_close(TAOS *taos);
typedef struct TAOS_BIND { typedef struct TAOS_BIND {
@ -88,6 +90,7 @@ TAOS_RES * taos_stmt_use_result(TAOS_STMT *stmt);
int taos_stmt_close(TAOS_STMT *stmt); int taos_stmt_close(TAOS_STMT *stmt);
DLL_EXPORT TAOS_RES *taos_query(TAOS *taos, const char *sql); DLL_EXPORT TAOS_RES *taos_query(TAOS *taos, const char *sql);
DLL_EXPORT TAOS_RES *taos_query_c(TAOS *taos, const char *sql, uint32_t sqlLen);
DLL_EXPORT TAOS_ROW taos_fetch_row(TAOS_RES *res); DLL_EXPORT TAOS_ROW taos_fetch_row(TAOS_RES *res);
DLL_EXPORT int taos_result_precision(TAOS_RES *res); // get the time precision of result DLL_EXPORT int taos_result_precision(TAOS_RES *res); // get the time precision of result
DLL_EXPORT void taos_free_result(TAOS_RES *res); DLL_EXPORT void taos_free_result(TAOS_RES *res);

46
src/inc/taosdef.h
View File

@ -160,7 +160,32 @@ extern tDataTypeDescriptor tDataTypeDesc[11];
#define POINTER_BYTES sizeof(void *) // 8 by default assert(sizeof(ptrdiff_t) == sizseof(void*) #define POINTER_BYTES sizeof(void *) // 8 by default assert(sizeof(ptrdiff_t) == sizseof(void*)
bool isValidDataType(int32_t type); bool isValidDataType(int32_t type);
bool isNull(const char *val, int32_t type); //bool isNull(const char *val, int32_t type);
static inline __attribute__((always_inline)) bool isNull(const char *val, int32_t type) {
switch (type) {
case TSDB_DATA_TYPE_BOOL:
return *(uint8_t *)val == TSDB_DATA_BOOL_NULL;
case TSDB_DATA_TYPE_TINYINT:
return *(uint8_t *)val == TSDB_DATA_TINYINT_NULL;
case TSDB_DATA_TYPE_SMALLINT:
return *(uint16_t *)val == TSDB_DATA_SMALLINT_NULL;
case TSDB_DATA_TYPE_INT:
return *(uint32_t *)val == TSDB_DATA_INT_NULL;
case TSDB_DATA_TYPE_BIGINT:
case TSDB_DATA_TYPE_TIMESTAMP:
return *(uint64_t *)val == TSDB_DATA_BIGINT_NULL;
case TSDB_DATA_TYPE_FLOAT:
return *(uint32_t *)val == TSDB_DATA_FLOAT_NULL;
case TSDB_DATA_TYPE_DOUBLE:
return *(uint64_t *)val == TSDB_DATA_DOUBLE_NULL;
case TSDB_DATA_TYPE_NCHAR:
return *(uint32_t*) varDataVal(val) == TSDB_DATA_NCHAR_NULL;
case TSDB_DATA_TYPE_BINARY:
return *(uint8_t *) varDataVal(val) == TSDB_DATA_BINARY_NULL;
default:
return false;
};
}
void setVardataNull(char* val, int32_t type); void setVardataNull(char* val, int32_t type);
void setNull(char *val, int32_t type, int32_t bytes); void setNull(char *val, int32_t type, int32_t bytes);
@ -249,8 +274,8 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
#define TSDB_DEFAULT_PAYLOAD_SIZE 5120 // default payload size, greater than PATH_MAX value #define TSDB_DEFAULT_PAYLOAD_SIZE 5120 // default payload size, greater than PATH_MAX value
#define TSDB_EXTRA_PAYLOAD_SIZE 128 // extra bytes for auth #define TSDB_EXTRA_PAYLOAD_SIZE 128 // extra bytes for auth
#define TSDB_CQ_SQL_SIZE 1024 #define TSDB_CQ_SQL_SIZE 1024
#define TSDB_MAX_VNODES 256 #define TSDB_MAX_VNODES 2048
#define TSDB_MIN_VNODES 50 #define TSDB_MIN_VNODES 256
#define TSDB_INVALID_VNODE_NUM 0 #define TSDB_INVALID_VNODE_NUM 0
#define TSDB_DNODE_ROLE_ANY 0 #define TSDB_DNODE_ROLE_ANY 0
@ -271,8 +296,9 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
#define TSDB_DEFAULT_TOTAL_BLOCKS 4 #define TSDB_DEFAULT_TOTAL_BLOCKS 4
#define TSDB_MIN_TABLES 4 #define TSDB_MIN_TABLES 4
#define TSDB_MAX_TABLES 200000 #define TSDB_MAX_TABLES 10000000
#define TSDB_DEFAULT_TABLES 1000 #define TSDB_DEFAULT_TABLES 1000000
#define TSDB_TABLES_STEP 1000
#define TSDB_MIN_DAYS_PER_FILE 1 #define TSDB_MIN_DAYS_PER_FILE 1
#define TSDB_MAX_DAYS_PER_FILE 3650 #define TSDB_MAX_DAYS_PER_FILE 3650
@ -306,9 +332,13 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
#define TSDB_MAX_WAL_LEVEL 2 #define TSDB_MAX_WAL_LEVEL 2
#define TSDB_DEFAULT_WAL_LEVEL 1 #define TSDB_DEFAULT_WAL_LEVEL 1
#define TSDB_MIN_REPLICA_NUM 1 #define TSDB_MIN_FSYNC_PERIOD 0
#define TSDB_MAX_REPLICA_NUM 3 #define TSDB_MAX_FSYNC_PERIOD 180000 // millisecond
#define TSDB_DEFAULT_REPLICA_NUM 1 #define TSDB_DEFAULT_FSYNC_PERIOD 3000 // three second
#define TSDB_MIN_DB_REPLICA_OPTION 1
#define TSDB_MAX_DB_REPLICA_OPTION 3
#define TSDB_DEFAULT_DB_REPLICA_OPTION 1
#define TSDB_MAX_JOIN_TABLE_NUM 5 #define TSDB_MAX_JOIN_TABLE_NUM 5
#define TSDB_MAX_UNION_CLAUSE 5 #define TSDB_MAX_UNION_CLAUSE 5

2
src/inc/taoserror.h
View File

@ -200,6 +200,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_CREATE_TB_MSG, 0, 0x060E, "tsdb inval
TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_TABLE_DATA_IN_MEM, 0, 0x060F, "tsdb no table data in memory skiplist") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_TABLE_DATA_IN_MEM, 0, 0x060F, "tsdb no table data in memory skiplist")
TAOS_DEFINE_ERROR(TSDB_CODE_TDB_FILE_ALREADY_EXISTS, 0, 0x0610, "tsdb file already exists") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_FILE_ALREADY_EXISTS, 0, 0x0610, "tsdb file already exists")
TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_RECONFIGURE, 0, 0x0611, "tsdb need to reconfigure table") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_RECONFIGURE, 0, 0x0611, "tsdb need to reconfigure table")
TAOS_DEFINE_ERROR(TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO, 0, 0x0612, "tsdb create table information")
// query // query
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_QHANDLE, 0, 0x0700, "query invalid handle") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_QHANDLE, 0, 0x0700, "query invalid handle")
@ -208,6 +209,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NO_DISKSPACE, 0, 0x0702, "query no d
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_OUT_OF_MEMORY, 0, 0x0703, "query out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_OUT_OF_MEMORY, 0, 0x0703, "query out of memory")
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_APP_ERROR, 0, 0x0704, "query app error") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_APP_ERROR, 0, 0x0704, "query app error")
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_DUP_JOIN_KEY, 0, 0x0705, "query duplicated join key") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_DUP_JOIN_KEY, 0, 0x0705, "query duplicated join key")
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_EXCEED_TAGS_LIMIT, 0, 0x0706, "query tag conditon too many")
// grant // grant
TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_EXPIRED, 0, 0x0800, "grant expired") TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_EXPIRED, 0, 0x0800, "grant expired")

11
src/inc/taosmsg.h
View File

@ -58,7 +58,7 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MD_DROP_VNODE, "drop-vnode" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MD_DROP_STABLE, "drop-stable" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MD_DROP_STABLE, "drop-stable" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MD_ALTER_STREAM, "alter-stream" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MD_ALTER_STREAM, "alter-stream" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MD_CONFIG_DNODE, "config-dnode" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MD_CONFIG_DNODE, "config-dnode" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY4, "dummy4" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MD_ALTER_VNODE, "alter-vnode" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY5, "dummy5" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY5, "dummy5" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY6, "dummy6" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY6, "dummy6" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY7, "dummy7" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY7, "dummy7" )
@ -515,6 +515,7 @@ typedef struct {
int32_t minRowsPerFileBlock; int32_t minRowsPerFileBlock;
int32_t maxRowsPerFileBlock; int32_t maxRowsPerFileBlock;
int32_t commitTime; int32_t commitTime;
int32_t fsyncPeriod;
uint8_t precision; // time resolution uint8_t precision; // time resolution
int8_t compression; int8_t compression;
int8_t walLevel; int8_t walLevel;
@ -562,9 +563,12 @@ typedef struct {
typedef struct { typedef struct {
int32_t numOfMnodes; // tsNumOfMnodes int32_t numOfMnodes; // tsNumOfMnodes
int32_t enableBalance; // tsEnableBalance
int32_t mnodeEqualVnodeNum; // tsMnodeEqualVnodeNum int32_t mnodeEqualVnodeNum; // tsMnodeEqualVnodeNum
int32_t offlineThreshold; // tsOfflineThreshold int32_t offlineThreshold; // tsOfflineThreshold
int32_t statusInterval; // tsStatusInterval int32_t statusInterval; // tsStatusInterval
int32_t maxtablesPerVnode;
int32_t maxVgroupsPerDb;
char arbitrator[TSDB_EP_LEN]; // tsArbitrator char arbitrator[TSDB_EP_LEN]; // tsArbitrator
char timezone[64]; // tsTimezone char timezone[64]; // tsTimezone
char locale[TSDB_LOCALE_LEN]; // tsLocale char locale[TSDB_LOCALE_LEN]; // tsLocale
@ -606,6 +610,7 @@ typedef struct {
int32_t minRowsPerFileBlock; int32_t minRowsPerFileBlock;
int32_t maxRowsPerFileBlock; int32_t maxRowsPerFileBlock;
int32_t commitTime; int32_t commitTime;
int32_t fsyncPeriod;
int8_t precision; int8_t precision;
int8_t compression; int8_t compression;
int8_t walLevel; int8_t walLevel;
@ -624,7 +629,7 @@ typedef struct {
char db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN]; char db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN];
SMDVnodeCfg cfg; SMDVnodeCfg cfg;
SMDVnodeDesc nodes[TSDB_MAX_REPLICA]; SMDVnodeDesc nodes[TSDB_MAX_REPLICA];
} SMDCreateVnodeMsg; } SMDCreateVnodeMsg, SMDAlterVnodeMsg;
typedef struct { typedef struct {
char tableId[TSDB_TABLE_ID_LEN]; char tableId[TSDB_TABLE_ID_LEN];
@ -644,7 +649,7 @@ typedef struct SCMSTableVgroupMsg {
typedef struct { typedef struct {
int32_t vgId; int32_t vgId;
int8_t numOfIps; int8_t numOfIps;
SIpAddr ipAddr[TSDB_MAX_REPLICA_NUM]; SIpAddr ipAddr[TSDB_MAX_REPLICA];
} SCMVgroupInfo; } SCMVgroupInfo;
typedef struct { typedef struct {

1
src/inc/tsdb.h
View File

@ -117,7 +117,6 @@ int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg);
int tsdbDropTable(TSDB_REPO_T *pRepo, STableId tableId); int tsdbDropTable(TSDB_REPO_T *pRepo, STableId tableId);
int tsdbUpdateTableTagValue(TSDB_REPO_T *repo, SUpdateTableTagValMsg *pMsg); int tsdbUpdateTableTagValue(TSDB_REPO_T *repo, SUpdateTableTagValMsg *pMsg);
TSKEY tsdbGetTableLastKey(TSDB_REPO_T *repo, uint64_t uid); TSKEY tsdbGetTableLastKey(TSDB_REPO_T *repo, uint64_t uid);
void tsdbStartStream(TSDB_REPO_T *repo);
uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_t eindex, int32_t *size); uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_t eindex, int32_t *size);

2
src/inc/tsync.h
View File

@ -79,7 +79,7 @@ typedef void (*FConfirmForward)(void *ahandle, void *mhandle, int32_t code);
typedef void (*FNotifyRole)(void *ahandle, int8_t role); typedef void (*FNotifyRole)(void *ahandle, int8_t role);
// when data file is synced successfully, notity app // when data file is synced successfully, notity app
typedef void (*FNotifyFileSynced)(void *ahandle, uint64_t fversion); typedef int (*FNotifyFileSynced)(void *ahandle, uint64_t fversion);
typedef struct { typedef struct {
int32_t vgId; // vgroup ID int32_t vgId; // vgroup ID

222
src/inc/ttokendef.h
View File

@ -110,117 +110,117 @@
#define TK_BLOCKS 92 #define TK_BLOCKS 92
#define TK_CTIME 93 #define TK_CTIME 93
#define TK_WAL 94 #define TK_WAL 94
#define TK_COMP 95 #define TK_FSYNC 95
#define TK_PRECISION 96 #define TK_COMP 96
#define TK_LP 97 #define TK_PRECISION 97
#define TK_RP 98 #define TK_LP 98
#define TK_TAGS 99 #define TK_RP 99
#define TK_USING 100 #define TK_TAGS 100
#define TK_AS 101 #define TK_USING 101
#define TK_COMMA 102 #define TK_AS 102
#define TK_NULL 103 #define TK_COMMA 103
#define TK_SELECT 104 #define TK_NULL 104
#define TK_UNION 105 #define TK_SELECT 105
#define TK_ALL 106 #define TK_UNION 106
#define TK_FROM 107 #define TK_ALL 107
#define TK_VARIABLE 108 #define TK_FROM 108
#define TK_INTERVAL 109 #define TK_VARIABLE 109
#define TK_FILL 110 #define TK_INTERVAL 110
#define TK_SLIDING 111 #define TK_FILL 111
#define TK_ORDER 112 #define TK_SLIDING 112
#define TK_BY 113 #define TK_ORDER 113
#define TK_ASC 114 #define TK_BY 114
#define TK_DESC 115 #define TK_ASC 115
#define TK_GROUP 116 #define TK_DESC 116
#define TK_HAVING 117 #define TK_GROUP 117
#define TK_LIMIT 118 #define TK_HAVING 118
#define TK_OFFSET 119 #define TK_LIMIT 119
#define TK_SLIMIT 120 #define TK_OFFSET 120
#define TK_SOFFSET 121 #define TK_SLIMIT 121
#define TK_WHERE 122 #define TK_SOFFSET 122
#define TK_NOW 123 #define TK_WHERE 123
#define TK_RESET 124 #define TK_NOW 124
#define TK_QUERY 125 #define TK_RESET 125
#define TK_ADD 126 #define TK_QUERY 126
#define TK_COLUMN 127 #define TK_ADD 127
#define TK_TAG 128 #define TK_COLUMN 128
#define TK_CHANGE 129 #define TK_TAG 129
#define TK_SET 130 #define TK_CHANGE 130
#define TK_KILL 131 #define TK_SET 131
#define TK_CONNECTION 132 #define TK_KILL 132
#define TK_STREAM 133 #define TK_CONNECTION 133
#define TK_COLON 134 #define TK_STREAM 134
#define TK_ABORT 135 #define TK_COLON 135
#define TK_AFTER 136 #define TK_ABORT 136
#define TK_ATTACH 137 #define TK_AFTER 137
#define TK_BEFORE 138 #define TK_ATTACH 138
#define TK_BEGIN 139 #define TK_BEFORE 139
#define TK_CASCADE 140 #define TK_BEGIN 140
#define TK_CLUSTER 141 #define TK_CASCADE 141
#define TK_CONFLICT 142 #define TK_CLUSTER 142
#define TK_COPY 143 #define TK_CONFLICT 143
#define TK_DEFERRED 144 #define TK_COPY 144
#define TK_DELIMITERS 145 #define TK_DEFERRED 145
#define TK_DETACH 146 #define TK_DELIMITERS 146
#define TK_EACH 147 #define TK_DETACH 147
#define TK_END 148 #define TK_EACH 148
#define TK_EXPLAIN 149 #define TK_END 149
#define TK_FAIL 150 #define TK_EXPLAIN 150
#define TK_FOR 151 #define TK_FAIL 151
#define TK_IGNORE 152 #define TK_FOR 152
#define TK_IMMEDIATE 153 #define TK_IGNORE 153
#define TK_INITIALLY 154 #define TK_IMMEDIATE 154
#define TK_INSTEAD 155 #define TK_INITIALLY 155
#define TK_MATCH 156 #define TK_INSTEAD 156
#define TK_KEY 157 #define TK_MATCH 157
#define TK_OF 158 #define TK_KEY 158
#define TK_RAISE 159 #define TK_OF 159
#define TK_REPLACE 160 #define TK_RAISE 160
#define TK_RESTRICT 161 #define TK_REPLACE 161
#define TK_ROW 162 #define TK_RESTRICT 162
#define TK_STATEMENT 163 #define TK_ROW 163
#define TK_TRIGGER 164 #define TK_STATEMENT 164
#define TK_VIEW 165 #define TK_TRIGGER 165
#define TK_COUNT 166 #define TK_VIEW 166
#define TK_SUM 167 #define TK_COUNT 167
#define TK_AVG 168 #define TK_SUM 168
#define TK_MIN 169 #define TK_AVG 169
#define TK_MAX 170 #define TK_MIN 170
#define TK_FIRST 171 #define TK_MAX 171
#define TK_LAST 172 #define TK_FIRST 172
#define TK_TOP 173 #define TK_LAST 173
#define TK_BOTTOM 174 #define TK_TOP 174
#define TK_STDDEV 175 #define TK_BOTTOM 175
#define TK_PERCENTILE 176 #define TK_STDDEV 176
#define TK_APERCENTILE 177 #define TK_PERCENTILE 177
#define TK_LEASTSQUARES 178 #define TK_APERCENTILE 178
#define TK_HISTOGRAM 179 #define TK_LEASTSQUARES 179
#define TK_DIFF 180 #define TK_HISTOGRAM 180
#define TK_SPREAD 181 #define TK_DIFF 181
#define TK_TWA 182 #define TK_SPREAD 182
#define TK_INTERP 183 #define TK_TWA 183
#define TK_LAST_ROW 184 #define TK_INTERP 184
#define TK_RATE 185 #define TK_LAST_ROW 185
#define TK_IRATE 186 #define TK_RATE 186
#define TK_SUM_RATE 187 #define TK_IRATE 187
#define TK_SUM_IRATE 188 #define TK_SUM_RATE 188
#define TK_AVG_RATE 189 #define TK_SUM_IRATE 189
#define TK_AVG_IRATE 190 #define TK_AVG_RATE 190
#define TK_TBID 191 #define TK_AVG_IRATE 191
#define TK_SEMI 192 #define TK_TBID 192
#define TK_NONE 193 #define TK_SEMI 193
#define TK_PREV 194 #define TK_NONE 194
#define TK_LINEAR 195 #define TK_PREV 195
#define TK_IMPORT 196 #define TK_LINEAR 196
#define TK_METRIC 197 #define TK_IMPORT 197
#define TK_TBNAME 198 #define TK_METRIC 198
#define TK_JOIN 199 #define TK_TBNAME 199
#define TK_METRICS 200 #define TK_JOIN 200
#define TK_STABLE 201 #define TK_METRICS 201
#define TK_INSERT 202 #define TK_STABLE 202
#define TK_INTO 203 #define TK_INSERT 203
#define TK_VALUES 204 #define TK_INTO 204
#define TK_VALUES 205
#define TK_SPACE 300 #define TK_SPACE 300
#define TK_COMMENT 301 #define TK_COMMENT 301

1
src/inc/twal.h
View File

@ -35,6 +35,7 @@ typedef struct {
typedef struct { typedef struct {
int8_t walLevel; // wal level int8_t walLevel; // wal level
int32_t fsyncPeriod; // millisecond
int8_t wals; // number of WAL files; int8_t wals; // number of WAL files;
int8_t keep; // keep the wal file when closed int8_t keep; // keep the wal file when closed
} SWalCfg; } SWalCfg;

16
src/inc/vnode.h
View File

@ -22,10 +22,10 @@ extern "C" {
typedef enum _VN_STATUS { typedef enum _VN_STATUS {
TAOS_VN_STATUS_INIT, TAOS_VN_STATUS_INIT,
TAOS_VN_STATUS_UPDATING,
TAOS_VN_STATUS_READY, TAOS_VN_STATUS_READY,
TAOS_VN_STATUS_CLOSING, TAOS_VN_STATUS_CLOSING,
TAOS_VN_STATUS_DELETING, TAOS_VN_STATUS_UPDATING,
TAOS_VN_STATUS_RESET,
} EVnStatus; } EVnStatus;
typedef struct { typedef struct {
@ -44,17 +44,13 @@ typedef struct {
int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg); int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg);
int32_t vnodeDrop(int32_t vgId); int32_t vnodeDrop(int32_t vgId);
int32_t vnodeOpen(int32_t vgId, char *rootDir); int32_t vnodeOpen(int32_t vgId, char *rootDir);
int32_t vnodeStartStream(int32_t vgId);
int32_t vnodeAlter(void *pVnode, SMDCreateVnodeMsg *pVnodeCfg); int32_t vnodeAlter(void *pVnode, SMDCreateVnodeMsg *pVnodeCfg);
int32_t vnodeClose(int32_t vgId); int32_t vnodeClose(int32_t vgId);
void vnodeRelease(void *pVnode); void* vnodeAcquire(int32_t vgId); // add refcount
void* vnodeAcquireVnode(int32_t vgId); // add refcount void* vnodeAcquireRqueue(int32_t vgId); // add refCount, get read queue
void* vnodeGetVnode(int32_t vgId); // keep refcount unchanged void* vnodeAcquireWqueue(int32_t vgId); // add recCount, get write queue
void vnodeRelease(void *pVnode); // dec refCount
void* vnodeAcquireRqueue(void *);
void* vnodeGetRqueue(void *);
void* vnodeGetWqueue(int32_t vgId);
void* vnodeGetWal(void *pVnode); void* vnodeGetWal(void *pVnode);
int32_t vnodeProcessWrite(void *pVnode, int qtype, void *pHead, void *item); int32_t vnodeProcessWrite(void *pVnode, int qtype, void *pHead, void *item);

4
src/kit/shell/src/shellImport.c
View File

@ -136,7 +136,7 @@ static void shellGetDirectoryFileList(char *inputDir)
static void shellSourceFile(TAOS *con, char *fptr) { static void shellSourceFile(TAOS *con, char *fptr) {
wordexp_t full_path; wordexp_t full_path;
int read_len = 0; int read_len = 0;
char * cmd = malloc(MAX_COMMAND_SIZE); char * cmd = malloc(tsMaxSQLStringLen);
size_t cmd_len = 0; size_t cmd_len = 0;
char * line = NULL; char * line = NULL;
size_t line_len = 0; size_t line_len = 0;
@ -185,7 +185,7 @@ static void shellSourceFile(TAOS *con, char *fptr) {
int lineNo = 0; int lineNo = 0;
while ((read_len = getline(&line, &line_len, f)) != -1) { while ((read_len = getline(&line, &line_len, f)) != -1) {
++lineNo; ++lineNo;
if (read_len >= MAX_COMMAND_SIZE) continue; if (read_len >= tsMaxSQLStringLen) continue;
line[--read_len] = '\0'; line[--read_len] = '\0';
if (read_len == 0 || isCommentLine(line)) { // line starts with # if (read_len == 0 || isCommentLine(line)) { // line starts with #

123
src/kit/taosdump/taosdump.c
View File

@ -179,8 +179,8 @@ static struct argp_option options[] = {
{"start-time", 'S', "START_TIME", 0, "Start time to dump.", 3}, {"start-time", 'S', "START_TIME", 0, "Start time to dump.", 3},
{"end-time", 'E', "END_TIME", 0, "End time to dump.", 3}, {"end-time", 'E', "END_TIME", 0, "End time to dump.", 3},
{"data-batch", 'N', "DATA_BATCH", 0, "Number of data point per insert statement. Default is 1.", 3}, {"data-batch", 'N', "DATA_BATCH", 0, "Number of data point per insert statement. Default is 1.", 3},
{"table-batch", 'T', "TABLE_BATCH", 0, "Number of table dumpout into one output file. Default is 1.", 3}, {"table-batch", 't', "TABLE_BATCH", 0, "Number of table dumpout into one output file. Default is 1.", 3},
{"thread_num", 't', "THREAD_NUM", 0, "Number of thread for dump in file. Default is 5.", 3}, {"thread_num", 'T', "THREAD_NUM", 0, "Number of thread for dump in file. Default is 5.", 3},
{"allow-sys", 'a', 0, 0, "Allow to dump sys database", 3}, {"allow-sys", 'a', 0, 0, "Allow to dump sys database", 3},
{0}}; {0}};
@ -304,10 +304,10 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
case 'N': case 'N':
arguments->data_batch = atoi(arg); arguments->data_batch = atoi(arg);
break; break;
case 'T': case 't':
arguments->table_batch = atoi(arg); arguments->table_batch = atoi(arg);
break; break;
case 't': case 'T':
arguments->thread_num = atoi(arg); arguments->thread_num = atoi(arg);
break; break;
case OPT_ABORT: case OPT_ABORT:
@ -406,7 +406,7 @@ int main(int argc, char *argv[]) {
printf("password: %s\n", tsArguments.password); printf("password: %s\n", tsArguments.password);
printf("port: %u\n", tsArguments.port); printf("port: %u\n", tsArguments.port);
printf("cversion: %s\n", tsArguments.cversion); printf("cversion: %s\n", tsArguments.cversion);
printf("mysqlFlag: %d", tsArguments.mysqlFlag); printf("mysqlFlag: %d\n", tsArguments.mysqlFlag);
printf("outpath: %s\n", tsArguments.outpath); printf("outpath: %s\n", tsArguments.outpath);
printf("inpath: %s\n", tsArguments.inpath); printf("inpath: %s\n", tsArguments.inpath);
printf("encode: %s\n", tsArguments.encode); printf("encode: %s\n", tsArguments.encode);
@ -821,7 +821,7 @@ _exit_failure:
return -1; return -1;
} }
int taosGetTableDes(char *table, STableDef *tableDes, TAOS* taosCon) { int taosGetTableDes(char *table, STableDef *tableDes, TAOS* taosCon, bool isSuperTable) {
TAOS_ROW row = NULL; TAOS_ROW row = NULL;
TAOS_RES *tmpResult = NULL; TAOS_RES *tmpResult = NULL;
int count = 0; int count = 0;
@ -832,6 +832,13 @@ int taosGetTableDes(char *table, STableDef *tableDes, TAOS* taosCon) {
return -1; return -1;
} }
char* tbuf = (char *)malloc(COMMAND_SIZE);
if (tbuf == NULL) {
fprintf(stderr, "failed to allocate memory\n");
free(tempCommand);
return -1;
}
sprintf(tempCommand, "describe %s", table); sprintf(tempCommand, "describe %s", table);
tmpResult = taos_query(taosCon, tempCommand); tmpResult = taos_query(taosCon, tempCommand);
@ -862,6 +869,92 @@ int taosGetTableDes(char *table, STableDef *tableDes, TAOS* taosCon) {
taos_free_result(tmpResult); taos_free_result(tmpResult);
tmpResult = NULL; tmpResult = NULL;
if (isSuperTable) {
free(tempCommand);
return count;
}
// if chidl-table have tag, using select tagName from table to get tagValue
for (int i = 0 ; i < count; i++) {
if (strcmp(tableDes->cols[i].note, "TAG") != 0) continue;
sprintf(tempCommand, "select %s from %s", tableDes->cols[i].field, table);
tmpResult = taos_query(taosCon, tempCommand);
code = taos_errno(tmpResult);
if (code != 0) {
fprintf(stderr, "failed to run command %s\n", tempCommand);
free(tempCommand);
taos_free_result(tmpResult);
return -1;
}
fields = taos_fetch_fields(tmpResult);
row = taos_fetch_row(tmpResult);
if (NULL == row) {
fprintf(stderr, " fetch failed to run command %s\n", tempCommand);
free(tempCommand);
taos_free_result(tmpResult);
return -1;
}
switch (fields[0].type) {
case TSDB_DATA_TYPE_BOOL:
sprintf(tableDes->cols[i].note, "%d", ((((int)(*((char *)row[0]))) == 1) ? 1 : 0));
break;
case TSDB_DATA_TYPE_TINYINT:
sprintf(tableDes->cols[i].note, "%d", (int)(*((char *)row[0])));
break;
case TSDB_DATA_TYPE_SMALLINT:
sprintf(tableDes->cols[i].note, "%d", (int)(*((short *)row[0])));
break;
case TSDB_DATA_TYPE_INT:
sprintf(tableDes->cols[i].note, "%d", *((int *)row[0]));
break;
case TSDB_DATA_TYPE_BIGINT:
sprintf(tableDes->cols[i].note, "%" PRId64 "", *((int64_t *)row[0]));
break;
case TSDB_DATA_TYPE_FLOAT:
sprintf(tableDes->cols[i].note, "%f", GET_FLOAT_VAL(row[0]));
break;
case TSDB_DATA_TYPE_DOUBLE:
sprintf(tableDes->cols[i].note, "%f", GET_DOUBLE_VAL(row[0]));
break;
case TSDB_DATA_TYPE_BINARY:
tableDes->cols[i].note[0] = '\'';
converStringToReadable((char *)row[0], fields[0].bytes, tbuf, COMMAND_SIZE);
char* pstr = stpcpy(&(tableDes->cols[i].note[1]), tbuf);
*(pstr++) = '\'';
break;
case TSDB_DATA_TYPE_NCHAR:
convertNCharToReadable((char *)row[0], fields[0].bytes, tbuf, COMMAND_SIZE);
sprintf(tableDes->cols[i].note, "\'%s\'", tbuf);
break;
case TSDB_DATA_TYPE_TIMESTAMP:
sprintf(tableDes->cols[i].note, "%" PRId64 "", *(int64_t *)row[0]);
#if 0
if (!arguments->mysqlFlag) {
sprintf(tableDes->cols[i].note, "%" PRId64 "", *(int64_t *)row[0]);
} else {
char buf[64] = "\0";
int64_t ts = *((int64_t *)row[0]);
time_t tt = (time_t)(ts / 1000);
struct tm *ptm = localtime(&tt);
strftime(buf, 64, "%y-%m-%d %H:%M:%S", ptm);
sprintf(tableDes->cols[i].note, "\'%s.%03d\'", buf, (int)(ts % 1000));
}
#endif
break;
default:
break;
}
taos_free_result(tmpResult);
tmpResult = NULL;
}
free(tempCommand); free(tempCommand);
return count; return count;
@ -886,23 +979,25 @@ int32_t taosDumpTable(char *table, char *metric, struct arguments *arguments, FI
memset(tableDes, 0, sizeof(STableDef) + sizeof(SColDes) * TSDB_MAX_COLUMNS); memset(tableDes, 0, sizeof(STableDef) + sizeof(SColDes) * TSDB_MAX_COLUMNS);
*/ */
count = taosGetTableDes(table, tableDes, taosCon); count = taosGetTableDes(table, tableDes, taosCon, false);
if (count < 0) { if (count < 0) {
free(tableDes); free(tableDes);
return -1; return -1;
} }
// create child-table using super-table
taosDumpCreateMTableClause(tableDes, metric, count, fp); taosDumpCreateMTableClause(tableDes, metric, count, fp);
} else { // dump table definition } else { // dump table definition
count = taosGetTableDes(table, tableDes, taosCon); count = taosGetTableDes(table, tableDes, taosCon, false);
if (count < 0) { if (count < 0) {
free(tableDes); free(tableDes);
return -1; return -1;
} }
// create normal-table or super-table
taosDumpCreateTableClause(tableDes, count, fp); taosDumpCreateTableClause(tableDes, count, fp);
} }
@ -1033,7 +1128,7 @@ int32_t taosDumpStable(char *table, FILE *fp, TAOS* taosCon) {
exit(-1); exit(-1);
} }
count = taosGetTableDes(table, tableDes, taosCon); count = taosGetTableDes(table, tableDes, taosCon, true);
if (count < 0) { if (count < 0) {
free(tableDes); free(tableDes);
@ -1083,7 +1178,6 @@ int32_t taosDumpCreateSuperTableClause(TAOS* taosCon, char* dbName, FILE *fp)
taos_free_result(tmpResult); taos_free_result(tmpResult);
exit(-1); exit(-1);
} }
taos_free_result(tmpResult);
TAOS_FIELD *fields = taos_fetch_fields(tmpResult); TAOS_FIELD *fields = taos_fetch_fields(tmpResult);
@ -1291,14 +1385,16 @@ void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols
if (counter != count_temp) { if (counter != count_temp) {
if (strcasecmp(tableDes->cols[counter].type, "binary") == 0 || if (strcasecmp(tableDes->cols[counter].type, "binary") == 0 ||
strcasecmp(tableDes->cols[counter].type, "nchar") == 0) { strcasecmp(tableDes->cols[counter].type, "nchar") == 0) {
pstr += sprintf(pstr, ", \'%s\'", tableDes->cols[counter].note); //pstr += sprintf(pstr, ", \'%s\'", tableDes->cols[counter].note);
pstr += sprintf(pstr, ", %s", tableDes->cols[counter].note);
} else { } else {
pstr += sprintf(pstr, ", %s", tableDes->cols[counter].note); pstr += sprintf(pstr, ", %s", tableDes->cols[counter].note);
} }
} else { } else {
if (strcasecmp(tableDes->cols[counter].type, "binary") == 0 || if (strcasecmp(tableDes->cols[counter].type, "binary") == 0 ||
strcasecmp(tableDes->cols[counter].type, "nchar") == 0) { strcasecmp(tableDes->cols[counter].type, "nchar") == 0) {
pstr += sprintf(pstr, "\'%s\'", tableDes->cols[counter].note); //pstr += sprintf(pstr, "\'%s\'", tableDes->cols[counter].note);
pstr += sprintf(pstr, "%s", tableDes->cols[counter].note);
} else { } else {
pstr += sprintf(pstr, "%s", tableDes->cols[counter].note); pstr += sprintf(pstr, "%s", tableDes->cols[counter].note);
} }
@ -1363,7 +1459,7 @@ int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS*
return -1; return -1;
} }
numFields = taos_field_count(taosCon); numFields = taos_field_count(tmpResult);
assert(numFields > 0); assert(numFields > 0);
TAOS_FIELD *fields = taos_fetch_fields(tmpResult); TAOS_FIELD *fields = taos_fetch_fields(tmpResult);
tbuf = (char *)malloc(COMMAND_SIZE); tbuf = (char *)malloc(COMMAND_SIZE);
@ -2015,6 +2111,7 @@ int taosDumpInOneFile(TAOS * taos, FILE* fp, char* fcharset, char* encode, c
} }
memcpy(cmd + cmd_len, line, read_len); memcpy(cmd + cmd_len, line, read_len);
cmd[read_len + cmd_len]= '\0';
if (queryDB(taos, cmd)) { if (queryDB(taos, cmd)) {
fprintf(stderr, "error sql: linenu:%d, file:%s\n", lineNo, fileName); fprintf(stderr, "error sql: linenu:%d, file:%s\n", lineNo, fileName);
} }

8
src/kit/taosmigrate/taosmigrateVnodeCfg.c
View File

@ -48,6 +48,7 @@ static int32_t saveVnodeCfg(SVnodeObj *pVnode, char* cfgFile)
len += snprintf(content + len, maxLen - len, " \"precision\": %d,\n", pVnode->tsdbCfg.precision); len += snprintf(content + len, maxLen - len, " \"precision\": %d,\n", pVnode->tsdbCfg.precision);
len += snprintf(content + len, maxLen - len, " \"compression\": %d,\n", pVnode->tsdbCfg.compression); len += snprintf(content + len, maxLen - len, " \"compression\": %d,\n", pVnode->tsdbCfg.compression);
len += snprintf(content + len, maxLen - len, " \"walLevel\": %d,\n", pVnode->walCfg.walLevel); len += snprintf(content + len, maxLen - len, " \"walLevel\": %d,\n", pVnode->walCfg.walLevel);
len += snprintf(content + len, maxLen - len, " \"fsync\": %d,\n", pVnode->walCfg.fsyncPeriod);
len += snprintf(content + len, maxLen - len, " \"replica\": %d,\n", pVnode->syncCfg.replica); len += snprintf(content + len, maxLen - len, " \"replica\": %d,\n", pVnode->syncCfg.replica);
len += snprintf(content + len, maxLen - len, " \"wals\": %d,\n", pVnode->walCfg.wals); len += snprintf(content + len, maxLen - len, " \"wals\": %d,\n", pVnode->walCfg.wals);
len += snprintf(content + len, maxLen - len, " \"quorum\": %d,\n", pVnode->syncCfg.quorum); len += snprintf(content + len, maxLen - len, " \"quorum\": %d,\n", pVnode->syncCfg.quorum);
@ -212,6 +213,13 @@ static int32_t readVnodeCfg(SVnodeObj *pVnode, char* cfgFile)
} }
pVnode->walCfg.walLevel = (int8_t) walLevel->valueint; pVnode->walCfg.walLevel = (int8_t) walLevel->valueint;
cJSON *fsyncPeriod = cJSON_GetObjectItem(root, "fsync");
if (!fsyncPeriod || fsyncPeriod->type != cJSON_Number) {
printf("vgId:%d, failed to read vnode cfg, fsyncPeriod not found\n", pVnode->vgId);
goto PARSE_OVER;
}
pVnode->walCfg.fsyncPeriod = fsyncPeriod->valueint;
cJSON *wals = cJSON_GetObjectItem(root, "wals"); cJSON *wals = cJSON_GetObjectItem(root, "wals");
if (!wals || wals->type != cJSON_Number) { if (!wals || wals->type != cJSON_Number) {
printf("vgId:%d, failed to read vnode cfg, wals not found\n", pVnode->vgId); printf("vgId:%d, failed to read vnode cfg, wals not found\n", pVnode->vgId);

3
src/mnode/inc/mnodeDb.h
View File

@ -44,10 +44,7 @@ void mnodeRemoveSuperTableFromDb(SDbObj *pDb);
void mnodeAddTableIntoDb(SDbObj *pDb); void mnodeAddTableIntoDb(SDbObj *pDb);
void mnodeRemoveTableFromDb(SDbObj *pDb); void mnodeRemoveTableFromDb(SDbObj *pDb);
void mnodeAddVgroupIntoDb(SVgObj *pVgroup); void mnodeAddVgroupIntoDb(SVgObj *pVgroup);
void mnodeAddVgroupIntoDbTail(SVgObj *pVgroup);
void mnodeRemoveVgroupFromDb(SVgObj *pVgroup); void mnodeRemoveVgroupFromDb(SVgObj *pVgroup);
void mnodeMoveVgroupToTail(SVgObj *pVgroup);
void mnodeMoveVgroupToHead(SVgObj *pVgroup);
#ifdef __cplusplus #ifdef __cplusplus
} }

12
src/mnode/inc/mnodeDef.h
View File

@ -145,10 +145,8 @@ typedef struct SVgObj {
int64_t totalStorage; int64_t totalStorage;
int64_t compStorage; int64_t compStorage;
int64_t pointsWritten; int64_t pointsWritten;
struct SVgObj *prev, *next;
struct SDbObj *pDb; struct SDbObj *pDb;
void * idPool; void * idPool;
SChildTableObj **tableList;
} SVgObj; } SVgObj;
typedef struct { typedef struct {
@ -162,6 +160,7 @@ typedef struct {
int32_t minRowsPerFileBlock; int32_t minRowsPerFileBlock;
int32_t maxRowsPerFileBlock; int32_t maxRowsPerFileBlock;
int32_t commitTime; int32_t commitTime;
int32_t fsyncPeriod;
int8_t precision; int8_t precision;
int8_t compression; int8_t compression;
int8_t walLevel; int8_t walLevel;
@ -183,9 +182,11 @@ typedef struct SDbObj {
int32_t numOfVgroups; int32_t numOfVgroups;
int32_t numOfTables; int32_t numOfTables;
int32_t numOfSuperTables; int32_t numOfSuperTables;
SVgObj *pHead; int32_t vgListSize;
SVgObj *pTail; int32_t vgListIndex;
SVgObj **vgList;
struct SAcctObj *pAcct; struct SAcctObj *pAcct;
pthread_mutex_t mutex;
} SDbObj; } SDbObj;
typedef struct SUserObj { typedef struct SUserObj {
@ -246,7 +247,8 @@ typedef struct {
int16_t offset[TSDB_MAX_COLUMNS]; int16_t offset[TSDB_MAX_COLUMNS];
int16_t bytes[TSDB_MAX_COLUMNS]; int16_t bytes[TSDB_MAX_COLUMNS];
int32_t numOfReads; int32_t numOfReads;
int8_t reserved0[2]; int8_t maxReplica;
int8_t reserved0[0];
uint16_t payloadLen; uint16_t payloadLen;
char payload[]; char payload[];
} SShowObj; } SShowObj;

1
src/mnode/inc/mnodeDnode.h
View File

@ -40,6 +40,7 @@ char* mnodeGetDnodeStatusStr(int32_t dnodeStatus);
void mgmtMonitorDnodeModule(); void mgmtMonitorDnodeModule();
int32_t mnodeGetDnodesNum(); int32_t mnodeGetDnodesNum();
int32_t mnodeGetOnlinDnodesCpuCoreNum();
int32_t mnodeGetOnlinDnodesNum(); int32_t mnodeGetOnlinDnodesNum();
void * mnodeGetNextDnode(void *pIter, SDnodeObj **pDnode); void * mnodeGetNextDnode(void *pIter, SDnodeObj **pDnode);
void mnodeIncDnodeRef(SDnodeObj *pDnode); void mnodeIncDnodeRef(SDnodeObj *pDnode);

16
src/mnode/inc/mnodeInt.h
View File

@ -29,17 +29,17 @@ extern int32_t sdbDebugFlag;
// mnode log function // mnode log function
#define mFatal(...) { if (mDebugFlag & DEBUG_FATAL) { taosPrintLog("MND FATAL ", 255, __VA_ARGS__); }} #define mFatal(...) { if (mDebugFlag & DEBUG_FATAL) { taosPrintLog("MND FATAL ", 255, __VA_ARGS__); }}
#define mError(...) { if (mDebugFlag & DEBUG_ERROR) { taosPrintLog("MND ERROR ", 255, __VA_ARGS__); }} #define mError(...) { if (mDebugFlag & DEBUG_ERROR) { taosPrintLog("MND ERROR ", 255, __VA_ARGS__); }}
#define mWarn(...) { if (mDebugFlag & DEBUG_WARN) { taosPrintLog("MND WARN ", 255, __VA_ARGS__); }} #define mWarn(...) { if (mDebugFlag & DEBUG_WARN) { taosPrintLog("MND WARN ", 255, __VA_ARGS__); }}
#define mInfo(...) { if (mDebugFlag & DEBUG_INFO) { taosPrintLog("MND INFO ", 255, __VA_ARGS__); }} #define mInfo(...) { if (mDebugFlag & DEBUG_INFO) { taosPrintLog("MND ", 255, __VA_ARGS__); }}
#define mDebug(...) { if (mDebugFlag & DEBUG_DEBUG) { taosPrintLog("MND DEBUG ", mDebugFlag, __VA_ARGS__); }} #define mDebug(...) { if (mDebugFlag & DEBUG_DEBUG) { taosPrintLog("MND ", mDebugFlag, __VA_ARGS__); }}
#define mTrace(...) { if (mDebugFlag & DEBUG_TRACE) { taosPrintLog("MND TRACE ", mDebugFlag, __VA_ARGS__); }} #define mTrace(...) { if (mDebugFlag & DEBUG_TRACE) { taosPrintLog("MND ", mDebugFlag, __VA_ARGS__); }}
#define sdbFatal(...) { if (sdbDebugFlag & DEBUG_FATAL) { taosPrintLog("SDB FATAL ", 255, __VA_ARGS__); }} #define sdbFatal(...) { if (sdbDebugFlag & DEBUG_FATAL) { taosPrintLog("SDB FATAL ", 255, __VA_ARGS__); }}
#define sdbError(...) { if (sdbDebugFlag & DEBUG_ERROR) { taosPrintLog("SDB ERROR ", 255, __VA_ARGS__); }} #define sdbError(...) { if (sdbDebugFlag & DEBUG_ERROR) { taosPrintLog("SDB ERROR ", 255, __VA_ARGS__); }}
#define sdbWarn(...) { if (sdbDebugFlag & DEBUG_WARN) { taosPrintLog("SDB WARN ", 255, __VA_ARGS__); }} #define sdbWarn(...) { if (sdbDebugFlag & DEBUG_WARN) { taosPrintLog("SDB WARN ", 255, __VA_ARGS__); }}
#define sdbInfo(...) { if (sdbDebugFlag & DEBUG_INFO) { taosPrintLog("SDB INFO ", 255, __VA_ARGS__); }} #define sdbInfo(...) { if (sdbDebugFlag & DEBUG_INFO) { taosPrintLog("SDB ", 255, __VA_ARGS__); }}
#define sdbDebug(...) { if (sdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("SDB DEBUG ", sdbDebugFlag, __VA_ARGS__); }} #define sdbDebug(...) { if (sdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("SDB ", sdbDebugFlag, __VA_ARGS__); }}
#define sdbTrace(...) { if (sdbDebugFlag & DEBUG_TRACE) { taosPrintLog("SDB TRACE ", sdbDebugFlag, __VA_ARGS__); }} #define sdbTrace(...) { if (sdbDebugFlag & DEBUG_TRACE) { taosPrintLog("SDB ", sdbDebugFlag, __VA_ARGS__); }}
#define mLError(...) { monitorSaveLog(2, __VA_ARGS__); mError(__VA_ARGS__) } #define mLError(...) { monitorSaveLog(2, __VA_ARGS__); mError(__VA_ARGS__) }
#define mLWarn(...) { monitorSaveLog(1, __VA_ARGS__); mWarn(__VA_ARGS__) } #define mLWarn(...) { monitorSaveLog(1, __VA_ARGS__); mWarn(__VA_ARGS__) }

1
src/mnode/inc/mnodeSdb.h
View File

@ -94,6 +94,7 @@ void sdbDecRef(void *thandle, void *pRow);
int64_t sdbGetNumOfRows(void *handle); int64_t sdbGetNumOfRows(void *handle);
int32_t sdbGetId(void *handle); int32_t sdbGetId(void *handle);
uint64_t sdbGetVersion(); uint64_t sdbGetVersion();
bool sdbCheckRowDeleted(void *thandle, void *pRow);
#ifdef __cplusplus #ifdef __cplusplus
} }

8
src/mnode/inc/mnodeVgroup.h
View File

@ -30,23 +30,23 @@ void mnodeDecVgroupRef(SVgObj *pVgroup);
void mnodeDropAllDbVgroups(SDbObj *pDropDb); void mnodeDropAllDbVgroups(SDbObj *pDropDb);
void mnodeSendDropAllDbVgroupsMsg(SDbObj *pDropDb); void mnodeSendDropAllDbVgroupsMsg(SDbObj *pDropDb);
void mnodeDropAllDnodeVgroups(SDnodeObj *pDropDnode); void mnodeDropAllDnodeVgroups(SDnodeObj *pDropDnode);
void mnodeUpdateAllDbVgroups(SDbObj *pAlterDb); //void mnodeUpdateAllDbVgroups(SDbObj *pAlterDb);
void * mnodeGetNextVgroup(void *pIter, SVgObj **pVgroup); void * mnodeGetNextVgroup(void *pIter, SVgObj **pVgroup);
void mnodeUpdateVgroup(SVgObj *pVgroup); void mnodeUpdateVgroup(SVgObj *pVgroup);
void mnodeUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *pDnode, SVnodeLoad *pVload); void mnodeUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *pDnode, SVnodeLoad *pVload);
void mnodeCheckUnCreatedVgroup(SDnodeObj *pDnode, SVnodeLoad *pVloads, int32_t openVnodes); void mnodeCheckUnCreatedVgroup(SDnodeObj *pDnode, SVnodeLoad *pVloads, int32_t openVnodes);
int32_t mnodeCreateVgroup(struct SMnodeMsg *pMsg, SDbObj *pDb); int32_t mnodeCreateVgroup(struct SMnodeMsg *pMsg);
void mnodeDropVgroup(SVgObj *pVgroup, void *ahandle); void mnodeDropVgroup(SVgObj *pVgroup, void *ahandle);
void mnodeAlterVgroup(SVgObj *pVgroup, void *ahandle); void mnodeAlterVgroup(SVgObj *pVgroup, void *ahandle);
SVgObj *mnodeGetAvailableVgroup(SDbObj *pDb); int32_t mnodeGetAvailableVgroup(struct SMnodeMsg *pMsg, SVgObj **pVgroup, int32_t *sid);
void mnodeAddTableIntoVgroup(SVgObj *pVgroup, SChildTableObj *pTable); void mnodeAddTableIntoVgroup(SVgObj *pVgroup, SChildTableObj *pTable);
void mnodeRemoveTableFromVgroup(SVgObj *pVgroup, SChildTableObj *pTable); void mnodeRemoveTableFromVgroup(SVgObj *pVgroup, SChildTableObj *pTable);
void mnodeSendCreateVnodeMsg(SVgObj *pVgroup, SRpcIpSet *ipSet, void *ahandle);
void mnodeSendDropVnodeMsg(int32_t vgId, SRpcIpSet *ipSet, void *ahandle); void mnodeSendDropVnodeMsg(int32_t vgId, SRpcIpSet *ipSet, void *ahandle);
void mnodeSendCreateVgroupMsg(SVgObj *pVgroup, void *ahandle); void mnodeSendCreateVgroupMsg(SVgObj *pVgroup, void *ahandle);
void mnodeSendAlterVgroupMsg(SVgObj *pVgroup);
SRpcIpSet mnodeGetIpSetFromVgroup(SVgObj *pVgroup); SRpcIpSet mnodeGetIpSetFromVgroup(SVgObj *pVgroup);
SRpcIpSet mnodeGetIpSetFromIp(char *ep); SRpcIpSet mnodeGetIpSetFromIp(char *ep);

129
src/mnode/src/mnodeDb.c
View File

@ -38,6 +38,7 @@
#include "mnodeUser.h" #include "mnodeUser.h"
#include "mnodeVgroup.h" #include "mnodeVgroup.h"
#define VG_LIST_SIZE 8
static void * tsDbSdb = NULL; static void * tsDbSdb = NULL;
static int32_t tsDbUpdateSize; static int32_t tsDbUpdateSize;
@ -50,8 +51,14 @@ static int32_t mnodeProcessCreateDbMsg(SMnodeMsg *pMsg);
static int32_t mnodeProcessAlterDbMsg(SMnodeMsg *pMsg); static int32_t mnodeProcessAlterDbMsg(SMnodeMsg *pMsg);
static int32_t mnodeProcessDropDbMsg(SMnodeMsg *pMsg); static int32_t mnodeProcessDropDbMsg(SMnodeMsg *pMsg);
static void mnodeDestroyDb(SDbObj *pDb) {
pthread_mutex_destroy(&pDb->mutex);
tfree(pDb->vgList);
tfree(pDb);
}
static int32_t mnodeDbActionDestroy(SSdbOper *pOper) { static int32_t mnodeDbActionDestroy(SSdbOper *pOper) {
tfree(pOper->pObj); mnodeDestroyDb(pOper->pObj);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -59,8 +66,9 @@ static int32_t mnodeDbActionInsert(SSdbOper *pOper) {
SDbObj *pDb = pOper->pObj; SDbObj *pDb = pOper->pObj;
SAcctObj *pAcct = mnodeGetAcct(pDb->acct); SAcctObj *pAcct = mnodeGetAcct(pDb->acct);
pDb->pHead = NULL; pthread_mutex_init(&pDb->mutex, NULL);
pDb->pTail = NULL; pDb->vgListSize = VG_LIST_SIZE;
pDb->vgList = calloc(pDb->vgListSize, sizeof(SVgObj *));
pDb->numOfVgroups = 0; pDb->numOfVgroups = 0;
pDb->numOfTables = 0; pDb->numOfTables = 0;
pDb->numOfSuperTables = 0; pDb->numOfSuperTables = 0;
@ -94,14 +102,15 @@ static int32_t mnodeDbActionDelete(SSdbOper *pOper) {
} }
static int32_t mnodeDbActionUpdate(SSdbOper *pOper) { static int32_t mnodeDbActionUpdate(SSdbOper *pOper) {
SDbObj *pDb = pOper->pObj; SDbObj *pNew = pOper->pObj;
SDbObj *pSaved = mnodeGetDb(pDb->name); SDbObj *pDb = mnodeGetDb(pNew->name);
if (pDb != pSaved) { if (pDb != NULL && pNew != pDb) {
memcpy(pSaved, pDb, pOper->rowSize); memcpy(pDb, pNew, pOper->rowSize);
free(pDb); free(pNew->vgList);
free(pNew);
} }
mnodeUpdateAllDbVgroups(pSaved); //mnodeUpdateAllDbVgroups(pDb);
mnodeDecDbRef(pSaved); mnodeDecDbRef(pDb);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -278,14 +287,14 @@ static int32_t mnodeCheckDbCfg(SDbCfg *pCfg) {
return TSDB_CODE_MND_INVALID_DB_OPTION; return TSDB_CODE_MND_INVALID_DB_OPTION;
} }
if (pCfg->replications < TSDB_MIN_REPLICA_NUM || pCfg->replications > TSDB_MAX_REPLICA_NUM) { if (pCfg->fsyncPeriod < TSDB_MIN_FSYNC_PERIOD || pCfg->fsyncPeriod > TSDB_MAX_FSYNC_PERIOD) {
mError("invalid db option replications:%d valid range: [%d, %d]", pCfg->replications, TSDB_MIN_REPLICA_NUM, mError("invalid db option fsyncPeriod:%d, valid range: [%d, %d]", pCfg->fsyncPeriod, TSDB_MIN_FSYNC_PERIOD, TSDB_MAX_FSYNC_PERIOD);
TSDB_MAX_REPLICA_NUM);
return TSDB_CODE_MND_INVALID_DB_OPTION; return TSDB_CODE_MND_INVALID_DB_OPTION;
} }
if (pCfg->walLevel < TSDB_MIN_WAL_LEVEL) { if (pCfg->replications < TSDB_MIN_DB_REPLICA_OPTION || pCfg->replications > TSDB_MAX_DB_REPLICA_OPTION) {
mError("invalid db option walLevel:%d must be greater than 0", pCfg->walLevel); mError("invalid db option replications:%d valid range: [%d, %d]", pCfg->replications, TSDB_MIN_DB_REPLICA_OPTION,
TSDB_MAX_DB_REPLICA_OPTION);
return TSDB_CODE_MND_INVALID_DB_OPTION; return TSDB_CODE_MND_INVALID_DB_OPTION;
} }
@ -309,6 +318,7 @@ static void mnodeSetDefaultDbCfg(SDbCfg *pCfg) {
if (pCfg->daysToKeep2 < 0) pCfg->daysToKeep2 = pCfg->daysToKeep; if (pCfg->daysToKeep2 < 0) pCfg->daysToKeep2 = pCfg->daysToKeep;
if (pCfg->minRowsPerFileBlock < 0) pCfg->minRowsPerFileBlock = tsMinRowsInFileBlock; if (pCfg->minRowsPerFileBlock < 0) pCfg->minRowsPerFileBlock = tsMinRowsInFileBlock;
if (pCfg->maxRowsPerFileBlock < 0) pCfg->maxRowsPerFileBlock = tsMaxRowsInFileBlock; if (pCfg->maxRowsPerFileBlock < 0) pCfg->maxRowsPerFileBlock = tsMaxRowsInFileBlock;
if (pCfg->fsyncPeriod <0) pCfg->fsyncPeriod = tsFsyncPeriod;
if (pCfg->commitTime < 0) pCfg->commitTime = tsCommitTime; if (pCfg->commitTime < 0) pCfg->commitTime = tsCommitTime;
if (pCfg->precision < 0) pCfg->precision = tsTimePrecision; if (pCfg->precision < 0) pCfg->precision = tsTimePrecision;
if (pCfg->compression < 0) pCfg->compression = tsCompression; if (pCfg->compression < 0) pCfg->compression = tsCompression;
@ -358,6 +368,7 @@ static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate, void *pMs
.daysToKeep2 = pCreate->daysToKeep2, .daysToKeep2 = pCreate->daysToKeep2,
.minRowsPerFileBlock = pCreate->minRowsPerFileBlock, .minRowsPerFileBlock = pCreate->minRowsPerFileBlock,
.maxRowsPerFileBlock = pCreate->maxRowsPerFileBlock, .maxRowsPerFileBlock = pCreate->maxRowsPerFileBlock,
.fsyncPeriod = pCreate->fsyncPeriod,
.commitTime = pCreate->commitTime, .commitTime = pCreate->commitTime,
.precision = pCreate->precision, .precision = pCreate->precision,
.compression = pCreate->compression, .compression = pCreate->compression,
@ -384,7 +395,7 @@ static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate, void *pMs
code = sdbInsertRow(&oper); code = sdbInsertRow(&oper);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
tfree(pDb); mnodeDestroyDb(pDb);
mLInfo("db:%s, failed to create, reason:%s", pDb->name, tstrerror(code)); mLInfo("db:%s, failed to create, reason:%s", pDb->name, tstrerror(code));
return code; return code;
} else { } else {
@ -421,45 +432,33 @@ void mnodePrintVgroups(SDbObj *pDb, char *oper) {
void mnodeAddVgroupIntoDb(SVgObj *pVgroup) { void mnodeAddVgroupIntoDb(SVgObj *pVgroup) {
SDbObj *pDb = pVgroup->pDb; SDbObj *pDb = pVgroup->pDb;
pVgroup->next = pDb->pHead; pthread_mutex_lock(&pDb->mutex);
pVgroup->prev = NULL; int32_t vgPos = pDb->numOfVgroups++;
if (vgPos >= pDb->vgListSize) {
pDb->vgList = realloc(pDb->vgList, pDb->vgListSize * 2 * sizeof(SVgObj *));
memset(pDb->vgList + pDb->vgListSize, 0, pDb->vgListSize * sizeof(SVgObj *));
pDb->vgListSize *= 2;
}
if (pDb->pHead) pDb->pHead->prev = pVgroup; pDb->vgList[vgPos] = pVgroup;
if (pDb->pTail == NULL) pDb->pTail = pVgroup; pthread_mutex_unlock(&pDb->mutex);
pDb->pHead = pVgroup;
pDb->numOfVgroups++;
}
void mnodeAddVgroupIntoDbTail(SVgObj *pVgroup) {
SDbObj *pDb = pVgroup->pDb;
pVgroup->next = NULL;
pVgroup->prev = pDb->pTail;
if (pDb->pTail) pDb->pTail->next = pVgroup;
if (pDb->pHead == NULL) pDb->pHead = pVgroup;
pDb->pTail = pVgroup;
pDb->numOfVgroups++;
} }
void mnodeRemoveVgroupFromDb(SVgObj *pVgroup) { void mnodeRemoveVgroupFromDb(SVgObj *pVgroup) {
SDbObj *pDb = pVgroup->pDb; SDbObj *pDb = pVgroup->pDb;
if (pVgroup->prev) pVgroup->prev->next = pVgroup->next;
if (pVgroup->next) pVgroup->next->prev = pVgroup->prev;
if (pVgroup->prev == NULL) pDb->pHead = pVgroup->next;
if (pVgroup->next == NULL) pDb->pTail = pVgroup->prev;
pDb->numOfVgroups--;
}
void mnodeMoveVgroupToTail(SVgObj *pVgroup) { pthread_mutex_lock(&pDb->mutex);
mnodeRemoveVgroupFromDb(pVgroup); for (int32_t v1 = 0; v1 < pDb->numOfVgroups; ++v1) {
mnodeAddVgroupIntoDbTail(pVgroup); if (pDb->vgList[v1] == pVgroup) {
} for (int32_t v2 = v1; v2 < pDb->numOfVgroups - 1; ++v2) {
pDb->vgList[v2] = pDb->vgList[v2 + 1];
}
pDb->numOfVgroups--;
break;
}
}
void mnodeMoveVgroupToHead(SVgObj *pVgroup) { pthread_mutex_unlock(&pDb->mutex);
mnodeRemoveVgroupFromDb(pVgroup);
mnodeAddVgroupIntoDb(pVgroup);
} }
void mnodeCleanupDbs() { void mnodeCleanupDbs() {
@ -531,11 +530,6 @@ static int32_t mnodeGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn
#ifndef __CLOUD_VERSION__ #ifndef __CLOUD_VERSION__
if (strcmp(pUser->user, TSDB_DEFAULT_USER) == 0) { if (strcmp(pUser->user, TSDB_DEFAULT_USER) == 0) {
#endif #endif
pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_INT;
strcpy(pSchema[cols].name, "maxtables");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 4; pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_INT; pSchema[cols].type = TSDB_DATA_TYPE_INT;
@ -561,18 +555,18 @@ static int32_t mnodeGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn
pSchema[cols].bytes = htons(pShow->bytes[cols]); pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++; cols++;
pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_INT;
strcpy(pSchema[cols].name, "ctime(Sec.)");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 1; pShow->bytes[cols] = 1;
pSchema[cols].type = TSDB_DATA_TYPE_TINYINT; pSchema[cols].type = TSDB_DATA_TYPE_TINYINT;
strcpy(pSchema[cols].name, "wallevel"); strcpy(pSchema[cols].name, "wallevel");
pSchema[cols].bytes = htons(pShow->bytes[cols]); pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++; cols++;
pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_INT;
strcpy(pSchema[cols].name, "fsync");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 1; pShow->bytes[cols] = 1;
pSchema[cols].type = TSDB_DATA_TYPE_TINYINT; pSchema[cols].type = TSDB_DATA_TYPE_TINYINT;
strcpy(pSchema[cols].name, "comp"); strcpy(pSchema[cols].name, "comp");
@ -676,10 +670,6 @@ static int32_t mnodeRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void
#ifndef __CLOUD_VERSION__ #ifndef __CLOUD_VERSION__
if (strcmp(pUser->user, TSDB_DEFAULT_USER) == 0) { if (strcmp(pUser->user, TSDB_DEFAULT_USER) == 0) {
#endif #endif
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int32_t *)pWrite = pDb->cfg.maxTables; // table num can be created should minus 1
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int32_t *)pWrite = pDb->cfg.cacheBlockSize; *(int32_t *)pWrite = pDb->cfg.cacheBlockSize;
cols++; cols++;
@ -697,11 +687,11 @@ static int32_t mnodeRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void
cols++; cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int32_t *)pWrite = pDb->cfg.commitTime; *(int8_t *)pWrite = pDb->cfg.walLevel;
cols++; cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int8_t *)pWrite = pDb->cfg.walLevel; *(int32_t *)pWrite = pDb->cfg.fsyncPeriod;
cols++; cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
@ -780,6 +770,7 @@ static int32_t mnodeProcessCreateDbMsg(SMnodeMsg *pMsg) {
pCreate->daysToKeep1 = htonl(pCreate->daysToKeep1); pCreate->daysToKeep1 = htonl(pCreate->daysToKeep1);
pCreate->daysToKeep2 = htonl(pCreate->daysToKeep2); pCreate->daysToKeep2 = htonl(pCreate->daysToKeep2);
pCreate->commitTime = htonl(pCreate->commitTime); pCreate->commitTime = htonl(pCreate->commitTime);
pCreate->fsyncPeriod = htonl(pCreate->fsyncPeriod);
pCreate->minRowsPerFileBlock = htonl(pCreate->minRowsPerFileBlock); pCreate->minRowsPerFileBlock = htonl(pCreate->minRowsPerFileBlock);
pCreate->maxRowsPerFileBlock = htonl(pCreate->maxRowsPerFileBlock); pCreate->maxRowsPerFileBlock = htonl(pCreate->maxRowsPerFileBlock);
@ -807,6 +798,7 @@ static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) {
int32_t minRows = htonl(pAlter->minRowsPerFileBlock); int32_t minRows = htonl(pAlter->minRowsPerFileBlock);
int32_t maxRows = htonl(pAlter->maxRowsPerFileBlock); int32_t maxRows = htonl(pAlter->maxRowsPerFileBlock);
int32_t commitTime = htonl(pAlter->commitTime); int32_t commitTime = htonl(pAlter->commitTime);
int32_t fsyncPeriod = htonl(pAlter->fsyncPeriod);
int8_t compression = pAlter->compression; int8_t compression = pAlter->compression;
int8_t walLevel = pAlter->walLevel; int8_t walLevel = pAlter->walLevel;
int8_t replications = pAlter->replications; int8_t replications = pAlter->replications;
@ -883,6 +875,11 @@ static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) {
terrno = TSDB_CODE_MND_INVALID_DB_OPTION; terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
} }
if (fsyncPeriod >= 0 && fsyncPeriod != pDb->cfg.fsyncPeriod) {
mError("db:%s, can't alter fsyncPeriod option", pDb->name);
terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
}
if (replications > 0 && replications != pDb->cfg.replications) { if (replications > 0 && replications != pDb->cfg.replications) {
mDebug("db:%s, replications:%d change to %d", pDb->name, pDb->cfg.replications, replications); mDebug("db:%s, replications:%d change to %d", pDb->name, pDb->cfg.replications, replications);
newCfg.replications = replications; newCfg.replications = replications;
@ -916,7 +913,7 @@ static int32_t mnodeAlterDbCb(SMnodeMsg *pMsg, int32_t code) {
pIter = mnodeGetNextVgroup(pIter, &pVgroup); pIter = mnodeGetNextVgroup(pIter, &pVgroup);
if (pVgroup == NULL) break; if (pVgroup == NULL) break;
if (pVgroup->pDb == pDb) { if (pVgroup->pDb == pDb) {
mnodeSendCreateVgroupMsg(pVgroup, NULL); mnodeSendAlterVgroupMsg(pVgroup);
} }
mnodeDecVgroupRef(pVgroup); mnodeDecVgroupRef(pVgroup);
} }

83
src/mnode/src/mnodeDnode.c
View File

@ -69,6 +69,7 @@ static int32_t mnodeDnodeActionInsert(SSdbOper *pOper) {
SDnodeObj *pDnode = pOper->pObj; SDnodeObj *pDnode = pOper->pObj;
if (pDnode->status != TAOS_DN_STATUS_DROPPING) { if (pDnode->status != TAOS_DN_STATUS_DROPPING) {
pDnode->status = TAOS_DN_STATUS_OFFLINE; pDnode->status = TAOS_DN_STATUS_OFFLINE;
pDnode->lastAccess = tsAccessSquence;
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -187,7 +188,27 @@ int32_t mnodeGetDnodesNum() {
return sdbGetNumOfRows(tsDnodeSdb); return sdbGetNumOfRows(tsDnodeSdb);
} }
int32_t mnodeGetOnlinDnodesNum(char *ep) { int32_t mnodeGetOnlinDnodesCpuCoreNum() {
SDnodeObj *pDnode = NULL;
void * pIter = NULL;
int32_t cpuCores = 0;
while (1) {
pIter = mnodeGetNextDnode(pIter, &pDnode);
if (pDnode == NULL) break;
if (pDnode->status != TAOS_DN_STATUS_OFFLINE) {
cpuCores += pDnode->numOfCores;
}
mnodeDecDnodeRef(pDnode);
}
sdbFreeIter(pIter);
if (cpuCores < 2) cpuCores = 2;
return cpuCores;
}
int32_t mnodeGetOnlinDnodesNum() {
SDnodeObj *pDnode = NULL; SDnodeObj *pDnode = NULL;
void * pIter = NULL; void * pIter = NULL;
int32_t onlineDnodes = 0; int32_t onlineDnodes = 0;
@ -248,18 +269,37 @@ void mnodeUpdateDnode(SDnodeObj *pDnode) {
} }
static int32_t mnodeProcessCfgDnodeMsg(SMnodeMsg *pMsg) { static int32_t mnodeProcessCfgDnodeMsg(SMnodeMsg *pMsg) {
if (strcmp(pMsg->pUser->user, TSDB_DEFAULT_USER) != 0) {
mError("failed to cfg dnode, no rights");
return TSDB_CODE_MND_NO_RIGHTS;
}
SCMCfgDnodeMsg *pCmCfgDnode = pMsg->rpcMsg.pCont; SCMCfgDnodeMsg *pCmCfgDnode = pMsg->rpcMsg.pCont;
if (pCmCfgDnode->ep[0] == 0) { if (pCmCfgDnode->ep[0] == 0) {
strcpy(pCmCfgDnode->ep, tsLocalEp); tstrncpy(pCmCfgDnode->ep, tsLocalEp, TSDB_EP_LEN);
} else { }
// TODO temporary disabled for compiling: strcpy(pCmCfgDnode->ep, pCmCfgDnode->ep);
}
if (strcmp(pMsg->pUser->user, TSDB_DEFAULT_USER) != 0) { int32_t dnodeId = 0;
return TSDB_CODE_MND_NO_RIGHTS; char* pos = strchr(pCmCfgDnode->ep, ':');
if (NULL == pos) {
dnodeId = strtol(pCmCfgDnode->ep, NULL, 10);
if (dnodeId <= 0 || dnodeId > 65536) {
mError("failed to cfg dnode, invalid dnodeId:%s", pCmCfgDnode->ep);
return TSDB_CODE_MND_DNODE_NOT_EXIST;
}
} }
SRpcIpSet ipSet = mnodeGetIpSetFromIp(pCmCfgDnode->ep); SRpcIpSet ipSet = mnodeGetIpSetFromIp(pCmCfgDnode->ep);
if (dnodeId != 0) {
SDnodeObj *pDnode = mnodeGetDnode(dnodeId);
if (pDnode == NULL) {
mError("failed to cfg dnode, invalid dnodeId:%d", dnodeId);
return TSDB_CODE_MND_DNODE_NOT_EXIST;
}
ipSet = mnodeGetIpSetFromIp(pDnode->dnodeEp);
mnodeDecDnodeRef(pDnode);
}
SMDCfgDnodeMsg *pMdCfgDnode = rpcMallocCont(sizeof(SMDCfgDnodeMsg)); SMDCfgDnodeMsg *pMdCfgDnode = rpcMallocCont(sizeof(SMDCfgDnodeMsg));
strcpy(pMdCfgDnode->ep, pCmCfgDnode->ep); strcpy(pMdCfgDnode->ep, pCmCfgDnode->ep);
strcpy(pMdCfgDnode->config, pCmCfgDnode->config); strcpy(pMdCfgDnode->config, pCmCfgDnode->config);
@ -271,9 +311,9 @@ static int32_t mnodeProcessCfgDnodeMsg(SMnodeMsg *pMsg) {
.pCont = pMdCfgDnode, .pCont = pMdCfgDnode,
.contLen = sizeof(SMDCfgDnodeMsg) .contLen = sizeof(SMDCfgDnodeMsg)
}; };
dnodeSendMsgToDnode(&ipSet, &rpcMdCfgDnodeMsg);
mInfo("dnode:%s, is configured by %s", pCmCfgDnode->ep, pMsg->pUser->user); mInfo("dnode:%s, is configured by %s", pCmCfgDnode->ep, pMsg->pUser->user);
dnodeSendMsgToDnode(&ipSet, &rpcMdCfgDnodeMsg);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -283,15 +323,18 @@ static void mnodeProcessCfgDnodeMsgRsp(SRpcMsg *rpcMsg) {
} }
static bool mnodeCheckClusterCfgPara(const SClusterCfg *clusterCfg) { static bool mnodeCheckClusterCfgPara(const SClusterCfg *clusterCfg) {
if (clusterCfg->numOfMnodes != tsNumOfMnodes) return false; if (clusterCfg->numOfMnodes != htonl(tsNumOfMnodes)) return false;
if (clusterCfg->mnodeEqualVnodeNum != tsMnodeEqualVnodeNum) return false; if (clusterCfg->enableBalance != htonl(tsEnableBalance)) return false;
if (clusterCfg->offlineThreshold != tsOfflineThreshold) return false; if (clusterCfg->mnodeEqualVnodeNum != htonl(tsMnodeEqualVnodeNum)) return false;
if (clusterCfg->statusInterval != tsStatusInterval) return false; if (clusterCfg->offlineThreshold != htonl(tsOfflineThreshold)) return false;
if (clusterCfg->statusInterval != htonl(tsStatusInterval)) return false;
if (clusterCfg->maxtablesPerVnode != htonl(tsMaxTablePerVnode)) return false;
if (clusterCfg->maxVgroupsPerDb != htonl(tsMaxVgroupsPerDb)) return false;
if (0 != strncasecmp(clusterCfg->arbitrator, tsArbitrator, strlen(tsArbitrator))) return false; if (0 != strncasecmp(clusterCfg->arbitrator, tsArbitrator, strlen(tsArbitrator))) return false;
if (0 != strncasecmp(clusterCfg->timezone, tsTimezone, strlen(tsTimezone))) return false; if (0 != strncasecmp(clusterCfg->timezone, tsTimezone, strlen(tsTimezone))) return false;
if (0 != strncasecmp(clusterCfg->locale, tsLocale, strlen(tsLocale))) return false; if (0 != strncasecmp(clusterCfg->locale, tsLocale, strlen(tsLocale))) return false;
if (0 != strncasecmp(clusterCfg->charset, tsCharset, strlen(tsCharset))) return false; if (0 != strncasecmp(clusterCfg->charset, tsCharset, strlen(tsCharset))) return false;
return true; return true;
} }
@ -376,7 +419,7 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
if (false == ret) { if (false == ret) {
mnodeDecDnodeRef(pDnode); mnodeDecDnodeRef(pDnode);
rpcFreeCont(pRsp); rpcFreeCont(pRsp);
mError("dnode %s cluster cfg parameters inconsistent", pStatus->dnodeEp); mError("dnode:%d, %s cluster cfg parameters inconsistent", pDnode->dnodeId, pStatus->dnodeEp);
return TSDB_CODE_MND_CLUSTER_CFG_INCONSISTENT; return TSDB_CODE_MND_CLUSTER_CFG_INCONSISTENT;
} }
@ -468,18 +511,22 @@ static int32_t mnodeDropDnodeByEp(char *ep, SMnodeMsg *pMsg) {
return TSDB_CODE_MND_DNODE_NOT_EXIST; return TSDB_CODE_MND_DNODE_NOT_EXIST;
} }
mnodeDecDnodeRef(pDnode);
if (strcmp(pDnode->dnodeEp, mnodeGetMnodeMasterEp()) == 0) { if (strcmp(pDnode->dnodeEp, mnodeGetMnodeMasterEp()) == 0) {
mError("dnode:%d, can't drop dnode:%s which is master", pDnode->dnodeId, ep); mError("dnode:%d, can't drop dnode:%s which is master", pDnode->dnodeId, ep);
mnodeDecDnodeRef(pDnode);
return TSDB_CODE_MND_NO_REMOVE_MASTER; return TSDB_CODE_MND_NO_REMOVE_MASTER;
} }
mInfo("dnode:%d, start to drop it", pDnode->dnodeId); mInfo("dnode:%d, start to drop it", pDnode->dnodeId);
#ifndef _SYNC #ifndef _SYNC
return mnodeDropDnode(pDnode, pMsg); int32_t code = mnodeDropDnode(pDnode, pMsg);
#else #else
return balanceDropDnode(pDnode); int32_t code = balanceDropDnode(pDnode);
#endif #endif
mnodeDecDnodeRef(pDnode);
return code;
} }
static int32_t mnodeProcessCreateDnodeMsg(SMnodeMsg *pMsg) { static int32_t mnodeProcessCreateDnodeMsg(SMnodeMsg *pMsg) {

2
src/mnode/src/mnodeProfile.c
View File

@ -115,7 +115,7 @@ SConnObj *mnodeAccquireConn(int32_t connId, char *user, uint32_t ip, uint16_t po
uint64_t expireTime = CONN_KEEP_TIME * 1000 + (uint64_t)taosGetTimestampMs(); uint64_t expireTime = CONN_KEEP_TIME * 1000 + (uint64_t)taosGetTimestampMs();
SConnObj *pConn = taosCacheUpdateExpireTimeByName(tsMnodeConnCache, &connId, sizeof(int32_t), expireTime); SConnObj *pConn = taosCacheUpdateExpireTimeByName(tsMnodeConnCache, &connId, sizeof(int32_t), expireTime);
if (pConn == NULL) { if (pConn == NULL) {
mError("connId:%d, is already destroyed, user:%s ip:%s:%u", connId, user, taosIpStr(ip), port); mDebug("connId:%d, is already destroyed, user:%s ip:%s:%u", connId, user, taosIpStr(ip), port);
return NULL; return NULL;
} }

53
src/mnode/src/mnodeSdb.c
View File

@ -170,7 +170,7 @@ static void *sdbGetTableFromId(int32_t tableId) {
} }
static int32_t sdbInitWal() { static int32_t sdbInitWal() {
SWalCfg walCfg = {.walLevel = 2, .wals = 2, .keep = 1}; SWalCfg walCfg = {.walLevel = 2, .wals = 2, .keep = 1, .fsyncPeriod = 0};
char temp[TSDB_FILENAME_LEN]; char temp[TSDB_FILENAME_LEN];
sprintf(temp, "%s/wal", tsMnodeDir); sprintf(temp, "%s/wal", tsMnodeDir);
tsSdbObj.wal = walOpen(temp, &walCfg); tsSdbObj.wal = walOpen(temp, &walCfg);
@ -252,20 +252,26 @@ static void sdbConfirmForward(void *ahandle, void *param, int32_t code) {
int32_t processedCount = atomic_add_fetch_32(&pOper->processedCount, 1); int32_t processedCount = atomic_add_fetch_32(&pOper->processedCount, 1);
if (processedCount <= 1) { if (processedCount <= 1) {
if (pMsg != NULL) { if (pMsg != NULL) {
sdbDebug("app:%p:%p, waiting for confirm this operation, count:%d", pMsg->rpcMsg.ahandle, pMsg, processedCount); sdbDebug("app:%p:%p, waiting for confirm this operation, count:%d result:%s", pMsg->rpcMsg.ahandle, pMsg,
processedCount, tstrerror(code));
} }
return; return;
} }
if (pMsg != NULL) { if (pMsg != NULL) {
sdbDebug("app:%p:%p, is confirmed and will do callback func", pMsg->rpcMsg.ahandle, pMsg); sdbDebug("app:%p:%p, is confirmed and will do callback func, result:%s", pMsg->rpcMsg.ahandle, pMsg,
tstrerror(code));
} }
if (pOper->cb != NULL) { if (pOper->cb != NULL) {
pOper->retCode = (*pOper->cb)(pMsg, pOper->retCode); pOper->retCode = (*pOper->cb)(pMsg, pOper->retCode);
} }
dnodeSendRpcMnodeWriteRsp(pMsg, pOper->retCode); dnodeSendRpcMnodeWriteRsp(pMsg, pOper->retCode);
// if ahandle, means this func is called by sdb write
if (ahandle == NULL) {
sdbDecRef(pOper->table, pOper->pObj);
}
taosFreeQitem(pOper); taosFreeQitem(pOper);
} }
@ -389,7 +395,7 @@ void sdbCleanUp() {
} }
void sdbIncRef(void *handle, void *pObj) { void sdbIncRef(void *handle, void *pObj) {
if (pObj == NULL) return; if (pObj == NULL || handle == NULL) return;
SSdbTable *pTable = handle; SSdbTable *pTable = handle;
int32_t * pRefCount = (int32_t *)(pObj + pTable->refCountPos); int32_t * pRefCount = (int32_t *)(pObj + pTable->refCountPos);
@ -398,7 +404,7 @@ void sdbIncRef(void *handle, void *pObj) {
} }
void sdbDecRef(void *handle, void *pObj) { void sdbDecRef(void *handle, void *pObj) {
if (pObj == NULL) return; if (pObj == NULL || handle == NULL) return;
SSdbTable *pTable = handle; SSdbTable *pTable = handle;
int32_t * pRefCount = (int32_t *)(pObj + pTable->refCountPos); int32_t * pRefCount = (int32_t *)(pObj + pTable->refCountPos);
@ -581,16 +587,26 @@ static int sdbWrite(void *param, void *data, int type) {
return sdbInsertHash(pTable, &oper); return sdbInsertHash(pTable, &oper);
} else if (action == SDB_ACTION_DELETE) { } else if (action == SDB_ACTION_DELETE) {
SSdbRow *rowMeta = sdbGetRowMeta(pTable, pHead->cont); SSdbRow *rowMeta = sdbGetRowMeta(pTable, pHead->cont);
assert(rowMeta != NULL && rowMeta->row != NULL); if (rowMeta == NULL || rowMeta->row == NULL) {
sdbError("table:%s, failed to get object:%s from wal while dispose delete action", pTable->tableName,
pHead->cont);
return TSDB_CODE_SUCCESS;
}
SSdbOper oper = {.table = pTable, .pObj = rowMeta->row}; SSdbOper oper = {.table = pTable, .pObj = rowMeta->row};
return sdbDeleteHash(pTable, &oper); return sdbDeleteHash(pTable, &oper);
} else if (action == SDB_ACTION_UPDATE) { } else if (action == SDB_ACTION_UPDATE) {
SSdbRow *rowMeta = sdbGetRowMeta(pTable, pHead->cont); SSdbRow *rowMeta = sdbGetRowMeta(pTable, pHead->cont);
assert(rowMeta != NULL && rowMeta->row != NULL); if (rowMeta == NULL || rowMeta->row == NULL) {
sdbError("table:%s, failed to get object:%s from wal while dispose update action", pTable->tableName,
pHead->cont);
return TSDB_CODE_SUCCESS;
}
SSdbOper oper = {.rowSize = pHead->len, .rowData = pHead->cont, .table = pTable}; SSdbOper oper = {.rowSize = pHead->len, .rowData = pHead->cont, .table = pTable};
code = (*pTable->decodeFp)(&oper); code = (*pTable->decodeFp)(&oper);
return sdbUpdateHash(pTable, &oper); return sdbUpdateHash(pTable, &oper);
} else { return TSDB_CODE_MND_INVALID_MSG_TYPE; } } else {
return TSDB_CODE_MND_INVALID_MSG_TYPE;
}
} }
int32_t sdbInsertRow(SSdbOper *pOper) { int32_t sdbInsertRow(SSdbOper *pOper) {
@ -647,6 +663,14 @@ int32_t sdbInsertRow(SSdbOper *pOper) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
bool sdbCheckRowDeleted(void *pTableInput, void *pRow) {
SSdbTable *pTable = pTableInput;
if (pTable == NULL) return false;
int8_t *updateEnd = pRow + pTable->refCountPos - 1;
return (*updateEnd == 1);
}
int32_t sdbDeleteRow(SSdbOper *pOper) { int32_t sdbDeleteRow(SSdbOper *pOper) {
SSdbTable *pTable = (SSdbTable *)pOper->table; SSdbTable *pTable = (SSdbTable *)pOper->table;
if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE; if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE;
@ -663,14 +687,18 @@ int32_t sdbDeleteRow(SSdbOper *pOper) {
return TSDB_CODE_MND_SDB_INVAID_META_ROW; return TSDB_CODE_MND_SDB_INVAID_META_ROW;
} }
sdbIncRef(pTable, pOper->pObj);
int32_t code = sdbDeleteHash(pTable, pOper); int32_t code = sdbDeleteHash(pTable, pOper);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
sdbError("table:%s, failed to delete from hash", pTable->tableName); sdbError("table:%s, failed to delete from hash", pTable->tableName);
sdbDecRef(pTable, pOper->pObj);
return code; return code;
} }
// just delete data from memory // just delete data from memory
if (pOper->type != SDB_OPER_GLOBAL) { if (pOper->type != SDB_OPER_GLOBAL) {
sdbDecRef(pTable, pOper->pObj);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -692,7 +720,6 @@ int32_t sdbDeleteRow(SSdbOper *pOper) {
pNewOper->pMsg, pTable->tableName, pOper->pObj, sdbGetKeyStrFromObj(pTable, pOper->pObj)); pNewOper->pMsg, pTable->tableName, pOper->pObj, sdbGetKeyStrFromObj(pTable, pOper->pObj));
} }
sdbIncRef(pNewOper->table, pNewOper->pObj);
taosWriteQitem(tsSdbWriteQueue, TAOS_QTYPE_RPC, pNewOper); taosWriteQitem(tsSdbWriteQueue, TAOS_QTYPE_RPC, pNewOper);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -968,10 +995,11 @@ static void *sdbWorkerFp(void *param) {
int32_t code = sdbWrite(pOper, pHead, type); int32_t code = sdbWrite(pOper, pHead, type);
if (code > 0) code = 0; if (code > 0) code = 0;
if (pOper) if (pOper) {
pOper->retCode = code; pOper->retCode = code;
else } else {
pHead->len = code; // hackway pHead->len = code; // hackway
}
} }
walFsync(tsSdbObj.wal); walFsync(tsSdbObj.wal);
@ -983,7 +1011,6 @@ static void *sdbWorkerFp(void *param) {
if (type == TAOS_QTYPE_RPC) { if (type == TAOS_QTYPE_RPC) {
pOper = (SSdbOper *)item; pOper = (SSdbOper *)item;
sdbDecRef(pOper->table, pOper->pObj);
sdbConfirmForward(NULL, pOper, pOper->retCode); sdbConfirmForward(NULL, pOper, pOper->retCode);
} else if (type == TAOS_QTYPE_FWD) { } else if (type == TAOS_QTYPE_FWD) {
pHead = (SWalHead *)item; pHead = (SWalHead *)item;

99
src/mnode/src/mnodeTable.c
View File

@ -72,7 +72,7 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg);
static int32_t mnodeProcessDropTableMsg(SMnodeMsg *mnodeMsg); static int32_t mnodeProcessDropTableMsg(SMnodeMsg *mnodeMsg);
static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg); static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg);
static void mnodeProcessDropSuperTableRsp(SRpcMsg *rpcMsg); static void mnodeProcessDropSuperTableRsp(SRpcMsg *rpcMsg);
static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg); static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg, bool needReturn);
static void mnodeProcessDropChildTableRsp(SRpcMsg *rpcMsg); static void mnodeProcessDropChildTableRsp(SRpcMsg *rpcMsg);
static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *mnodeMsg); static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *mnodeMsg);
@ -319,15 +319,6 @@ static int32_t mnodeChildTableActionRestored() {
continue; continue;
} }
if (pVgroup->tableList == NULL) {
mError("ctable:%s, vgId:%d tableList is null", pTable->info.tableId, pTable->vgId);
pTable->vgId = 0;
SSdbOper desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .table = tsChildTableSdb};
sdbDeleteRow(&desc);
mnodeDecTableRef(pTable);
continue;
}
if (pTable->info.type == TSDB_CHILD_TABLE) { if (pTable->info.type == TSDB_CHILD_TABLE) {
SSuperTableObj *pSuperTable = mnodeGetSuperTableByUid(pTable->suid); SSuperTableObj *pSuperTable = mnodeGetSuperTableByUid(pTable->suid);
if (pSuperTable == NULL) { if (pSuperTable == NULL) {
@ -385,7 +376,7 @@ static void mnodeCleanupChildTables() {
} }
static void mnodeAddTableIntoStable(SSuperTableObj *pStable, SChildTableObj *pCtable) { static void mnodeAddTableIntoStable(SSuperTableObj *pStable, SChildTableObj *pCtable) {
pStable->numOfTables++; atomic_add_fetch_32(&pStable->numOfTables, 1);
if (pStable->vgHash == NULL) { if (pStable->vgHash == NULL) {
pStable->vgHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false); pStable->vgHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false);
@ -394,18 +385,22 @@ static void mnodeAddTableIntoStable(SSuperTableObj *pStable, SChildTableObj *pCt
if (pStable->vgHash != NULL) { if (pStable->vgHash != NULL) {
if (taosHashGet(pStable->vgHash, &pCtable->vgId, sizeof(pCtable->vgId)) == NULL) { if (taosHashGet(pStable->vgHash, &pCtable->vgId, sizeof(pCtable->vgId)) == NULL) {
taosHashPut(pStable->vgHash, &pCtable->vgId, sizeof(pCtable->vgId), &pCtable->vgId, sizeof(pCtable->vgId)); taosHashPut(pStable->vgHash, &pCtable->vgId, sizeof(pCtable->vgId), &pCtable->vgId, sizeof(pCtable->vgId));
mDebug("table:%s, vgId:%d is put into stable vgList, sizeOfVgList:%d", pStable->info.tableId, pCtable->vgId,
(int32_t)taosHashGetSize(pStable->vgHash));
} }
} }
} }
static void mnodeRemoveTableFromStable(SSuperTableObj *pStable, SChildTableObj *pCtable) { static void mnodeRemoveTableFromStable(SSuperTableObj *pStable, SChildTableObj *pCtable) {
pStable->numOfTables--; atomic_sub_fetch_32(&pStable->numOfTables, 1);
if (pStable->vgHash == NULL) return; if (pStable->vgHash == NULL) return;
SVgObj *pVgroup = mnodeGetVgroup(pCtable->vgId); SVgObj *pVgroup = mnodeGetVgroup(pCtable->vgId);
if (pVgroup == NULL) { if (pVgroup == NULL) {
taosHashRemove(pStable->vgHash, (char *)&pCtable->vgId, sizeof(pCtable->vgId)); taosHashRemove(pStable->vgHash, (char *)&pCtable->vgId, sizeof(pCtable->vgId));
mDebug("table:%s, vgId:%d is remove from stable vgList, sizeOfVgList:%d", pStable->info.tableId, pCtable->vgId,
(int32_t)taosHashGetSize(pStable->vgHash));
} }
mnodeDecVgroupRef(pVgroup); mnodeDecVgroupRef(pVgroup);
} }
@ -757,11 +752,15 @@ static int32_t mnodeProcessDropTableMsg(SMnodeMsg *pMsg) {
} }
if (pMsg->pTable->type == TSDB_SUPER_TABLE) { if (pMsg->pTable->type == TSDB_SUPER_TABLE) {
mInfo("app:%p:%p, table:%s, start to drop stable", pMsg->rpcMsg.ahandle, pMsg, pDrop->tableId); SSuperTableObj *pSTable = (SSuperTableObj *)pMsg->pTable;
mInfo("app:%p:%p, table:%s, start to drop stable, uid:%" PRIu64 ", numOfChildTables:%d, sizeOfVgList:%d",
pMsg->rpcMsg.ahandle, pMsg, pDrop->tableId, pSTable->uid, pSTable->numOfTables, (int32_t)taosHashGetSize(pSTable->vgHash));
return mnodeProcessDropSuperTableMsg(pMsg); return mnodeProcessDropSuperTableMsg(pMsg);
} else { } else {
mInfo("app:%p:%p, table:%s, start to drop ctable", pMsg->rpcMsg.ahandle, pMsg, pDrop->tableId); SChildTableObj *pCTable = (SChildTableObj *)pMsg->pTable;
return mnodeProcessDropChildTableMsg(pMsg); mInfo("app:%p:%p, table:%s, start to drop ctable, vgId:%d sid:%d uid:%" PRIu64, pMsg->rpcMsg.ahandle, pMsg,
pDrop->tableId, pCTable->vgId, pCTable->sid, pCTable->uid);
return mnodeProcessDropChildTableMsg(pMsg, true);
} }
} }
@ -808,7 +807,7 @@ static int32_t mnodeCreateSuperTableCb(SMnodeMsg *pMsg, int32_t code) {
assert(pTable); assert(pTable);
if (code == TSDB_CODE_SUCCESS) { if (code == TSDB_CODE_SUCCESS) {
mLInfo("stable:%s, is created in sdb", pTable->info.tableId); mLInfo("stable:%s, is created in sdb, uid:%" PRIu64, pTable->info.tableId, pTable->uid);
} else { } else {
mError("app:%p:%p, stable:%s, failed to create in sdb, reason:%s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, mError("app:%p:%p, stable:%s, failed to create in sdb, reason:%s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId,
tstrerror(code)); tstrerror(code));
@ -896,7 +895,7 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) {
if (pMsg == NULL) return TSDB_CODE_MND_APP_ERROR; if (pMsg == NULL) return TSDB_CODE_MND_APP_ERROR;
SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable; SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable;
if (pStable->numOfTables != 0) { if (pStable->vgHash != NULL /*pStable->numOfTables != 0*/) {
SHashMutableIterator *pIter = taosHashCreateIter(pStable->vgHash); SHashMutableIterator *pIter = taosHashCreateIter(pStable->vgHash);
while (taosHashIterNext(pIter)) { while (taosHashIterNext(pIter)) {
int32_t *pVgId = taosHashIterGet(pIter); int32_t *pVgId = taosHashIterGet(pIter);
@ -1600,8 +1599,8 @@ static int32_t mnodeDoCreateChildTableCb(SMnodeMsg *pMsg, int32_t code) {
assert(pTable); assert(pTable);
if (code == TSDB_CODE_SUCCESS) { if (code == TSDB_CODE_SUCCESS) {
mDebug("app:%p:%p, table:%s, create table in sid:%d, uid:%" PRIu64, pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, mDebug("app:%p:%p, table:%s, created in mnode, vgId:%d sid:%d, uid:%" PRIu64 ", result:%s", pMsg->rpcMsg.ahandle,
pTable->sid, pTable->uid); pMsg, pTable->info.tableId, pTable->vgId, pTable->sid, pTable->uid, tstrerror(code));
} else { } else {
mError("app:%p:%p, table:%s, failed to create table sid:%d, uid:%" PRIu64 ", reason:%s", pMsg->rpcMsg.ahandle, pMsg, mError("app:%p:%p, table:%s, failed to create table sid:%d, uid:%" PRIu64 ", reason:%s", pMsg->rpcMsg.ahandle, pMsg,
pTable->info.tableId, pTable->sid, pTable->uid, tstrerror(code)); pTable->info.tableId, pTable->sid, pTable->uid, tstrerror(code));
@ -1730,27 +1729,25 @@ static int32_t mnodeProcessCreateChildTableMsg(SMnodeMsg *pMsg) {
return code; return code;
} }
SVgObj *pVgroup = mnodeGetAvailableVgroup(pMsg->pDb);
if (pVgroup == NULL) {
mDebug("app:%p:%p, table:%s, start to create a new vgroup", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId);
return mnodeCreateVgroup(pMsg, pMsg->pDb);
}
if (pMsg->retry == 0) { if (pMsg->retry == 0) {
if (pMsg->pTable == NULL) { if (pMsg->pTable == NULL) {
int32_t sid = taosAllocateId(pVgroup->idPool); SVgObj *pVgroup = NULL;
if (sid <= 0) { int32_t sid = 0;
mDebug("app:%p:%p, table:%s, no enough sid in vgId:%d", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId, code = mnodeGetAvailableVgroup(pMsg, &pVgroup, &sid);
pVgroup->vgId); if (code != TSDB_CODE_SUCCESS) {
return mnodeCreateVgroup(pMsg, pMsg->pDb); mDebug("app:%p:%p, table:%s, failed to get available vgroup, reason:%s", pMsg->rpcMsg.ahandle, pMsg,
pCreate->tableId, tstrerror(code));
return code;
} }
if (pMsg->pVgroup == NULL) { if (pMsg->pVgroup != NULL) {
pMsg->pVgroup = pVgroup; mnodeDecVgroupRef(pMsg->pVgroup);
mnodeIncVgroupRef(pVgroup);
} }
mDebug("app:%p:%p, table:%s, create table in vgroup, vgId:%d sid:%d", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId, pMsg->pVgroup = pVgroup;
mnodeIncVgroupRef(pVgroup);
mDebug("app:%p:%p, table:%s, allocated in vgroup, vgId:%d sid:%d", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId,
pVgroup->vgId, sid); pVgroup->vgId, sid);
return mnodeDoCreateChildTable(pMsg, sid); return mnodeDoCreateChildTable(pMsg, sid);
@ -1769,7 +1766,7 @@ static int32_t mnodeProcessCreateChildTableMsg(SMnodeMsg *pMsg) {
} }
} }
static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg) { static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg, bool needReturn) {
SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable; SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable;
if (pMsg->pVgroup == NULL) pMsg->pVgroup = mnodeGetVgroup(pTable->vgId); if (pMsg->pVgroup == NULL) pMsg->pVgroup = mnodeGetVgroup(pTable->vgId);
if (pMsg->pVgroup == NULL) { if (pMsg->pVgroup == NULL) {
@ -1793,7 +1790,9 @@ static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg) {
SRpcIpSet ipSet = mnodeGetIpSetFromVgroup(pMsg->pVgroup); SRpcIpSet ipSet = mnodeGetIpSetFromVgroup(pMsg->pVgroup);
mInfo("app:%p:%p, table:%s, send drop ctable msg", pMsg->rpcMsg.ahandle, pMsg, pDrop->tableId); mInfo("app:%p:%p, table:%s, send drop ctable msg, vgId:%d sid:%d uid:%" PRIu64, pMsg->rpcMsg.ahandle, pMsg,
pDrop->tableId, pTable->vgId, pTable->sid, pTable->uid);
SRpcMsg rpcMsg = { SRpcMsg rpcMsg = {
.ahandle = pMsg, .ahandle = pMsg,
.pCont = pDrop, .pCont = pDrop,
@ -1802,6 +1801,8 @@ static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg) {
.msgType = TSDB_MSG_TYPE_MD_DROP_TABLE .msgType = TSDB_MSG_TYPE_MD_DROP_TABLE
}; };
if (!needReturn) rpcMsg.ahandle = NULL;
dnodeSendMsgToDnode(&ipSet, &rpcMsg); dnodeSendMsgToDnode(&ipSet, &rpcMsg);
return TSDB_CODE_MND_ACTION_IN_PROGRESS; return TSDB_CODE_MND_ACTION_IN_PROGRESS;
@ -2125,7 +2126,7 @@ static void mnodeDropAllChildTablesInStable(SSuperTableObj *pStable) {
int32_t numOfTables = 0; int32_t numOfTables = 0;
SChildTableObj *pTable = NULL; SChildTableObj *pTable = NULL;
mInfo("stable:%s, all child tables(%d) will dropped from sdb", pStable->info.tableId, numOfTables); mInfo("stable:%s, all child tables:%d will dropped from sdb", pStable->info.tableId, pStable->numOfTables);
while (1) { while (1) {
pIter = mnodeGetNextChildTable(pIter, &pTable); pIter = mnodeGetNextChildTable(pIter, &pTable);
@ -2149,6 +2150,7 @@ static void mnodeDropAllChildTablesInStable(SSuperTableObj *pStable) {
mInfo("stable:%s, all child tables:%d is dropped from sdb", pStable->info.tableId, numOfTables); mInfo("stable:%s, all child tables:%d is dropped from sdb", pStable->info.tableId, numOfTables);
} }
#if 0
static SChildTableObj* mnodeGetTableByPos(int32_t vnode, int32_t sid) { static SChildTableObj* mnodeGetTableByPos(int32_t vnode, int32_t sid) {
SVgObj *pVgroup = mnodeGetVgroup(vnode); SVgObj *pVgroup = mnodeGetVgroup(vnode);
if (pVgroup == NULL) return NULL; if (pVgroup == NULL) return NULL;
@ -2159,8 +2161,11 @@ static SChildTableObj* mnodeGetTableByPos(int32_t vnode, int32_t sid) {
mnodeDecVgroupRef(pVgroup); mnodeDecVgroupRef(pVgroup);
return pTable; return pTable;
} }
#endif
static int32_t mnodeProcessTableCfgMsg(SMnodeMsg *pMsg) { static int32_t mnodeProcessTableCfgMsg(SMnodeMsg *pMsg) {
return TSDB_CODE_COM_OPS_NOT_SUPPORT;
#if 0
SDMConfigTableMsg *pCfg = pMsg->rpcMsg.pCont; SDMConfigTableMsg *pCfg = pMsg->rpcMsg.pCont;
pCfg->dnodeId = htonl(pCfg->dnodeId); pCfg->dnodeId = htonl(pCfg->dnodeId);
pCfg->vgId = htonl(pCfg->vgId); pCfg->vgId = htonl(pCfg->vgId);
@ -2184,6 +2189,7 @@ static int32_t mnodeProcessTableCfgMsg(SMnodeMsg *pMsg) {
pMsg->rpcRsp.rsp = pCreate; pMsg->rpcRsp.rsp = pCreate;
pMsg->rpcRsp.len = htonl(pCreate->contLen); pMsg->rpcRsp.len = htonl(pCreate->contLen);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
#endif
} }
// handle drop child response // handle drop child response
@ -2195,12 +2201,15 @@ static void mnodeProcessDropChildTableRsp(SRpcMsg *rpcMsg) {
SChildTableObj *pTable = (SChildTableObj *)mnodeMsg->pTable; SChildTableObj *pTable = (SChildTableObj *)mnodeMsg->pTable;
assert(pTable); assert(pTable);
mInfo("app:%p:%p, table:%s, drop table rsp received, thandle:%p result:%s", mnodeMsg->rpcMsg.ahandle, mnodeMsg,
pTable->info.tableId, mnodeMsg->rpcMsg.handle, tstrerror(rpcMsg->code)); mInfo("app:%p:%p, table:%s, drop table rsp received, vgId:%d sid:%d uid:%" PRIu64 ", thandle:%p result:%s",
mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, pTable->vgId, pTable->sid, pTable->uid,
mnodeMsg->rpcMsg.handle, tstrerror(rpcMsg->code));
if (rpcMsg->code != TSDB_CODE_SUCCESS) { if (rpcMsg->code != TSDB_CODE_SUCCESS) {
mError("app:%p:%p, table:%s, failed to drop in dnode, reason:%s", mnodeMsg->rpcMsg.ahandle, mnodeMsg, mError("app:%p:%p, table:%s, failed to drop in dnode, vgId:%d sid:%d uid:%" PRIu64 ", reason:%s",
pTable->info.tableId, tstrerror(rpcMsg->code)); mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, pTable->vgId, pTable->sid, pTable->uid,
tstrerror(rpcMsg->code));
dnodeSendRpcMnodeWriteRsp(mnodeMsg, rpcMsg->code); dnodeSendRpcMnodeWriteRsp(mnodeMsg, rpcMsg->code);
return; return;
} }
@ -2247,6 +2256,14 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) {
SChildTableObj *pTable = (SChildTableObj *)mnodeMsg->pTable; SChildTableObj *pTable = (SChildTableObj *)mnodeMsg->pTable;
assert(pTable); assert(pTable);
// If the table is deleted by another thread during creation, stop creating and send drop msg to vnode
if (sdbCheckRowDeleted(tsChildTableSdb, pTable)) {
mDebug("app:%p:%p, table:%s, create table rsp received, but a deleting opertion incoming, vgId:%d sid:%d uid:%" PRIu64,
mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, pTable->vgId, pTable->sid, pTable->uid);
mnodeProcessDropChildTableMsg(mnodeMsg, false);
rpcMsg->code = TSDB_CODE_SUCCESS;
}
if (rpcMsg->code == TSDB_CODE_SUCCESS || rpcMsg->code == TSDB_CODE_TDB_TABLE_ALREADY_EXIST) { if (rpcMsg->code == TSDB_CODE_SUCCESS || rpcMsg->code == TSDB_CODE_TDB_TABLE_ALREADY_EXIST) {
SCMCreateTableMsg *pCreate = mnodeMsg->rpcMsg.pCont; SCMCreateTableMsg *pCreate = mnodeMsg->rpcMsg.pCont;
if (pCreate->getMeta) { if (pCreate->getMeta) {

388
src/mnode/src/mnodeVgroup.c
View File

@ -40,30 +40,41 @@
typedef enum { typedef enum {
TAOS_VG_STATUS_READY, TAOS_VG_STATUS_READY,
TAOS_VG_STATUS_DROPPING TAOS_VG_STATUS_DROPPING,
TAOS_VG_STATUS_CREATING,
TAOS_VG_STATUS_UPDATING,
} EVgroupStatus; } EVgroupStatus;
char* vgroupStatus[] = {
"ready",
"dropping",
"creating",
"updating"
};
static void *tsVgroupSdb = NULL; static void *tsVgroupSdb = NULL;
static int32_t tsVgUpdateSize = 0; static int32_t tsVgUpdateSize = 0;
static int32_t mnodeAllocVgroupIdPool(SVgObj *pInputVgroup);
static int32_t mnodeGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mnodeGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
static int32_t mnodeRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pConn); static int32_t mnodeRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pConn);
static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg); static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg);
static void mnodeProcessAlterVnodeRsp(SRpcMsg *rpcMsg);
static void mnodeProcessDropVnodeRsp(SRpcMsg *rpcMsg); static void mnodeProcessDropVnodeRsp(SRpcMsg *rpcMsg);
static int32_t mnodeProcessVnodeCfgMsg(SMnodeMsg *pMsg) ; static int32_t mnodeProcessVnodeCfgMsg(SMnodeMsg *pMsg) ;
static void mnodeSendDropVgroupMsg(SVgObj *pVgroup, void *ahandle); static void mnodeSendDropVgroupMsg(SVgObj *pVgroup, void *ahandle);
static int32_t mnodeVgroupActionDestroy(SSdbOper *pOper) { static void mnodeDestroyVgroup(SVgObj *pVgroup) {
SVgObj *pVgroup = pOper->pObj;
if (pVgroup->idPool) { if (pVgroup->idPool) {
taosIdPoolCleanUp(pVgroup->idPool); taosIdPoolCleanUp(pVgroup->idPool);
pVgroup->idPool = NULL; pVgroup->idPool = NULL;
} }
if (pVgroup->tableList) {
tfree(pVgroup->tableList);
}
tfree(pOper->pObj); tfree(pVgroup);
}
static int32_t mnodeVgroupActionDestroy(SSdbOper *pOper) {
mnodeDestroyVgroup(pOper->pObj);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -82,21 +93,10 @@ static int32_t mnodeVgroupActionInsert(SSdbOper *pOper) {
} }
pVgroup->pDb = pDb; pVgroup->pDb = pDb;
pVgroup->prev = NULL; pVgroup->status = TAOS_VG_STATUS_CREATING;
pVgroup->next = NULL;
pVgroup->accessState = TSDB_VN_ALL_ACCCESS; pVgroup->accessState = TSDB_VN_ALL_ACCCESS;
if (mnodeAllocVgroupIdPool(pVgroup) < 0) {
int32_t size = sizeof(SChildTableObj *) * pDb->cfg.maxTables; mError("vgId:%d, failed to init idpool for vgroups", pVgroup->vgId);
pVgroup->tableList = calloc(pDb->cfg.maxTables, sizeof(SChildTableObj *));
if (pVgroup->tableList == NULL) {
mError("vgId:%d, failed to malloc(size:%d) for the tableList of vgroups", pVgroup->vgId, size);
return -1;
}
pVgroup->idPool = taosInitIdPool(pDb->cfg.maxTables);
if (pVgroup->idPool == NULL) {
mError("vgId:%d, failed to taosInitIdPool for vgroups", pVgroup->vgId);
tfree(pVgroup->tableList);
return -1; return -1;
} }
@ -134,20 +134,6 @@ static int32_t mnodeVgroupActionDelete(SSdbOper *pOper) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static void mnodeVgroupUpdateIdPool(SVgObj *pVgroup) {
int32_t oldTables = taosIdPoolMaxSize(pVgroup->idPool);
SDbObj *pDb = pVgroup->pDb;
if (pDb != NULL) {
if (pDb->cfg.maxTables != oldTables) {
mInfo("vgId:%d tables change from %d to %d", pVgroup->vgId, oldTables, pDb->cfg.maxTables);
taosUpdateIdPool(pVgroup->idPool, pDb->cfg.maxTables);
int32_t size = sizeof(SChildTableObj *) * pDb->cfg.maxTables;
pVgroup->tableList = (SChildTableObj **)realloc(pVgroup->tableList, size);
memset(pVgroup->tableList + oldTables, 0, (pDb->cfg.maxTables - oldTables) * sizeof(SChildTableObj *));
}
}
}
static int32_t mnodeVgroupActionUpdate(SSdbOper *pOper) { static int32_t mnodeVgroupActionUpdate(SSdbOper *pOper) {
SVgObj *pNew = pOper->pObj; SVgObj *pNew = pOper->pObj;
SVgObj *pVgroup = mnodeGetVgroup(pNew->vgId); SVgObj *pVgroup = mnodeGetVgroup(pNew->vgId);
@ -174,7 +160,6 @@ static int32_t mnodeVgroupActionUpdate(SSdbOper *pOper) {
free(pNew); free(pNew);
} }
mnodeVgroupUpdateIdPool(pVgroup);
// reset vgid status on vgroup changed // reset vgid status on vgroup changed
mDebug("vgId:%d, reset sync status to unsynced", pVgroup->vgId); mDebug("vgId:%d, reset sync status to unsynced", pVgroup->vgId);
@ -243,6 +228,7 @@ int32_t mnodeInitVgroups() {
mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_VGROUP, mnodeGetVgroupMeta); mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_VGROUP, mnodeGetVgroupMeta);
mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_VGROUP, mnodeRetrieveVgroups); mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_VGROUP, mnodeRetrieveVgroups);
mnodeAddPeerRspHandle(TSDB_MSG_TYPE_MD_CREATE_VNODE_RSP, mnodeProcessCreateVnodeRsp); mnodeAddPeerRspHandle(TSDB_MSG_TYPE_MD_CREATE_VNODE_RSP, mnodeProcessCreateVnodeRsp);
mnodeAddPeerRspHandle(TSDB_MSG_TYPE_MD_ALTER_VNODE_RSP, mnodeProcessAlterVnodeRsp);
mnodeAddPeerRspHandle(TSDB_MSG_TYPE_MD_DROP_VNODE_RSP, mnodeProcessDropVnodeRsp); mnodeAddPeerRspHandle(TSDB_MSG_TYPE_MD_DROP_VNODE_RSP, mnodeProcessDropVnodeRsp);
mnodeAddPeerMsgHandle(TSDB_MSG_TYPE_DM_CONFIG_VNODE, mnodeProcessVnodeCfgMsg); mnodeAddPeerMsgHandle(TSDB_MSG_TYPE_DM_CONFIG_VNODE, mnodeProcessVnodeCfgMsg);
@ -273,7 +259,7 @@ void mnodeUpdateVgroup(SVgObj *pVgroup) {
if (sdbUpdateRow(&oper) != TSDB_CODE_SUCCESS) { if (sdbUpdateRow(&oper) != TSDB_CODE_SUCCESS) {
mError("vgId:%d, failed to update vgroup", pVgroup->vgId); mError("vgId:%d, failed to update vgroup", pVgroup->vgId);
} }
mnodeSendCreateVgroupMsg(pVgroup, NULL); mnodeSendAlterVgroupMsg(pVgroup);
} }
/* /*
@ -297,10 +283,17 @@ void mnodeCheckUnCreatedVgroup(SDnodeObj *pDnode, SVnodeLoad *pVloads, int32_t o
pNextV++; pNextV++;
} }
if (i == openVnodes && pVgroup->status == TAOS_VG_STATUS_READY) { if (i == openVnodes) {
mnodeSendCreateVgroupMsg(pVgroup, NULL); if (pVgroup->status == TAOS_VG_STATUS_CREATING || pVgroup->status == TAOS_VG_STATUS_DROPPING) {
mDebug("vgId:%d, not exist in dnode:%d and status is %s, do nothing", pVgroup->vgId, pDnode->dnodeId,
vgroupStatus[pVgroup->status]);
} else {
mDebug("vgId:%d, not exist in dnode:%d and status is %s, send create msg", pVgroup->vgId, pDnode->dnodeId,
vgroupStatus[pVgroup->status]);
mnodeSendCreateVgroupMsg(pVgroup, NULL);
}
} }
mnodeDecVgroupRef(pVgroup); mnodeDecVgroupRef(pVgroup);
} }
@ -340,12 +333,136 @@ void mnodeUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *pDnode, SVnodeLoad *pVl
mError("dnode:%d, vgId:%d, vnode cfgVersion:%d repica:%d not match with mnode cfgVersion:%d replica:%d", mError("dnode:%d, vgId:%d, vnode cfgVersion:%d repica:%d not match with mnode cfgVersion:%d replica:%d",
pDnode->dnodeId, pVload->vgId, pVload->cfgVersion, pVload->replica, pVgroup->pDb->cfgVersion, pDnode->dnodeId, pVload->vgId, pVload->cfgVersion, pVload->replica, pVgroup->pDb->cfgVersion,
pVgroup->numOfVnodes); pVgroup->numOfVnodes);
mnodeSendCreateVgroupMsg(pVgroup, NULL); mnodeSendAlterVgroupMsg(pVgroup);
} }
} }
SVgObj *mnodeGetAvailableVgroup(SDbObj *pDb) { static int32_t mnodeAllocVgroupIdPool(SVgObj *pInputVgroup) {
return pDb->pHead; SDbObj *pDb = pInputVgroup->pDb;
if (pDb == NULL) return TSDB_CODE_MND_APP_ERROR;
int32_t minIdPoolSize = TSDB_MAX_TABLES;
int32_t maxIdPoolSize = tsMinTablePerVnode;
for (int32_t v = 0; v < pDb->numOfVgroups; ++v) {
SVgObj *pVgroup = pDb->vgList[v];
if (pVgroup == NULL) continue;
int32_t idPoolSize = taosIdPoolMaxSize(pVgroup->idPool);
minIdPoolSize = MIN(minIdPoolSize, idPoolSize);
maxIdPoolSize = MAX(maxIdPoolSize, idPoolSize);
}
// new vgroup
if (pInputVgroup->idPool == NULL) {
pInputVgroup->idPool = taosInitIdPool(maxIdPoolSize);
if (pInputVgroup->idPool == NULL) {
mError("vgId:%d, failed to init idPool for vgroup, size:%d", pInputVgroup->vgId, maxIdPoolSize);
return TSDB_CODE_MND_OUT_OF_MEMORY;
} else {
mDebug("vgId:%d, init idPool for vgroup, size:%d", pInputVgroup->vgId, maxIdPoolSize);
return TSDB_CODE_SUCCESS;
}
}
// realloc all vgroups in db
int32_t newIdPoolSize;
if (minIdPoolSize * 4 < tsTableIncStepPerVnode) {
newIdPoolSize = minIdPoolSize * 4;
} else {
newIdPoolSize = ((minIdPoolSize / tsTableIncStepPerVnode) + 1) * tsTableIncStepPerVnode;
}
if (newIdPoolSize > tsMaxTablePerVnode) {
if (minIdPoolSize >= tsMaxTablePerVnode) {
mError("db:%s, minIdPoolSize:%d newIdPoolSize:%d larger than maxTablesPerVnode:%d", pDb->name, minIdPoolSize, newIdPoolSize,
tsMaxTablePerVnode);
return TSDB_CODE_MND_NO_ENOUGH_DNODES;
} else {
newIdPoolSize = tsMaxTablePerVnode;
}
}
for (int32_t v = 0; v < pDb->numOfVgroups; ++v) {
SVgObj *pVgroup = pDb->vgList[v];
if (pVgroup == NULL) continue;
int32_t oldIdPoolSize = taosIdPoolMaxSize(pVgroup->idPool);
if (newIdPoolSize == oldIdPoolSize) continue;
if (taosUpdateIdPool(pVgroup->idPool, newIdPoolSize) < 0) {
mError("vgId:%d, failed to update idPoolSize from %d to %d", pVgroup->vgId, oldIdPoolSize, newIdPoolSize);
return TSDB_CODE_MND_NO_ENOUGH_DNODES;
} else {
mDebug("vgId:%d, idPoolSize update from %d to %d", pVgroup->vgId, oldIdPoolSize, newIdPoolSize);
}
}
return TSDB_CODE_SUCCESS;
}
int32_t mnodeGetAvailableVgroup(SMnodeMsg *pMsg, SVgObj **ppVgroup, int32_t *pSid) {
SDbObj *pDb = pMsg->pDb;
pthread_mutex_lock(&pDb->mutex);
for (int32_t v = 0; v < pDb->numOfVgroups; ++v) {
int vgIndex = (v + pDb->vgListIndex) % pDb->numOfVgroups;
SVgObj *pVgroup = pDb->vgList[vgIndex];
if (pVgroup == NULL) {
mError("db:%s, index:%d vgroup is null", pDb->name, vgIndex);
pthread_mutex_unlock(&pDb->mutex);
return TSDB_CODE_MND_APP_ERROR;
}
int32_t sid = taosAllocateId(pVgroup->idPool);
if (sid <= 0) {
mDebug("app:%p:%p, db:%s, no enough sid in vgId:%d", pMsg->rpcMsg.ahandle, pMsg, pDb->name, pVgroup->vgId);
continue;
}
*pSid = sid;
*ppVgroup = pVgroup;
pDb->vgListIndex = vgIndex;
pthread_mutex_unlock(&pDb->mutex);
return TSDB_CODE_SUCCESS;
}
int maxVgroupsPerDb = tsMaxVgroupsPerDb;
if (maxVgroupsPerDb <= 0) {
maxVgroupsPerDb = mnodeGetOnlinDnodesCpuCoreNum();
maxVgroupsPerDb = MAX(maxVgroupsPerDb, 2);
}
if (pDb->numOfVgroups < maxVgroupsPerDb) {
mDebug("app:%p:%p, db:%s, try to create a new vgroup, numOfVgroups:%d maxVgroupsPerDb:%d", pMsg->rpcMsg.ahandle, pMsg,
pDb->name, pDb->numOfVgroups, maxVgroupsPerDb);
pthread_mutex_unlock(&pDb->mutex);
int32_t code = mnodeCreateVgroup(pMsg);
if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) return code;
}
SVgObj *pVgroup = pDb->vgList[0];
if (pVgroup == NULL) return TSDB_CODE_MND_NO_ENOUGH_DNODES;
int32_t code = mnodeAllocVgroupIdPool(pVgroup);
if (code != TSDB_CODE_SUCCESS) {
pthread_mutex_unlock(&pDb->mutex);
return code;
}
int32_t sid = taosAllocateId(pVgroup->idPool);
if (sid <= 0) {
mError("app:%p:%p, db:%s, no enough sid in vgId:%d", pMsg->rpcMsg.ahandle, pMsg, pDb->name, pVgroup->vgId);
pthread_mutex_unlock(&pDb->mutex);
return TSDB_CODE_MND_NO_ENOUGH_DNODES;
}
*pSid = sid;
*ppVgroup = pVgroup;
pDb->vgListIndex = 0;
pthread_mutex_unlock(&pDb->mutex);
return TSDB_CODE_SUCCESS;
} }
void *mnodeGetNextVgroup(void *pIter, SVgObj **pVgroup) { void *mnodeGetNextVgroup(void *pIter, SVgObj **pVgroup) {
@ -363,6 +480,10 @@ static int32_t mnodeCreateVgroupCb(SMnodeMsg *pMsg, int32_t code) {
SSdbOper desc = {.type = SDB_OPER_GLOBAL, .pObj = pVgroup, .table = tsVgroupSdb}; SSdbOper desc = {.type = SDB_OPER_GLOBAL, .pObj = pVgroup, .table = tsVgroupSdb};
sdbDeleteRow(&desc); sdbDeleteRow(&desc);
return code; return code;
} else {
pVgroup->status = TAOS_VG_STATUS_READY;
SSdbOper desc = {.type = SDB_OPER_GLOBAL, .pObj = pVgroup, .table = tsVgroupSdb};
sdbUpdateRow(&desc);
} }
mInfo("app:%p:%p, vgId:%d, is created in mnode, db:%s replica:%d", pMsg->rpcMsg.ahandle, pMsg, pVgroup->vgId, mInfo("app:%p:%p, vgId:%d, is created in mnode, db:%s replica:%d", pMsg->rpcMsg.ahandle, pMsg, pVgroup->vgId,
@ -373,13 +494,16 @@ static int32_t mnodeCreateVgroupCb(SMnodeMsg *pMsg, int32_t code) {
} }
pMsg->expected = pVgroup->numOfVnodes; pMsg->expected = pVgroup->numOfVnodes;
pMsg->successed = 0;
pMsg->received = 0;
mnodeSendCreateVgroupMsg(pVgroup, pMsg); mnodeSendCreateVgroupMsg(pVgroup, pMsg);
return TSDB_CODE_MND_ACTION_IN_PROGRESS; return TSDB_CODE_MND_ACTION_IN_PROGRESS;
} }
int32_t mnodeCreateVgroup(SMnodeMsg *pMsg, SDbObj *pDb) { int32_t mnodeCreateVgroup(SMnodeMsg *pMsg) {
if (pMsg == NULL) return TSDB_CODE_MND_APP_ERROR; if (pMsg == NULL) return TSDB_CODE_MND_APP_ERROR;
SDbObj *pDb = pMsg->pDb;
SVgObj *pVgroup = (SVgObj *)calloc(1, sizeof(SVgObj)); SVgObj *pVgroup = (SVgObj *)calloc(1, sizeof(SVgObj));
tstrncpy(pVgroup->dbName, pDb->name, TSDB_ACCT_LEN + TSDB_DB_NAME_LEN); tstrncpy(pVgroup->dbName, pDb->name, TSDB_ACCT_LEN + TSDB_DB_NAME_LEN);
@ -407,7 +531,7 @@ int32_t mnodeCreateVgroup(SMnodeMsg *pMsg, SDbObj *pDb) {
int32_t code = sdbInsertRow(&oper); int32_t code = sdbInsertRow(&oper);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
pMsg->pVgroup = NULL; pMsg->pVgroup = NULL;
tfree(pVgroup); mnodeDestroyVgroup(pVgroup);
} else { } else {
code = TSDB_CODE_MND_ACTION_IN_PROGRESS; code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
} }
@ -435,7 +559,7 @@ void mnodeCleanupVgroups() {
tsVgroupSdb = NULL; tsVgroupSdb = NULL;
} }
int32_t mnodeGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { static int32_t mnodeGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
SDbObj *pDb = mnodeGetDb(pShow->db); SDbObj *pDb = mnodeGetDb(pShow->db);
if (pDb == NULL) { if (pDb == NULL) {
return TSDB_CODE_MND_DB_NOT_SELECTED; return TSDB_CODE_MND_DB_NOT_SELECTED;
@ -461,29 +585,27 @@ int32_t mnodeGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
pSchema[cols].bytes = htons(pShow->bytes[cols]); pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++; cols++;
int32_t maxReplica = 0; pShow->bytes[cols] = 4;
SVgObj *pVgroup = NULL; pSchema[cols].type = TSDB_DATA_TYPE_INT;
STableObj *pTable = NULL; strcpy(pSchema[cols].name, "poolSize");
if (pShow->payloadLen > 0 ) { pSchema[cols].bytes = htons(pShow->bytes[cols]);
pTable = mnodeGetTable(pShow->payload); cols++;
if (NULL == pTable || pTable->type == TSDB_SUPER_TABLE) {
mnodeDecTableRef(pTable); pShow->bytes[cols] = 4;
return TSDB_CODE_MND_INVALID_TABLE_NAME; pSchema[cols].type = TSDB_DATA_TYPE_INT;
} strcpy(pSchema[cols].name, "onlineVnodes");
mnodeDecTableRef(pTable); pSchema[cols].bytes = htons(pShow->bytes[cols]);
pVgroup = mnodeGetVgroup(((SChildTableObj*)pTable)->vgId); cols++;
if (NULL == pVgroup) return TSDB_CODE_MND_INVALID_TABLE_NAME;
mnodeDecVgroupRef(pVgroup); pShow->maxReplica = 1;
maxReplica = pVgroup->numOfVnodes > maxReplica ? pVgroup->numOfVnodes : maxReplica; for (int32_t v = 0; v < pDb->numOfVgroups; ++v) {
} else { SVgObj *pVgroup = pDb->vgList[v];
SVgObj *pVgroup = pDb->pHead; if (pVgroup != NULL) {
while (pVgroup != NULL) { pShow->maxReplica = pVgroup->numOfVnodes > pShow->maxReplica ? pVgroup->numOfVnodes : pShow->maxReplica;
maxReplica = pVgroup->numOfVnodes > maxReplica ? pVgroup->numOfVnodes : maxReplica;
pVgroup = pVgroup->next;
} }
} }
for (int32_t i = 0; i < maxReplica; ++i) { for (int32_t i = 0; i < pShow->maxReplica; ++i) {
pShow->bytes[cols] = 2; pShow->bytes[cols] = 2;
pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
strcpy(pSchema[cols].name, "dnode"); strcpy(pSchema[cols].name, "dnode");
@ -507,48 +629,54 @@ int32_t mnodeGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
pShow->numOfColumns = cols; pShow->numOfColumns = cols;
pShow->offset[0] = 0; pShow->offset[0] = 0;
for (int32_t i = 1; i < cols; ++i) pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; for (int32_t i = 1; i < cols; ++i) {
pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1];
pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
if (NULL == pTable) {
pShow->numOfRows = pDb->numOfVgroups;
pShow->pIter = pDb->pHead;
} else {
pShow->numOfRows = 1;
pShow->pIter = pVgroup;
} }
mnodeDecDbRef(pDb); pShow->numOfRows = pDb->numOfVgroups;
pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
mnodeDecDbRef(pDb);
return 0; return 0;
} }
int32_t mnodeRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pConn) { static bool mnodeFilterVgroups(SVgObj *pVgroup, STableObj *pTable) {
if (NULL == pTable || pTable->type == TSDB_SUPER_TABLE) {
return true;
}
SChildTableObj *pCTable = (SChildTableObj *)pTable;
if (pVgroup->vgId == pCTable->vgId) {
return true;
} else {
return false;
}
}
static int32_t mnodeRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pConn) {
int32_t numOfRows = 0; int32_t numOfRows = 0;
SVgObj *pVgroup = NULL; SVgObj *pVgroup = NULL;
int32_t maxReplica = 0;
int32_t cols = 0; int32_t cols = 0;
char * pWrite; char * pWrite;
SDbObj *pDb = mnodeGetDb(pShow->db); SDbObj *pDb = mnodeGetDb(pShow->db);
if (pDb == NULL) return 0; if (pDb == NULL) return 0;
if (pDb->status != TSDB_DB_STATUS_READY) { if (pDb->status != TSDB_DB_STATUS_READY) {
mError("db:%s, status:%d, in dropping", pDb->name, pDb->status); mError("db:%s, status:%d, in dropping", pDb->name, pDb->status);
return 0; return 0;
} }
pVgroup = pDb->pHead; STableObj *pTable = NULL;
while (pVgroup != NULL) { if (pShow->payloadLen > 0 ) {
maxReplica = pVgroup->numOfVnodes > maxReplica ? pVgroup->numOfVnodes : maxReplica; pTable = mnodeGetTable(pShow->payload);
pVgroup = pVgroup->next;
} }
while (numOfRows < rows) { while (numOfRows < rows) {
pVgroup = (SVgObj *) pShow->pIter; pShow->pIter = mnodeGetNextVgroup(pShow->pIter, &pVgroup);
if (pVgroup == NULL) break; if (pVgroup == NULL) break;
pShow->pIter = (void *) pVgroup->next; if (pVgroup->pDb != pDb) continue;
if (!mnodeFilterVgroups(pVgroup, pTable)) continue;
cols = 0; cols = 0;
@ -560,7 +688,22 @@ int32_t mnodeRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pC
*(int32_t *) pWrite = pVgroup->numOfTables; *(int32_t *) pWrite = pVgroup->numOfTables;
cols++; cols++;
for (int32_t i = 0; i < maxReplica; ++i) { pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int32_t *)pWrite = taosIdPoolMaxSize(pVgroup->idPool);
cols++;
int32_t onlineVnodes = 0;
for (int32_t i = 0; i < pShow->maxReplica; ++i) {
if (pVgroup->vnodeGid[i].role == TAOS_SYNC_ROLE_SLAVE || pVgroup->vnodeGid[i].role == TAOS_SYNC_ROLE_MASTER) {
onlineVnodes++;
}
}
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int32_t *)pWrite = onlineVnodes;
cols++;
for (int32_t i = 0; i < pShow->maxReplica; ++i) {
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int16_t *) pWrite = pVgroup->vnodeGid[i].dnodeId; *(int16_t *) pWrite = pVgroup->vnodeGid[i].dnodeId;
cols++; cols++;
@ -588,41 +731,39 @@ int32_t mnodeRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pC
} }
} }
mnodeDecVgroupRef(pVgroup);
numOfRows++; numOfRows++;
} }
pShow->numOfReads += numOfRows; pShow->numOfReads += numOfRows;
mnodeDecTableRef(pTable);
mnodeDecDbRef(pDb); mnodeDecDbRef(pDb);
return numOfRows; return numOfRows;
} }
void mnodeAddTableIntoVgroup(SVgObj *pVgroup, SChildTableObj *pTable) { void mnodeAddTableIntoVgroup(SVgObj *pVgroup, SChildTableObj *pTable) {
if (pTable->sid >= 1 && pVgroup->tableList[pTable->sid - 1] == NULL) { int32_t idPoolSize = taosIdPoolMaxSize(pVgroup->idPool);
pVgroup->tableList[pTable->sid - 1] = pTable; if (pTable->sid > idPoolSize) {
taosIdPoolMarkStatus(pVgroup->idPool, pTable->sid); mnodeAllocVgroupIdPool(pVgroup);
pVgroup->numOfTables++;
}
if (pVgroup->numOfTables >= pVgroup->pDb->cfg.maxTables) {
mnodeMoveVgroupToTail(pVgroup);
} }
mnodeIncVgroupRef(pVgroup); if (pTable->sid >= 1) {
taosIdPoolMarkStatus(pVgroup->idPool, pTable->sid);
pVgroup->numOfTables++;
mnodeIncVgroupRef(pVgroup);
}
} }
void mnodeRemoveTableFromVgroup(SVgObj *pVgroup, SChildTableObj *pTable) { void mnodeRemoveTableFromVgroup(SVgObj *pVgroup, SChildTableObj *pTable) {
if (pTable->sid >= 1 && pVgroup->tableList[pTable->sid - 1] != NULL) { if (pTable->sid >= 1) {
pVgroup->tableList[pTable->sid - 1] = NULL;
taosFreeId(pVgroup->idPool, pTable->sid); taosFreeId(pVgroup->idPool, pTable->sid);
pVgroup->numOfTables--; pVgroup->numOfTables--;
mnodeDecVgroupRef(pVgroup);
} }
mnodeMoveVgroupToHead(pVgroup);
mnodeDecVgroupRef(pVgroup);
} }
SMDCreateVnodeMsg *mnodeBuildCreateVnodeMsg(SVgObj *pVgroup) { static SMDCreateVnodeMsg *mnodeBuildVnodeMsg(SVgObj *pVgroup) {
SDbObj *pDb = pVgroup->pDb; SDbObj *pDb = pVgroup->pDb;
if (pDb == NULL) return NULL; if (pDb == NULL) return NULL;
@ -630,19 +771,23 @@ SMDCreateVnodeMsg *mnodeBuildCreateVnodeMsg(SVgObj *pVgroup) {
if (pVnode == NULL) return NULL; if (pVnode == NULL) return NULL;
strcpy(pVnode->db, pVgroup->dbName); strcpy(pVnode->db, pVgroup->dbName);
int32_t maxTables = taosIdPoolMaxSize(pVgroup->idPool);
//TODO: dynamic alloc tables in tsdb
maxTables = MAX(10000, tsMaxTablePerVnode);
SMDVnodeCfg *pCfg = &pVnode->cfg; SMDVnodeCfg *pCfg = &pVnode->cfg;
pCfg->vgId = htonl(pVgroup->vgId); pCfg->vgId = htonl(pVgroup->vgId);
pCfg->cfgVersion = htonl(pDb->cfgVersion); pCfg->cfgVersion = htonl(pDb->cfgVersion);
pCfg->cacheBlockSize = htonl(pDb->cfg.cacheBlockSize); pCfg->cacheBlockSize = htonl(pDb->cfg.cacheBlockSize);
pCfg->totalBlocks = htonl(pDb->cfg.totalBlocks); pCfg->totalBlocks = htonl(pDb->cfg.totalBlocks);
pCfg->maxTables = htonl(pDb->cfg.maxTables + 1); pCfg->maxTables = htonl(maxTables + 1);
pCfg->daysPerFile = htonl(pDb->cfg.daysPerFile); pCfg->daysPerFile = htonl(pDb->cfg.daysPerFile);
pCfg->daysToKeep = htonl(pDb->cfg.daysToKeep); pCfg->daysToKeep = htonl(pDb->cfg.daysToKeep);
pCfg->daysToKeep1 = htonl(pDb->cfg.daysToKeep1); pCfg->daysToKeep1 = htonl(pDb->cfg.daysToKeep1);
pCfg->daysToKeep2 = htonl(pDb->cfg.daysToKeep2); pCfg->daysToKeep2 = htonl(pDb->cfg.daysToKeep2);
pCfg->minRowsPerFileBlock = htonl(pDb->cfg.minRowsPerFileBlock); pCfg->minRowsPerFileBlock = htonl(pDb->cfg.minRowsPerFileBlock);
pCfg->maxRowsPerFileBlock = htonl(pDb->cfg.maxRowsPerFileBlock); pCfg->maxRowsPerFileBlock = htonl(pDb->cfg.maxRowsPerFileBlock);
pCfg->fsyncPeriod = htonl(pDb->cfg.fsyncPeriod);
pCfg->commitTime = htonl(pDb->cfg.commitTime); pCfg->commitTime = htonl(pDb->cfg.commitTime);
pCfg->precision = pDb->cfg.precision; pCfg->precision = pDb->cfg.precision;
pCfg->compression = pDb->cfg.compression; pCfg->compression = pDb->cfg.compression;
@ -685,8 +830,31 @@ SRpcIpSet mnodeGetIpSetFromIp(char *ep) {
return ipSet; return ipSet;
} }
void mnodeSendCreateVnodeMsg(SVgObj *pVgroup, SRpcIpSet *ipSet, void *ahandle) { static void mnodeSendAlterVnodeMsg(SVgObj *pVgroup, SRpcIpSet *ipSet) {
SMDCreateVnodeMsg *pCreate = mnodeBuildCreateVnodeMsg(pVgroup); SMDAlterVnodeMsg *pAlter = mnodeBuildVnodeMsg(pVgroup);
SRpcMsg rpcMsg = {
.ahandle = NULL,
.pCont = pAlter,
.contLen = pAlter ? sizeof(SMDAlterVnodeMsg) : 0,
.code = 0,
.msgType = TSDB_MSG_TYPE_MD_ALTER_VNODE
};
dnodeSendMsgToDnode(ipSet, &rpcMsg);
}
void mnodeSendAlterVgroupMsg(SVgObj *pVgroup) {
mDebug("vgId:%d, send alter all vnodes msg, numOfVnodes:%d db:%s", pVgroup->vgId, pVgroup->numOfVnodes,
pVgroup->dbName);
for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
SRpcIpSet ipSet = mnodeGetIpSetFromIp(pVgroup->vnodeGid[i].pDnode->dnodeEp);
mDebug("vgId:%d, index:%d, send alter vnode msg to dnode %s", pVgroup->vgId, i,
pVgroup->vnodeGid[i].pDnode->dnodeEp);
mnodeSendAlterVnodeMsg(pVgroup, &ipSet);
}
}
static void mnodeSendCreateVnodeMsg(SVgObj *pVgroup, SRpcIpSet *ipSet, void *ahandle) {
SMDCreateVnodeMsg *pCreate = mnodeBuildVnodeMsg(pVgroup);
SRpcMsg rpcMsg = { SRpcMsg rpcMsg = {
.ahandle = ahandle, .ahandle = ahandle,
.pCont = pCreate, .pCont = pCreate,
@ -708,6 +876,10 @@ void mnodeSendCreateVgroupMsg(SVgObj *pVgroup, void *ahandle) {
} }
} }
static void mnodeProcessAlterVnodeRsp(SRpcMsg *rpcMsg) {
mDebug("alter vnode rsp received");
}
static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) { static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) {
if (rpcMsg->ahandle == NULL) return; if (rpcMsg->ahandle == NULL) return;
@ -724,6 +896,8 @@ static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) {
pVgroup->vgId, tstrerror(rpcMsg->code), mnodeMsg->received, mnodeMsg->successed, mnodeMsg->expected, pVgroup->vgId, tstrerror(rpcMsg->code), mnodeMsg->received, mnodeMsg->successed, mnodeMsg->expected,
mnodeMsg->rpcMsg.handle, rpcMsg->ahandle); mnodeMsg->rpcMsg.handle, rpcMsg->ahandle);
assert(mnodeMsg->received <= mnodeMsg->expected);
if (mnodeMsg->received != mnodeMsg->expected) return; if (mnodeMsg->received != mnodeMsg->expected) return;
if (mnodeMsg->received == mnodeMsg->successed) { if (mnodeMsg->received == mnodeMsg->successed) {
@ -857,9 +1031,10 @@ void mnodeDropAllDnodeVgroups(SDnodeObj *pDropDnode) {
sdbFreeIter(pIter); sdbFreeIter(pIter);
mInfo("dnode:%d, all vgroups is dropped from sdb", pDropDnode->dnodeId); mInfo("dnode:%d, all vgroups:%d is dropped from sdb", pDropDnode->dnodeId, numOfVgroups);
} }
#if 0
void mnodeUpdateAllDbVgroups(SDbObj *pAlterDb) { void mnodeUpdateAllDbVgroups(SDbObj *pAlterDb) {
void * pIter = NULL; void * pIter = NULL;
SVgObj *pVgroup = NULL; SVgObj *pVgroup = NULL;
@ -881,6 +1056,7 @@ void mnodeUpdateAllDbVgroups(SDbObj *pAlterDb) {
mInfo("db:%s, all vgroups is updated in sdb", pAlterDb->name); mInfo("db:%s, all vgroups is updated in sdb", pAlterDb->name);
} }
#endif
void mnodeDropAllDbVgroups(SDbObj *pDropDb) { void mnodeDropAllDbVgroups(SDbObj *pDropDb) {
void * pIter = NULL; void * pIter = NULL;

2
src/plugins/http/inc/httpCode.h
View File

@ -105,7 +105,7 @@
#define HTTP_OP_VALUE_TYPE 79 #define HTTP_OP_VALUE_TYPE 79
//tgf //tgf
#define HTTP_TG_STABLE_NOT_EXIST 80 #define HTTP_TG_STABLE_NOT_EXIST 80
extern char *httpMsg[]; extern char *httpMsg[];

3
src/plugins/http/inc/httpInt.h
View File

@ -61,6 +61,9 @@
#define HTTP_CHECK_BODY_CONTINUE 0 #define HTTP_CHECK_BODY_CONTINUE 0
#define HTTP_CHECK_BODY_SUCCESS 1 #define HTTP_CHECK_BODY_SUCCESS 1
#define HTTP_READ_DATA_SUCCESS 0
#define HTTP_READ_DATA_FAILED 1
#define HTTP_WRITE_RETRY_TIMES 500 #define HTTP_WRITE_RETRY_TIMES 500
#define HTTP_WRITE_WAIT_TIME_MS 5 #define HTTP_WRITE_WAIT_TIME_MS 5
#define HTTP_EXPIRED_TIME 60000 #define HTTP_EXPIRED_TIME 60000

10
src/plugins/http/inc/httpLog.h
View File

@ -22,10 +22,10 @@ extern int32_t httpDebugFlag;
#define httpFatal(...) { if (httpDebugFlag & DEBUG_FATAL) { taosPrintLog("HTP FATAL ", 255, __VA_ARGS__); }} #define httpFatal(...) { if (httpDebugFlag & DEBUG_FATAL) { taosPrintLog("HTP FATAL ", 255, __VA_ARGS__); }}
#define httpError(...) { if (httpDebugFlag & DEBUG_ERROR) { taosPrintLog("HTP ERROR ", 255, __VA_ARGS__); }} #define httpError(...) { if (httpDebugFlag & DEBUG_ERROR) { taosPrintLog("HTP ERROR ", 255, __VA_ARGS__); }}
#define httpWarn(...) { if (httpDebugFlag & DEBUG_WARN) { taosPrintLog("HTP WARN ", 255, __VA_ARGS__); }} #define httpWarn(...) { if (httpDebugFlag & DEBUG_WARN) { taosPrintLog("HTP WARN ", 255, __VA_ARGS__); }}
#define httpInfo(...) { if (httpDebugFlag & DEBUG_INFO) { taosPrintLog("HTP INFO ", 255, __VA_ARGS__); }} #define httpInfo(...) { if (httpDebugFlag & DEBUG_INFO) { taosPrintLog("HTP ", 255, __VA_ARGS__); }}
#define httpDebug(...) { if (httpDebugFlag & DEBUG_DEBUG) { taosPrintLog("HTP DEBUG ", httpDebugFlag, __VA_ARGS__); }} #define httpDebug(...) { if (httpDebugFlag & DEBUG_DEBUG) { taosPrintLog("HTP ", httpDebugFlag, __VA_ARGS__); }}
#define httpTrace(...) { if (httpDebugFlag & DEBUG_TRACE) { taosPrintLog("HTP TRACE ", httpDebugFlag, __VA_ARGS__); }} #define httpTrace(...) { if (httpDebugFlag & DEBUG_TRACE) { taosPrintLog("HTP ", httpDebugFlag, __VA_ARGS__); }}
#define httpTraceL(...){ if (httpDebugFlag & DEBUG_TRACE) { taosPrintLongString("HTP TRACE ", httpDebugFlag, __VA_ARGS__); }} #define httpTraceL(...){ if (httpDebugFlag & DEBUG_TRACE) { taosPrintLongString("HTP ", httpDebugFlag, __VA_ARGS__); }}
#endif #endif

2
src/plugins/http/inc/httpServer.h
View File

@ -23,6 +23,6 @@ void httpCleanUpConnect();
void *httpInitServer(char *ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle); void *httpInitServer(char *ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle);
void httpCleanUpServer(HttpServer *pServer); void httpCleanUpServer(HttpServer *pServer);
bool httpReadDataImp(HttpContext *pContext); int httpReadDataImp(HttpContext *pContext);
#endif #endif

2
src/plugins/http/src/httpContext.c
View File

@ -137,7 +137,7 @@ void httpReleaseContext(HttpContext *pContext) {
assert(refCount >= 0); assert(refCount >= 0);
HttpContext **ppContext = pContext->ppContext; HttpContext **ppContext = pContext->ppContext;
httpDebug("context:%p, is releasd, data:%p refCount:%d", pContext, ppContext, refCount); httpDebug("context:%p, is released, data:%p refCount:%d", pContext, ppContext, refCount);
if (tsHttpServer.contextCache != NULL) { if (tsHttpServer.contextCache != NULL) {
taosCacheRelease(tsHttpServer.contextCache, (void **)(&ppContext), false); taosCacheRelease(tsHttpServer.contextCache, (void **)(&ppContext), false);

25
src/plugins/http/src/httpHandle.c
View File

@ -60,6 +60,7 @@ bool httpParseURL(HttpContext* pContext) {
char* pSeek; char* pSeek;
char* pEnd = strchr(pParser->pLast, ' '); char* pEnd = strchr(pParser->pLast, ' ');
if (pEnd == NULL) { if (pEnd == NULL) {
httpSendErrorResp(pContext, HTTP_UNSUPPORT_URL);
return false; return false;
} }
@ -275,14 +276,14 @@ bool httpParseChunkedBody(HttpContext* pContext, HttpParser* pParser, bool test)
return true; return true;
} }
bool httpReadChunkedBody(HttpContext* pContext, HttpParser* pParser) { int httpReadChunkedBody(HttpContext* pContext, HttpParser* pParser) {
bool parsedOk = httpParseChunkedBody(pContext, pParser, true); bool parsedOk = httpParseChunkedBody(pContext, pParser, true);
if (parsedOk) { if (parsedOk) {
httpParseChunkedBody(pContext, pParser, false); httpParseChunkedBody(pContext, pParser, false);
return HTTP_CHECK_BODY_SUCCESS; return HTTP_CHECK_BODY_SUCCESS;
} else { } else {
httpTrace("context:%p, fd:%d, ip:%s, chunked body not finished, continue read", pContext, pContext->fd, pContext->ipstr); httpTrace("context:%p, fd:%d, ip:%s, chunked body not finished, continue read", pContext, pContext->fd, pContext->ipstr);
if (!httpReadDataImp(pContext)) { if (httpReadDataImp(pContext) != HTTP_READ_DATA_SUCCESS) {
httpError("context:%p, fd:%d, ip:%s, read chunked request error", pContext, pContext->fd, pContext->ipstr); httpError("context:%p, fd:%d, ip:%s, read chunked request error", pContext, pContext->fd, pContext->ipstr);
return HTTP_CHECK_BODY_ERROR; return HTTP_CHECK_BODY_ERROR;
} else { } else {
@ -296,7 +297,6 @@ int httpReadUnChunkedBody(HttpContext* pContext, HttpParser* pParser) {
if (dataReadLen > pParser->data.len) { if (dataReadLen > pParser->data.len) {
httpError("context:%p, fd:%d, ip:%s, un-chunked body length invalid, read size:%d dataReadLen:%d > pContext->data.len:%d", httpError("context:%p, fd:%d, ip:%s, un-chunked body length invalid, read size:%d dataReadLen:%d > pContext->data.len:%d",
pContext, pContext->fd, pContext->ipstr, pContext->parser.bufsize, dataReadLen, pParser->data.len); pContext, pContext->fd, pContext->ipstr, pContext->parser.bufsize, dataReadLen, pParser->data.len);
httpSendErrorResp(pContext, HTTP_PARSE_BODY_ERROR);
return HTTP_CHECK_BODY_ERROR; return HTTP_CHECK_BODY_ERROR;
} else if (dataReadLen < pParser->data.len) { } else if (dataReadLen < pParser->data.len) {
httpTrace("context:%p, fd:%d, ip:%s, un-chunked body not finished, read size:%d dataReadLen:%d < pContext->data.len:%d, continue read", httpTrace("context:%p, fd:%d, ip:%s, un-chunked body not finished, read size:%d dataReadLen:%d < pContext->data.len:%d, continue read",
@ -358,20 +358,13 @@ bool httpParseRequest(HttpContext* pContext) {
} }
int httpCheckReadCompleted(HttpContext* pContext) { int httpCheckReadCompleted(HttpContext* pContext) {
HttpParser *pParser = &pContext->parser; HttpParser* pParser = &pContext->parser;
if (pContext->httpChunked == HTTP_UNCUNKED) {
int ret = httpReadUnChunkedBody(pContext, pParser);
if (ret != HTTP_CHECK_BODY_SUCCESS) {
return ret;
}
} else {
int ret = httpReadChunkedBody(pContext, pParser);
if (ret != HTTP_CHECK_BODY_SUCCESS) {
return ret;
}
}
return HTTP_CHECK_BODY_SUCCESS; if (pContext->httpChunked == HTTP_UNCUNKED) {
return httpReadUnChunkedBody(pContext, pParser);
} else {
return httpReadChunkedBody(pContext, pParser);
}
} }
bool httpDecodeRequest(HttpContext* pContext) { bool httpDecodeRequest(HttpContext* pContext) {

21
src/plugins/http/src/httpServer.c
View File

@ -69,7 +69,7 @@ void httpCleanUpConnect() {
httpDebug("http server:%s is cleaned up", pServer->label); httpDebug("http server:%s is cleaned up", pServer->label);
} }
bool httpReadDataImp(HttpContext *pContext) { int httpReadDataImp(HttpContext *pContext) {
HttpParser *pParser = &pContext->parser; HttpParser *pParser = &pContext->parser;
while (pParser->bufsize <= (HTTP_BUFFER_SIZE - HTTP_STEP_SIZE)) { while (pParser->bufsize <= (HTTP_BUFFER_SIZE - HTTP_STEP_SIZE)) {
@ -85,8 +85,7 @@ bool httpReadDataImp(HttpContext *pContext) {
} else { } else {
httpError("context:%p, fd:%d, ip:%s, read from socket error:%d, close connect", httpError("context:%p, fd:%d, ip:%s, read from socket error:%d, close connect",
pContext, pContext->fd, pContext->ipstr, errno); pContext, pContext->fd, pContext->ipstr, errno);
httpReleaseContext(pContext); return HTTP_READ_DATA_FAILED;
return false;
} }
} else { } else {
pParser->bufsize += nread; pParser->bufsize += nread;
@ -95,15 +94,13 @@ bool httpReadDataImp(HttpContext *pContext) {
if (pParser->bufsize >= (HTTP_BUFFER_SIZE - HTTP_STEP_SIZE)) { if (pParser->bufsize >= (HTTP_BUFFER_SIZE - HTTP_STEP_SIZE)) {
httpError("context:%p, fd:%d, ip:%s, thread:%s, request big than:%d", httpError("context:%p, fd:%d, ip:%s, thread:%s, request big than:%d",
pContext, pContext->fd, pContext->ipstr, pContext->pThread->label, HTTP_BUFFER_SIZE); pContext, pContext->fd, pContext->ipstr, pContext->pThread->label, HTTP_BUFFER_SIZE);
httpSendErrorResp(pContext, HTTP_REQUSET_TOO_BIG); return HTTP_REQUSET_TOO_BIG;
httpNotifyContextClose(pContext);
return false;
} }
} }
pParser->buffer[pParser->bufsize] = 0; pParser->buffer[pParser->bufsize] = 0;
return true; return HTTP_READ_DATA_SUCCESS;
} }
static bool httpDecompressData(HttpContext *pContext) { static bool httpDecompressData(HttpContext *pContext) {
@ -141,8 +138,14 @@ static bool httpReadData(HttpContext *pContext) {
httpInitContext(pContext); httpInitContext(pContext);
} }
if (!httpReadDataImp(pContext)) { int32_t code = httpReadDataImp(pContext);
httpNotifyContextClose(pContext); if (code != HTTP_READ_DATA_SUCCESS) {
if (code == HTTP_READ_DATA_FAILED) {
httpReleaseContext(pContext);
} else {
httpSendErrorResp(pContext, code);
httpNotifyContextClose(pContext);
}
return false; return false;
} }

18
src/plugins/http/src/httpSql.c
View File

@ -47,6 +47,10 @@ void httpProcessMultiSqlRetrieveCallBack(void *param, TAOS_RES *result, int numO
} }
} }
// if (tscResultsetFetchCompleted(result)) {
// isContinue = false;
// }
if (isContinue) { if (isContinue) {
// retrieve next batch of rows // retrieve next batch of rows
httpDebug("context:%p, fd:%d, ip:%s, user:%s, process pos:%d, continue retrieve, numOfRows:%d, sql:%s", httpDebug("context:%p, fd:%d, ip:%s, user:%s, process pos:%d, continue retrieve, numOfRows:%d, sql:%s",
@ -75,7 +79,8 @@ void httpProcessMultiSqlCallBack(void *param, TAOS_RES *result, int code) {
HttpContext *pContext = (HttpContext *)param; HttpContext *pContext = (HttpContext *)param;
if (pContext == NULL) return; if (pContext == NULL) return;
HttpSqlCmds * multiCmds = pContext->multiCmds; code = taos_errno(result);
HttpSqlCmds *multiCmds = pContext->multiCmds;
HttpEncodeMethod *encode = pContext->encodeMethod; HttpEncodeMethod *encode = pContext->encodeMethod;
HttpSqlCmd *singleCmd = multiCmds->cmds + multiCmds->pos; HttpSqlCmd *singleCmd = multiCmds->cmds + multiCmds->pos;
@ -109,8 +114,8 @@ void httpProcessMultiSqlCallBack(void *param, TAOS_RES *result, int code) {
return; return;
} }
int num_fields = taos_field_count(result); bool isUpdate = tscIsUpdateQuery(result);
if (num_fields == 0) { if (isUpdate) {
// not select or show commands // not select or show commands
int affectRows = taos_affected_rows(result); int affectRows = taos_affected_rows(result);
httpDebug("context:%p, fd:%d, ip:%s, user:%s, process pos:%d, affect rows:%d, sql:%s", httpDebug("context:%p, fd:%d, ip:%s, user:%s, process pos:%d, affect rows:%d, sql:%s",
@ -221,9 +226,9 @@ void httpProcessSingleSqlRetrieveCallBack(void *param, TAOS_RES *result, int num
if (numOfRows < 0) { if (numOfRows < 0) {
httpError("context:%p, fd:%d, ip:%s, user:%s, retrieve failed, code:%s", pContext, pContext->fd, pContext->ipstr, httpError("context:%p, fd:%d, ip:%s, user:%s, retrieve failed, code:%s", pContext, pContext->fd, pContext->ipstr,
pContext->user, tstrerror(numOfRows)); pContext->user, tstrerror(numOfRows));
} }
taos_free_result(result); taos_free_result(result);
if (encode->stopJsonFp) { if (encode->stopJsonFp) {
(encode->stopJsonFp)(pContext, &pContext->singleCmd); (encode->stopJsonFp)(pContext, &pContext->singleCmd);
@ -238,6 +243,7 @@ void httpProcessSingleSqlCallBack(void *param, TAOS_RES *result, int unUsedCode)
if (pContext == NULL) return; if (pContext == NULL) return;
int32_t code = taos_errno(result); int32_t code = taos_errno(result);
HttpEncodeMethod *encode = pContext->encodeMethod; HttpEncodeMethod *encode = pContext->encodeMethod;
if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {

8
src/plugins/monitor/src/monitorMain.c
View File

@ -30,10 +30,10 @@
#define monitorFatal(...) { if (monitorDebugFlag & DEBUG_FATAL) { taosPrintLog("MON FATAL ", 255, __VA_ARGS__); }} #define monitorFatal(...) { if (monitorDebugFlag & DEBUG_FATAL) { taosPrintLog("MON FATAL ", 255, __VA_ARGS__); }}
#define monitorError(...) { if (monitorDebugFlag & DEBUG_ERROR) { taosPrintLog("MON ERROR ", 255, __VA_ARGS__); }} #define monitorError(...) { if (monitorDebugFlag & DEBUG_ERROR) { taosPrintLog("MON ERROR ", 255, __VA_ARGS__); }}
#define monitorWarn(...) { if (monitorDebugFlag & DEBUG_WARN) { taosPrintLog("MON WARN ", 255, __VA_ARGS__); }} #define monitorWarn(...) { if (monitorDebugFlag & DEBUG_WARN) { taosPrintLog("MON WARN ", 255, __VA_ARGS__); }}
#define monitorInfo(...) { if (monitorDebugFlag & DEBUG_INFO) { taosPrintLog("MON INFO ", 255, __VA_ARGS__); }} #define monitorInfo(...) { if (monitorDebugFlag & DEBUG_INFO) { taosPrintLog("MON ", 255, __VA_ARGS__); }}
#define monitorDebug(...) { if (monitorDebugFlag & DEBUG_DEBUG) { taosPrintLog("MON DEBUG ", monitorDebugFlag, __VA_ARGS__); }} #define monitorDebug(...) { if (monitorDebugFlag & DEBUG_DEBUG) { taosPrintLog("MON ", monitorDebugFlag, __VA_ARGS__); }}
#define monitorTrace(...) { if (monitorDebugFlag & DEBUG_TRACE) { taosPrintLog("MON TRACE ", monitorDebugFlag, __VA_ARGS__); }} #define monitorTrace(...) { if (monitorDebugFlag & DEBUG_TRACE) { taosPrintLog("MON ", monitorDebugFlag, __VA_ARGS__); }}
#define SQL_LENGTH 1024 #define SQL_LENGTH 1024
#define LOG_LEN_STR 100 #define LOG_LEN_STR 100

8
src/plugins/mqtt/inc/mqttLog.h
View File

@ -22,9 +22,9 @@ extern int32_t mqttDebugFlag;
#define mqttFatal(...) { if (mqttDebugFlag & DEBUG_FATAL) { taosPrintLog("MQT FATAL ", 255, __VA_ARGS__); }} #define mqttFatal(...) { if (mqttDebugFlag & DEBUG_FATAL) { taosPrintLog("MQT FATAL ", 255, __VA_ARGS__); }}
#define mqttError(...) { if (mqttDebugFlag & DEBUG_ERROR) { taosPrintLog("MQT ERROR ", 255, __VA_ARGS__); }} #define mqttError(...) { if (mqttDebugFlag & DEBUG_ERROR) { taosPrintLog("MQT ERROR ", 255, __VA_ARGS__); }}
#define mqttWarn(...) { if (mqttDebugFlag & DEBUG_WARN) { taosPrintLog("MQT WARN ", 255, __VA_ARGS__); }} #define mqttWarn(...) { if (mqttDebugFlag & DEBUG_WARN) { taosPrintLog("MQT WARN ", 255, __VA_ARGS__); }}
#define mqttInfo(...) { if (mqttDebugFlag & DEBUG_INFO) { taosPrintLog("MQT INFO ", 255, __VA_ARGS__); }} #define mqttInfo(...) { if (mqttDebugFlag & DEBUG_INFO) { taosPrintLog("MQT ", 255, __VA_ARGS__); }}
#define mqttDebug(...) { if (mqttDebugFlag & DEBUG_DEBUG) { taosPrintLog("MQT DEBUG ", mqttDebugFlag, __VA_ARGS__); }} #define mqttDebug(...) { if (mqttDebugFlag & DEBUG_DEBUG) { taosPrintLog("MQT ", mqttDebugFlag, __VA_ARGS__); }}
#define mqttTrace(...) { if (mqttDebugFlag & DEBUG_TRACE) { taosPrintLog("MQT TRACE ", mqttDebugFlag, __VA_ARGS__); }} #define mqttTrace(...) { if (mqttDebugFlag & DEBUG_TRACE) { taosPrintLog("MQT ", mqttDebugFlag, __VA_ARGS__); }}
#endif #endif

6
src/query/inc/qExecutor.h
View File

@ -24,7 +24,7 @@
#include "qtsbuf.h" #include "qtsbuf.h"
#include "taosdef.h" #include "taosdef.h"
#include "tarray.h" #include "tarray.h"
#include "tref.h" #include "tlockfree.h"
#include "tsdb.h" #include "tsdb.h"
#include "tsqlfunction.h" #include "tsqlfunction.h"
#include "query.h" #include "query.h"
@ -121,6 +121,7 @@ typedef struct SQueryCostInfo {
uint32_t loadBlockStatis; uint32_t loadBlockStatis;
uint32_t discardBlocks; uint32_t discardBlocks;
uint64_t elapsedTime; uint64_t elapsedTime;
uint64_t ioTime;
uint64_t computTime; uint64_t computTime;
} SQueryCostInfo; } SQueryCostInfo;
@ -192,7 +193,6 @@ typedef struct SQInfo {
int32_t offset; // offset in group result set of subgroup, todo refactor int32_t offset; // offset in group result set of subgroup, todo refactor
SArray* arrTableIdInfo; SArray* arrTableIdInfo;
T_REF_DECLARE()
/* /*
* the query is executed position on which meter of the whole list. * the query is executed position on which meter of the whole list.
* when the index reaches the last one of the list, it means the query is completed. * when the index reaches the last one of the list, it means the query is completed.
@ -201,8 +201,6 @@ typedef struct SQInfo {
*/ */
int32_t tableIndex; int32_t tableIndex;
int32_t numOfGroupResultPages; int32_t numOfGroupResultPages;
_qinfo_free_fn_t freeFn; //todo remove it
void* pBuf; // allocated buffer for STableQueryInfo, sizeof(STableQueryInfo)*numOfTables; void* pBuf; // allocated buffer for STableQueryInfo, sizeof(STableQueryInfo)*numOfTables;
} SQInfo; } SQInfo;

2
src/query/inc/qUtil.h
View File

@ -37,7 +37,7 @@ SWindowResult *getWindowResult(SWindowResInfo *pWindowResInfo, int32_t slot);
#define curTimeWindow(_winres) ((_winres)->curIndex) #define curTimeWindow(_winres) ((_winres)->curIndex)
bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot); bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot);
void createQueryResultInfo(SQuery *pQuery, SWindowResult *pResultRow, bool isSTableQuery, SPosInfo *posInfo, size_t interBufSize); int32_t createQueryResultInfo(SQuery *pQuery, SWindowResult *pResultRow, bool isSTableQuery, SPosInfo *posInfo, size_t interBufSize);
char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int32_t columnIndex, SWindowResult *pResult); char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int32_t columnIndex, SWindowResult *pResult);

1
src/query/inc/qsqlparser.h
View File

@ -116,6 +116,7 @@ typedef struct SCreateDBInfo {
int32_t daysPerFile; int32_t daysPerFile;
int32_t minRowsPerBlock; int32_t minRowsPerBlock;
int32_t maxRowsPerBlock; int32_t maxRowsPerBlock;
int32_t fsyncPeriod;
int64_t commitTime; int64_t commitTime;
int32_t walLevel; int32_t walLevel;
int32_t compressionLevel; int32_t compressionLevel;

2
src/query/inc/qtsbuf.h
View File

@ -107,7 +107,7 @@ STSBuf* tsBufCreate(bool autoDelete, int32_t order);
STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete); STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete);
STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_t len, int32_t tsOrder); STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_t len, int32_t tsOrder);
void* tsBufDestory(STSBuf* pTSBuf); void* tsBufDestroy(STSBuf* pTSBuf);
void tsBufAppend(STSBuf* pTSBuf, int32_t vnodeId, int64_t tag, const char* pData, int32_t len); void tsBufAppend(STSBuf* pTSBuf, int32_t vnodeId, int64_t tag, const char* pData, int32_t len);
int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf, int32_t vnodeIdx); int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf, int32_t vnodeIdx);

8
src/query/inc/queryLog.h
View File

@ -27,10 +27,10 @@ extern int32_t tscEmbedded;
#define qFatal(...) { if (qDebugFlag & DEBUG_FATAL) { taosPrintLog("QRY FATAL ", 255, __VA_ARGS__); }} #define qFatal(...) { if (qDebugFlag & DEBUG_FATAL) { taosPrintLog("QRY FATAL ", 255, __VA_ARGS__); }}
#define qError(...) { if (qDebugFlag & DEBUG_ERROR) { taosPrintLog("QRY ERROR ", 255, __VA_ARGS__); }} #define qError(...) { if (qDebugFlag & DEBUG_ERROR) { taosPrintLog("QRY ERROR ", 255, __VA_ARGS__); }}
#define qWarn(...) { if (qDebugFlag & DEBUG_WARN) { taosPrintLog("QRY WARN ", 255, __VA_ARGS__); }} #define qWarn(...) { if (qDebugFlag & DEBUG_WARN) { taosPrintLog("QRY WARN ", 255, __VA_ARGS__); }}
#define qInfo(...) { if (qDebugFlag & DEBUG_INFO) { taosPrintLog("QRY INFO ", 255, __VA_ARGS__); }} #define qInfo(...) { if (qDebugFlag & DEBUG_INFO) { taosPrintLog("QRY ", 255, __VA_ARGS__); }}
#define qDebug(...) { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLog("QRY DEBUG ", qDebugFlag, __VA_ARGS__); }} #define qDebug(...) { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }}
#define qTrace(...) { if (qDebugFlag & DEBUG_TRACE) { taosPrintLog("QRY TRACE ", qDebugFlag, __VA_ARGS__); }} #define qTrace(...) { if (qDebugFlag & DEBUG_TRACE) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }}
#ifdef __cplusplus #ifdef __cplusplus
} }

3
src/query/inc/sql.y
View File

@ -221,6 +221,7 @@ maxrows(Y) ::= MAXROWS INTEGER(X). { Y = X; }
blocks(Y) ::= BLOCKS INTEGER(X). { Y = X; } blocks(Y) ::= BLOCKS INTEGER(X). { Y = X; }
ctime(Y) ::= CTIME INTEGER(X). { Y = X; } ctime(Y) ::= CTIME INTEGER(X). { Y = X; }
wal(Y) ::= WAL INTEGER(X). { Y = X; } wal(Y) ::= WAL INTEGER(X). { Y = X; }
fsync(Y) ::= FSYNC INTEGER(X). { Y = X; }
comp(Y) ::= COMP INTEGER(X). { Y = X; } comp(Y) ::= COMP INTEGER(X). { Y = X; }
prec(Y) ::= PRECISION STRING(X). { Y = X; } prec(Y) ::= PRECISION STRING(X). { Y = X; }
@ -236,6 +237,7 @@ db_optr(Y) ::= db_optr(Z) maxrows(X). { Y = Z; Y.maxRowsPerBlock = strtod
db_optr(Y) ::= db_optr(Z) blocks(X). { Y = Z; Y.numOfBlocks = strtol(X.z, NULL, 10); } db_optr(Y) ::= db_optr(Z) blocks(X). { Y = Z; Y.numOfBlocks = strtol(X.z, NULL, 10); }
db_optr(Y) ::= db_optr(Z) ctime(X). { Y = Z; Y.commitTime = strtol(X.z, NULL, 10); } db_optr(Y) ::= db_optr(Z) ctime(X). { Y = Z; Y.commitTime = strtol(X.z, NULL, 10); }
db_optr(Y) ::= db_optr(Z) wal(X). { Y = Z; Y.walLevel = strtol(X.z, NULL, 10); } db_optr(Y) ::= db_optr(Z) wal(X). { Y = Z; Y.walLevel = strtol(X.z, NULL, 10); }
db_optr(Y) ::= db_optr(Z) fsync(X). { Y = Z; Y.fsyncPeriod = strtol(X.z, NULL, 10); }
db_optr(Y) ::= db_optr(Z) comp(X). { Y = Z; Y.compressionLevel = strtol(X.z, NULL, 10); } db_optr(Y) ::= db_optr(Z) comp(X). { Y = Z; Y.compressionLevel = strtol(X.z, NULL, 10); }
db_optr(Y) ::= db_optr(Z) prec(X). { Y = Z; Y.precision = X; } db_optr(Y) ::= db_optr(Z) prec(X). { Y = Z; Y.precision = X; }
db_optr(Y) ::= db_optr(Z) keep(X). { Y = Z; Y.keep = X; } db_optr(Y) ::= db_optr(Z) keep(X). { Y = Z; Y.keep = X; }
@ -249,6 +251,7 @@ alter_db_optr(Y) ::= alter_db_optr(Z) keep(X). { Y = Z; Y.keep = X; }
alter_db_optr(Y) ::= alter_db_optr(Z) blocks(X). { Y = Z; Y.numOfBlocks = strtol(X.z, NULL, 10); } alter_db_optr(Y) ::= alter_db_optr(Z) blocks(X). { Y = Z; Y.numOfBlocks = strtol(X.z, NULL, 10); }
alter_db_optr(Y) ::= alter_db_optr(Z) comp(X). { Y = Z; Y.compressionLevel = strtol(X.z, NULL, 10); } alter_db_optr(Y) ::= alter_db_optr(Z) comp(X). { Y = Z; Y.compressionLevel = strtol(X.z, NULL, 10); }
alter_db_optr(Y) ::= alter_db_optr(Z) wal(X). { Y = Z; Y.walLevel = strtol(X.z, NULL, 10); } alter_db_optr(Y) ::= alter_db_optr(Z) wal(X). { Y = Z; Y.walLevel = strtol(X.z, NULL, 10); }
alter_db_optr(Y) ::= alter_db_optr(Z) fsync(X). { Y = Z; Y.fsyncPeriod = strtod(X.z, NULL, 10); }
%type typename {TAOS_FIELD} %type typename {TAOS_FIELD}
typename(A) ::= ids(X). { typename(A) ::= ids(X). {

29
src/query/inc/tsqlfunction.h
View File

@ -125,7 +125,8 @@ typedef struct SArithmeticSupport {
} SArithmeticSupport; } SArithmeticSupport;
typedef struct SQLPreAggVal { typedef struct SQLPreAggVal {
bool isSet; bool isSet; // statistics info set or not
bool dataBlockLoaded; // data block is loaded or not
SDataStatis statis; SDataStatis statis;
} SQLPreAggVal; } SQLPreAggVal;
@ -187,7 +188,7 @@ typedef struct SQLFunctionCtx {
} SQLFunctionCtx; } SQLFunctionCtx;
typedef struct SQLAggFuncElem { typedef struct SQLAggFuncElem {
char aName[TSDB_FUNCTIONS_NAME_MAX_LENGTH]; char aName[TSDB_FUNCTIONS_NAME_MAX_LENGTH];
uint8_t nAggIdx; // index of function in aAggs uint8_t nAggIdx; // index of function in aAggs
int8_t stableFuncId; // transfer function for super table query int8_t stableFuncId; // transfer function for super table query
@ -224,25 +225,14 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI
#define IS_SINGLEOUTPUT(x) (((x)&TSDB_FUNCSTATE_SO) != 0) #define IS_SINGLEOUTPUT(x) (((x)&TSDB_FUNCSTATE_SO) != 0)
#define IS_OUTER_FORWARD(x) (((x)&TSDB_FUNCSTATE_OF) != 0) #define IS_OUTER_FORWARD(x) (((x)&TSDB_FUNCSTATE_OF) != 0)
/*
* the status of one block, used in metric query. all blocks are mixed together,
* we need the status to decide if one block is a first/end/inter block of one meter
*/
enum {
BLK_FILE_BLOCK = 0x1,
BLK_BLOCK_LOADED = 0x2,
BLK_CACHE_BLOCK = 0x4, // in case of cache block, block must be loaded
};
/* determine the real data need to calculated the result */ /* determine the real data need to calculated the result */
enum { enum {
BLK_DATA_NO_NEEDED = 0x0, BLK_DATA_NO_NEEDED = 0x0,
BLK_DATA_STATIS_NEEDED = 0x1, BLK_DATA_STATIS_NEEDED = 0x1,
BLK_DATA_ALL_NEEDED = 0x3, BLK_DATA_ALL_NEEDED = 0x3,
BLK_DATA_DISCARD = 0x4, // discard current data block since it is not qualified for filter
}; };
#define SET_DATA_BLOCK_NOT_LOADED(x) ((x) &= (~BLK_BLOCK_LOADED));
typedef struct STwaInfo { typedef struct STwaInfo {
TSKEY lastKey; TSKEY lastKey;
int8_t hasResult; // flag to denote has value int8_t hasResult; // flag to denote has value
@ -264,12 +254,9 @@ typedef struct STwaInfo {
/* global sql function array */ /* global sql function array */
extern struct SQLAggFuncElem aAggs[]; extern struct SQLAggFuncElem aAggs[];
/* compatible check array list */ extern int32_t functionCompatList[]; // compatible check array list
extern int32_t funcCompatDefList[];
bool top_bot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, char *minval, char *maxval); bool topbot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, const char *minval, const char *maxval);
bool stableQueryFunctChanged(int32_t funcId);
void resetResultInfo(SResultInfo *pResInfo); void resetResultInfo(SResultInfo *pResInfo);
void setResultInfoBuf(SResultInfo *pResInfo, int32_t size, bool superTable, char* buf); void setResultInfoBuf(SResultInfo *pResInfo, int32_t size, bool superTable, char* buf);

642
src/query/src/qExecutor.c
View File

File diff suppressed because it is too large Load Diff

16
src/query/src/qUtil.c
View File

@ -50,9 +50,15 @@ int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, SQueryRuntimeEnv *pRun
// use the pointer arraylist // use the pointer arraylist
pWindowResInfo->pResult = calloc(threshold, sizeof(SWindowResult)); pWindowResInfo->pResult = calloc(threshold, sizeof(SWindowResult));
if (pWindowResInfo->pResult == NULL) {
return TSDB_CODE_QRY_OUT_OF_MEMORY;
}
for (int32_t i = 0; i < pWindowResInfo->capacity; ++i) { for (int32_t i = 0; i < pWindowResInfo->capacity; ++i) {
SPosInfo posInfo = {-1, -1}; SPosInfo posInfo = {-1, -1};
createQueryResultInfo(pRuntimeEnv->pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, &posInfo, pRuntimeEnv->interBufSize); int32_t code = createQueryResultInfo(pRuntimeEnv->pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, &posInfo, pRuntimeEnv->interBufSize);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -76,9 +82,11 @@ void cleanupTimeWindowInfo(SWindowResInfo *pWindowResInfo, int32_t numOfCols) {
return; return;
} }
for (int32_t i = 0; i < pWindowResInfo->capacity; ++i) { if (pWindowResInfo->pResult != NULL) {
SWindowResult *pResult = &pWindowResInfo->pResult[i]; for (int32_t i = 0; i < pWindowResInfo->capacity; ++i) {
destroyTimeWindowRes(pResult, numOfCols); SWindowResult *pResult = &pWindowResInfo->pResult[i];
destroyTimeWindowRes(pResult, numOfCols);
}
} }
taosHashCleanup(pWindowResInfo->hashList); taosHashCleanup(pWindowResInfo->hashList);

6
src/query/src/qast.c
View File

@ -962,10 +962,13 @@ static UNUSED_FUNC char* exception_strdup(const char* str) {
static tExprNode* exprTreeFromBinaryImpl(SBufferReader* br) { static tExprNode* exprTreeFromBinaryImpl(SBufferReader* br) {
int32_t anchor = CLEANUP_GET_ANCHOR(); int32_t anchor = CLEANUP_GET_ANCHOR();
if (CLEANUP_EXCEED_LIMIT()) {
THROW(TSDB_CODE_QRY_EXCEED_TAGS_LIMIT);
return NULL;
}
tExprNode* pExpr = exception_calloc(1, sizeof(tExprNode)); tExprNode* pExpr = exception_calloc(1, sizeof(tExprNode));
CLEANUP_PUSH_VOID_PTR_PTR(true, tExprNodeDestroy, pExpr, NULL); CLEANUP_PUSH_VOID_PTR_PTR(true, tExprNodeDestroy, pExpr, NULL);
pExpr->nodeType = tbufReadUint8(br); pExpr->nodeType = tbufReadUint8(br);
if (pExpr->nodeType == TSQL_NODE_VALUE) { if (pExpr->nodeType == TSQL_NODE_VALUE) {
@ -995,7 +998,6 @@ static tExprNode* exprTreeFromBinaryImpl(SBufferReader* br) {
pExpr->_node.hasPK = tbufReadUint8(br); pExpr->_node.hasPK = tbufReadUint8(br);
pExpr->_node.pLeft = exprTreeFromBinaryImpl(br); pExpr->_node.pLeft = exprTreeFromBinaryImpl(br);
pExpr->_node.pRight = exprTreeFromBinaryImpl(br); pExpr->_node.pRight = exprTreeFromBinaryImpl(br);
assert(pExpr->_node.pLeft != NULL && pExpr->_node.pRight != NULL); assert(pExpr->_node.pLeft != NULL && pExpr->_node.pRight != NULL);
} }

1
src/query/src/qparserImpl.c
View File

@ -896,6 +896,7 @@ void setDefaultCreateDbOption(SCreateDBInfo *pDBInfo) {
pDBInfo->compressionLevel = -1; pDBInfo->compressionLevel = -1;
pDBInfo->walLevel = -1; pDBInfo->walLevel = -1;
pDBInfo->fsyncPeriod = -1;
pDBInfo->commitTime = -1; pDBInfo->commitTime = -1;
pDBInfo->maxTablesPerVnode = -1; pDBInfo->maxTablesPerVnode = -1;

10
src/query/src/qtokenizer.c
View File

@ -124,6 +124,7 @@ static SKeyword keywordTable[] = {
{"CACHE", TK_CACHE}, {"CACHE", TK_CACHE},
{"CTIME", TK_CTIME}, {"CTIME", TK_CTIME},
{"WAL", TK_WAL}, {"WAL", TK_WAL},
{"FSYNC", TK_FSYNC},
{"COMP", TK_COMP}, {"COMP", TK_COMP},
{"PRECISION", TK_PRECISION}, {"PRECISION", TK_PRECISION},
{"LP", TK_LP}, {"LP", TK_LP},
@ -509,10 +510,11 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenType) {
for (i = 1; isdigit(z[i]); i++) { for (i = 1; isdigit(z[i]); i++) {
} }
/* here is the 1a/2s/3m/9y */ /* here is the 1u/1a/2s/3m/9y */
if ((z[i] == 'a' || z[i] == 's' || z[i] == 'm' || z[i] == 'h' || z[i] == 'd' || z[i] == 'n' || z[i] == 'y' || if ((z[i] == 'u' || z[i] == 'a' || z[i] == 's' || z[i] == 'm' || z[i] == 'h' || z[i] == 'd' || z[i] == 'n' ||
z[i] == 'w' || z[i] == 'A' || z[i] == 'S' || z[i] == 'M' || z[i] == 'H' || z[i] == 'D' || z[i] == 'N' || z[i] == 'y' || z[i] == 'w' ||
z[i] == 'Y' || z[i] == 'W') && z[i] == 'U' || z[i] == 'A' || z[i] == 'S' || z[i] == 'M' || z[i] == 'H' || z[i] == 'D' || z[i] == 'N' ||
z[i] == 'Y' || z[i] == 'W') &&
(isIdChar[(uint8_t)z[i + 1]] == 0)) { (isIdChar[(uint8_t)z[i + 1]] == 0)) {
*tokenType = TK_VARIABLE; *tokenType = TK_VARIABLE;
i += 1; i += 1;

18
src/query/src/qtsbuf.c
View File

@ -79,7 +79,7 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) {
pTSBuf->numOfAlloc = header.numOfVnode; pTSBuf->numOfAlloc = header.numOfVnode;
STSVnodeBlockInfoEx* tmp = realloc(pTSBuf->pData, sizeof(STSVnodeBlockInfoEx) * pTSBuf->numOfAlloc); STSVnodeBlockInfoEx* tmp = realloc(pTSBuf->pData, sizeof(STSVnodeBlockInfoEx) * pTSBuf->numOfAlloc);
if (tmp == NULL) { if (tmp == NULL) {
tsBufDestory(pTSBuf); tsBufDestroy(pTSBuf);
return NULL; return NULL;
} }
@ -92,7 +92,7 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) {
pTSBuf->tsOrder = header.tsOrder; pTSBuf->tsOrder = header.tsOrder;
if (pTSBuf->tsOrder != TSDB_ORDER_ASC && pTSBuf->tsOrder != TSDB_ORDER_DESC) { if (pTSBuf->tsOrder != TSDB_ORDER_ASC && pTSBuf->tsOrder != TSDB_ORDER_DESC) {
// tscError("invalid order info in buf:%d", pTSBuf->tsOrder); // tscError("invalid order info in buf:%d", pTSBuf->tsOrder);
tsBufDestory(pTSBuf); tsBufDestroy(pTSBuf);
return NULL; return NULL;
} }
@ -100,7 +100,7 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) {
STSVnodeBlockInfo* buf = (STSVnodeBlockInfo*)calloc(1, infoSize); STSVnodeBlockInfo* buf = (STSVnodeBlockInfo*)calloc(1, infoSize);
if (buf == NULL) { if (buf == NULL) {
tsBufDestory(pTSBuf); tsBufDestroy(pTSBuf);
return NULL; return NULL;
} }
@ -120,7 +120,7 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) {
struct stat fileStat; struct stat fileStat;
if (fstat(fileno(pTSBuf->f), &fileStat) != 0) { if (fstat(fileno(pTSBuf->f), &fileStat) != 0) {
tsBufDestory(pTSBuf); tsBufDestroy(pTSBuf);
return NULL; return NULL;
} }
@ -137,7 +137,7 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) {
return pTSBuf; return pTSBuf;
} }
void* tsBufDestory(STSBuf* pTSBuf) { void* tsBufDestroy(STSBuf* pTSBuf) {
if (pTSBuf == NULL) { if (pTSBuf == NULL) {
return NULL; return NULL;
} }
@ -920,13 +920,13 @@ static STSBuf* allocResForTSBuf(STSBuf* pTSBuf) {
pTSBuf->numOfAlloc = INITIAL_VNODEINFO_SIZE; pTSBuf->numOfAlloc = INITIAL_VNODEINFO_SIZE;
pTSBuf->pData = calloc(pTSBuf->numOfAlloc, sizeof(STSVnodeBlockInfoEx)); pTSBuf->pData = calloc(pTSBuf->numOfAlloc, sizeof(STSVnodeBlockInfoEx));
if (pTSBuf->pData == NULL) { if (pTSBuf->pData == NULL) {
tsBufDestory(pTSBuf); tsBufDestroy(pTSBuf);
return NULL; return NULL;
} }
pTSBuf->tsData.rawBuf = malloc(MEM_BUF_SIZE); pTSBuf->tsData.rawBuf = malloc(MEM_BUF_SIZE);
if (pTSBuf->tsData.rawBuf == NULL) { if (pTSBuf->tsData.rawBuf == NULL) {
tsBufDestory(pTSBuf); tsBufDestroy(pTSBuf);
return NULL; return NULL;
} }
@ -936,13 +936,13 @@ static STSBuf* allocResForTSBuf(STSBuf* pTSBuf) {
pTSBuf->assistBuf = malloc(MEM_BUF_SIZE); pTSBuf->assistBuf = malloc(MEM_BUF_SIZE);
if (pTSBuf->assistBuf == NULL) { if (pTSBuf->assistBuf == NULL) {
tsBufDestory(pTSBuf); tsBufDestroy(pTSBuf);
return NULL; return NULL;
} }
pTSBuf->block.payload = malloc(MEM_BUF_SIZE); pTSBuf->block.payload = malloc(MEM_BUF_SIZE);
if (pTSBuf->block.payload == NULL) { if (pTSBuf->block.payload == NULL) {
tsBufDestory(pTSBuf); tsBufDestroy(pTSBuf);
return NULL; return NULL;
} }

2253
src/query/src/sql.c
View File

File diff suppressed because it is too large Load Diff

16
src/query/tests/tsBufTest.cpp
View File

@ -47,7 +47,7 @@ void simpleTest() {
EXPECT_EQ(pTSBuf->tsData.len, 0); EXPECT_EQ(pTSBuf->tsData.len, 0);
EXPECT_EQ(pTSBuf->block.numOfElem, num); EXPECT_EQ(pTSBuf->block.numOfElem, num);
tsBufDestory(pTSBuf); tsBufDestroy(pTSBuf);
} }
// one large list of ts, the ts list need to be split into several small blocks // one large list of ts, the ts list need to be split into several small blocks
@ -71,7 +71,7 @@ void largeTSTest() {
EXPECT_EQ(pTSBuf->tsData.len, 0); EXPECT_EQ(pTSBuf->tsData.len, 0);
EXPECT_EQ(pTSBuf->block.numOfElem, num); EXPECT_EQ(pTSBuf->block.numOfElem, num);
tsBufDestory(pTSBuf); tsBufDestroy(pTSBuf);
} }
void multiTagsTest() { void multiTagsTest() {
@ -101,7 +101,7 @@ void multiTagsTest() {
EXPECT_EQ(pTSBuf->tsData.len, 0); EXPECT_EQ(pTSBuf->tsData.len, 0);
EXPECT_EQ(pTSBuf->block.numOfElem, num); EXPECT_EQ(pTSBuf->block.numOfElem, num);
tsBufDestory(pTSBuf); tsBufDestroy(pTSBuf);
} }
void multiVnodeTagsTest() { void multiVnodeTagsTest() {
@ -139,7 +139,7 @@ void multiVnodeTagsTest() {
EXPECT_EQ(pTSBuf->tsData.len, 0); EXPECT_EQ(pTSBuf->tsData.len, 0);
EXPECT_EQ(pTSBuf->block.numOfElem, num); EXPECT_EQ(pTSBuf->block.numOfElem, num);
tsBufDestory(pTSBuf); tsBufDestroy(pTSBuf);
} }
void loadDataTest() { void loadDataTest() {
@ -386,8 +386,8 @@ void mergeDiffVnodeBufferTest() {
tsBufDisplay(pTSBuf1); tsBufDisplay(pTSBuf1);
tsBufDestory(pTSBuf2); tsBufDestroy(pTSBuf2);
tsBufDestory(pTSBuf1); tsBufDestroy(pTSBuf1);
} }
void mergeIdenticalVnodeBufferTest() { void mergeIdenticalVnodeBufferTest() {
@ -432,8 +432,8 @@ void mergeIdenticalVnodeBufferTest() {
printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag, elem.ts); printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag, elem.ts);
} }
tsBufDestory(pTSBuf1); tsBufDestroy(pTSBuf1);
tsBufDestory(pTSBuf2); tsBufDestroy(pTSBuf2);
} }
} // namespace } // namespace

8
src/rpc/inc/rpcLog.h
View File

@ -27,10 +27,10 @@ extern int32_t tscEmbedded;
#define tFatal(...) { if (rpcDebugFlag & DEBUG_FATAL) { taosPrintLog("RPC FATAL ", tscEmbedded ? 255 : rpcDebugFlag, __VA_ARGS__); }} #define tFatal(...) { if (rpcDebugFlag & DEBUG_FATAL) { taosPrintLog("RPC FATAL ", tscEmbedded ? 255 : rpcDebugFlag, __VA_ARGS__); }}
#define tError(...) { if (rpcDebugFlag & DEBUG_ERROR) { taosPrintLog("RPC ERROR ", tscEmbedded ? 255 : rpcDebugFlag, __VA_ARGS__); }} #define tError(...) { if (rpcDebugFlag & DEBUG_ERROR) { taosPrintLog("RPC ERROR ", tscEmbedded ? 255 : rpcDebugFlag, __VA_ARGS__); }}
#define tWarn(...) { if (rpcDebugFlag & DEBUG_WARN) { taosPrintLog("RPC WARN ", tscEmbedded ? 255 : rpcDebugFlag, __VA_ARGS__); }} #define tWarn(...) { if (rpcDebugFlag & DEBUG_WARN) { taosPrintLog("RPC WARN ", tscEmbedded ? 255 : rpcDebugFlag, __VA_ARGS__); }}
#define tInfo(...) { if (rpcDebugFlag & DEBUG_INFO) { taosPrintLog("RPC INFO ", tscEmbedded ? 255 : rpcDebugFlag, __VA_ARGS__); }} #define tInfo(...) { if (rpcDebugFlag & DEBUG_INFO) { taosPrintLog("RPC ", tscEmbedded ? 255 : rpcDebugFlag, __VA_ARGS__); }}
#define tDebug(...) { if (rpcDebugFlag & DEBUG_DEBUG) { taosPrintLog("RPC DEBUG ", rpcDebugFlag, __VA_ARGS__); }} #define tDebug(...) { if (rpcDebugFlag & DEBUG_DEBUG) { taosPrintLog("RPC ", rpcDebugFlag, __VA_ARGS__); }}
#define tTrace(...) { if (rpcDebugFlag & DEBUG_TRACE) { taosPrintLog("RPC TRACE ", rpcDebugFlag, __VA_ARGS__); }} #define tTrace(...) { if (rpcDebugFlag & DEBUG_TRACE) { taosPrintLog("RPC ", rpcDebugFlag, __VA_ARGS__); }}
#define tDump(x, y) { if (rpcDebugFlag & DEBUG_DUMP) { taosDumpData((unsigned char *)x, y); }} #define tDump(x, y) { if (rpcDebugFlag & DEBUG_DUMP) { taosDumpData((unsigned char *)x, y); }}
#ifdef __cplusplus #ifdef __cplusplus

85
src/tsdb/inc/tsdbMain.h
View File

@ -21,11 +21,10 @@
#include "tkvstore.h" #include "tkvstore.h"
#include "tlist.h" #include "tlist.h"
#include "tlog.h" #include "tlog.h"
#include "tref.h" #include "tlockfree.h"
#include "tsdb.h" #include "tsdb.h"
#include "tskiplist.h" #include "tskiplist.h"
#include "tutil.h" #include "tutil.h"
#include "trwlatch.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
@ -35,10 +34,10 @@ extern int tsdbDebugFlag;
#define tsdbFatal(...) { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TDB FATAL ", 255, __VA_ARGS__); }} #define tsdbFatal(...) { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TDB FATAL ", 255, __VA_ARGS__); }}
#define tsdbError(...) { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TDB ERROR ", 255, __VA_ARGS__); }} #define tsdbError(...) { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TDB ERROR ", 255, __VA_ARGS__); }}
#define tsdbWarn(...) { if (tsdbDebugFlag & DEBUG_WARN) { taosPrintLog("TDB WARN ", 255, __VA_ARGS__); }} #define tsdbWarn(...) { if (tsdbDebugFlag & DEBUG_WARN) { taosPrintLog("TDB WARN ", 255, __VA_ARGS__); }}
#define tsdbInfo(...) { if (tsdbDebugFlag & DEBUG_INFO) { taosPrintLog("TDB INFO ", 255, __VA_ARGS__); }} #define tsdbInfo(...) { if (tsdbDebugFlag & DEBUG_INFO) { taosPrintLog("TDB ", 255, __VA_ARGS__); }}
#define tsdbDebug(...) { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TDB DEBUG ", tsdbDebugFlag, __VA_ARGS__); }} #define tsdbDebug(...) { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }}
#define tsdbTrace(...) { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB TRACE ", tsdbDebugFlag, __VA_ARGS__); }} #define tsdbTrace(...) { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }}
#define TSDB_MAX_TABLE_SCHEMAS 16 #define TSDB_MAX_TABLE_SCHEMAS 16
#define TSDB_FILE_HEAD_SIZE 512 #define TSDB_FILE_HEAD_SIZE 512
@ -96,6 +95,11 @@ typedef struct {
} STsdbBufPool; } STsdbBufPool;
// ------------------ tsdbMemTable.c // ------------------ tsdbMemTable.c
typedef struct {
STable * pTable;
SSkipListIterator *pIter;
} SCommitIter;
typedef struct { typedef struct {
uint64_t uid; uint64_t uid;
TSKEY keyFirst; TSKEY keyFirst;
@ -206,10 +210,10 @@ typedef struct {
int64_t offset : 63; int64_t offset : 63;
int32_t algorithm : 8; int32_t algorithm : 8;
int32_t numOfRows : 24; int32_t numOfRows : 24;
int32_t sversion;
int32_t len; int32_t len;
int32_t keyLen; // key column length, keyOffset = offset+sizeof(SCompData)+sizeof(SCompCol)*numOfCols
int16_t numOfSubBlocks; int16_t numOfSubBlocks;
int16_t numOfCols; int16_t numOfCols; // not including timestamp column
TSKEY keyFirst; TSKEY keyFirst;
TSKEY keyLast; TSKEY keyLast;
} SCompBlock; } SCompBlock;
@ -377,6 +381,24 @@ int tsdbUnRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable);
int tsdbTakeMemSnapshot(STsdbRepo* pRepo, SMemTable** pMem, SMemTable** pIMem); int tsdbTakeMemSnapshot(STsdbRepo* pRepo, SMemTable** pMem, SMemTable** pIMem);
void* tsdbAllocBytes(STsdbRepo* pRepo, int bytes); void* tsdbAllocBytes(STsdbRepo* pRepo, int bytes);
int tsdbAsyncCommit(STsdbRepo* pRepo); int tsdbAsyncCommit(STsdbRepo* pRepo);
int tsdbLoadDataFromCache(STable* pTable, SSkipListIterator* pIter, TSKEY maxKey, int maxRowsToRead, SDataCols* pCols,
TSKEY* filterKeys, int nFilterKeys);
static FORCE_INLINE SDataRow tsdbNextIterRow(SSkipListIterator* pIter) {
if (pIter == NULL) return NULL;
SSkipListNode* node = tSkipListIterGet(pIter);
if (node == NULL) return NULL;
return SL_GET_NODE_DATA(node);
}
static FORCE_INLINE TSKEY tsdbNextIterKey(SSkipListIterator* pIter) {
SDataRow row = tsdbNextIterRow(pIter);
if (row == NULL) return -1;
return dataRowKey(row);
}
// ------------------ tsdbFile.c // ------------------ tsdbFile.c
#define TSDB_KEY_FILEID(key, daysPerFile, precision) ((key) / tsMsPerDay[(precision)] / (daysPerFile)) #define TSDB_KEY_FILEID(key, daysPerFile, precision) ((key) / tsMsPerDay[(precision)] / (daysPerFile))
@ -421,25 +443,36 @@ void tsdbRemoveFileGroup(STsdbRepo* pRepo, SFileGroup* pFGroup);
#define helperType(h) (h)->type #define helperType(h) (h)->type
#define helperRepo(h) (h)->pRepo #define helperRepo(h) (h)->pRepo
#define helperState(h) (h)->state #define helperState(h) (h)->state
#define TSDB_NLAST_FILE_OPENED(h) ((h)->files.nLastF.fd > 0)
int tsdbInitReadHelper(SRWHelper* pHelper, STsdbRepo* pRepo); int tsdbInitReadHelper(SRWHelper* pHelper, STsdbRepo* pRepo);
int tsdbInitWriteHelper(SRWHelper* pHelper, STsdbRepo* pRepo); int tsdbInitWriteHelper(SRWHelper* pHelper, STsdbRepo* pRepo);
void tsdbDestroyHelper(SRWHelper* pHelper); void tsdbDestroyHelper(SRWHelper* pHelper);
void tsdbResetHelper(SRWHelper* pHelper); void tsdbResetHelper(SRWHelper* pHelper);
int tsdbSetAndOpenHelperFile(SRWHelper* pHelper, SFileGroup* pGroup); int tsdbSetAndOpenHelperFile(SRWHelper* pHelper, SFileGroup* pGroup);
int tsdbCloseHelperFile(SRWHelper* pHelper, bool hasError); int tsdbCloseHelperFile(SRWHelper* pHelper, bool hasError);
void tsdbSetHelperTable(SRWHelper* pHelper, STable* pTable, STsdbRepo* pRepo); void tsdbSetHelperTable(SRWHelper* pHelper, STable* pTable, STsdbRepo* pRepo);
int tsdbWriteDataBlock(SRWHelper* pHelper, SDataCols* pDataCols); int tsdbCommitTableData(SRWHelper* pHelper, SCommitIter* pCommitIter, SDataCols* pDataCols, TSKEY maxKey);
int tsdbMoveLastBlockIfNeccessary(SRWHelper* pHelper); int tsdbMoveLastBlockIfNeccessary(SRWHelper* pHelper);
int tsdbWriteCompInfo(SRWHelper* pHelper); int tsdbWriteCompInfo(SRWHelper* pHelper);
int tsdbWriteCompIdx(SRWHelper* pHelper); int tsdbWriteCompIdx(SRWHelper* pHelper);
int tsdbLoadCompIdx(SRWHelper* pHelper, void* target); int tsdbLoadCompIdx(SRWHelper* pHelper, void* target);
int tsdbLoadCompInfo(SRWHelper* pHelper, void* target); int tsdbLoadCompInfo(SRWHelper* pHelper, void* target);
int tsdbLoadCompData(SRWHelper* phelper, SCompBlock* pcompblock, void* target); int tsdbLoadCompData(SRWHelper* phelper, SCompBlock* pcompblock, void* target);
void tsdbGetDataStatis(SRWHelper* pHelper, SDataStatis* pStatis, int numOfCols); void tsdbGetDataStatis(SRWHelper* pHelper, SDataStatis* pStatis, int numOfCols);
int tsdbLoadBlockDataCols(SRWHelper* pHelper, SCompBlock* pCompBlock, SCompInfo* pCompInfo, int16_t* colIds, int tsdbLoadBlockDataCols(SRWHelper* pHelper, SCompBlock* pCompBlock, SCompInfo* pCompInfo, int16_t* colIds,
int numOfColIds); int numOfColIds);
int tsdbLoadBlockData(SRWHelper* pHelper, SCompBlock* pCompBlock, SCompInfo* pCompInfo); int tsdbLoadBlockData(SRWHelper* pHelper, SCompBlock* pCompBlock, SCompInfo* pCompInfo);
static FORCE_INLINE int compTSKEY(const void* key1, const void* key2) {
if (*(TSKEY*)key1 > *(TSKEY*)key2) {
return 1;
} else if (*(TSKEY*)key1 == *(TSKEY*)key2) {
return 0;
} else {
return -1;
}
}
// ------------------ tsdbMain.c // ------------------ tsdbMain.c
#define REPO_ID(r) (r)->config.tsdbId #define REPO_ID(r) (r)->config.tsdbId

43
src/tsdb/src/tsdbMain.c
View File

@ -69,6 +69,8 @@ static int tsdbEncodeCfg(void **buf, STsdbCfg *pCfg);
static void * tsdbDecodeCfg(void *buf, STsdbCfg *pCfg); static void * tsdbDecodeCfg(void *buf, STsdbCfg *pCfg);
static int tsdbCheckTableSchema(STsdbRepo *pRepo, SSubmitBlk *pBlock, STable *pTable); static int tsdbCheckTableSchema(STsdbRepo *pRepo, SSubmitBlk *pBlock, STable *pTable);
static int tsdbScanAndConvertSubmitMsg(STsdbRepo *pRepo, SSubmitMsg *pMsg); static int tsdbScanAndConvertSubmitMsg(STsdbRepo *pRepo, SSubmitMsg *pMsg);
static void tsdbStartStream(STsdbRepo *pRepo);
static void tsdbStopStream(STsdbRepo *pRepo);
// Function declaration // Function declaration
int32_t tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg) { int32_t tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg) {
@ -127,6 +129,7 @@ TSDB_REPO_T *tsdbOpenRepo(char *rootDir, STsdbAppH *pAppH) {
goto _err; goto _err;
} }
tsdbStartStream(pRepo);
// pRepo->state = TSDB_REPO_STATE_ACTIVE; // pRepo->state = TSDB_REPO_STATE_ACTIVE;
tsdbDebug("vgId:%d open tsdb repository succeed!", REPO_ID(pRepo)); tsdbDebug("vgId:%d open tsdb repository succeed!", REPO_ID(pRepo));
@ -145,6 +148,8 @@ void tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit) {
STsdbRepo *pRepo = (STsdbRepo *)repo; STsdbRepo *pRepo = (STsdbRepo *)repo;
int vgId = REPO_ID(pRepo); int vgId = REPO_ID(pRepo);
tsdbStopStream(repo);
if (toCommit) { if (toCommit) {
tsdbAsyncCommit(pRepo); tsdbAsyncCommit(pRepo);
if (pRepo->commit) pthread_join(pRepo->commitThread, NULL); if (pRepo->commit) pthread_join(pRepo->commitThread, NULL);
@ -265,19 +270,6 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_
return magic; return magic;
} }
void tsdbStartStream(TSDB_REPO_T *repo) {
STsdbRepo *pRepo = (STsdbRepo *)repo;
STsdbMeta *pMeta = pRepo->tsdbMeta;
for (int i = 0; i < pRepo->config.maxTables; i++) {
STable *pTable = pMeta->tables[i];
if (pTable && pTable->type == TSDB_STREAM_TABLE) {
pTable->cqhandle = (*pRepo->appH.cqCreateFunc)(pRepo->appH.cqH, TABLE_UID(pTable), TABLE_TID(pTable), pTable->sql,
tsdbGetTableSchemaImpl(pTable, false, false, -1));
}
}
}
STsdbCfg *tsdbGetCfg(const TSDB_REPO_T *repo) { STsdbCfg *tsdbGetCfg(const TSDB_REPO_T *repo) {
ASSERT(repo != NULL); ASSERT(repo != NULL);
return &((STsdbRepo *)repo)->config; return &((STsdbRepo *)repo)->config;
@ -1120,4 +1112,27 @@ TSKEY tsdbGetTableLastKey(TSDB_REPO_T *repo, uint64_t uid) {
return TSDB_GET_TABLE_LAST_KEY(pTable); return TSDB_GET_TABLE_LAST_KEY(pTable);
} }
#endif #endif
static void tsdbStartStream(STsdbRepo *pRepo) {
STsdbMeta *pMeta = pRepo->tsdbMeta;
for (int i = 0; i < pRepo->config.maxTables; i++) {
STable *pTable = pMeta->tables[i];
if (pTable && pTable->type == TSDB_STREAM_TABLE) {
pTable->cqhandle = (*pRepo->appH.cqCreateFunc)(pRepo->appH.cqH, TABLE_UID(pTable), TABLE_TID(pTable), pTable->sql,
tsdbGetTableSchemaImpl(pTable, false, false, -1));
}
}
}
static void tsdbStopStream(STsdbRepo *pRepo) {
STsdbMeta *pMeta = pRepo->tsdbMeta;
for (int i = 0; i < pRepo->config.maxTables; i++) {
STable *pTable = pMeta->tables[i];
if (pTable && pTable->type == TSDB_STREAM_TABLE) {
(*pRepo->appH.cqDropFunc)(pTable->cqhandle);
}
}
}

163
src/tsdb/src/tsdbMemTable.c
View File

@ -18,11 +18,6 @@
#define TSDB_DATA_SKIPLIST_LEVEL 5 #define TSDB_DATA_SKIPLIST_LEVEL 5
typedef struct {
STable * pTable;
SSkipListIterator *pIter;
} SCommitIter;
static FORCE_INLINE STsdbBufBlock *tsdbGetCurrBufBlock(STsdbRepo *pRepo); static FORCE_INLINE STsdbBufBlock *tsdbGetCurrBufBlock(STsdbRepo *pRepo);
static void tsdbFreeBytes(STsdbRepo *pRepo, void *ptr, int bytes); static void tsdbFreeBytes(STsdbRepo *pRepo, void *ptr, int bytes);
@ -34,14 +29,11 @@ static char * tsdbGetTsTupleKey(const void *data);
static void * tsdbCommitData(void *arg); static void * tsdbCommitData(void *arg);
static int tsdbCommitMeta(STsdbRepo *pRepo); static int tsdbCommitMeta(STsdbRepo *pRepo);
static void tsdbEndCommit(STsdbRepo *pRepo); static void tsdbEndCommit(STsdbRepo *pRepo);
static TSKEY tsdbNextIterKey(SCommitIter *pIter);
static int tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey); static int tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey);
static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHelper *pHelper, SDataCols *pDataCols); static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHelper *pHelper, SDataCols *pDataCols);
static void tsdbGetFidKeyRange(int daysPerFile, int8_t precision, int fileId, TSKEY *minKey, TSKEY *maxKey); static void tsdbGetFidKeyRange(int daysPerFile, int8_t precision, int fileId, TSKEY *minKey, TSKEY *maxKey);
static SCommitIter *tsdbCreateTableIters(STsdbRepo *pRepo); static SCommitIter *tsdbCreateCommitIters(STsdbRepo *pRepo);
static void tsdbDestroyTableIters(SCommitIter *iters, int maxTables); static void tsdbDestroyCommitIters(SCommitIter *iters, int maxTables);
static int tsdbReadRowsFromCache(STsdbMeta *pMeta, STable *pTable, SSkipListIterator *pIter, TSKEY maxKey,
int maxRowsToRead, SDataCols *pCols);
// ---------------- INTERNAL FUNCTIONS ---------------- // ---------------- INTERNAL FUNCTIONS ----------------
int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) {
@ -252,6 +244,66 @@ int tsdbAsyncCommit(STsdbRepo *pRepo) {
return 0; return 0;
} }
int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols,
TSKEY *filterKeys, int nFilterKeys) {
ASSERT(maxRowsToRead > 0 && nFilterKeys >= 0);
if (pIter == NULL) return 0;
STSchema *pSchema = NULL;
int numOfRows = 0;
TSKEY keyNext = 0;
int filterIter = 0;
if (nFilterKeys != 0) { // for filter purpose
ASSERT(filterKeys != NULL);
keyNext = tsdbNextIterKey(pIter);
if (keyNext < 0 || keyNext > maxKey) return numOfRows;
void *ptr = taosbsearch((void *)(&keyNext), (void *)filterKeys, nFilterKeys, sizeof(TSKEY), compTSKEY, TD_GE);
filterIter = (ptr == NULL) ? nFilterKeys : (POINTER_DISTANCE(ptr, filterKeys) / sizeof(TSKEY));
}
do {
if (numOfRows >= maxRowsToRead) break;
SDataRow row = tsdbNextIterRow(pIter);
if (row == NULL) break;
keyNext = dataRowKey(row);
if (keyNext < 0 || keyNext > maxKey) break;
bool keyFiltered = false;
if (nFilterKeys != 0) {
while (true) {
if (filterIter >= nFilterKeys) break;
if (keyNext == filterKeys[filterIter]) {
keyFiltered = true;
filterIter++;
break;
} else if (keyNext < filterKeys[filterIter]) {
break;
} else {
filterIter++;
}
}
}
if (!keyFiltered) {
if (pCols) {
if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) {
pSchema = tsdbGetTableSchemaImpl(pTable, false, false, dataRowVersion(row));
if (pSchema == NULL) {
ASSERT(0);
}
}
tdAppendDataRowToDataCol(row, pSchema, pCols);
}
numOfRows++;
}
} while (tSkipListIterNext(pIter));
return numOfRows;
}
// ---------------- LOCAL FUNCTIONS ---------------- // ---------------- LOCAL FUNCTIONS ----------------
static FORCE_INLINE STsdbBufBlock *tsdbGetCurrBufBlock(STsdbRepo *pRepo) { static FORCE_INLINE STsdbBufBlock *tsdbGetCurrBufBlock(STsdbRepo *pRepo) {
ASSERT(pRepo != NULL); ASSERT(pRepo != NULL);
@ -378,7 +430,7 @@ static void *tsdbCommitData(void *arg) {
// Create the iterator to read from cache // Create the iterator to read from cache
if (pMem->numOfRows > 0) { if (pMem->numOfRows > 0) {
iters = tsdbCreateTableIters(pRepo); iters = tsdbCreateCommitIters(pRepo);
if (iters == NULL) { if (iters == NULL) {
tsdbError("vgId:%d failed to create commit iterator since %s", REPO_ID(pRepo), tstrerror(terrno)); tsdbError("vgId:%d failed to create commit iterator since %s", REPO_ID(pRepo), tstrerror(terrno));
goto _exit; goto _exit;
@ -418,7 +470,7 @@ static void *tsdbCommitData(void *arg) {
_exit: _exit:
tdFreeDataCols(pDataCols); tdFreeDataCols(pDataCols);
tsdbDestroyTableIters(iters, pCfg->maxTables); tsdbDestroyCommitIters(iters, pCfg->maxTables);
tsdbDestroyHelper(&whelper); tsdbDestroyHelper(&whelper);
tsdbEndCommit(pRepo); tsdbEndCommit(pRepo);
tsdbInfo("vgId:%d commit over", pRepo->config.tsdbId); tsdbInfo("vgId:%d commit over", pRepo->config.tsdbId);
@ -479,19 +531,9 @@ static void tsdbEndCommit(STsdbRepo *pRepo) {
if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_OVER); if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_OVER);
} }
static TSKEY tsdbNextIterKey(SCommitIter *pIter) {
if (pIter == NULL) return -1;
SSkipListNode *node = tSkipListIterGet(pIter->pIter);
if (node == NULL) return -1;
SDataRow row = SL_GET_NODE_DATA(node);
return dataRowKey(row);
}
static int tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey) { static int tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey) {
for (int i = 0; i < nIters; i++) { for (int i = 0; i < nIters; i++) {
TSKEY nextKey = tsdbNextIterKey(iters + i); TSKEY nextKey = tsdbNextIterKey((iters + i)->pIter);
if (nextKey > 0 && (nextKey >= minKey && nextKey <= maxKey)) return 1; if (nextKey > 0 && (nextKey >= minKey && nextKey <= maxKey)) return 1;
} }
return 0; return 0;
@ -504,7 +546,6 @@ static void tsdbGetFidKeyRange(int daysPerFile, int8_t precision, int fileId, TS
static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHelper *pHelper, SDataCols *pDataCols) { static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHelper *pHelper, SDataCols *pDataCols) {
char * dataDir = NULL; char * dataDir = NULL;
STsdbMeta * pMeta = pRepo->tsdbMeta;
STsdbCfg * pCfg = &pRepo->config; STsdbCfg * pCfg = &pRepo->config;
STsdbFileH *pFileH = pRepo->tsdbFileH; STsdbFileH *pFileH = pRepo->tsdbFileH;
SFileGroup *pGroup = NULL; SFileGroup *pGroup = NULL;
@ -549,33 +590,13 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe
if (pIter->pIter != NULL) { if (pIter->pIter != NULL) {
tdInitDataCols(pDataCols, tsdbGetTableSchemaImpl(pIter->pTable, false, false, -1)); tdInitDataCols(pDataCols, tsdbGetTableSchemaImpl(pIter->pTable, false, false, -1));
int maxRowsToRead = pCfg->maxRowsPerFileBlock * 4 / 5; if (tsdbCommitTableData(pHelper, pIter, pDataCols, maxKey) < 0) {
int nLoop = 0; taosRUnLockLatch(&(pIter->pTable->latch));
while (true) { tsdbError("vgId:%d failed to write data of table %s tid %d uid %" PRIu64 " since %s", REPO_ID(pRepo),
int rowsRead = tsdbReadRowsFromCache(pMeta, pIter->pTable, pIter->pIter, maxKey, maxRowsToRead, pDataCols); TABLE_CHAR_NAME(pIter->pTable), TABLE_TID(pIter->pTable), TABLE_UID(pIter->pTable),
ASSERT(rowsRead >= 0); tstrerror(terrno));
if (pDataCols->numOfRows == 0) break; goto _err;
nLoop++;
ASSERT(dataColsKeyFirst(pDataCols) >= minKey && dataColsKeyFirst(pDataCols) <= maxKey);
ASSERT(dataColsKeyLast(pDataCols) >= minKey && dataColsKeyLast(pDataCols) <= maxKey);
int rowsWritten = tsdbWriteDataBlock(pHelper, pDataCols);
ASSERT(rowsWritten != 0);
if (rowsWritten < 0) {
taosRUnLockLatch(&(pIter->pTable->latch));
tsdbError("vgId:%d failed to write data block to table %s tid %d uid %" PRIu64 " since %s", REPO_ID(pRepo),
TABLE_CHAR_NAME(pIter->pTable), TABLE_TID(pIter->pTable), TABLE_UID(pIter->pTable),
tstrerror(terrno));
goto _err;
}
ASSERT(rowsWritten <= pDataCols->numOfRows);
tdPopDataColsPoints(pDataCols, rowsWritten);
maxRowsToRead = pCfg->maxRowsPerFileBlock * 4 / 5 - pDataCols->numOfRows;
} }
ASSERT(pDataCols->numOfRows == 0);
} }
taosRUnLockLatch(&(pIter->pTable->latch)); taosRUnLockLatch(&(pIter->pTable->latch));
@ -615,7 +636,7 @@ _err:
return -1; return -1;
} }
static SCommitIter *tsdbCreateTableIters(STsdbRepo *pRepo) { static SCommitIter *tsdbCreateCommitIters(STsdbRepo *pRepo) {
STsdbCfg * pCfg = &(pRepo->config); STsdbCfg * pCfg = &(pRepo->config);
SMemTable *pMem = pRepo->imem; SMemTable *pMem = pRepo->imem;
STsdbMeta *pMeta = pRepo->tsdbMeta; STsdbMeta *pMeta = pRepo->tsdbMeta;
@ -645,21 +666,18 @@ static SCommitIter *tsdbCreateTableIters(STsdbRepo *pRepo) {
goto _err; goto _err;
} }
if (!tSkipListIterNext(iters[i].pIter)) { tSkipListIterNext(iters[i].pIter);
terrno = TSDB_CODE_TDB_NO_TABLE_DATA_IN_MEM;
goto _err;
}
} }
} }
return iters; return iters;
_err: _err:
tsdbDestroyTableIters(iters, pCfg->maxTables); tsdbDestroyCommitIters(iters, pCfg->maxTables);
return NULL; return NULL;
} }
static void tsdbDestroyTableIters(SCommitIter *iters, int maxTables) { static void tsdbDestroyCommitIters(SCommitIter *iters, int maxTables) {
if (iters == NULL) return; if (iters == NULL) return;
for (int i = 1; i < maxTables; i++) { for (int i = 1; i < maxTables; i++) {
@ -670,35 +688,4 @@ static void tsdbDestroyTableIters(SCommitIter *iters, int maxTables) {
} }
free(iters); free(iters);
}
static int tsdbReadRowsFromCache(STsdbMeta *pMeta, STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols) {
ASSERT(maxRowsToRead > 0);
if (pIter == NULL) return 0;
STSchema *pSchema = NULL;
int numOfRows = 0;
do {
if (numOfRows >= maxRowsToRead) break;
SSkipListNode *node = tSkipListIterGet(pIter);
if (node == NULL) break;
SDataRow row = SL_GET_NODE_DATA(node);
if (dataRowKey(row) > maxKey) break;
if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) {
pSchema = tsdbGetTableSchemaImpl(pTable, true, false, dataRowVersion(row));
if (pSchema == NULL) {
// TODO: deal with the error here
ASSERT(0);
}
}
tdAppendDataRowToDataCol(row, pSchema, pCols);
numOfRows++;
} while (tSkipListIterNext(pIter));
return numOfRows;
} }

41
src/tsdb/src/tsdbMeta.c
View File

@ -57,8 +57,30 @@ int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg) {
STable * super = NULL; STable * super = NULL;
STable * table = NULL; STable * table = NULL;
int newSuper = 0; int newSuper = 0;
int tid = pCfg->tableId.tid;
STable * pTable = NULL;
STable *pTable = tsdbGetTableByUid(pMeta, pCfg->tableId.uid); if (tid < 0 || tid >= pRepo->config.maxTables) {
tsdbError("vgId:%d failed to create table since invalid tid %d", REPO_ID(pRepo), tid);
terrno = TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO;
goto _err;
}
if (pMeta->tables[tid] != NULL) {
if (TABLE_UID(pMeta->tables[tid]) == pCfg->tableId.uid) {
tsdbError("vgId:%d table %s already exists, tid %d uid %" PRId64, REPO_ID(pRepo), TABLE_CHAR_NAME(pTable),
TABLE_TID(pTable), TABLE_UID(pTable));
return TSDB_CODE_TDB_TABLE_ALREADY_EXIST;
} else {
tsdbError("vgId:%d table %s at tid %d uid %" PRIu64
" exists, replace it with new table, this can be not reasonable",
REPO_ID(pRepo), TABLE_CHAR_NAME(pMeta->tables[tid]), TABLE_TID(pMeta->tables[tid]),
TABLE_UID(pMeta->tables[tid]));
tsdbDropTable(pRepo, pMeta->tables[tid]->tableId);
}
}
pTable = tsdbGetTableByUid(pMeta, pCfg->tableId.uid);
if (pTable != NULL) { if (pTable != NULL) {
tsdbError("vgId:%d table %s already exists, tid %d uid %" PRId64, REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), tsdbError("vgId:%d table %s already exists, tid %d uid %" PRId64, REPO_ID(pRepo), TABLE_CHAR_NAME(pTable),
TABLE_TID(pTable), TABLE_UID(pTable)); TABLE_TID(pTable), TABLE_UID(pTable));
@ -72,10 +94,10 @@ int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg) {
super = tsdbNewTable(pCfg, true); super = tsdbNewTable(pCfg, true);
if (super == NULL) goto _err; if (super == NULL) goto _err;
} else { } else {
// TODO if (TABLE_TYPE(super) != TSDB_SUPER_TABLE || TABLE_UID(super) != pCfg->superUid) {
if (super->type != TSDB_SUPER_TABLE) return -1; terrno = TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO;
if (super->tableId.uid != pCfg->superUid) return -1; goto _err;
// tsdbUpdateTable(pRepo, super, pCfg); }
} }
} }
@ -705,6 +727,9 @@ static STable *tsdbNewTable(STableCfg *pCfg, bool isSuper) {
T_REF_INC(pTable); T_REF_INC(pTable);
tsdbTrace("table %s tid %d uid %" PRIu64 " is created", TABLE_CHAR_NAME(pTable), TABLE_TID(pTable),
TABLE_UID(pTable));
return pTable; return pTable;
_err: _err:
@ -714,7 +739,9 @@ _err:
static void tsdbFreeTable(STable *pTable) { static void tsdbFreeTable(STable *pTable) {
if (pTable) { if (pTable) {
if (pTable->name != NULL) tsdbDebug("table %s is destroyed", TABLE_CHAR_NAME(pTable)); if (pTable->name != NULL)
tsdbTrace("table %s tid %d uid %" PRIu64 " is freed", TABLE_CHAR_NAME(pTable), TABLE_TID(pTable),
TABLE_UID(pTable));
tfree(TABLE_NAME(pTable)); tfree(TABLE_NAME(pTable));
if (TABLE_TYPE(pTable) != TSDB_CHILD_TABLE) { if (TABLE_TYPE(pTable) != TSDB_CHILD_TABLE) {
for (int i = 0; i < TSDB_MAX_TABLE_SCHEMAS; i++) { for (int i = 0; i < TSDB_MAX_TABLE_SCHEMAS; i++) {
@ -782,7 +809,7 @@ static int tsdbAddTableToMeta(STsdbRepo *pRepo, STable *pTable, bool addIdx, boo
tsdbGetTableSchemaImpl(pTable, false, false, -1)); tsdbGetTableSchemaImpl(pTable, false, false, -1));
} }
tsdbTrace("vgId:%d table %s tid %d uid %" PRIu64 " is added to meta", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), tsdbDebug("vgId:%d table %s tid %d uid %" PRIu64 " is added to meta", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable),
TABLE_TID(pTable), TABLE_UID(pTable)); TABLE_TID(pTable), TABLE_UID(pTable));
return 0; return 0;

820
src/tsdb/src/tsdbRWHelper.c
View File

File diff suppressed because it is too large Load Diff

151
src/tsdb/src/tsdbRead.c
View File

@ -110,9 +110,10 @@ typedef struct STsdbQueryHandle {
SFileGroupIter fileIter; SFileGroupIter fileIter;
SRWHelper rhelper; SRWHelper rhelper;
STableBlockInfo* pDataBlockInfo; STableBlockInfo* pDataBlockInfo;
int32_t allocSize; // allocated data block size
SMemTable* mem; // mem-table SMemTable* mem; // mem-table
SMemTable* imem; // imem-table, acquired from snapshot SMemTable* imem; // imem-table, acquired from snapshot
SArray* defaultLoadColumn;// default load column
SDataBlockLoadInfo dataBlockLoadInfo; /* record current block load information */ SDataBlockLoadInfo dataBlockLoadInfo; /* record current block load information */
SLoadCompBlockInfo compBlockLoadInfo; /* record current compblock information in SQuery */ SLoadCompBlockInfo compBlockLoadInfo; /* record current compblock information in SQuery */
} STsdbQueryHandle; } STsdbQueryHandle;
@ -136,6 +137,34 @@ static void tsdbInitCompBlockLoadInfo(SLoadCompBlockInfo* pCompBlockLoadInfo) {
pCompBlockLoadInfo->fileId = -1; pCompBlockLoadInfo->fileId = -1;
} }
static SArray* getColumnIdList(STsdbQueryHandle* pQueryHandle) {
size_t numOfCols = QH_GET_NUM_OF_COLS(pQueryHandle);
assert(numOfCols <= TSDB_MAX_COLUMNS);
SArray* pIdList = taosArrayInit(numOfCols, sizeof(int16_t));
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i);
taosArrayPush(pIdList, &pCol->info.colId);
}
return pIdList;
}
static SArray* getDefaultLoadColumns(STsdbQueryHandle* pQueryHandle, bool loadTS) {
SArray* pLocalIdList = getColumnIdList(pQueryHandle);
// check if the primary time stamp column needs to load
int16_t colId = *(int16_t*)taosArrayGet(pLocalIdList, 0);
// the primary timestamp column does not be included in the the specified load column list, add it
if (loadTS && colId != 0) {
int16_t columnId = 0;
taosArrayInsert(pLocalIdList, 0, &columnId);
}
return pLocalIdList;
}
TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, void* qinfo) { TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, void* qinfo) {
STsdbQueryHandle* pQueryHandle = calloc(1, sizeof(STsdbQueryHandle)); STsdbQueryHandle* pQueryHandle = calloc(1, sizeof(STsdbQueryHandle));
pQueryHandle->order = pCond->order; pQueryHandle->order = pCond->order;
@ -148,8 +177,12 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab
pQueryHandle->activeIndex = 0; // current active table index pQueryHandle->activeIndex = 0; // current active table index
pQueryHandle->qinfo = qinfo; pQueryHandle->qinfo = qinfo;
pQueryHandle->outputCapacity = ((STsdbRepo*)tsdb)->config.maxRowsPerFileBlock; pQueryHandle->outputCapacity = ((STsdbRepo*)tsdb)->config.maxRowsPerFileBlock;
pQueryHandle->allocSize = 0;
tsdbInitReadHelper(&pQueryHandle->rhelper, (STsdbRepo*) tsdb);
if (tsdbInitReadHelper(&pQueryHandle->rhelper, (STsdbRepo*) tsdb) != 0) {
free(pQueryHandle);
return NULL;
}
tsdbTakeMemSnapshot(pQueryHandle->pTsdb, &pQueryHandle->mem, &pQueryHandle->imem); tsdbTakeMemSnapshot(pQueryHandle->pTsdb, &pQueryHandle->mem, &pQueryHandle->imem);
size_t sizeOfGroup = taosArrayGetSize(groupList->pGroupList); size_t sizeOfGroup = taosArrayGetSize(groupList->pGroupList);
@ -195,7 +228,9 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab
taosArrayPush(pQueryHandle->pTableCheckInfo, &info); taosArrayPush(pQueryHandle->pTableCheckInfo, &info);
} }
} }
pQueryHandle->defaultLoadColumn = getDefaultLoadColumns(pQueryHandle, true);
tsdbDebug("%p total numOfTable:%zu in query", pQueryHandle, taosArrayGetSize(pQueryHandle->pTableCheckInfo)); tsdbDebug("%p total numOfTable:%zu in query", pQueryHandle, taosArrayGetSize(pQueryHandle->pTableCheckInfo));
tsdbInitDataBlockLoadInfo(&pQueryHandle->dataBlockLoadInfo); tsdbInitDataBlockLoadInfo(&pQueryHandle->dataBlockLoadInfo);
@ -206,11 +241,11 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab
TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, void* qinfo) { TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, void* qinfo) {
STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo); STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo);
if (pQueryHandle != NULL) {
pQueryHandle->type = TSDB_QUERY_TYPE_LAST; pQueryHandle->type = TSDB_QUERY_TYPE_LAST;
pQueryHandle->order = TSDB_ORDER_DESC; pQueryHandle->order = TSDB_ORDER_DESC;
changeQueryHandleForLastrowQuery(pQueryHandle);
changeQueryHandleForLastrowQuery(pQueryHandle); }
return pQueryHandle; return pQueryHandle;
} }
@ -232,9 +267,10 @@ SArray* tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle) {
TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TSDB_REPO_T *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, void* qinfo) { TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TSDB_REPO_T *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, void* qinfo) {
STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo); STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo);
if (pQueryHandle != NULL) {
pQueryHandle->type = TSDB_QUERY_TYPE_EXTERNAL; pQueryHandle->type = TSDB_QUERY_TYPE_EXTERNAL;
changeQueryHandleForInterpQuery(pQueryHandle); changeQueryHandleForInterpQuery(pQueryHandle);
}
return pQueryHandle; return pQueryHandle;
} }
@ -546,33 +582,7 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo
.tid = (_checkInfo)->tableId.tid, \ .tid = (_checkInfo)->tableId.tid, \
.uid = (_checkInfo)->tableId.uid}) .uid = (_checkInfo)->tableId.uid})
static SArray* getColumnIdList(STsdbQueryHandle* pQueryHandle) {
size_t numOfCols = QH_GET_NUM_OF_COLS(pQueryHandle);
assert(numOfCols <= TSDB_MAX_COLUMNS);
SArray* pIdList = taosArrayInit(numOfCols, sizeof(int16_t));
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i);
taosArrayPush(pIdList, &pCol->info.colId);
}
return pIdList;
}
static SArray* getDefaultLoadColumns(STsdbQueryHandle* pQueryHandle, bool loadTS) {
SArray* pLocalIdList = getColumnIdList(pQueryHandle);
// check if the primary time stamp column needs to load
int16_t colId = *(int16_t*)taosArrayGet(pLocalIdList, 0);
// the primary timestamp column does not be included in the the specified load column list, add it
if (loadTS && colId != 0) {
int16_t columnId = 0;
taosArrayInsert(pLocalIdList, 0, &columnId);
}
return pLocalIdList;
}
static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo) { static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo) {
STsdbRepo *pRepo = pQueryHandle->pTsdb; STsdbRepo *pRepo = pQueryHandle->pTsdb;
@ -584,8 +594,6 @@ static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlo
data->uid = pCheckInfo->pTableObj->tableId.uid; data->uid = pCheckInfo->pTableObj->tableId.uid;
bool blockLoaded = false; bool blockLoaded = false;
SArray* sa = getDefaultLoadColumns(pQueryHandle, true);
int64_t st = taosGetTimestampUs(); int64_t st = taosGetTimestampUs();
if (pCheckInfo->pDataCols == NULL) { if (pCheckInfo->pDataCols == NULL) {
@ -613,7 +621,6 @@ static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlo
assert(pCols->numOfRows != 0 && pCols->numOfRows <= pBlock->numOfRows); assert(pCols->numOfRows != 0 && pCols->numOfRows <= pBlock->numOfRows);
pBlock->numOfRows = pCols->numOfRows; pBlock->numOfRows = pCols->numOfRows;
taosArrayDestroy(sa);
tfree(data); tfree(data);
int64_t et = taosGetTimestampUs() - st; int64_t et = taosGetTimestampUs() - st;
@ -656,12 +663,8 @@ static void handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBlock*
return; return;
} }
SArray* sa = getDefaultLoadColumns(pQueryHandle, true);
doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo); doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo);
doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock, sa); doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock, pQueryHandle->defaultLoadColumn);
taosArrayDestroy(sa);
} else { } else {
/* /*
* no data in cache, only load data from file * no data in cache, only load data from file
@ -681,14 +684,12 @@ static void handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBlock*
} }
static bool loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo) { static bool loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo) {
SArray* sa = getDefaultLoadColumns(pQueryHandle, true);
SQueryFilePos* cur = &pQueryHandle->cur; SQueryFilePos* cur = &pQueryHandle->cur;
if (ASCENDING_TRAVERSE(pQueryHandle->order)) { if (ASCENDING_TRAVERSE(pQueryHandle->order)) {
// query ended in current block // query ended in current block
if (pQueryHandle->window.ekey < pBlock->keyLast || pCheckInfo->lastKey > pBlock->keyFirst) { if (pQueryHandle->window.ekey < pBlock->keyLast || pCheckInfo->lastKey > pBlock->keyFirst) {
if (!doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo)) { if (!doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo)) {
taosArrayDestroy(sa);
return false; return false;
} }
@ -702,7 +703,7 @@ static bool loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock
cur->pos = 0; cur->pos = 0;
} }
doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock, sa); doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock, pQueryHandle->defaultLoadColumn);
} else { // the whole block is loaded in to buffer } else { // the whole block is loaded in to buffer
handleDataMergeIfNeeded(pQueryHandle, pBlock, pCheckInfo); handleDataMergeIfNeeded(pQueryHandle, pBlock, pCheckInfo);
} }
@ -719,13 +720,12 @@ static bool loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock
cur->pos = pBlock->numOfRows - 1; cur->pos = pBlock->numOfRows - 1;
} }
doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock, sa); doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock, pQueryHandle->defaultLoadColumn);
} else { } else {
handleDataMergeIfNeeded(pQueryHandle, pBlock, pCheckInfo); handleDataMergeIfNeeded(pQueryHandle, pBlock, pCheckInfo);
} }
} }
taosArrayDestroy(sa);
return pQueryHandle->realNumOfRows > 0; return pQueryHandle->realNumOfRows > 0;
} }
@ -1241,22 +1241,30 @@ static int32_t dataBlockOrderCompar(const void* pLeft, const void* pRight, void*
STableBlockInfo* pRightBlockInfoEx = &pSupporter->pDataBlockInfo[rightTableIndex][rightTableBlockIndex]; STableBlockInfo* pRightBlockInfoEx = &pSupporter->pDataBlockInfo[rightTableIndex][rightTableBlockIndex];
// assert(pLeftBlockInfoEx->compBlock->offset != pRightBlockInfoEx->compBlock->offset); // assert(pLeftBlockInfoEx->compBlock->offset != pRightBlockInfoEx->compBlock->offset);
#if 0 // TODO: temporarily comment off requested by Dr. Liao
if (pLeftBlockInfoEx->compBlock->offset == pRightBlockInfoEx->compBlock->offset && if (pLeftBlockInfoEx->compBlock->offset == pRightBlockInfoEx->compBlock->offset &&
pLeftBlockInfoEx->compBlock->last == pRightBlockInfoEx->compBlock->last) { pLeftBlockInfoEx->compBlock->last == pRightBlockInfoEx->compBlock->last) {
tsdbError("error in header file, two block with same offset:%" PRId64, (int64_t)pLeftBlockInfoEx->compBlock->offset); tsdbError("error in header file, two block with same offset:%" PRId64, (int64_t)pLeftBlockInfoEx->compBlock->offset);
} }
#endif
return pLeftBlockInfoEx->compBlock->offset > pRightBlockInfoEx->compBlock->offset ? 1 : -1; return pLeftBlockInfoEx->compBlock->offset > pRightBlockInfoEx->compBlock->offset ? 1 : -1;
} }
static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numOfBlocks, int32_t* numOfAllocBlocks) { static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numOfBlocks, int32_t* numOfAllocBlocks) {
char* tmp = realloc(pQueryHandle->pDataBlockInfo, sizeof(STableBlockInfo) * numOfBlocks); size_t size = sizeof(STableBlockInfo) * numOfBlocks;
if (tmp == NULL) {
return TSDB_CODE_TDB_OUT_OF_MEMORY; if (pQueryHandle->allocSize < size) {
pQueryHandle->allocSize = size;
char* tmp = realloc(pQueryHandle->pDataBlockInfo, pQueryHandle->allocSize);
if (tmp == NULL) {
return TSDB_CODE_TDB_OUT_OF_MEMORY;
}
pQueryHandle->pDataBlockInfo = (STableBlockInfo*) tmp;
} }
pQueryHandle->pDataBlockInfo = (STableBlockInfo*) tmp; memset(pQueryHandle->pDataBlockInfo, 0, size);
memset(pQueryHandle->pDataBlockInfo, 0, sizeof(STableBlockInfo) * numOfBlocks);
*numOfAllocBlocks = numOfBlocks; *numOfAllocBlocks = numOfBlocks;
int32_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); int32_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
@ -1492,9 +1500,8 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
return false; return false;
} }
SArray* sa = getDefaultLoadColumns(pQueryHandle, true);
/*SDataBlockInfo* pBlockInfo =*/ tsdbRetrieveDataBlockInfo(pHandle, &blockInfo); /*SDataBlockInfo* pBlockInfo =*/ tsdbRetrieveDataBlockInfo(pHandle, &blockInfo);
/*SArray *pDataBlock = */tsdbRetrieveDataBlock(pHandle, sa); /*SArray *pDataBlock = */tsdbRetrieveDataBlock(pHandle, pQueryHandle->defaultLoadColumn);
if (pQueryHandle->cur.win.ekey == pQueryHandle->window.skey) { if (pQueryHandle->cur.win.ekey == pQueryHandle->window.skey) {
// data already retrieve, discard other data rows and return // data already retrieve, discard other data rows and return
@ -1508,7 +1515,6 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
pQueryHandle->window = pQueryHandle->cur.win; pQueryHandle->window = pQueryHandle->cur.win;
pQueryHandle->cur.rows = 1; pQueryHandle->cur.rows = 1;
pQueryHandle->type = TSDB_QUERY_TYPE_EXTERNAL; pQueryHandle->type = TSDB_QUERY_TYPE_EXTERNAL;
taosArrayDestroy(sa);
return true; return true;
} else { } else {
STsdbQueryHandle* pSecQueryHandle = calloc(1, sizeof(STsdbQueryHandle)); STsdbQueryHandle* pSecQueryHandle = calloc(1, sizeof(STsdbQueryHandle));
@ -1522,7 +1528,10 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
pSecQueryHandle->activeIndex = 0; pSecQueryHandle->activeIndex = 0;
pSecQueryHandle->outputCapacity = ((STsdbRepo*)pSecQueryHandle->pTsdb)->config.maxRowsPerFileBlock; pSecQueryHandle->outputCapacity = ((STsdbRepo*)pSecQueryHandle->pTsdb)->config.maxRowsPerFileBlock;
tsdbInitReadHelper(&pSecQueryHandle->rhelper, (STsdbRepo*) pSecQueryHandle->pTsdb); if (tsdbInitReadHelper(&pSecQueryHandle->rhelper, (STsdbRepo*) pSecQueryHandle->pTsdb) != 0) {
free(pSecQueryHandle);
return false;
}
tsdbTakeMemSnapshot(pSecQueryHandle->pTsdb, &pSecQueryHandle->mem, &pSecQueryHandle->imem); tsdbTakeMemSnapshot(pSecQueryHandle->pTsdb, &pSecQueryHandle->mem, &pSecQueryHandle->imem);
// allocate buffer in order to load data blocks from file // allocate buffer in order to load data blocks from file
@ -1565,7 +1574,7 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
assert(ret); assert(ret);
/*SDataBlockInfo* pBlockInfo =*/ tsdbRetrieveDataBlockInfo((void*) pSecQueryHandle, &blockInfo); /*SDataBlockInfo* pBlockInfo =*/ tsdbRetrieveDataBlockInfo((void*) pSecQueryHandle, &blockInfo);
/*SArray *pDataBlock = */tsdbRetrieveDataBlock((void*) pSecQueryHandle, sa); /*SArray *pDataBlock = */tsdbRetrieveDataBlock((void*) pSecQueryHandle, pSecQueryHandle->defaultLoadColumn);
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i); SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i);
@ -1606,7 +1615,9 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
} }
// TODO: opt by consider the scan order // TODO: opt by consider the scan order
return doHasDataInBuffer(pQueryHandle); bool ret = doHasDataInBuffer(pQueryHandle);
terrno = TSDB_CODE_SUCCESS;
return ret;
} }
void changeQueryHandleForLastrowQuery(TsdbQueryHandleT pqHandle) { void changeQueryHandleForLastrowQuery(TsdbQueryHandleT pqHandle) {
@ -1801,7 +1812,8 @@ int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataSta
} }
tsdbLoadCompData(&pHandle->rhelper, pBlockInfo->compBlock, NULL); tsdbLoadCompData(&pHandle->rhelper, pBlockInfo->compBlock, NULL);
// todo opt perf
size_t numOfCols = QH_GET_NUM_OF_COLS(pHandle); size_t numOfCols = QH_GET_NUM_OF_COLS(pHandle);
for(int32_t i = 0; i < numOfCols; ++i) { for(int32_t i = 0; i < numOfCols; ++i) {
SDataStatis* st = &pHandle->statis[i]; SDataStatis* st = &pHandle->statis[i];
@ -1820,6 +1832,13 @@ int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataSta
if (pHandle->statis[i].numOfNull == -1) { // set the column data are all NULL if (pHandle->statis[i].numOfNull == -1) { // set the column data are all NULL
pHandle->statis[i].numOfNull = pBlockInfo->compBlock->numOfRows; pHandle->statis[i].numOfNull = pBlockInfo->compBlock->numOfRows;
} }
// todo opt perf
SColumnInfo* pColInfo = taosArrayGet(pHandle->pColumns, i);
if (pColInfo->type == TSDB_DATA_TYPE_TIMESTAMP) {
pHandle->statis[i].min = pBlockInfo->compBlock->keyFirst;
pHandle->statis[i].max = pBlockInfo->compBlock->keyLast;
}
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -2193,7 +2212,7 @@ int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, const char* pT
int32_t ret = TSDB_CODE_SUCCESS; int32_t ret = TSDB_CODE_SUCCESS;
tExprNode* expr = NULL; tExprNode* expr = NULL;
TRY(32) { TRY(TSDB_MAX_TAGS) {
expr = exprTreeFromTableName(tbnameCond); expr = exprTreeFromTableName(tbnameCond);
if (expr == NULL) { if (expr == NULL) {
expr = exprTreeFromBinary(pTagCond, len); expr = exprTreeFromBinary(pTagCond, len);
@ -2217,7 +2236,8 @@ int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, const char* pT
} CATCH( code ) { } CATCH( code ) {
CLEANUP_EXECUTE(); CLEANUP_EXECUTE();
ret = code; terrno = code;
goto _error;
// TODO: more error handling // TODO: more error handling
} END_TRY } END_TRY
@ -2333,6 +2353,7 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) {
} }
taosArrayDestroy(pQueryHandle->pColumns); taosArrayDestroy(pQueryHandle->pColumns);
taosArrayDestroy(pQueryHandle->defaultLoadColumn);
tfree(pQueryHandle->pDataBlockInfo); tfree(pQueryHandle->pDataBlockInfo);
tfree(pQueryHandle->statis); tfree(pQueryHandle->statis);

3
src/util/inc/exception.h
View File

@ -78,6 +78,7 @@ void cleanupPush_int_ptr ( bool failOnly, void* func, void* arg );
int32_t cleanupGetActionCount(); int32_t cleanupGetActionCount();
void cleanupExecuteTo( int32_t anchor, bool failed ); void cleanupExecuteTo( int32_t anchor, bool failed );
void cleanupExecute( SExceptionNode* node, bool failed ); void cleanupExecute( SExceptionNode* node, bool failed );
bool cleanupExceedLimit();
#define CLEANUP_PUSH_VOID_PTR_PTR( failOnly, func, arg1, arg2 ) cleanupPush_void_ptr_ptr( (failOnly), (void*)(func), (void*)(arg1), (void*)(arg2) ) #define CLEANUP_PUSH_VOID_PTR_PTR( failOnly, func, arg1, arg2 ) cleanupPush_void_ptr_ptr( (failOnly), (void*)(func), (void*)(arg1), (void*)(arg2) )
#define CLEANUP_PUSH_VOID_PTR_BOOL( failOnly, func, arg1, arg2 ) cleanupPush_void_ptr_bool( (failOnly), (void*)(func), (void*)(arg1), (bool)(arg2) ) #define CLEANUP_PUSH_VOID_PTR_BOOL( failOnly, func, arg1, arg2 ) cleanupPush_void_ptr_bool( (failOnly), (void*)(func), (void*)(arg1), (bool)(arg2) )
@ -91,7 +92,7 @@ void cleanupExecute( SExceptionNode* node, bool failed );
#define CLEANUP_GET_ANCHOR() cleanupGetActionCount() #define CLEANUP_GET_ANCHOR() cleanupGetActionCount()
#define CLEANUP_EXECUTE_TO( anchor, failed ) cleanupExecuteTo( (anchor), (failed) ) #define CLEANUP_EXECUTE_TO( anchor, failed ) cleanupExecuteTo( (anchor), (failed) )
#define CLEANUP_EXCEED_LIMIT() cleanupExceedLimit()
//////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////
// functions & macros for exception handling // functions & macros for exception handling

5
src/util/inc/tarray.h
View File

@ -23,14 +23,13 @@ extern "C" {
#include "os.h" #include "os.h"
#define TARRAY_MIN_SIZE 8 #define TARRAY_MIN_SIZE 8
#define TARRAY_GET_ELEM(array, index) ((array)->pData + (index) * (array)->elemSize) #define TARRAY_GET_ELEM(array, index) ((void*)((char*)((array)->pData) + (index) * (array)->elemSize))
typedef struct SArray { typedef struct SArray {
size_t size; size_t size;
size_t capacity; size_t capacity;
size_t elemSize; size_t elemSize;
void* pData;
void* pData;
} SArray; } SArray;
/** /**

23
src/util/inc/tcache.h
View File

@ -21,7 +21,7 @@ extern "C" {
#endif #endif
#include "os.h" #include "os.h"
#include "tref.h" #include "tlockfree.h"
#include "hash.h" #include "hash.h"
typedef void (*__cache_free_fn_t)(void*); typedef void (*__cache_free_fn_t)(void*);
@ -33,17 +33,20 @@ typedef struct SCacheStatis {
int64_t refreshCount; int64_t refreshCount;
} SCacheStatis; } SCacheStatis;
struct STrashElem;
typedef struct SCacheDataNode { typedef struct SCacheDataNode {
uint64_t addedTime; // the added time when this element is added or updated into cache uint64_t addedTime; // the added time when this element is added or updated into cache
uint64_t lifespan; // expiredTime expiredTime when this element should be remove from cache uint64_t lifespan; // life duration when this element should be remove from cache
uint64_t signature; uint64_t expireTime; // expire time
uint32_t size; // allocated size for current SCacheDataNode uint64_t signature;
struct STrashElem *pTNodeHeader; // point to trash node head
uint16_t keySize: 15; // max key size: 32kb
bool inTrashCan: 1;// denote if it is in trash or not
uint32_t size; // allocated size for current SCacheDataNode
T_REF_DECLARE() T_REF_DECLARE()
uint16_t keySize: 15; // max key size: 32kb char *key;
bool inTrashCan: 1;// denote if it is in trash or not char data[];
int32_t extendFactor; // number of life span extend
char *key;
char data[];
} SCacheDataNode; } SCacheDataNode;
typedef struct STrashElem { typedef struct STrashElem {

13
src/util/inc/tfile.h
View File

@ -19,13 +19,14 @@
#ifdef TAOS_RANDOM_FILE_FAIL #ifdef TAOS_RANDOM_FILE_FAIL
void taosSetRandomFileFailFactor(int factor); void taosSetRandomFileFailFactor(int factor);
ssize_t taos_tread(int fd, void *buf, size_t count); void taosSetRandomFileFailOutput(const char *path);
ssize_t taos_twrite(int fd, void *buf, size_t count); ssize_t taos_tread(int fd, void *buf, size_t count, const char *file, uint32_t line);
off_t taos_lseek(int fd, off_t offset, int whence); ssize_t taos_twrite(int fd, void *buf, size_t count, const char *file, uint32_t line);
off_t taos_lseek(int fd, off_t offset, int whence, const char *file, uint32_t line);
#define tread(fd, buf, count) taos_tread(fd, buf, count) #define tread(fd, buf, count) taos_tread(fd, buf, count, __FILE__, __LINE__)
#define twrite(fd, buf, count) taos_twrite(fd, buf, count) #define twrite(fd, buf, count) taos_twrite(fd, buf, count, __FILE__, __LINE__)
#define lseek(fd, offset, whence) taos_lseek(fd, offset, whence) #define lseek(fd, offset, whence) taos_lseek(fd, offset, whence, __FILE__, __LINE__)
#endif // TAOS_RANDOM_FILE_FAIL #endif // TAOS_RANDOM_FILE_FAIL

56
src/util/inc/tref.h → src/util/inc/tlockfree.h
View File

@ -12,12 +12,17 @@
* You should have received a copy of the GNU Affero General Public License * You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#ifndef __TD_LOCK_FREE_H__
#ifndef TDENGINE_TREF_H #define __TD_LOCK_FREE_H__
#define TDENGINE_TREF_H
#include "os.h" #include "os.h"
#ifdef __cplusplus
extern "C" {
#endif
// reference counting
typedef void (*_ref_fn_t)(const void* pObj); typedef void (*_ref_fn_t)(const void* pObj);
#define T_REF_DECLARE() \ #define T_REF_DECLARE() \
@ -55,4 +60,47 @@ typedef void (*_ref_fn_t)(const void* pObj);
#define T_REF_VAL_GET(x) (x)->_ref.val #define T_REF_VAL_GET(x) (x)->_ref.val
#endif // TDENGINE_TREF_H
// single writer multiple reader lock
typedef int32_t SRWLatch;
void taosInitRWLatch(SRWLatch *pLatch);
void taosWLockLatch(SRWLatch *pLatch);
void taosWUnLockLatch(SRWLatch *pLatch);
void taosRLockLatch(SRWLatch *pLatch);
void taosRUnLockLatch(SRWLatch *pLatch);
// copy on read
#define taosCorBeginRead(x) for (uint32_t i_ = 1; 1; ++i_) { \
int32_t old_ = atomic_load_32(x); \
if (old_ & 0x00000001) { \
if (i_ % 1000 == 0) { \
sched_yield(); \
} \
continue; \
}
#define taosCorEndRead(x) \
if (atomic_load_32(x) == old_) { \
break; \
} \
}
#define taosCorBeginWrite(x) taosCorBeginRead(x) \
if (atomic_val_compare_exchange_32((x), old_, old_ + 1) != old_) { \
continue; \
}
#define taosCorEndWrite(x) atomic_add_fetch_32((x), 1); \
break; \
}
#ifdef __cplusplus
}
#endif
#endif

36
src/util/inc/trwlatch.h
View File

@ -1,36 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef __TD_RWLATCH_H__
#define __TD_RWLATCH_H__
#ifdef __cplusplus
extern "C" {
#endif
#include <stdint.h>
typedef int32_t SRWLatch;
void taosInitRWLatch(SRWLatch *pLatch);
void taosWLockLatch(SRWLatch *pLatch);
void taosWUnLockLatch(SRWLatch *pLatch);
void taosRLockLatch(SRWLatch *pLatch);
void taosRUnLockLatch(SRWLatch *pLatch);
#ifdef __cplusplus
}
#endif
#endif

2
src/util/inc/tutil.h
View File

@ -133,6 +133,8 @@ char **strsplit(char *src, const char *delim, int32_t *num);
char* strtolower(char *dst, const char *src); char* strtolower(char *dst, const char *src);
char* strntolower(char *dst, const char *src, int32_t n);
int64_t strnatoi(char *num, int32_t len); int64_t strnatoi(char *num, int32_t len);
//char* strreplace(const char* str, const char* pattern, const char* rep); //char* strreplace(const char* str, const char* pattern, const char* rep);

3
src/util/src/exception.c
View File

@ -147,3 +147,6 @@ void cleanupExecuteTo( int32_t anchor, bool failed ) {
void cleanupExecute( SExceptionNode* node, bool failed ) { void cleanupExecute( SExceptionNode* node, bool failed ) {
doExecuteCleanup( node, 0, failed ); doExecuteCleanup( node, 0, failed );
} }
bool cleanupExceedLimit() {
return expList->numCleanupAction >= expList->maxCleanupAction;
}

Some files were not shown because too many files have changed in this diff Show More