xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch 'develop' into hotfix/TD-884

This commit is contained in:
dengyihao 2020-07-18 21:34:52 +08:00 committed by GitHub
parent 3c4e620f31 f28c39da67
commit 599df650ae
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
207 changed files with 2594 additions and 2046 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

92
importSampleData/app/main.go
View File

@ -7,7 +7,6 @@ import (
"encoding/json" "encoding/json"
"flag" "flag"
"fmt" "fmt"
"github.com/taosdata/TDengine/importSampleData/import"
"hash/crc32" "hash/crc32"
"io" "io"
"log" "log"
@ -18,6 +17,8 @@ import (
"sync" "sync"
"time" "time"
dataimport "github.com/taosdata/TDengine/importSampleData/import"
_ "github.com/taosdata/driver-go/taosSql" _ "github.com/taosdata/driver-go/taosSql"
) )
@ -26,7 +27,7 @@ const (
DATETIME = "datetime" DATETIME = "datetime"
MILLISECOND = "millisecond" MILLISECOND = "millisecond"
DEFAULT_STARTTIME int64 = -1 DEFAULT_STARTTIME int64 = -1
DEFAULT_INTERVAL int64 = 1*1000 DEFAULT_INTERVAL int64 = 1 * 1000
DEFAULT_DELAY int64 = -1 DEFAULT_DELAY int64 = -1
DEFAULT_STATISTIC_TABLE = "statistic" DEFAULT_STATISTIC_TABLE = "statistic"
@ -41,7 +42,6 @@ const (
) )
var ( var (
cfg string cfg string
cases string cases string
hnum int hnum int
@ -127,7 +127,7 @@ func init() {
if db == "" { if db == "" {
//db = "go" //db = "go"
db = fmt.Sprintf("test_%s",time.Now().Format("20060102")) db = fmt.Sprintf("test_%s", time.Now().Format("20060102"))
} }
if auto == 1 && len(starttimestr) == 0 { if auto == 1 && len(starttimestr) == 0 {
@ -141,7 +141,7 @@ func init() {
} }
startTime = t.UnixNano() / 1e6 // as millisecond startTime = t.UnixNano() / 1e6 // as millisecond
}else{ } else {
startTime = DEFAULT_STARTTIME startTime = DEFAULT_STARTTIME
} }
@ -179,7 +179,7 @@ func main() {
_, exists := superTableConfigMap[caseConfig.Stname] _, exists := superTableConfigMap[caseConfig.Stname]
if !exists { if !exists {
superTableConfigMap[caseConfig.Stname] = &superTableConfig{config:caseConfig} superTableConfigMap[caseConfig.Stname] = &superTableConfig{config: caseConfig}
} else { } else {
log.Fatalf("the stname of case %s already exist.\n", caseConfig.Stname) log.Fatalf("the stname of case %s already exist.\n", caseConfig.Stname)
} }
@ -259,27 +259,27 @@ func main() {
total := getTotalRows(successRows) total := getTotalRows(successRows)
log.Printf("finished insert %d rows, used %d ms, speed %d rows/s", total, usedTime/1e6, total * 1e9 / int64(usedTime)) log.Printf("finished insert %d rows, used %d ms, speed %d rows/s", total, usedTime/1e6, total*1e3/usedTime.Milliseconds())
if vnum == 0 { if vnum == 0 {
// continue waiting for insert data // continue waiting for insert data
wait := make(chan string) wait := make(chan string)
v := <- wait v := <-wait
log.Printf("program receive %s, exited.\n", v) log.Printf("program receive %s, exited.\n", v)
}else{ } else {
timeTicker.Stop() timeTicker.Stop()
} }
} }
func staticSpeed(){ func staticSpeed() {
connection := getConnection() connection := getConnection()
defer connection.Close() defer connection.Close()
if save == 1 { if save == 1 {
connection.Exec("use " + db) connection.Exec("use " + db)
_, err := connection.Exec("create table if not exists " + saveTable +"(ts timestamp, speed int)") _, err := connection.Exec("create table if not exists " + saveTable + "(ts timestamp, speed int)")
if err != nil { if err != nil {
log.Fatalf("create %s Table error: %s\n", saveTable, err) log.Fatalf("create %s Table error: %s\n", saveTable, err)
} }
@ -308,7 +308,7 @@ func staticSpeed(){
} }
func getTotalRows(successRows []int64) int64{ func getTotalRows(successRows []int64) int64 {
var total int64 = 0 var total int64 = 0
for j := 0; j < len(successRows); j++ { for j := 0; j < len(successRows); j++ {
total += successRows[j] total += successRows[j]
@ -316,18 +316,18 @@ func getTotalRows(successRows []int64) int64{
return total return total
} }
func getSuperTableTimeConfig(fileRows dataRows) (start, cycleTime, avgInterval int64){ func getSuperTableTimeConfig(fileRows dataRows) (start, cycleTime, avgInterval int64) {
if auto == 1 { if auto == 1 {
// use auto generate data time // use auto generate data time
start = startTime start = startTime
avgInterval = interval avgInterval = interval
maxTableRows := normalizationDataWithSameInterval(fileRows, avgInterval) maxTableRows := normalizationDataWithSameInterval(fileRows, avgInterval)
cycleTime = maxTableRows * avgInterval + avgInterval cycleTime = maxTableRows*avgInterval + avgInterval
} else { } else {
// use the sample data primary timestamp // use the sample data primary timestamp
sort.Sort(fileRows)// sort the file data by the primarykey sort.Sort(fileRows) // sort the file data by the primarykey
minTime := getPrimaryKey(fileRows.rows[0][fileRows.config.Timestamp]) minTime := getPrimaryKey(fileRows.rows[0][fileRows.config.Timestamp])
maxTime := getPrimaryKey(fileRows.rows[len(fileRows.rows)-1][fileRows.config.Timestamp]) maxTime := getPrimaryKey(fileRows.rows[len(fileRows.rows)-1][fileRows.config.Timestamp])
@ -340,8 +340,8 @@ func getSuperTableTimeConfig(fileRows dataRows) (start, cycleTime, avgInterval i
if minTime == maxTime { if minTime == maxTime {
avgInterval = interval avgInterval = interval
cycleTime = tableNum * avgInterval + avgInterval cycleTime = tableNum*avgInterval + avgInterval
}else{ } else {
avgInterval = (maxTime - minTime) / int64(len(fileRows.rows)) * tableNum avgInterval = (maxTime - minTime) / int64(len(fileRows.rows)) * tableNum
cycleTime = maxTime - minTime + avgInterval cycleTime = maxTime - minTime + avgInterval
} }
@ -350,11 +350,11 @@ func getSuperTableTimeConfig(fileRows dataRows) (start, cycleTime, avgInterval i
return return
} }
func createStatisticTable(){ func createStatisticTable() {
connection := getConnection() connection := getConnection()
defer connection.Close() defer connection.Close()
_, err := connection.Exec("create table if not exist " + db + "."+ saveTable +"(ts timestamp, speed int)") _, err := connection.Exec("create table if not exist " + db + "." + saveTable + "(ts timestamp, speed int)")
if err != nil { if err != nil {
log.Fatalf("createStatisticTable error: %s\n", err) log.Fatalf("createStatisticTable error: %s\n", err)
} }
@ -389,12 +389,12 @@ func createSubTable(subTableMaps map[string]*dataRows) {
buffers.WriteString(" using ") buffers.WriteString(" using ")
buffers.WriteString(superTableName) buffers.WriteString(superTableName)
buffers.WriteString(" tags(") buffers.WriteString(" tags(")
for _, tag := range subTableMaps[subTableName].config.Tags{ for _, tag := range subTableMaps[subTableName].config.Tags {
tagValue := fmt.Sprintf("%v", tagValues[strings.ToLower(tag.Name)]) tagValue := fmt.Sprintf("%v", tagValues[strings.ToLower(tag.Name)])
buffers.WriteString("'" + tagValue + "'") buffers.WriteString("'" + tagValue + "'")
buffers.WriteString(",") buffers.WriteString(",")
} }
buffers.Truncate(buffers.Len()-1) buffers.Truncate(buffers.Len() - 1)
buffers.WriteString(")") buffers.WriteString(")")
createTableSql := buffers.String() createTableSql := buffers.String()
@ -451,14 +451,14 @@ func createSuperTable(superTableConfigMap map[string]*superTableConfig) {
buffer.WriteString(field.Name + " " + field.Type + ",") buffer.WriteString(field.Name + " " + field.Type + ",")
} }
buffer.Truncate(buffer.Len()-1) buffer.Truncate(buffer.Len() - 1)
buffer.WriteString(") tags( ") buffer.WriteString(") tags( ")
for _, tag := range superTableConf.config.Tags { for _, tag := range superTableConf.config.Tags {
buffer.WriteString(tag.Name + " " + tag.Type + ",") buffer.WriteString(tag.Name + " " + tag.Type + ",")
} }
buffer.Truncate(buffer.Len()-1) buffer.Truncate(buffer.Len() - 1)
buffer.WriteString(")") buffer.WriteString(")")
createSql := buffer.String() createSql := buffer.String()
@ -477,14 +477,13 @@ func getScaleSubTableName(subTableName string, hnum int) string {
if hnum == 0 { if hnum == 0 {
return subTableName return subTableName
} }
return fmt.Sprintf( "%s_%d", subTableName, hnum) return fmt.Sprintf("%s_%d", subTableName, hnum)
} }
func getSuperTableName(stname string) string { func getSuperTableName(stname string) string {
return SUPERTABLE_PREFIX + stname return SUPERTABLE_PREFIX + stname
} }
/** /**
* normalizationData , and return the num of subTables * normalizationData , and return the num of subTables
*/ */
@ -510,7 +509,7 @@ func normalizationData(fileRows dataRows, minTime int64) int64 {
} }
tableNum++ tableNum++
}else{ } else {
value.rows = append(value.rows, row) value.rows = append(value.rows, row)
} }
} }
@ -518,7 +517,7 @@ func normalizationData(fileRows dataRows, minTime int64) int64 {
} }
// return the maximum table rows // return the maximum table rows
func normalizationDataWithSameInterval(fileRows dataRows, avgInterval int64) int64{ func normalizationDataWithSameInterval(fileRows dataRows, avgInterval int64) int64 {
// subTableMap // subTableMap
currSubTableMap := make(map[string]*dataRows) currSubTableMap := make(map[string]*dataRows)
for _, row := range fileRows.rows { for _, row := range fileRows.rows {
@ -537,7 +536,7 @@ func normalizationDataWithSameInterval(fileRows dataRows, avgInterval int64) int
rows: []map[string]interface{}{row}, rows: []map[string]interface{}{row},
config: fileRows.config, config: fileRows.config,
} }
}else{ } else {
row[fileRows.config.Timestamp] = int64(len(value.rows)) * avgInterval row[fileRows.config.Timestamp] = int64(len(value.rows)) * avgInterval
value.rows = append(value.rows, row) value.rows = append(value.rows, row)
} }
@ -545,7 +544,7 @@ func normalizationDataWithSameInterval(fileRows dataRows, avgInterval int64) int
} }
var maxRows, tableRows int = 0, 0 var maxRows, tableRows int = 0, 0
for tableName := range currSubTableMap{ for tableName := range currSubTableMap {
tableRows = len(currSubTableMap[tableName].rows) tableRows = len(currSubTableMap[tableName].rows)
subTableMap[tableName] = currSubTableMap[tableName] // add to global subTableMap subTableMap[tableName] = currSubTableMap[tableName] // add to global subTableMap
if tableRows > maxRows { if tableRows > maxRows {
@ -556,12 +555,10 @@ func normalizationDataWithSameInterval(fileRows dataRows, avgInterval int64) int
return int64(maxRows) return int64(maxRows)
} }
func getSubTableName(subTableValue string, superTableName string) string { func getSubTableName(subTableValue string, superTableName string) string {
return SUBTABLE_PREFIX + subTableValue + "_" + superTableName return SUBTABLE_PREFIX + subTableValue + "_" + superTableName
} }
func insertData(threadIndex, start, end int, wg *sync.WaitGroup, successRows []int64) { func insertData(threadIndex, start, end int, wg *sync.WaitGroup, successRows []int64) {
connection := getConnection() connection := getConnection()
defer connection.Close() defer connection.Close()
@ -591,9 +588,9 @@ func insertData(threadIndex, start, end int, wg *sync.WaitGroup, successRows []
var tableEndTime int64 var tableEndTime int64
if vnum == 0 { if vnum == 0 {
// need continue generate data // need continue generate data
tableEndTime = time.Now().UnixNano()/1e6 tableEndTime = time.Now().UnixNano() / 1e6
}else { } else {
tableEndTime = tableStartTime + superTableConf.cycleTime * int64(vnum) - superTableConf.avgInterval tableEndTime = tableStartTime + superTableConf.cycleTime*int64(vnum) - superTableConf.avgInterval
} }
insertRows := scaleTableMap[tableName].insertRows insertRows := scaleTableMap[tableName].insertRows
@ -603,7 +600,7 @@ func insertData(threadIndex, start, end int, wg *sync.WaitGroup, successRows []
rowIndex := insertRows % subTableRows rowIndex := insertRows % subTableRows
currentRow := subTableInfo.rows[rowIndex] currentRow := subTableInfo.rows[rowIndex]
currentTime := getPrimaryKey(currentRow[subTableInfo.config.Timestamp]) + loopNum * superTableConf.cycleTime + tableStartTime currentTime := getPrimaryKey(currentRow[subTableInfo.config.Timestamp]) + loopNum*superTableConf.cycleTime + tableStartTime
if currentTime <= tableEndTime { if currentTime <= tableEndTime {
// append // append
@ -618,7 +615,7 @@ func insertData(threadIndex, start, end int, wg *sync.WaitGroup, successRows []
buffers.WriteString(",") buffers.WriteString(",")
// fieldNum := len(subTableInfo.config.Fields) // fieldNum := len(subTableInfo.config.Fields)
for _,field := range subTableInfo.config.Fields { for _, field := range subTableInfo.config.Fields {
buffers.WriteString(getFieldValue(currentRow[strings.ToLower(field.Name)])) buffers.WriteString(getFieldValue(currentRow[strings.ToLower(field.Name)]))
buffers.WriteString(",") buffers.WriteString(",")
// if( i != fieldNum -1){ // if( i != fieldNum -1){
@ -626,7 +623,7 @@ func insertData(threadIndex, start, end int, wg *sync.WaitGroup, successRows []
// } // }
} }
buffers.Truncate(buffers.Len()-1) buffers.Truncate(buffers.Len() - 1)
buffers.WriteString(") ") buffers.WriteString(") ")
appendRows++ appendRows++
@ -645,7 +642,7 @@ func insertData(threadIndex, start, end int, wg *sync.WaitGroup, successRows []
lastTableName = "" lastTableName = ""
appendRows = 0 appendRows = 0
} }
}else { } else {
// finished insert current table // finished insert current table
break break
} }
@ -676,7 +673,7 @@ func insertData(threadIndex, start, end int, wg *sync.WaitGroup, successRows []
break break
} }
if(num == 0){ if num == 0 {
wg.Done() //finished insert history data wg.Done() //finished insert history data
num++ num++
} }
@ -691,7 +688,7 @@ func insertData(threadIndex, start, end int, wg *sync.WaitGroup, successRows []
} }
func buildSql(rows []tableRows) string{ func buildSql(rows []tableRows) string {
var lastTableName string var lastTableName string
@ -709,7 +706,7 @@ func buildSql(rows []tableRows) string{
if lastTableName == row.tableName { if lastTableName == row.tableName {
buffers.WriteString(row.value) buffers.WriteString(row.value)
}else { } else {
buffers.WriteString(" ") buffers.WriteString(" ")
buffers.WriteString(row.tableName) buffers.WriteString(row.tableName)
buffers.WriteString(" values") buffers.WriteString(" values")
@ -722,7 +719,7 @@ func buildSql(rows []tableRows) string{
return inserSql return inserSql
} }
func buildRow(tableName string, currentTime int64, subTableInfo *dataRows, currentRow map[string]interface{}) tableRows{ func buildRow(tableName string, currentTime int64, subTableInfo *dataRows, currentRow map[string]interface{}) tableRows {
tableRows := tableRows{tableName: tableName} tableRows := tableRows{tableName: tableName}
@ -732,12 +729,12 @@ func buildRow(tableName string, currentTime int64, subTableInfo *dataRows, curre
buffers.WriteString(fmt.Sprintf("%v", currentTime)) buffers.WriteString(fmt.Sprintf("%v", currentTime))
buffers.WriteString(",") buffers.WriteString(",")
for _,field := range subTableInfo.config.Fields { for _, field := range subTableInfo.config.Fields {
buffers.WriteString(getFieldValue(currentRow[strings.ToLower(field.Name)])) buffers.WriteString(getFieldValue(currentRow[strings.ToLower(field.Name)]))
buffers.WriteString(",") buffers.WriteString(",")
} }
buffers.Truncate(buffers.Len()-1) buffers.Truncate(buffers.Len() - 1)
buffers.WriteString(")") buffers.WriteString(")")
insertSql := buffers.String() insertSql := buffers.String()
@ -764,7 +761,7 @@ func getFieldValue(fieldValue interface{}) string {
return fmt.Sprintf("'%v'", fieldValue) return fmt.Sprintf("'%v'", fieldValue)
} }
func getConnection() *sql.DB{ func getConnection() *sql.DB {
db, err := sql.Open(DRIVER_NAME, dataSourceName) db, err := sql.Open(DRIVER_NAME, dataSourceName)
if err != nil { if err != nil {
panic(err) panic(err)
@ -772,7 +769,6 @@ func getConnection() *sql.DB{
return db return db
} }
func getSubTableNameValue(suffix interface{}) string { func getSubTableNameValue(suffix interface{}) string {
return fmt.Sprintf("%v", suffix) return fmt.Sprintf("%v", suffix)
} }
@ -950,7 +946,7 @@ func parseMillisecond(str interface{}, layout string) int64 {
log.Println(err) log.Println(err)
return -1 return -1
} }
return t.UnixNano()/1e6 return t.UnixNano() / 1e6
} }
// lowerMapKey transfer all the map key to lowercase // lowerMapKey transfer all the map key to lowercase
@ -1009,7 +1005,7 @@ func checkUserCaseConfig(caseName string, caseConfig *dataimport.CaseConfig) {
if i < len(caseConfig.Fields)-1 { if i < len(caseConfig.Fields)-1 {
// delete middle item, a = a[:i+copy(a[i:], a[i+1:])] // delete middle item, a = a[:i+copy(a[i:], a[i+1:])]
caseConfig.Fields = caseConfig.Fields[:i+copy(caseConfig.Fields[i:], caseConfig.Fields[i+1:])] caseConfig.Fields = caseConfig.Fields[:i+copy(caseConfig.Fields[i:], caseConfig.Fields[i+1:])]
}else { } else {
// delete the last item // delete the last item
caseConfig.Fields = caseConfig.Fields[:len(caseConfig.Fields)-1] caseConfig.Fields = caseConfig.Fields[:len(caseConfig.Fields)-1]
} }

6
src/client/inc/tscUtil.h
View File

@ -263,10 +263,12 @@ int16_t tscGetJoinTagColIdByUid(STagCond* pTagCond, uint64_t uid);
void tscPrintSelectClause(SSqlObj* pSql, int32_t subClauseIndex); void tscPrintSelectClause(SSqlObj* pSql, int32_t subClauseIndex);
bool hasMoreVnodesToTry(SSqlObj *pSql); bool hasMoreVnodesToTry(SSqlObj *pSql);
bool hasMoreClauseToTry(SSqlObj* pSql);
void tscTryQueryNextVnode(SSqlObj *pSql, __async_cb_func_t fp); void tscTryQueryNextVnode(SSqlObj *pSql, __async_cb_func_t fp);
void tscAsyncQuerySingleRowForNextVnode(void *param, TAOS_RES *tres, int numOfRows); void tscAsyncQuerySingleRowForNextVnode(void *param, TAOS_RES *tres, int numOfRows);
void tscTryQueryNextClause(SSqlObj* pSql, void (*queryFp)()); void tscTryQueryNextClause(SSqlObj* pSql, __async_cb_func_t fp);
int tscSetMgmtIpListFromCfg(const char *first, const char *second); int tscSetMgmtEpSetFromCfg(const char *first, const char *second);
void* malloc_throw(size_t size); void* malloc_throw(size_t size);
void* calloc_throw(size_t nmemb, size_t size); void* calloc_throw(size_t nmemb, size_t size);

8
src/client/inc/tsclient.h
View File

@ -314,7 +314,7 @@ typedef struct SSqlObj {
char * sqlstr; char * sqlstr;
char retry; char retry;
char maxRetry; char maxRetry;
SRpcIpSet ipList; SRpcEpSet epSet;
char listed; char listed;
tsem_t rspSem; tsem_t rspSem;
SSqlCmd cmd; SSqlCmd cmd;
@ -358,7 +358,7 @@ void tscInitMsgsFp();
int tsParseSql(SSqlObj *pSql, bool initial); int tsParseSql(SSqlObj *pSql, bool initial);
void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet); void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet);
int tscProcessSql(SSqlObj *pSql); int tscProcessSql(SSqlObj *pSql);
int tscRenewTableMeta(SSqlObj *pSql, char *tableId); int tscRenewTableMeta(SSqlObj *pSql, char *tableId);
@ -430,7 +430,6 @@ static FORCE_INLINE void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pField
int32_t bytes = pInfo->pSqlExpr->resBytes; int32_t bytes = pInfo->pSqlExpr->resBytes;
char* pData = pRes->data + pInfo->pSqlExpr->offset * pRes->numOfRows + bytes * pRes->row; char* pData = pRes->data + pInfo->pSqlExpr->offset * pRes->numOfRows + bytes * pRes->row;
if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BINARY) { if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BINARY) {
int32_t realLen = varDataLen(pData); int32_t realLen = varDataLen(pData);
assert(realLen <= bytes - VARSTR_HEADER_SIZE); assert(realLen <= bytes - VARSTR_HEADER_SIZE);
@ -465,7 +464,8 @@ extern void * tscQhandle;
extern int tscKeepConn[]; extern int tscKeepConn[];
extern int tsInsertHeadSize; extern int tsInsertHeadSize;
extern int tscNumOfThreads; extern int tscNumOfThreads;
extern SRpcCorIpSet tscMgmtIpSet;
extern SRpcCorEpSet tscMgmtEpSet;
extern int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo); extern int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo);

27
src/client/src/tscAsync.c
View File

@ -169,7 +169,11 @@ static void tscProcessAsyncRetrieveImpl(void *param, TAOS_RES *tres, int numOfRo
pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH;
} }
if (pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE) {
tscFetchDatablockFromSubquery(pSql);
} else {
tscProcessSql(pSql); tscProcessSql(pSql);
}
} }
/* /*
@ -474,9 +478,8 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
// in case of insert, redo parsing the sql string and build new submit data block for two reasons: // in case of insert, redo parsing the sql string and build new submit data block for two reasons:
// 1. the table Id(tid & uid) may have been update, the submit block needs to be updated accordingly. // 1. the table Id(tid & uid) may have been update, the submit block needs to be updated accordingly.
// 2. vnode may need the schema information along with submit block to update its local table schema. // 2. vnode may need the schema information along with submit block to update its local table schema.
if (pCmd->command == TSDB_SQL_INSERT) { if (pCmd->command == TSDB_SQL_INSERT || pCmd->command == TSDB_SQL_SELECT) {
tscDebug("%p redo parse sql string to build submit block", pSql); tscDebug("%p redo parse sql string and proceed", pSql);
pCmd->parseFinished = false; pCmd->parseFinished = false;
tscResetSqlCmdObj(pCmd); tscResetSqlCmdObj(pCmd);
@ -488,6 +491,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
goto _error; goto _error;
} }
if (pCmd->command == TSDB_SQL_INSERT) {
/* /*
* Discard previous built submit blocks, and then parse the sql string again and build up all submit blocks, * Discard previous built submit blocks, and then parse the sql string again and build up all submit blocks,
* and send the required submit block according to index value in supporter to server. * and send the required submit block according to index value in supporter to server.
@ -495,22 +499,9 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
pSql->fp = pSql->fetchFp; // restore the fp pSql->fp = pSql->fetchFp; // restore the fp
tscHandleInsertRetry(pSql); tscHandleInsertRetry(pSql);
} else if (pCmd->command == TSDB_SQL_SELECT) { // in case of other query type, continue } else if (pCmd->command == TSDB_SQL_SELECT) { // in case of other query type, continue
tscDebug("%p redo parse sql string and proceed", pSql);
//tscDebug("before %p fp:%p, fetchFp:%p", pSql, pSql->fp, pSql->fetchFp);
pCmd->parseFinished = false;
tscResetSqlCmdObj(pCmd);
//tscDebug("after %p fp:%p, fetchFp:%p", pSql, pSql->fp, pSql->fetchFp);
code = tsParseSql(pSql, true);
if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
return;
} else if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
tscProcessSql(pSql); tscProcessSql(pSql);
} else { // in all other cases, simple retry }
}else { // in all other cases, simple retry
tscProcessSql(pSql); tscProcessSql(pSql);
} }

20
src/client/src/tscFunctionImpl.c
View File

@ -1481,7 +1481,7 @@ static bool first_last_function_setup(SQLFunctionCtx *pCtx) {
// todo opt for null block // todo opt for null block
static void first_function(SQLFunctionCtx *pCtx) { static void first_function(SQLFunctionCtx *pCtx) {
if (pCtx->order == TSDB_ORDER_DESC) { if (pCtx->order == TSDB_ORDER_DESC || pCtx->preAggVals.dataBlockLoaded == false) {
return; return;
} }
@ -1550,28 +1550,17 @@ static void first_data_assign_impl(SQLFunctionCtx *pCtx, char *pData, int32_t in
* to decide if the value is earlier than current intermediate result * to decide if the value is earlier than current intermediate result
*/ */
static void first_dist_function(SQLFunctionCtx *pCtx) { static void first_dist_function(SQLFunctionCtx *pCtx) {
assert(pCtx->size > 0);
if (pCtx->size == 0) {
return;
}
/* /*
* do not to check data in the following cases: * do not to check data in the following cases:
* 1. data block that are not loaded * 1. data block that are not loaded
* 2. scan data files in desc order * 2. scan data files in desc order
*/ */
if (pCtx->order == TSDB_ORDER_DESC) { if (pCtx->order == TSDB_ORDER_DESC || pCtx->preAggVals.dataBlockLoaded == false) {
return; return;
} }
int32_t notNullElems = 0; int32_t notNullElems = 0;
// data block is discard, not loaded, do not need to check it
if (!pCtx->preAggVals.dataBlockLoaded) {
return;
}
// find the first not null value // find the first not null value
for (int32_t i = 0; i < pCtx->size; ++i) { for (int32_t i = 0; i < pCtx->size; ++i) {
char *data = GET_INPUT_CHAR_INDEX(pCtx, i); char *data = GET_INPUT_CHAR_INDEX(pCtx, i);
@ -1655,7 +1644,7 @@ static void first_dist_func_second_merge(SQLFunctionCtx *pCtx) {
* least one data in this block that is not null.(TODO opt for this case) * least one data in this block that is not null.(TODO opt for this case)
*/ */
static void last_function(SQLFunctionCtx *pCtx) { static void last_function(SQLFunctionCtx *pCtx) {
if (pCtx->order != pCtx->param[0].i64Key) { if (pCtx->order != pCtx->param[0].i64Key || pCtx->preAggVals.dataBlockLoaded == false) {
return; return;
} }
@ -2303,8 +2292,9 @@ static void top_func_second_merge(SQLFunctionCtx *pCtx) {
// the intermediate result is binary, we only use the output data type // the intermediate result is binary, we only use the output data type
for (int32_t i = 0; i < pInput->num; ++i) { for (int32_t i = 0; i < pInput->num; ++i) {
int16_t type = (pCtx->outputType == TSDB_DATA_TYPE_FLOAT)? TSDB_DATA_TYPE_DOUBLE:pCtx->outputType;
do_top_function_add(pOutput, pCtx->param[0].i64Key, &pInput->res[i]->v.i64Key, pInput->res[i]->timestamp, do_top_function_add(pOutput, pCtx->param[0].i64Key, &pInput->res[i]->v.i64Key, pInput->res[i]->timestamp,
pCtx->outputType, &pCtx->tagInfo, pInput->res[i]->pTags, pCtx->currentStage); type, &pCtx->tagInfo, pInput->res[i]->pTags, pCtx->currentStage);
} }
SET_VAL(pCtx, pInput->num, pOutput->num); SET_VAL(pCtx, pInput->num, pOutput->num);

40
src/client/src/tscSQLParser.c
View File

@ -485,7 +485,6 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) {
} }
case TSDB_SQL_SELECT: { case TSDB_SQL_SELECT: {
assert(pCmd->numOfClause == 1);
const char* msg1 = "columns in select clause not identical"; const char* msg1 = "columns in select clause not identical";
for (int32_t i = pCmd->numOfClause; i < pInfo->subclauseInfo.numOfClause; ++i) { for (int32_t i = pCmd->numOfClause; i < pInfo->subclauseInfo.numOfClause; ++i) {
@ -496,16 +495,19 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) {
} }
assert(pCmd->numOfClause == pInfo->subclauseInfo.numOfClause); assert(pCmd->numOfClause == pInfo->subclauseInfo.numOfClause);
for (int32_t i = 0; i < pInfo->subclauseInfo.numOfClause; ++i) { for (int32_t i = pCmd->clauseIndex; i < pInfo->subclauseInfo.numOfClause; ++i) {
SQuerySQL* pQuerySql = pInfo->subclauseInfo.pClause[i]; SQuerySQL* pQuerySql = pInfo->subclauseInfo.pClause[i];
tscTrace("%p start to parse %dth subclause, total:%d", pSql, i, pInfo->subclauseInfo.numOfClause);
if ((code = doCheckForQuery(pSql, pQuerySql, i)) != TSDB_CODE_SUCCESS) { if ((code = doCheckForQuery(pSql, pQuerySql, i)) != TSDB_CODE_SUCCESS) {
return code; return code;
} }
tscPrintSelectClause(pSql, i); tscPrintSelectClause(pSql, i);
pCmd->clauseIndex += 1;
} }
// restore the clause index
pCmd->clauseIndex = 0;
// set the command/global limit parameters from the first subclause to the sqlcmd object // set the command/global limit parameters from the first subclause to the sqlcmd object
SQueryInfo* pQueryInfo1 = tscGetQueryInfoDetail(pCmd, 0); SQueryInfo* pQueryInfo1 = tscGetQueryInfoDetail(pCmd, 0);
pCmd->command = pQueryInfo1->command; pCmd->command = pQueryInfo1->command;
@ -1385,6 +1387,11 @@ static int32_t doAddProjectionExprAndResultFields(SQueryInfo* pQueryInfo, SColum
return numOfTotalColumns; return numOfTotalColumns;
} }
static void tscInsertPrimaryTSSourceColumn(SQueryInfo* pQueryInfo, SColumnIndex* pIndex) {
SColumnIndex tsCol = {.tableIndex = pIndex->tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX};
tscColumnListInsert(pQueryInfo->colList, &tsCol);
}
int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExprItem* pItem) { int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExprItem* pItem) {
const char* msg0 = "invalid column name"; const char* msg0 = "invalid column name";
const char* msg1 = "tag for normal table query is not allowed"; const char* msg1 = "tag for normal table query is not allowed";
@ -1427,6 +1434,8 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t
addProjectQueryCol(pQueryInfo, startPos, &index, pItem); addProjectQueryCol(pQueryInfo, startPos, &index, pItem);
} }
tscInsertPrimaryTSSourceColumn(pQueryInfo, &index);
} else { } else {
return TSDB_CODE_TSC_INVALID_SQL; return TSDB_CODE_TSC_INVALID_SQL;
} }
@ -1499,8 +1508,8 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
switch (optr) { switch (optr) {
case TK_COUNT: { case TK_COUNT: {
if (pItem->pNode->pParam != NULL && pItem->pNode->pParam->nExpr != 1) {
/* more than one parameter for count() function */ /* more than one parameter for count() function */
if (pItem->pNode->pParam != NULL && pItem->pNode->pParam->nExpr != 1) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
} }
@ -1551,11 +1560,12 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
} }
} else { // count(*) is equalled to count(primary_timestamp_key) } else { // count(*) is equalled to count(primary_timestamp_key)
index = (SColumnIndex){0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; index = (SColumnIndex){0, PRIMARYKEY_TIMESTAMP_COL_INDEX};
int32_t size = tDataTypeDesc[TSDB_DATA_TYPE_BIGINT].nSize; int32_t size = tDataTypeDesc[TSDB_DATA_TYPE_BIGINT].nSize;
pExpr = tscSqlExprAppend(pQueryInfo, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, size, false); pExpr = tscSqlExprAppend(pQueryInfo, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, size, false);
} }
pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex);
memset(pExpr->aliasName, 0, tListLen(pExpr->aliasName)); memset(pExpr->aliasName, 0, tListLen(pExpr->aliasName));
getColumnName(pItem, pExpr->aliasName, sizeof(pExpr->aliasName) - 1); getColumnName(pItem, pExpr->aliasName, sizeof(pExpr->aliasName) - 1);
@ -1570,9 +1580,8 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
} }
// the time stamp may be always needed // the time stamp may be always needed
if (index.tableIndex > 0 && index.tableIndex < tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) { if (index.tableIndex < tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) {
SColumnIndex tsCol = {.tableIndex = index.tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX}; tscInsertPrimaryTSSourceColumn(pQueryInfo, &index);
tscColumnListInsert(pQueryInfo->colList, &tsCol);
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -1683,9 +1692,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
} }
} }
SColumnIndex tsCol = {.tableIndex = index.tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX}; tscInsertPrimaryTSSourceColumn(pQueryInfo, &index);
tscColumnListInsert(pQueryInfo->colList, &tsCol);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
case TK_FIRST: case TK_FIRST:
@ -2348,9 +2355,9 @@ bool validateIpAddress(const char* ip, size_t size) {
strncpy(tmp, ip, size); strncpy(tmp, ip, size);
in_addr_t ipAddr = inet_addr(tmp); in_addr_t epAddr = inet_addr(tmp);
return ipAddr != INADDR_NONE; return epAddr != INADDR_NONE;
} }
int32_t tscTansformSQLFuncForSTableQuery(SQueryInfo* pQueryInfo) { int32_t tscTansformSQLFuncForSTableQuery(SQueryInfo* pQueryInfo) {
@ -5862,6 +5869,8 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) {
pTableMetaInfo = tscAddEmptyMetaInfo(pQueryInfo); pTableMetaInfo = tscAddEmptyMetaInfo(pQueryInfo);
} }
assert(pCmd->clauseIndex == index);
// too many result columns not support order by in query // too many result columns not support order by in query
if (pQuerySql->pSelection->nExpr > TSDB_MAX_COLUMNS) { if (pQuerySql->pSelection->nExpr > TSDB_MAX_COLUMNS) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg8); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg8);
@ -5975,12 +5984,11 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) {
pQueryInfo->window.ekey = pQueryInfo->window.ekey / 1000; pQueryInfo->window.ekey = pQueryInfo->window.ekey / 1000;
} }
} else { // set the time rang } else { // set the time rang
pQueryInfo->window.skey = TSKEY_INITIAL_VAL; pQueryInfo->window = TSWINDOW_INITIALIZER;
pQueryInfo->window.ekey = INT64_MAX;
} }
// user does not specified the query time window, twa is not allowed in such case. // user does not specified the query time window, twa is not allowed in such case.
if ((pQueryInfo->window.skey == 0 || pQueryInfo->window.ekey == INT64_MAX || if ((pQueryInfo->window.skey == INT64_MIN || pQueryInfo->window.ekey == INT64_MAX ||
(pQueryInfo->window.ekey == INT64_MAX / 1000 && tinfo.precision == TSDB_TIME_PRECISION_MILLI)) && tscIsTWAQuery(pQueryInfo)) { (pQueryInfo->window.ekey == INT64_MAX / 1000 && tinfo.precision == TSDB_TIME_PRECISION_MILLI)) && tscIsTWAQuery(pQueryInfo)) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg9); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg9);
} }

118
src/client/src/tscServer.c
View File

@ -30,8 +30,8 @@
#define TSC_MGMT_VNODE 999 #define TSC_MGMT_VNODE 999
SRpcCorIpSet tscMgmtIpSet; SRpcCorEpSet tscMgmtEpSet;
SRpcIpSet tscDnodeIpSet; SRpcEpSet tscDnodeEpSet;
int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo) = {0}; int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo) = {0};
@ -45,18 +45,18 @@ void tscSaveSubscriptionProgress(void* sub);
static int32_t minMsgSize() { return tsRpcHeadSize + 100; } static int32_t minMsgSize() { return tsRpcHeadSize + 100; }
static void tscSetDnodeIpList(SSqlObj* pSql, SCMVgroupInfo* pVgroupInfo) { static void tscSetDnodeEpSet(SSqlObj* pSql, SCMVgroupInfo* pVgroupInfo) {
SRpcIpSet* pIpList = &pSql->ipList; SRpcEpSet* pEpSet = &pSql->epSet;
pIpList->inUse = 0; pEpSet->inUse = 0;
if (pVgroupInfo == NULL) { if (pVgroupInfo == NULL) {
pIpList->numOfIps = 0; pEpSet->numOfEps = 0;
return; return;
} }
pIpList->numOfIps = pVgroupInfo->numOfIps; pEpList->numOfEps = pVgroupInfo->numOfEps;
for(int32_t i = 0; i < pVgroupInfo->numOfIps; ++i) { for(int32_t i = 0; i < pVgroupInfo->numOfEps; ++i) {
strcpy(pIpList->fqdn[i], pVgroupInfo->ipAddr[i].fqdn); strcpy(pEpList->fqdn[i], pVgroupInfo->epAddr[i].fqdn);
pIpList->port[i] = pVgroupInfo->ipAddr[i].port; pEpList->port[i] = pVgroupInfo->epAddr[i].port;
} }
} }
void tscIpSetCopy(SRpcIpSet *dst, SRpcIpSet *src) { void tscIpSetCopy(SRpcIpSet *dst, SRpcIpSet *src) {
@ -118,21 +118,21 @@ static void tscUpdateVgroupInfo(SSqlObj *pObj, SRpcIpSet *pIpSet) {
taosCorBeginWrite(&pVgroupInfo->version); taosCorBeginWrite(&pVgroupInfo->version);
//TODO(dengyihao), dont care vgid //TODO(dengyihao), dont care vgid
pVgroupInfo->inUse = pIpSet->inUse; pVgroupInfo->inUse = pEpSet->inUse;
pVgroupInfo->numOfIps = pIpSet->numOfIps; pVgroupInfo->numOfEps = pEpSet->numOfEps;
for (int32_t i = 0; pVgroupInfo->numOfIps; i++) { for (int32_t i = 0; pVgroupInfo->numOfEps; i++) {
strncpy(pVgroupInfo->ipAddr[i].fqdn, pIpSet->fqdn[i], TSDB_FQDN_LEN); strncpy(pVgroupInfo->epAddr[i].fqdn, pEpSet->fqdn[i], TSDB_FQDN_LEN);
pVgroupInfo->ipAddr[i].port = pIpSet->port[i]; pVgroupInfo->epAddr[i].port = pEpSet->port[i];
} }
taosCorEndWrite(&pVgroupInfo->version); taosCorEndWrite(&pVgroupInfo->version);
} }
void tscPrintMgmtIp() { void tscPrintMgmtEp() {
SRpcIpSet dump; SRpcEpSet dump;
tscDumpMgmtIpSet(&dump); tscDumpMgmtEpSet(&dump);
if (dump.numOfIps <= 0) { if (dump.numOfEps <= 0) {
tscError("invalid mnode IP list:%d", dump.numOfIps); tscError("invalid mnode EP list:%d", dump.numOfEPs);
} else { } else {
for (int i = 0; i < dump.numOfIps; ++i) { for (int i = 0; i < dump.numOfEps; ++i) {
tscDebug("mnode index:%d %s:%d", i, dump.fqdn[i], dump.port[i]); tscDebug("mnode index:%d %s:%d", i, dump.fqdn[i], dump.port[i]);
} }
} }
@ -148,9 +148,9 @@ void tscPrintMgmtIp() {
UNUSED_FUNC UNUSED_FUNC
static int32_t tscGetMgmtConnMaxRetryTimes() { static int32_t tscGetMgmtConnMaxRetryTimes() {
int32_t factor = 2; int32_t factor = 2;
SRpcIpSet dump; SRpcEpSet dump;
tscDumpMgmtIpSet(&dump); tscDumpMgmtEpSet(&dump);
return dump.numOfIps * factor; return dump.numOfEps * factor;
} }
void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) {
@ -166,10 +166,10 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) {
if (code == 0) { if (code == 0) {
SCMHeartBeatRsp *pRsp = (SCMHeartBeatRsp *)pRes->pRsp; SCMHeartBeatRsp *pRsp = (SCMHeartBeatRsp *)pRes->pRsp;
SRpcIpSet * pIpList = &pRsp->ipList; SRpcEpSet * pEpList = &pRsp->epList;
if (pIpList->numOfIps > 0) { if (pEpList->numOfEps > 0) {
tscIpSetHtons(pIpList); tscEpSetHtons(pEpList);
tscUpdateMgmtIpList(pIpList); tscUpdateMgmtEpList(pEpList);
} }
pSql->pTscObj->connId = htonl(pRsp->connId); pSql->pTscObj->connId = htonl(pRsp->connId);
@ -242,7 +242,7 @@ int tscSendMsgToServer(SSqlObj *pSql) {
// set the mgmt ip list // set the mgmt ip list
if (pSql->cmd.command >= TSDB_SQL_MGMT) { if (pSql->cmd.command >= TSDB_SQL_MGMT) {
tscDumpMgmtIpSet(&pSql->ipList); tscDumpMgmtEpSet(&pSql->epList);
} }
memcpy(pMsg, pSql->cmd.payload, pSql->cmd.payloadLen); memcpy(pMsg, pSql->cmd.payload, pSql->cmd.payloadLen);
@ -260,11 +260,11 @@ int tscSendMsgToServer(SSqlObj *pSql) {
// Otherwise, the pSql object may have been released already during the response function, which is // Otherwise, the pSql object may have been released already during the response function, which is
// processMsgFromServer function. In the meanwhile, the assignment of the rpc context to sql object will absolutely // processMsgFromServer function. In the meanwhile, the assignment of the rpc context to sql object will absolutely
// cause crash. // cause crash.
rpcSendRequest(pObj->pDnodeConn, &pSql->ipList, &rpcMsg); rpcSendRequest(pObj->pDnodeConn, &pSql->epSet, &rpcMsg);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet) { void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) {
SSqlObj *pSql = (SSqlObj *)rpcMsg->ahandle; SSqlObj *pSql = (SSqlObj *)rpcMsg->ahandle;
if (pSql == NULL || pSql->signature != pSql) { if (pSql == NULL || pSql->signature != pSql) {
tscError("%p sql is already released", pSql); tscError("%p sql is already released", pSql);
@ -293,14 +293,14 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet) {
return; return;
} }
if (pIpSet) { if (pEpSet) {
//SRpcIpSet dump; //SRpcEpSet dump;
tscIpSetHtons(pIpSet); tscEpSetHtons(pEpSet);
if (tscIpSetIsEqual(&pSql->ipList, pIpSet)) { if (tscEpSetIsEqual(&pSql->epList, pEpSet)) {
if(pCmd->command < TSDB_SQL_MGMT) { if(pCmd->command < TSDB_SQL_MGMT) {
tscUpdateVgroupInfo(pSql, pIpSet); tscUpdateVgroupInfo(pSql, pEpSet);
} else { } else {
tscUpdateMgmtIpList(pIpSet); tscUpdateMgmtEpList(pEpSet);
} }
} }
} }
@ -484,8 +484,9 @@ int tscProcessSql(SSqlObj *pSql) {
return pSql->res.code; return pSql->res.code;
} }
} else if (pCmd->command < TSDB_SQL_LOCAL) { } else if (pCmd->command < TSDB_SQL_LOCAL) {
//tscDumpMgmtIpSet(&pSql->ipList);
} else { //pSql->epSet = tscMgmtEpSet;
} else { // local handler
return (*tscProcessMsgRsp[pCmd->command])(pSql); return (*tscProcessMsgRsp[pCmd->command])(pSql);
} }
@ -538,7 +539,7 @@ int tscBuildFetchMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SRetrieveTableMsg *pRetrieveMsg = (SRetrieveTableMsg *) pSql->cmd.payload; SRetrieveTableMsg *pRetrieveMsg = (SRetrieveTableMsg *) pSql->cmd.payload;
pRetrieveMsg->qhandle = htobe64(pSql->res.qhandle); pRetrieveMsg->qhandle = htobe64(pSql->res.qhandle);
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, pSql->cmd.clauseIndex);
pRetrieveMsg->free = htons(pQueryInfo->type); pRetrieveMsg->free = htons(pQueryInfo->type);
// todo valid the vgroupId at the client side // todo valid the vgroupId at the client side
@ -588,10 +589,10 @@ int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
// pSql->cmd.payloadLen is set during copying data into payload // pSql->cmd.payloadLen is set during copying data into payload
pSql->cmd.msgType = TSDB_MSG_TYPE_SUBMIT; pSql->cmd.msgType = TSDB_MSG_TYPE_SUBMIT;
tscDumpIpSetFromVgroupInfo(&pTableMeta->corVgroupInfo, &pSql->ipList); tscDumpEpSetFromVgroupInfo(&pTableMeta->corVgroupInfo, &pSql->epList);
tscDebug("%p build submit msg, vgId:%d numOfTables:%d numberOfIP:%d", pSql, vgId, pSql->cmd.numOfTablesInSubmit, tscDebug("%p build submit msg, vgId:%d numOfTables:%d numberOfEP:%d", pSql, vgId, pSql->cmd.numOfTablesInSubmit,
pSql->ipList.numOfIps); pSql->epSet.numOfEps);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -630,7 +631,8 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char
} else { } else {
pVgroupInfo = &pTableMeta->vgroupInfo; pVgroupInfo = &pTableMeta->vgroupInfo;
} }
tscSetDnodeIpList(pSql, pVgroupInfo); tscSetDnodeEpList(pSql, pVgroupInfo);
if (pVgroupInfo != NULL) { if (pVgroupInfo != NULL) {
pQueryMsg->head.vgId = htonl(pVgroupInfo->vgId); pQueryMsg->head.vgId = htonl(pVgroupInfo->vgId);
} }
@ -642,7 +644,7 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char
pQueryMsg->numOfTables = htonl(1); // set the number of tables pQueryMsg->numOfTables = htonl(1); // set the number of tables
pMsg += sizeof(STableIdInfo); pMsg += sizeof(STableIdInfo);
} else { // it is a subquery of the super table query, this IP info is acquired from vgroupInfo } else { // it is a subquery of the super table query, this EP info is acquired from vgroupInfo
int32_t index = pTableMetaInfo->vgroupIndex; int32_t index = pTableMetaInfo->vgroupIndex;
int32_t numOfVgroups = taosArrayGetSize(pTableMetaInfo->pVgroupTables); int32_t numOfVgroups = taosArrayGetSize(pTableMetaInfo->pVgroupTables);
assert(index >= 0 && index < numOfVgroups); assert(index >= 0 && index < numOfVgroups);
@ -652,7 +654,7 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char
SVgroupTableInfo* pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, index); SVgroupTableInfo* pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, index);
// set the vgroup info // set the vgroup info
tscSetDnodeIpList(pSql, &pTableIdList->vgInfo); tscSetDnodeEpList(pSql, &pTableIdList->vgInfo);
pQueryMsg->head.vgId = htonl(pTableIdList->vgInfo.vgId); pQueryMsg->head.vgId = htonl(pTableIdList->vgInfo.vgId);
int32_t numOfTables = taosArrayGetSize(pTableIdList->itemList); int32_t numOfTables = taosArrayGetSize(pTableIdList->itemList);
@ -1385,7 +1387,7 @@ int tscBuildUpdateTagMsg(SSqlObj* pSql, SSqlInfo *pInfo) {
SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
tscDumpIpSetFromVgroupInfo(&pTableMetaInfo->pTableMeta->corVgroupInfo, &pSql->ipList); tscDumpEpSetFromVgroupInfo(&pTableMetaInfo->pTableMeta->corVgroupInfo, &pSql->epList);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -1720,8 +1722,8 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) {
pMetaMsg->contLen = htons(pMetaMsg->contLen); pMetaMsg->contLen = htons(pMetaMsg->contLen);
pMetaMsg->numOfColumns = htons(pMetaMsg->numOfColumns); pMetaMsg->numOfColumns = htons(pMetaMsg->numOfColumns);
if (pMetaMsg->sid < 0 || pMetaMsg->vgroup.numOfIps < 0) { if (pMetaMsg->sid < 0 || pMetaMsg->vgroup.numOfEps < 0) {
tscError("invalid meter vgId:%d, sid%d", pMetaMsg->vgroup.numOfIps, pMetaMsg->sid); tscError("invalid meter vgId:%d, sid%d", pMetaMsg->vgroup.numOfEps, pMetaMsg->sid);
return TSDB_CODE_TSC_INVALID_VALUE; return TSDB_CODE_TSC_INVALID_VALUE;
} }
@ -1735,8 +1737,8 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) {
return TSDB_CODE_TSC_INVALID_VALUE; return TSDB_CODE_TSC_INVALID_VALUE;
} }
for (int i = 0; i < pMetaMsg->vgroup.numOfIps; ++i) { for (int i = 0; i < pMetaMsg->vgroup.numOfEps; ++i) {
pMetaMsg->vgroup.ipAddr[i].port = htons(pMetaMsg->vgroup.ipAddr[i].port); pMetaMsg->vgroup.epAddr[i].port = htons(pMetaMsg->vgroup.epAddr[i].port);
} }
SSchema* pSchema = pMetaMsg->schema; SSchema* pSchema = pMetaMsg->schema;
@ -1912,9 +1914,11 @@ int tscProcessSTableVgroupRsp(SSqlObj *pSql) {
//just init, no need to lock //just init, no need to lock
SCMVgroupInfo *pVgroups = &pInfo->vgroupList->vgroups[j]; SCMVgroupInfo *pVgroups = &pInfo->vgroupList->vgroups[j];
pVgroups->vgId = htonl(pVgroups->vgId); pVgroups->vgId = htonl(pVgroups->vgId);
assert(pVgroups->numOfIps >= 1); assert(pVgroups->numOfEps >= 1);
for (int32_t k = 0; k < pVgroups->numOfIps; ++k) {
pVgroups->ipAddr[k].port = htons(pVgroups->ipAddr[k].port); for (int32_t k = 0; k < pVgroups->numOfEps; ++k) {
pVgroups->epAddr[k].port = htons(pVgroups->epAddr[k].port);
} }
pMsg += size; pMsg += size;
@ -2007,9 +2011,9 @@ int tscProcessConnectRsp(SSqlObj *pSql) {
assert(len <= sizeof(pObj->db)); assert(len <= sizeof(pObj->db));
tstrncpy(pObj->db, temp, sizeof(pObj->db)); tstrncpy(pObj->db, temp, sizeof(pObj->db));
if (pConnect->ipList.numOfIps > 0) { if (pConnect->epList.numOfEps > 0) {
tscIpSetHtons(&pConnect->ipList); tscEpSetHtons(&pConnect->epList);
tscUpdateMgmtIpList(&pConnect->ipList); tscUpdateMgmtEpList(&pConnect->epList);
} }
strcpy(pObj->sversion, pConnect->serverVersion); strcpy(pObj->sversion, pConnect->serverVersion);

6
src/client/src/tscSql.c
View File

@ -62,8 +62,8 @@ SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con
} }
if (ip) { if (ip) {
if (tscSetMgmtIpListFromCfg(ip, NULL) < 0) return NULL; if (tscSetMgmtEpListFromCfg(ip, NULL) < 0) return NULL;
if (port) tscMgmtIpSet.ipSet.port[0] = port; if (port) tscMgmtEpSet.epSet.port[0] = port;
} }
void *pDnodeConn = NULL; void *pDnodeConn = NULL;
@ -424,7 +424,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) {
} }
// current data set are exhausted, fetch more data from node // current data set are exhausted, fetch more data from node
if (pRes->row >= pRes->numOfRows && (pRes->completed != true || hasMoreVnodesToTry(pSql)) && if (pRes->row >= pRes->numOfRows && (pRes->completed != true || hasMoreVnodesToTry(pSql) || hasMoreClauseToTry(pSql)) &&
(pCmd->command == TSDB_SQL_RETRIEVE || (pCmd->command == TSDB_SQL_RETRIEVE ||
pCmd->command == TSDB_SQL_RETRIEVE_LOCALMERGE || pCmd->command == TSDB_SQL_RETRIEVE_LOCALMERGE ||
pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE || pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE ||

192
src/client/src/tscSubquery.c
View File

@ -458,7 +458,7 @@ void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArr
break; break;
} }
} }
assert(info.vgInfo.numOfIps != 0); assert(info.vgInfo.numOfEps != 0);
vgTables = taosArrayInit(4, sizeof(STableIdInfo)); vgTables = taosArrayInit(4, sizeof(STableIdInfo));
info.itemList = vgTables; info.itemList = vgTables;
@ -848,13 +848,14 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR
SSqlRes* pRes = &pSql->res; SSqlRes* pRes = &pSql->res;
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
// TODO put to async res?
if (taos_errno(pSql) != TSDB_CODE_SUCCESS) { if (taos_errno(pSql) != TSDB_CODE_SUCCESS) {
assert(numOfRows == taos_errno(pSql)); assert(numOfRows == taos_errno(pSql));
pParentSql->res.code = numOfRows; pParentSql->res.code = numOfRows;
tscError("%p retrieve failed, index:%d, code:%s", pSql, pSupporter->subqueryIndex, tstrerror(numOfRows)); tscError("%p retrieve failed, index:%d, code:%s", pSql, pSupporter->subqueryIndex, tstrerror(numOfRows));
tscQueueAsyncRes(pParentSql);
return;
} }
if (numOfRows >= 0) { if (numOfRows >= 0) {
@ -941,14 +942,7 @@ void tscFetchDatablockFromSubquery(SSqlObj* pSql) {
SSqlRes *pRes = &pSub->res; SSqlRes *pRes = &pSub->res;
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSub->cmd, 0); SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSub->cmd, 0);
// STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
// if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) {
// if (pRes->row >= pRes->numOfRows && pTableMetaInfo->vgroupIndex < pTableMetaInfo->vgroupList->numOfVgroups &&
// (!tscHasReachLimitation(pQueryInfo, pRes)) && !pRes->completed) {
// numOfFetch++;
// }
// } else {
if (!tscHasReachLimitation(pQueryInfo, pRes)) { if (!tscHasReachLimitation(pQueryInfo, pRes)) {
if (pRes->row >= pRes->numOfRows) { if (pRes->row >= pRes->numOfRows) {
hasData = false; hasData = false;
@ -963,9 +957,7 @@ void tscFetchDatablockFromSubquery(SSqlObj* pSql) {
break; break;
} }
} }
} }
// }
// has data remains in client side, and continue to return data to app // has data remains in client side, and continue to return data to app
if (hasData) { if (hasData) {
@ -1026,7 +1018,7 @@ void tscSetupOutputColumnIndex(SSqlObj* pSql) {
SSqlCmd* pCmd = &pSql->cmd; SSqlCmd* pCmd = &pSql->cmd;
SSqlRes* pRes = &pSql->res; SSqlRes* pRes = &pSql->res;
tscDebug("%p all subquery response, retrieve data", pSql); tscDebug("%p all subquery response, retrieve data for subclause:%d", pSql, pCmd->clauseIndex);
// the column transfer support struct has been built // the column transfer support struct has been built
if (pRes->pColumnIndex != NULL) { if (pRes->pColumnIndex != NULL) {
@ -1195,7 +1187,10 @@ int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter
pNew->cmd.numOfCols = 0; pNew->cmd.numOfCols = 0;
pNewQueryInfo->intervalTime = 0; pNewQueryInfo->intervalTime = 0;
memset(&pNewQueryInfo->limit, 0, sizeof(SLimitVal)); pSupporter->limit = pNewQueryInfo->limit;
pNewQueryInfo->limit.limit = -1;
pNewQueryInfo->limit.offset = 0;
// backup the data and clear it in the sqlcmd object // backup the data and clear it in the sqlcmd object
pSupporter->groupbyExpr = pNewQueryInfo->groupbyExpr; pSupporter->groupbyExpr = pNewQueryInfo->groupbyExpr;
@ -1605,8 +1600,8 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p
// data in from current vnode is stored in cache and disk // data in from current vnode is stored in cache and disk
uint32_t numOfRowsFromSubquery = trsupport->pExtMemBuffer[idx]->numOfTotalElems + trsupport->localBuffer->num; uint32_t numOfRowsFromSubquery = trsupport->pExtMemBuffer[idx]->numOfTotalElems + trsupport->localBuffer->num;
tscDebug("%p sub:%p all data retrieved from ip:%s, vgId:%d, numOfRows:%d, orderOfSub:%d", pParentSql, pSql, tscDebug("%p sub:%p all data retrieved from ep:%s, vgId:%d, numOfRows:%d, orderOfSub:%d", pParentSql, pSql,
pTableMetaInfo->vgroupList->vgroups[0].ipAddr[0].fqdn, pTableMetaInfo->vgroupList->vgroups[0].vgId, pTableMetaInfo->vgroupList->vgroups[0].epAddr[0].fqdn, pTableMetaInfo->vgroupList->vgroups[0].vgId,
numOfRowsFromSubquery, idx); numOfRowsFromSubquery, idx);
tColModelCompact(pDesc->pColumnModel, trsupport->localBuffer, pDesc->pColumnModel->capacity); tColModelCompact(pDesc->pColumnModel, trsupport->localBuffer, pDesc->pColumnModel->capacity);
@ -1724,8 +1719,8 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR
assert(pRes->numOfRows == numOfRows); assert(pRes->numOfRows == numOfRows);
int64_t num = atomic_add_fetch_64(&pState->numOfRetrievedRows, numOfRows); int64_t num = atomic_add_fetch_64(&pState->numOfRetrievedRows, numOfRows);
tscDebug("%p sub:%p retrieve numOfRows:%" PRId64 " totalNumOfRows:%" PRIu64 " from ip:%s, orderOfSub:%d", pParentSql, pSql, tscDebug("%p sub:%p retrieve numOfRows:%" PRId64 " totalNumOfRows:%" PRIu64 " from ep:%s, orderOfSub:%d", pParentSql, pSql,
pRes->numOfRows, pState->numOfRetrievedRows, pSql->ipList.fqdn[pSql->ipList.inUse], idx); pRes->numOfRows, pState->numOfRetrievedRows, pSql->epSet.fqdn[pSql->epSet.inUse], idx);
if (num > tsMaxNumOfOrderedResults && tscIsProjectionQueryOnSTable(pQueryInfo, 0)) { if (num > tsMaxNumOfOrderedResults && tscIsProjectionQueryOnSTable(pQueryInfo, 0)) {
tscError("%p sub:%p num of OrderedRes is too many, max allowed:%" PRId32 " , current:%" PRId64, tscError("%p sub:%p num of OrderedRes is too many, max allowed:%" PRId32 " , current:%" PRId64,
@ -1833,8 +1828,8 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) {
return; return;
} }
tscTrace("%p sub:%p query complete, ip:%s, vgId:%d, orderOfSub:%d, retrieve data", trsupport->pParentSql, pSql, tscTrace("%p sub:%p query complete, ep:%s, vgId:%d, orderOfSub:%d, retrieve data", trsupport->pParentSql, pSql,
pVgroup->ipAddr[0].fqdn, pVgroup->vgId, trsupport->subqueryIndex); pVgroup->epAddr[0].fqdn, pVgroup->vgId, trsupport->subqueryIndex);
if (pSql->res.qhandle == 0) { // qhandle is NULL, code is TSDB_CODE_SUCCESS means no results generated from this vnode if (pSql->res.qhandle == 0) { // qhandle is NULL, code is TSDB_CODE_SUCCESS means no results generated from this vnode
tscRetrieveFromDnodeCallBack(param, pSql, 0); tscRetrieveFromDnodeCallBack(param, pSql, 0);
@ -1982,88 +1977,119 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) {
return TSDB_CODE_TSC_OUT_OF_MEMORY; return TSDB_CODE_TSC_OUT_OF_MEMORY;
} }
static char* getResultBlockPosition(SSqlCmd* pCmd, SSqlRes* pRes, int32_t columnIndex, int16_t* bytes) {
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
SFieldSupInfo* pInfo = (SFieldSupInfo*) TARRAY_GET_ELEM(pQueryInfo->fieldsInfo.pSupportInfo, columnIndex);
assert(pInfo->pSqlExpr != NULL);
*bytes = pInfo->pSqlExpr->resBytes;
char* pData = pRes->data + pInfo->pSqlExpr->offset * pRes->numOfRows;
return pData;
}
static void doBuildResFromSubqueries(SSqlObj* pSql) {
SSqlRes* pRes = &pSql->res;
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, pSql->cmd.clauseIndex);
int32_t numOfRes = INT32_MAX;
for (int32_t i = 0; i < pSql->numOfSubs; ++i) {
if (pSql->pSubs[i] == NULL) {
continue;
}
numOfRes = MIN(numOfRes, pSql->pSubs[i]->res.numOfRows);
}
int32_t totalSize = tscGetResRowLength(pQueryInfo->exprList);
pRes->pRsp = realloc(pRes->pRsp, numOfRes * totalSize);
pRes->data = pRes->pRsp;
char* data = pRes->data;
int16_t bytes = 0;
size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo);
for(int32_t i = 0; i < numOfExprs; ++i) {
SColumnIndex* pIndex = &pRes->pColumnIndex[i];
SSqlRes *pRes1 = &pSql->pSubs[pIndex->tableIndex]->res;
SSqlCmd *pCmd1 = &pSql->pSubs[pIndex->tableIndex]->cmd;
char* pData = getResultBlockPosition(pCmd1, pRes1, pIndex->columnIndex, &bytes);
memcpy(data, pData, bytes * numOfRes);
data += bytes * numOfRes;
pRes1->row = numOfRes;
}
pRes->numOfRows = numOfRes;
pRes->numOfClauseTotal += numOfRes;
}
void tscBuildResFromSubqueries(SSqlObj *pSql) { void tscBuildResFromSubqueries(SSqlObj *pSql) {
SSqlRes *pRes = &pSql->res; SSqlRes* pRes = &pSql->res;
if (pRes->code != TSDB_CODE_SUCCESS) { if (pRes->code != TSDB_CODE_SUCCESS) {
tscQueueAsyncRes(pSql); tscQueueAsyncRes(pSql);
return; return;
} }
while (1) {
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, pSql->cmd.clauseIndex);
size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo);
if (pRes->tsrow == NULL) { if (pRes->tsrow == NULL) {
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, pSql->cmd.clauseIndex);
size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo);
pRes->tsrow = calloc(numOfExprs, POINTER_BYTES); pRes->tsrow = calloc(numOfExprs, POINTER_BYTES);
pRes->buffer = calloc(numOfExprs, POINTER_BYTES);
pRes->length = calloc(numOfExprs, sizeof(int32_t)); pRes->length = calloc(numOfExprs, sizeof(int32_t));
tscRestoreSQLFuncForSTableQuery(pQueryInfo);
} }
bool success = false; while (1) {
if (pRes->row < pRes->numOfRows) {
int32_t numOfTableHasRes = 0; assert(0);
for (int32_t i = 0; i < pSql->numOfSubs; ++i) {
if (pSql->pSubs[i] != NULL) {
numOfTableHasRes++;
}
} }
if (numOfTableHasRes >= 2) { // do merge result doBuildResFromSubqueries(pSql);
success = (doSetResultRowData(pSql->pSubs[0], false) != NULL) && (doSetResultRowData(pSql->pSubs[1], false) != NULL);
} else { // only one subquery
SSqlObj *pSub = pSql->pSubs[0];
if (pSub == NULL) {
pSub = pSql->pSubs[1];
}
success = (doSetResultRowData(pSub, false) != NULL);
}
if (success) { // current row of final output has been built, return to app
for (int32_t i = 0; i < numOfExprs; ++i) {
SColumnIndex* pIndex = &pRes->pColumnIndex[i];
SSqlRes *pRes1 = &pSql->pSubs[pIndex->tableIndex]->res;
pRes->tsrow[i] = pRes1->tsrow[pIndex->columnIndex];
pRes->length[i] = pRes1->length[pIndex->columnIndex];
}
pRes->numOfClauseTotal++;
break;
} else { // continue retrieve data from vnode
if (!tscHasRemainDataInSubqueryResultSet(pSql)) {
tscDebug("%p at least one subquery exhausted, free all other %d subqueries", pSql, pSql->numOfSubs - 1);
SSubqueryState *pState = NULL;
// free all sub sqlobj
for (int32_t i = 0; i < pSql->numOfSubs; ++i) {
SSqlObj *pChildObj = pSql->pSubs[i];
if (pChildObj == NULL) {
continue;
}
SJoinSupporter *pSupporter = (SJoinSupporter *)pChildObj->param;
pState = pSupporter->pState;
tscDestroyJoinSupporter(pChildObj->param);
taos_free_result(pChildObj);
}
free(pState);
pRes->completed = true; // set query completed
sem_post(&pSql->rspSem); sem_post(&pSql->rspSem);
return; return;
}
// continue retrieve data from vnode
// if (!tscHasRemainDataInSubqueryResultSet(pSql)) {
// tscDebug("%p at least one subquery exhausted, free all other %d subqueries", pSql, pSql->numOfSubs - 1);
// SSubqueryState* pState = NULL;
//
// // free all sub sqlobj
// for (int32_t i = 0; i < pSql->numOfSubs; ++i) {
// SSqlObj* pChildObj = pSql->pSubs[i];
// if (pChildObj == NULL) {
// continue;
// }
//
// SJoinSupporter* pSupporter = (SJoinSupporter*)pChildObj->param;
// pState = pSupporter->pState;
//
// tscDestroyJoinSupporter(pChildObj->param);
// taos_free_result(pChildObj);
// }
//
// free(pState);
//
// pRes->completed = true; // set query completed
// sem_post(&pSql->rspSem);
// return;
// }
tscFetchDatablockFromSubquery(pSql); tscFetchDatablockFromSubquery(pSql);
if (pRes->code != TSDB_CODE_SUCCESS) { if (pRes->code != TSDB_CODE_SUCCESS) {
return; return;
} }
} }
}
if (pSql->res.code == TSDB_CODE_SUCCESS) { if (pSql->res.code == TSDB_CODE_SUCCESS) {
(*pSql->fp)(pSql->param, pSql, 0); (*pSql->fp)(pSql->param, pSql, pRes->numOfRows);
} else { } else {
tscQueueAsyncRes(pSql); tscQueueAsyncRes(pSql);
} }
@ -2117,14 +2143,6 @@ void **doSetResultRowData(SSqlObj *pSql, bool finalResult) {
assert(pRes->row >= 0 && pRes->row <= pRes->numOfRows); assert(pRes->row >= 0 && pRes->row <= pRes->numOfRows);
if(pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE) {
if (pRes->completed) {
tfree(pRes->tsrow);
}
return pRes->tsrow;
}
if (pRes->row >= pRes->numOfRows) { // all the results has returned to invoker if (pRes->row >= pRes->numOfRows) { // all the results has returned to invoker
tfree(pRes->tsrow); tfree(pRes->tsrow);
return pRes->tsrow; return pRes->tsrow;
@ -2182,7 +2200,7 @@ void **doSetResultRowData(SSqlObj *pSql, bool finalResult) {
return pRes->tsrow; return pRes->tsrow;
} }
static bool tscHasRemainDataInSubqueryResultSet(SSqlObj *pSql) { static UNUSED_FUNC bool tscHasRemainDataInSubqueryResultSet(SSqlObj *pSql) {
bool hasData = true; bool hasData = true;
SSqlCmd *pCmd = &pSql->cmd; SSqlCmd *pCmd = &pSql->cmd;

7
src/client/src/tscSystem.c
View File

@ -41,8 +41,7 @@ int tscNumOfThreads;
static pthread_once_t tscinit = PTHREAD_ONCE_INIT; static pthread_once_t tscinit = PTHREAD_ONCE_INIT;
void taosInitNote(int numOfNoteLines, int maxNotes, char* lable); void taosInitNote(int numOfNoteLines, int maxNotes, char* lable);
//void tscUpdateIpSet(void *ahandle, SRpcIpSet *pIpSet); //void tscUpdateEpSet(void *ahandle, SRpcEpSet *pEpSet);
void tscCheckDiskUsage(void *UNUSED_PARAM(para), void* UNUSED_PARAM(param)) { void tscCheckDiskUsage(void *UNUSED_PARAM(para), void* UNUSED_PARAM(param)) {
taosGetDisk(); taosGetDisk();
@ -117,8 +116,8 @@ void taos_init_imp() {
taosInitNote(tsNumOfLogLines / 10, 1, (char*)"tsc_note"); taosInitNote(tsNumOfLogLines / 10, 1, (char*)"tsc_note");
} }
if (tscSetMgmtIpListFromCfg(tsFirst, tsSecond) < 0) { if (tscSetMgmtEpSetFromCfg(tsFirst, tsSecond) < 0) {
tscError("failed to init mnode IP list"); tscError("failed to init mnode EP list");
return; return;
} }

34
src/client/src/tscUtil.c
View File

@ -1994,6 +1994,10 @@ bool hasMoreVnodesToTry(SSqlObj* pSql) {
(!tscHasReachLimitation(pQueryInfo, pRes)) && (pTableMetaInfo->vgroupIndex < numOfVgroups - 1); (!tscHasReachLimitation(pQueryInfo, pRes)) && (pTableMetaInfo->vgroupIndex < numOfVgroups - 1);
} }
bool hasMoreClauseToTry(SSqlObj* pSql) {
return pSql->cmd.clauseIndex < pSql->cmd.numOfClause - 1;
}
void tscTryQueryNextVnode(SSqlObj* pSql, __async_cb_func_t fp) { void tscTryQueryNextVnode(SSqlObj* pSql, __async_cb_func_t fp) {
SSqlCmd* pCmd = &pSql->cmd; SSqlCmd* pCmd = &pSql->cmd;
SSqlRes* pRes = &pSql->res; SSqlRes* pRes = &pSql->res;
@ -2050,7 +2054,7 @@ void tscTryQueryNextVnode(SSqlObj* pSql, __async_cb_func_t fp) {
} }
} }
void tscTryQueryNextClause(SSqlObj* pSql, void (*queryFp)()) { void tscTryQueryNextClause(SSqlObj* pSql, __async_cb_func_t fp) {
SSqlCmd* pCmd = &pSql->cmd; SSqlCmd* pCmd = &pSql->cmd;
SSqlRes* pRes = &pSql->res; SSqlRes* pRes = &pSql->res;
@ -2070,17 +2074,13 @@ void tscTryQueryNextClause(SSqlObj* pSql, void (*queryFp)()) {
tfree(pSql->pSubs); tfree(pSql->pSubs);
pSql->numOfSubs = 0; pSql->numOfSubs = 0;
pSql->fp = fp;
if (pSql->fp != NULL) {
pSql->fp = queryFp;
assert(queryFp != NULL);
}
tscDebug("%p try data in the next subclause:%d, total subclause:%d", pSql, pCmd->clauseIndex, pCmd->numOfClause); tscDebug("%p try data in the next subclause:%d, total subclause:%d", pSql, pCmd->clauseIndex, pCmd->numOfClause);
if (pCmd->command > TSDB_SQL_LOCAL) { if (pCmd->command > TSDB_SQL_LOCAL) {
tscProcessLocalCmd(pSql); tscProcessLocalCmd(pSql);
} else { } else {
tscProcessSql(pSql); tscDoQuery(pSql);
} }
} }
@ -2145,20 +2145,20 @@ char* strdup_throw(const char* str) {
return p; return p;
} }
int tscSetMgmtIpListFromCfg(const char *first, const char *second) { int tscSetMgmtEpListFromCfg(const char *first, const char *second) {
// init mgmt ip set // init mgmt ip set
tscMgmtIpSet.version = 0; tscMgmtEpSet.version = 0;
SRpcIpSet *mgmtIpSet = &(tscMgmtIpSet.ipSet); SRpcEpSet *mgmtEpSet = &(tscMgmtEpSet.epSet);
mgmtIpSet->numOfIps = 0; mgmtEpSet->numOfEps = 0;
mgmtIpSet->inUse = 0; mgmtEpSet->inUse = 0;
if (first && first[0] != 0) { if (first && first[0] != 0) {
if (strlen(first) >= TSDB_EP_LEN) { if (strlen(first) >= TSDB_EP_LEN) {
terrno = TSDB_CODE_TSC_INVALID_FQDN; terrno = TSDB_CODE_TSC_INVALID_FQDN;
return -1; return -1;
} }
taosGetFqdnPortFromEp(first, mgmtIpSet->fqdn[mgmtIpSet->numOfIps], &(mgmtIpSet->port[mgmtIpSet->numOfIps])); taosGetFqdnPortFromEp(first, mgmtEpSet->fqdn[mgmtEpSet->numOfEps], &(mgmtEpSet->port[mgmtEpSet->numOfEps]));
mgmtIpSet->numOfIps++; mgmtEpSet->numOfEps++;
} }
if (second && second[0] != 0) { if (second && second[0] != 0) {
@ -2166,11 +2166,11 @@ int tscSetMgmtIpListFromCfg(const char *first, const char *second) {
terrno = TSDB_CODE_TSC_INVALID_FQDN; terrno = TSDB_CODE_TSC_INVALID_FQDN;
return -1; return -1;
} }
taosGetFqdnPortFromEp(second, mgmtIpSet->fqdn[mgmtIpSet->numOfIps], &(mgmtIpSet->port[mgmtIpSet->numOfIps])); taosGetFqdnPortFromEp(second, mgmtEpSet->fqdn[mgmtEpSet->numOfEps], &(mgmtEpSet->port[mgmtEpSet->numOfEps]));
mgmtIpSet->numOfIps++; mgmtEpSet->numOfEps++;
} }
if (mgmtIpSet->numOfIps == 0) { if (mgmtEpSet->numOfEps == 0) {
terrno = TSDB_CODE_TSC_INVALID_FQDN; terrno = TSDB_CODE_TSC_INVALID_FQDN;
return -1; return -1;
} }

2
src/common/inc/tglobal.h
View File

@ -30,8 +30,6 @@ extern uint16_t tsDnodeShellPort;
extern uint16_t tsDnodeDnodePort; extern uint16_t tsDnodeDnodePort;
extern uint16_t tsSyncPort; extern uint16_t tsSyncPort;
extern int32_t tsStatusInterval; extern int32_t tsStatusInterval;
extern int16_t tsNumOfVnodesPerCore;
extern int16_t tsNumOfTotalVnodes;
extern int32_t tsNumOfMnodes; extern int32_t tsNumOfMnodes;
extern int32_t tsEnableVnodeBak; extern int32_t tsEnableVnodeBak;

40
src/common/src/tglobal.c
View File

@ -38,12 +38,9 @@ uint16_t tsDnodeShellPort = 6030; // udp[6035-6039] tcp[6035]
uint16_t tsDnodeDnodePort = 6035; // udp/tcp uint16_t tsDnodeDnodePort = 6035; // udp/tcp
uint16_t tsSyncPort = 6040; uint16_t tsSyncPort = 6040;
int32_t tsStatusInterval = 1; // second int32_t tsStatusInterval = 1; // second
int16_t tsNumOfVnodesPerCore = 32;
int16_t tsNumOfTotalVnodes = TSDB_INVALID_VNODE_NUM;
int32_t tsNumOfMnodes = 3; int32_t tsNumOfMnodes = 3;
int32_t tsEnableVnodeBak = 1; int32_t tsEnableVnodeBak = 1;
// common // common
int32_t tsRpcTimer = 1000; int32_t tsRpcTimer = 1000;
int32_t tsRpcMaxTime = 600; // seconds; int32_t tsRpcMaxTime = 600; // seconds;
@ -199,6 +196,9 @@ int32_t sDebugFlag = 135;
int32_t wDebugFlag = 135; int32_t wDebugFlag = 135;
int32_t tsdbDebugFlag = 131; int32_t tsdbDebugFlag = 131;
int32_t (*monitorStartSystemFp)() = NULL;
void (*monitorStopSystemFp)() = NULL;
static pthread_once_t tsInitGlobalCfgOnce = PTHREAD_ONCE_INIT; static pthread_once_t tsInitGlobalCfgOnce = PTHREAD_ONCE_INIT;
void taosSetAllDebugFlag() { void taosSetAllDebugFlag() {
@ -248,11 +248,17 @@ bool taosCfgDynamicOptions(char *msg) {
*((int32_t *)cfg->ptr) = vint; *((int32_t *)cfg->ptr) = vint;
if (strncasecmp(cfg->option, "monitor", olen) == 0) { if (strncasecmp(cfg->option, "monitor", olen) == 0) {
// if (0 == vint) { if (1 == vint) {
// monitorStartSystem(); if (monitorStartSystemFp) {
// } else { (*monitorStartSystemFp)();
// monitorStopSystem(); uInfo("monitor is enabled");
// } }
} else {
if (monitorStopSystemFp) {
(*monitorStopSystemFp)();
uInfo("monitor is disabled");
}
}
return true; return true;
} }
@ -393,16 +399,6 @@ static void doInitGlobalConfig() {
cfg.unitType = TAOS_CFG_UTYPE_NONE; cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg); taosInitConfigOption(cfg);
cfg.option = "numOfTotalVnodes";
cfg.ptr = &tsNumOfTotalVnodes;
cfg.valType = TAOS_CFG_VTYPE_INT16;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
cfg.minValue = 0;
cfg.maxValue = TSDB_MAX_VNODES;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "numOfMnodes"; cfg.option = "numOfMnodes";
cfg.ptr = &tsNumOfMnodes; cfg.ptr = &tsNumOfMnodes;
cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.valType = TAOS_CFG_VTYPE_INT32;
@ -444,7 +440,7 @@ static void doInitGlobalConfig() {
taosInitConfigOption(cfg); taosInitConfigOption(cfg);
// 0-any; 1-mnode; 2-vnode // 0-any; 1-mnode; 2-vnode
cfg.option = "alternativeRole"; cfg.option = "role";
cfg.ptr = &tsAlternativeRole; cfg.ptr = &tsAlternativeRole;
cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
@ -1271,12 +1267,6 @@ bool taosCheckGlobalCfg() {
tsNumOfCores = 1; tsNumOfCores = 1;
} }
if (tsNumOfTotalVnodes == TSDB_INVALID_VNODE_NUM) {
tsNumOfTotalVnodes = tsNumOfCores * tsNumOfVnodesPerCore;
tsNumOfTotalVnodes = tsNumOfTotalVnodes > TSDB_MAX_VNODES ? TSDB_MAX_VNODES : tsNumOfTotalVnodes;
tsNumOfTotalVnodes = tsNumOfTotalVnodes < TSDB_MIN_VNODES ? TSDB_MIN_VNODES : tsNumOfTotalVnodes;
}
// todo refactor // todo refactor
tsVersion = 0; tsVersion = 0;
for (int i = 0; i < 10; i++) { for (int i = 0; i < 10; i++) {

1
src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNode.java
View File

@ -208,7 +208,6 @@ public class TDNode {
setCfgConfig("mnodeEqualVnodeNum", "0"); setCfgConfig("mnodeEqualVnodeNum", "0");
setCfgConfig("walLevel", "1"); setCfgConfig("walLevel", "1");
setCfgConfig("statusInterval", "1"); setCfgConfig("statusInterval", "1");
setCfgConfig("numOfTotalVnodes", "64");
setCfgConfig("numOfMnodes", "3"); setCfgConfig("numOfMnodes", "3");
setCfgConfig("numOfThreadsPerCore", "2.0"); setCfgConfig("numOfThreadsPerCore", "2.0");
setCfgConfig("monitor", "0"); setCfgConfig("monitor", "0");

6
src/cq/src/cqMain.c
View File

@ -103,9 +103,6 @@ void cqClose(void *handle) {
SCqContext *pContext = handle; SCqContext *pContext = handle;
if (handle == NULL) return; if (handle == NULL) return;
taosTmrCleanUp(pContext->tmrCtrl);
pContext->tmrCtrl = NULL;
// stop all CQs // stop all CQs
cqStop(pContext); cqStop(pContext);
@ -125,6 +122,9 @@ void cqClose(void *handle) {
pthread_mutex_destroy(&pContext->mutex); pthread_mutex_destroy(&pContext->mutex);
taosTmrCleanUp(pContext->tmrCtrl);
pContext->tmrCtrl = NULL;
cTrace("vgId:%d, CQ is closed", pContext->vgId); cTrace("vgId:%d, CQ is closed", pContext->vgId);
free(pContext); free(pContext);
} }

4
src/dnode/inc/dnodeMgmt.h
View File

@ -35,8 +35,8 @@ void* dnodeGetVnodeTsdb(void *pVnode);
void dnodeReleaseVnode(void *pVnode); void dnodeReleaseVnode(void *pVnode);
void dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell); void dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell);
void dnodeGetMnodeIpSetForPeer(void *ipSet); void dnodeGetMnodeEpSetForPeer(void *epSet);
void dnodeGetMnodeIpSetForShell(void *ipSet); void dnodeGetMnodeEpSetForShell(void *epSet);
#ifdef __cplusplus #ifdef __cplusplus
} }

105
src/dnode/src/dnodeMgmt.c
View File

@ -52,7 +52,7 @@ void * tsDnodeTmr = NULL;
static void * tsStatusTimer = NULL; static void * tsStatusTimer = NULL;
static uint32_t tsRebootTime; static uint32_t tsRebootTime;
static SRpcIpSet tsDMnodeIpSet = {0}; static SRpcEpSet tsDMnodeEpSet = {0};
static SDMMnodeInfos tsDMnodeInfos = {0}; static SDMMnodeInfos tsDMnodeInfos = {0};
static SDMDnodeCfg tsDnodeCfg = {0}; static SDMDnodeCfg tsDnodeCfg = {0};
static taos_qset tsMgmtQset = NULL; static taos_qset tsMgmtQset = NULL;
@ -90,21 +90,21 @@ int32_t dnodeInitMgmt() {
tsRebootTime = taosGetTimestampSec(); tsRebootTime = taosGetTimestampSec();
if (!dnodeReadMnodeInfos()) { if (!dnodeReadMnodeInfos()) {
memset(&tsDMnodeIpSet, 0, sizeof(SRpcIpSet)); memset(&tsDMnodeEpSet, 0, sizeof(SRpcEpSet));
memset(&tsDMnodeInfos, 0, sizeof(SDMMnodeInfos)); memset(&tsDMnodeInfos, 0, sizeof(SDMMnodeInfos));
tsDMnodeIpSet.numOfIps = 1; tsDMnodeEpSet.numOfEps = 1;
taosGetFqdnPortFromEp(tsFirst, tsDMnodeIpSet.fqdn[0], &tsDMnodeIpSet.port[0]); taosGetFqdnPortFromEp(tsFirst, tsDMnodeEpSet.fqdn[0], &tsDMnodeEpSet.port[0]);
if (strcmp(tsSecond, tsFirst) != 0) { if (strcmp(tsSecond, tsFirst) != 0) {
tsDMnodeIpSet.numOfIps = 2; tsDMnodeEpSet.numOfEps = 2;
taosGetFqdnPortFromEp(tsSecond, tsDMnodeIpSet.fqdn[1], &tsDMnodeIpSet.port[1]); taosGetFqdnPortFromEp(tsSecond, tsDMnodeEpSet.fqdn[1], &tsDMnodeEpSet.port[1]);
} }
} else { } else {
tsDMnodeIpSet.inUse = tsDMnodeInfos.inUse; tsDMnodeEpSet.inUse = tsDMnodeInfos.inUse;
tsDMnodeIpSet.numOfIps = tsDMnodeInfos.nodeNum; tsDMnodeEpSet.numOfEps = tsDMnodeInfos.nodeNum;
for (int32_t i = 0; i < tsDMnodeInfos.nodeNum; i++) { for (int32_t i = 0; i < tsDMnodeInfos.nodeNum; i++) {
taosGetFqdnPortFromEp(tsDMnodeInfos.nodeInfos[i].nodeEp, tsDMnodeIpSet.fqdn[i], &tsDMnodeIpSet.port[i]); taosGetFqdnPortFromEp(tsDMnodeInfos.nodeInfos[i].nodeEp, tsDMnodeEpSet.fqdn[i], &tsDMnodeEpSet.port[i]);
} }
} }
@ -450,27 +450,27 @@ static int32_t dnodeProcessConfigDnodeMsg(SRpcMsg *pMsg) {
return taosCfgDynamicOptions(pCfg->config); return taosCfgDynamicOptions(pCfg->config);
} }
void dnodeUpdateMnodeIpSetForPeer(SRpcIpSet *pIpSet) { void dnodeUpdateMnodeEpSetForPeer(SRpcEpSet *pEpSet) {
dInfo("mnode IP list for is changed, numOfIps:%d inUse:%d", pIpSet->numOfIps, pIpSet->inUse); dInfo("mnode EP list for is changed, numOfEps:%d inUse:%d", pEpSet->numOfEps, pEpSet->inUse);
for (int i = 0; i < pIpSet->numOfIps; ++i) { for (int i = 0; i < pEpSet->numOfEps; ++i) {
pIpSet->port[i] -= TSDB_PORT_DNODEDNODE; pEpSet->port[i] -= TSDB_PORT_DNODEDNODE;
dInfo("mnode index:%d %s:%u", i, pIpSet->fqdn[i], pIpSet->port[i]) dInfo("mnode index:%d %s:%u", i, pEpSet->fqdn[i], pEpSet->port[i])
} }
tsDMnodeIpSet = *pIpSet; tsDMnodeEpSet = *pEpSet;
} }
void dnodeGetMnodeIpSetForPeer(void *ipSetRaw) { void dnodeGetMnodeEpSetForPeer(void *epSetRaw) {
SRpcIpSet *ipSet = ipSetRaw; SRpcEpSet *epSet = epSetRaw;
*ipSet = tsDMnodeIpSet; *epSet = tsDMnodeEpSet;
for (int i=0; i<ipSet->numOfIps; ++i) for (int i=0; i<epSet->numOfEps; ++i)
ipSet->port[i] += TSDB_PORT_DNODEDNODE; epSet->port[i] += TSDB_PORT_DNODEDNODE;
} }
void dnodeGetMnodeIpSetForShell(void *ipSetRaw) { void dnodeGetMnodeEpSetForShell(void *epSetRaw) {
SRpcIpSet *ipSet = ipSetRaw; SRpcEpSet *epSet = epSetRaw;
*ipSet = tsDMnodeIpSet; *epSet = tsDMnodeEpSet;
} }
static void dnodeProcessStatusRsp(SRpcMsg *pMsg) { static void dnodeProcessStatusRsp(SRpcMsg *pMsg) {
@ -536,10 +536,10 @@ static void dnodeUpdateMnodeInfos(SDMMnodeInfos *pMnodes) {
dInfo("mnode index:%d, %s", tsDMnodeInfos.nodeInfos[i].nodeId, tsDMnodeInfos.nodeInfos[i].nodeEp); dInfo("mnode index:%d, %s", tsDMnodeInfos.nodeInfos[i].nodeId, tsDMnodeInfos.nodeInfos[i].nodeEp);
} }
tsDMnodeIpSet.inUse = tsDMnodeInfos.inUse; tsDMnodeEpSet.inUse = tsDMnodeInfos.inUse;
tsDMnodeIpSet.numOfIps = tsDMnodeInfos.nodeNum; tsDMnodeEpSet.numOfEps = tsDMnodeInfos.nodeNum;
for (int32_t i = 0; i < tsDMnodeInfos.nodeNum; i++) { for (int32_t i = 0; i < tsDMnodeInfos.nodeNum; i++) {
taosGetFqdnPortFromEp(tsDMnodeInfos.nodeInfos[i].nodeEp, tsDMnodeIpSet.fqdn[i], &tsDMnodeIpSet.port[i]); taosGetFqdnPortFromEp(tsDMnodeInfos.nodeInfos[i].nodeEp, tsDMnodeEpSet.fqdn[i], &tsDMnodeEpSet.port[i]);
} }
dnodeSaveMnodeInfos(); dnodeSaveMnodeInfos();
@ -549,10 +549,10 @@ static void dnodeUpdateMnodeInfos(SDMMnodeInfos *pMnodes) {
static bool dnodeReadMnodeInfos() { static bool dnodeReadMnodeInfos() {
char ipFile[TSDB_FILENAME_LEN*2] = {0}; char ipFile[TSDB_FILENAME_LEN*2] = {0};
sprintf(ipFile, "%s/mnodeIpList.json", tsDnodeDir); sprintf(ipFile, "%s/mnodeEpSet.json", tsDnodeDir);
FILE *fp = fopen(ipFile, "r"); FILE *fp = fopen(ipFile, "r");
if (!fp) { if (!fp) {
dDebug("failed to read mnodeIpList.json, file not exist"); dDebug("failed to read mnodeEpSet.json, file not exist");
return false; return false;
} }
@ -563,40 +563,40 @@ static bool dnodeReadMnodeInfos() {
if (len <= 0) { if (len <= 0) {
free(content); free(content);
fclose(fp); fclose(fp);
dError("failed to read mnodeIpList.json, content is null"); dError("failed to read mnodeEpSet.json, content is null");
return false; return false;
} }
content[len] = 0; content[len] = 0;
cJSON* root = cJSON_Parse(content); cJSON* root = cJSON_Parse(content);
if (root == NULL) { if (root == NULL) {
dError("failed to read mnodeIpList.json, invalid json format"); dError("failed to read mnodeEpSet.json, invalid json format");
goto PARSE_OVER; goto PARSE_OVER;
} }
cJSON* inUse = cJSON_GetObjectItem(root, "inUse"); cJSON* inUse = cJSON_GetObjectItem(root, "inUse");
if (!inUse || inUse->type != cJSON_Number) { if (!inUse || inUse->type != cJSON_Number) {
dError("failed to read mnodeIpList.json, inUse not found"); dError("failed to read mnodeEpSet.json, inUse not found");
goto PARSE_OVER; goto PARSE_OVER;
} }
tsDMnodeInfos.inUse = inUse->valueint; tsDMnodeInfos.inUse = inUse->valueint;
cJSON* nodeNum = cJSON_GetObjectItem(root, "nodeNum"); cJSON* nodeNum = cJSON_GetObjectItem(root, "nodeNum");
if (!nodeNum || nodeNum->type != cJSON_Number) { if (!nodeNum || nodeNum->type != cJSON_Number) {
dError("failed to read mnodeIpList.json, nodeNum not found"); dError("failed to read mnodeEpSet.json, nodeNum not found");
goto PARSE_OVER; goto PARSE_OVER;
} }
tsDMnodeInfos.nodeNum = nodeNum->valueint; tsDMnodeInfos.nodeNum = nodeNum->valueint;
cJSON* nodeInfos = cJSON_GetObjectItem(root, "nodeInfos"); cJSON* nodeInfos = cJSON_GetObjectItem(root, "nodeInfos");
if (!nodeInfos || nodeInfos->type != cJSON_Array) { if (!nodeInfos || nodeInfos->type != cJSON_Array) {
dError("failed to read mnodeIpList.json, nodeInfos not found"); dError("failed to read mnodeEpSet.json, nodeInfos not found");
goto PARSE_OVER; goto PARSE_OVER;
} }
int size = cJSON_GetArraySize(nodeInfos); int size = cJSON_GetArraySize(nodeInfos);
if (size != tsDMnodeInfos.nodeNum) { if (size != tsDMnodeInfos.nodeNum) {
dError("failed to read mnodeIpList.json, nodeInfos size not matched"); dError("failed to read mnodeEpSet.json, nodeInfos size not matched");
goto PARSE_OVER; goto PARSE_OVER;
} }
@ -606,14 +606,14 @@ static bool dnodeReadMnodeInfos() {
cJSON *nodeId = cJSON_GetObjectItem(nodeInfo, "nodeId"); cJSON *nodeId = cJSON_GetObjectItem(nodeInfo, "nodeId");
if (!nodeId || nodeId->type != cJSON_Number) { if (!nodeId || nodeId->type != cJSON_Number) {
dError("failed to read mnodeIpList.json, nodeId not found"); dError("failed to read mnodeEpSet.json, nodeId not found");
goto PARSE_OVER; goto PARSE_OVER;
} }
tsDMnodeInfos.nodeInfos[i].nodeId = nodeId->valueint; tsDMnodeInfos.nodeInfos[i].nodeId = nodeId->valueint;
cJSON *nodeEp = cJSON_GetObjectItem(nodeInfo, "nodeEp"); cJSON *nodeEp = cJSON_GetObjectItem(nodeInfo, "nodeEp");
if (!nodeEp || nodeEp->type != cJSON_String || nodeEp->valuestring == NULL) { if (!nodeEp || nodeEp->type != cJSON_String || nodeEp->valuestring == NULL) {
dError("failed to read mnodeIpList.json, nodeName not found"); dError("failed to read mnodeEpSet.json, nodeName not found");
goto PARSE_OVER; goto PARSE_OVER;
} }
strncpy(tsDMnodeInfos.nodeInfos[i].nodeEp, nodeEp->valuestring, TSDB_EP_LEN); strncpy(tsDMnodeInfos.nodeInfos[i].nodeEp, nodeEp->valuestring, TSDB_EP_LEN);
@ -621,7 +621,7 @@ static bool dnodeReadMnodeInfos() {
ret = true; ret = true;
dInfo("read mnode iplist successed, numOfIps:%d inUse:%d", tsDMnodeInfos.nodeNum, tsDMnodeInfos.inUse); dInfo("read mnode epSet successed, numOfEps:%d inUse:%d", tsDMnodeInfos.nodeNum, tsDMnodeInfos.inUse);
for (int32_t i = 0; i < tsDMnodeInfos.nodeNum; i++) { for (int32_t i = 0; i < tsDMnodeInfos.nodeNum; i++) {
dInfo("mnode:%d, %s", tsDMnodeInfos.nodeInfos[i].nodeId, tsDMnodeInfos.nodeInfos[i].nodeEp); dInfo("mnode:%d, %s", tsDMnodeInfos.nodeInfos[i].nodeId, tsDMnodeInfos.nodeInfos[i].nodeEp);
} }
@ -635,7 +635,7 @@ PARSE_OVER:
static void dnodeSaveMnodeInfos() { static void dnodeSaveMnodeInfos() {
char ipFile[TSDB_FILENAME_LEN] = {0}; char ipFile[TSDB_FILENAME_LEN] = {0};
sprintf(ipFile, "%s/mnodeIpList.json", tsDnodeDir); sprintf(ipFile, "%s/mnodeEpSet.json", tsDnodeDir);
FILE *fp = fopen(ipFile, "w"); FILE *fp = fopen(ipFile, "w");
if (!fp) return; if (!fp) return;
@ -663,11 +663,11 @@ static void dnodeSaveMnodeInfos() {
fclose(fp); fclose(fp);
free(content); free(content);
dInfo("save mnode iplist successed"); dInfo("save mnode epSet successed");
} }
char *dnodeGetMnodeMasterEp() { char *dnodeGetMnodeMasterEp() {
return tsDMnodeInfos.nodeInfos[tsDMnodeIpSet.inUse].nodeEp; return tsDMnodeInfos.nodeInfos[tsDMnodeEpSet.inUse].nodeEp;
} }
void* dnodeGetMnodeInfos() { void* dnodeGetMnodeInfos() {
@ -699,7 +699,6 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) {
pStatus->dnodeId = htonl(tsDnodeCfg.dnodeId); pStatus->dnodeId = htonl(tsDnodeCfg.dnodeId);
strcpy(pStatus->dnodeEp, tsLocalEp); strcpy(pStatus->dnodeEp, tsLocalEp);
pStatus->lastReboot = htonl(tsRebootTime); pStatus->lastReboot = htonl(tsRebootTime);
pStatus->numOfTotalVnodes = htons((uint16_t) tsNumOfTotalVnodes);
pStatus->numOfCores = htons((uint16_t) tsNumOfCores); pStatus->numOfCores = htons((uint16_t) tsNumOfCores);
pStatus->diskAvailable = tsAvailDataDirGB; pStatus->diskAvailable = tsAvailDataDirGB;
pStatus->alternativeRole = (uint8_t) tsAlternativeRole; pStatus->alternativeRole = (uint8_t) tsAlternativeRole;
@ -727,9 +726,9 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) {
.msgType = TSDB_MSG_TYPE_DM_STATUS .msgType = TSDB_MSG_TYPE_DM_STATUS
}; };
SRpcIpSet ipSet; SRpcEpSet epSet;
dnodeGetMnodeIpSetForPeer(&ipSet); dnodeGetMnodeEpSetForPeer(&epSet);
dnodeSendMsgToDnode(&ipSet, &rpcMsg); dnodeSendMsgToDnode(&epSet, &rpcMsg);
} }
static bool dnodeReadDnodeCfg() { static bool dnodeReadDnodeCfg() {
@ -818,20 +817,20 @@ void dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell) {
SRpcConnInfo connInfo = {0}; SRpcConnInfo connInfo = {0};
rpcGetConnInfo(rpcMsg->handle, &connInfo); rpcGetConnInfo(rpcMsg->handle, &connInfo);
SRpcIpSet ipSet = {0}; SRpcEpSet epSet = {0};
if (forShell) { if (forShell) {
dnodeGetMnodeIpSetForShell(&ipSet); dnodeGetMnodeEpSetForShell(&epSet);
} else { } else {
dnodeGetMnodeIpSetForPeer(&ipSet); dnodeGetMnodeEpSetForPeer(&epSet);
} }
dDebug("msg:%s will be redirected, dnodeIp:%s user:%s, numOfIps:%d inUse:%d", taosMsg[rpcMsg->msgType], dDebug("msg:%s will be redirected, dnodeIp:%s user:%s, numOfEps:%d inUse:%d", taosMsg[rpcMsg->msgType],
taosIpStr(connInfo.clientIp), connInfo.user, ipSet.numOfIps, ipSet.inUse); taosIpStr(connInfo.clientIp), connInfo.user, epSet.numOfEps, epSet.inUse);
for (int i = 0; i < ipSet.numOfIps; ++i) { for (int i = 0; i < epSet.numOfEps; ++i) {
dDebug("mnode index:%d %s:%d", i, ipSet.fqdn[i], ipSet.port[i]); dDebug("mnode index:%d %s:%d", i, epSet.fqdn[i], epSet.port[i]);
ipSet.port[i] = htons(ipSet.port[i]); epSet.port[i] = htons(epSet.port[i]);
} }
rpcSendRedirectRsp(rpcMsg->handle, &ipSet); rpcSendRedirectRsp(rpcMsg->handle, &epSet);
} }

24
src/dnode/src/dnodePeer.c
View File

@ -29,11 +29,11 @@
#include "dnodeVWrite.h" #include "dnodeVWrite.h"
#include "dnodeMPeer.h" #include "dnodeMPeer.h"
extern void dnodeUpdateMnodeIpSetForPeer(SRpcIpSet *pIpSet); extern void dnodeUpdateMnodeEpSetForPeer(SRpcEpSet *pEpSet);
static void (*dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *); static void (*dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *);
static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcIpSet *); static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcEpSet *);
static void (*dnodeProcessRspMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *rpcMsg); static void (*dnodeProcessRspMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *rpcMsg);
static void dnodeProcessRspFromDnode(SRpcMsg *pMsg, SRpcIpSet *pIpSet); static void dnodeProcessRspFromDnode(SRpcMsg *pMsg, SRpcEpSet *pEpSet);
static void *tsDnodeServerRpc = NULL; static void *tsDnodeServerRpc = NULL;
static void *tsDnodeClientRpc = NULL; static void *tsDnodeClientRpc = NULL;
@ -83,7 +83,7 @@ void dnodeCleanupServer() {
} }
} }
static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcIpSet *pIpSet) { static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
SRpcMsg rspMsg = { SRpcMsg rspMsg = {
.handle = pMsg->handle, .handle = pMsg->handle,
.pCont = NULL, .pCont = NULL,
@ -148,9 +148,9 @@ void dnodeCleanupClient() {
} }
} }
static void dnodeProcessRspFromDnode(SRpcMsg *pMsg, SRpcIpSet *pIpSet) { static void dnodeProcessRspFromDnode(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
if (pMsg->msgType == TSDB_MSG_TYPE_DM_STATUS_RSP && pIpSet) { if (pMsg->msgType == TSDB_MSG_TYPE_DM_STATUS_RSP && pEpSet) {
dnodeUpdateMnodeIpSetForPeer(pIpSet); dnodeUpdateMnodeEpSetForPeer(pEpSet);
} }
if (dnodeProcessRspMsgFp[pMsg->msgType]) { if (dnodeProcessRspMsgFp[pMsg->msgType]) {
@ -166,12 +166,12 @@ void dnodeAddClientRspHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg)) {
dnodeProcessRspMsgFp[msgType] = fp; dnodeProcessRspMsgFp[msgType] = fp;
} }
void dnodeSendMsgToDnode(SRpcIpSet *ipSet, SRpcMsg *rpcMsg) { void dnodeSendMsgToDnode(SRpcEpSet *epSet, SRpcMsg *rpcMsg) {
rpcSendRequest(tsDnodeClientRpc, ipSet, rpcMsg); rpcSendRequest(tsDnodeClientRpc, epSet, rpcMsg);
} }
void dnodeSendMsgToDnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp) { void dnodeSendMsgToDnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp) {
SRpcIpSet ipSet = {0}; SRpcEpSet epSet = {0};
dnodeGetMnodeIpSetForPeer(&ipSet); dnodeGetMnodeEpSetForPeer(&epSet);
rpcSendRecv(tsDnodeClientRpc, &ipSet, rpcMsg, rpcRsp); rpcSendRecv(tsDnodeClientRpc, &epSet, rpcMsg, rpcRsp);
} }

4
src/dnode/src/dnodeShell.c
View File

@ -31,7 +31,7 @@
#include "dnodeShell.h" #include "dnodeShell.h"
static void (*dnodeProcessShellMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *); static void (*dnodeProcessShellMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *);
static void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcIpSet *); static void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *);
static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char *secret, char *ckey); static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char *secret, char *ckey);
static void * tsDnodeShellRpc = NULL; static void * tsDnodeShellRpc = NULL;
static int32_t tsDnodeQueryReqNum = 0; static int32_t tsDnodeQueryReqNum = 0;
@ -108,7 +108,7 @@ void dnodeCleanupShell() {
} }
} }
void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcIpSet *pIpSet) { void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
SRpcMsg rpcMsg = { SRpcMsg rpcMsg = {
.handle = pMsg->handle, .handle = pMsg->handle,
.pCont = NULL, .pCont = NULL,

6
src/inc/dnode.h
View File

@ -39,13 +39,13 @@ SDnodeStatisInfo dnodeGetStatisInfo();
bool dnodeIsFirstDeploy(); bool dnodeIsFirstDeploy();
char * dnodeGetMnodeMasterEp(); char * dnodeGetMnodeMasterEp();
void dnodeGetMnodeIpSetForPeer(void *ipSet); void dnodeGetMnodeEpSetForPeer(void *epSet);
void dnodeGetMnodeIpSetForShell(void *ipSet); void dnodeGetMnodeEpSetForShell(void *epSet);
void * dnodeGetMnodeInfos(); void * dnodeGetMnodeInfos();
int32_t dnodeGetDnodeId(); int32_t dnodeGetDnodeId();
void dnodeAddClientRspHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg)); void dnodeAddClientRspHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg));
void dnodeSendMsgToDnode(SRpcIpSet *ipSet, SRpcMsg *rpcMsg); void dnodeSendMsgToDnode(SRpcEpSet *epSet, SRpcMsg *rpcMsg);
void dnodeSendMsgToDnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp); void dnodeSendMsgToDnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp);
void *dnodeSendCfgTableToRecv(int32_t vgId, int32_t sid); void *dnodeSendCfgTableToRecv(int32_t vgId, int32_t sid);

5
src/inc/taosdef.h
View File

@ -274,9 +274,8 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
#define TSDB_DEFAULT_PAYLOAD_SIZE 5120 // default payload size, greater than PATH_MAX value #define TSDB_DEFAULT_PAYLOAD_SIZE 5120 // default payload size, greater than PATH_MAX value
#define TSDB_EXTRA_PAYLOAD_SIZE 128 // extra bytes for auth #define TSDB_EXTRA_PAYLOAD_SIZE 128 // extra bytes for auth
#define TSDB_CQ_SQL_SIZE 1024 #define TSDB_CQ_SQL_SIZE 1024
#define TSDB_MIN_VNODES 64
#define TSDB_MAX_VNODES 2048 #define TSDB_MAX_VNODES 2048
#define TSDB_MIN_VNODES 256
#define TSDB_INVALID_VNODE_NUM 0
#define TSDB_DNODE_ROLE_ANY 0 #define TSDB_DNODE_ROLE_ANY 0
#define TSDB_DNODE_ROLE_MGMT 1 #define TSDB_DNODE_ROLE_MGMT 1
@ -293,7 +292,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
#define TSDB_MIN_TOTAL_BLOCKS 2 #define TSDB_MIN_TOTAL_BLOCKS 2
#define TSDB_MAX_TOTAL_BLOCKS 10000 #define TSDB_MAX_TOTAL_BLOCKS 10000
#define TSDB_DEFAULT_TOTAL_BLOCKS 4 #define TSDB_DEFAULT_TOTAL_BLOCKS 6
#define TSDB_MIN_TABLES 4 #define TSDB_MIN_TABLES 4
#define TSDB_MAX_TABLES 10000000 #define TSDB_MAX_TABLES 10000000

14
src/inc/taosmsg.h
View File

@ -176,7 +176,7 @@ extern char *taosMsg[];
typedef struct { typedef struct {
char fqdn[TSDB_FQDN_LEN]; char fqdn[TSDB_FQDN_LEN];
uint16_t port; uint16_t port;
} SIpAddr; } SEpAddr;
typedef struct { typedef struct {
int32_t numOfVnodes; int32_t numOfVnodes;
@ -306,7 +306,7 @@ typedef struct {
int8_t reserved1; int8_t reserved1;
int8_t reserved2; int8_t reserved2;
int32_t connId; int32_t connId;
SRpcIpSet ipList; SRpcEpSet epSet;
} SCMConnectRsp; } SCMConnectRsp;
typedef struct { typedef struct {
@ -581,12 +581,12 @@ typedef struct {
char dnodeEp[TSDB_EP_LEN]; char dnodeEp[TSDB_EP_LEN];
uint32_t moduleStatus; uint32_t moduleStatus;
uint32_t lastReboot; // time stamp for last reboot uint32_t lastReboot; // time stamp for last reboot
uint16_t numOfTotalVnodes; // from config file uint16_t reserve1; // from config file
uint16_t openVnodes; uint16_t openVnodes;
uint16_t numOfCores; uint16_t numOfCores;
float diskAvailable; // GB float diskAvailable; // GB
uint8_t alternativeRole; uint8_t alternativeRole;
uint8_t reserve[15]; uint8_t reserve2[15];
SClusterCfg clusterCfg; SClusterCfg clusterCfg;
SVnodeLoad load[]; SVnodeLoad load[];
} SDMStatusMsg; } SDMStatusMsg;
@ -648,8 +648,8 @@ typedef struct SCMSTableVgroupMsg {
typedef struct { typedef struct {
int32_t vgId; int32_t vgId;
int8_t numOfIps; int8_t numOfEps;
SIpAddr ipAddr[TSDB_MAX_REPLICA]; SEpAddr epAddr[TSDB_MAX_REPLICA];
} SCMVgroupInfo; } SCMVgroupInfo;
typedef struct { typedef struct {
@ -753,7 +753,7 @@ typedef struct {
uint32_t onlineDnodes; uint32_t onlineDnodes;
uint32_t connId; uint32_t connId;
int8_t killConnection; int8_t killConnection;
SRpcIpSet ipList; SRpcEpSet epSet;
} SCMHeartBeatRsp; } SCMHeartBeatRsp;
typedef struct { typedef struct {

14
src/inc/trpc.h
View File

@ -28,12 +28,12 @@ extern "C" {
extern int tsRpcHeadSize; extern int tsRpcHeadSize;
typedef struct SRpcIpSet { typedef struct SRpcEpSet {
int8_t inUse; int8_t inUse;
int8_t numOfIps; int8_t numOfEps;
uint16_t port[TSDB_MAX_REPLICA]; uint16_t port[TSDB_MAX_REPLICA];
char fqdn[TSDB_MAX_REPLICA][TSDB_FQDN_LEN]; char fqdn[TSDB_MAX_REPLICA][TSDB_FQDN_LEN];
} SRpcIpSet; } SRpcEpSet;
typedef struct SRpcCorIpSet { typedef struct SRpcCorIpSet {
int32_t version; int32_t version;
@ -72,7 +72,7 @@ typedef struct SRpcInit {
char *ckey; // ciphering key char *ckey; // ciphering key
// call back to process incoming msg, code shall be ignored by server app // call back to process incoming msg, code shall be ignored by server app
void (*cfp)(SRpcMsg *, SRpcIpSet *); void (*cfp)(SRpcMsg *, SRpcEpSet *);
// call back to retrieve the client auth info, for server app only // call back to retrieve the client auth info, for server app only
int (*afp)(char *tableId, char *spi, char *encrypt, char *secret, char *ckey); int (*afp)(char *tableId, char *spi, char *encrypt, char *secret, char *ckey);
@ -83,11 +83,11 @@ void rpcClose(void *);
void *rpcMallocCont(int contLen); void *rpcMallocCont(int contLen);
void rpcFreeCont(void *pCont); void rpcFreeCont(void *pCont);
void *rpcReallocCont(void *ptr, int contLen); void *rpcReallocCont(void *ptr, int contLen);
void rpcSendRequest(void *thandle, const SRpcIpSet *pIpSet, SRpcMsg *pMsg); void rpcSendRequest(void *thandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg);
void rpcSendResponse(const SRpcMsg *pMsg); void rpcSendResponse(const SRpcMsg *pMsg);
void rpcSendRedirectRsp(void *pConn, const SRpcIpSet *pIpSet); void rpcSendRedirectRsp(void *pConn, const SRpcEpSet *pEpSet);
int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo); int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo);
void rpcSendRecv(void *shandle, SRpcIpSet *pIpSet, SRpcMsg *pReq, SRpcMsg *pRsp); void rpcSendRecv(void *shandle, SRpcEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp);
int rpcReportProgress(void *pConn, char *pCont, int contLen); int rpcReportProgress(void *pConn, char *pCont, int contLen);
void rpcCancelRequest(void *pContext); void rpcCancelRequest(void *pContext);

8
src/inc/tsdb.h
View File

@ -46,7 +46,6 @@ typedef struct {
int (*eventCallBack)(void *); int (*eventCallBack)(void *);
void *(*cqCreateFunc)(void *handle, uint64_t uid, int sid, char *sqlStr, STSchema *pSchema); void *(*cqCreateFunc)(void *handle, uint64_t uid, int sid, char *sqlStr, STSchema *pSchema);
void (*cqDropFunc)(void *handle); void (*cqDropFunc)(void *handle);
void *(*configFunc)(int32_t vgId, int32_t sid);
} STsdbAppH; } STsdbAppH;
// --------- TSDB REPOSITORY CONFIGURATION DEFINITION // --------- TSDB REPOSITORY CONFIGURATION DEFINITION
@ -175,11 +174,6 @@ typedef struct {
SHashObj *map; // speedup acquire the tableQueryInfo from STableId SHashObj *map; // speedup acquire the tableQueryInfo from STableId
} STableGroupInfo; } STableGroupInfo;
typedef struct SQueryRowCond {
int32_t rel;
TSKEY ts;
} SQueryRowCond;
/** /**
* Get the data block iterator, starting from position according to the query condition * Get the data block iterator, starting from position according to the query condition
* *
@ -276,7 +270,7 @@ int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T *tsdb, uint64_t uid, const char *pT
* destory the created table group list, which is generated by tag query * destory the created table group list, which is generated by tag query
* @param pGroupList * @param pGroupList
*/ */
void tsdbDestoryTableGroup(STableGroupInfo *pGroupList); void tsdbDestroyTableGroup(STableGroupInfo *pGroupList);
/** /**
* create the table group result including only one table, used to handle the normal table query * create the table group result including only one table, used to handle the normal table query

8
src/kit/taosmigrate/taosmigrate.c
View File

@ -210,10 +210,10 @@ int32_t main(int32_t argc, char *argv[]) {
(void)snprintf(mnodeWal, TSDB_FILENAME_LEN*2, "%s/mnode/wal/wal0", arguments.dataDir); (void)snprintf(mnodeWal, TSDB_FILENAME_LEN*2, "%s/mnode/wal/wal0", arguments.dataDir);
walModWalFile(mnodeWal); walModWalFile(mnodeWal);
// 2. modfiy dnode config: mnodeIpList.json // 2. modfiy dnode config: mnodeEpSet.json
char dnodeIpList[TSDB_FILENAME_LEN*2] = {0}; char dnodeEpSet[TSDB_FILENAME_LEN*2] = {0};
(void)snprintf(dnodeIpList, TSDB_FILENAME_LEN*2, "%s/dnode/mnodeIpList.json", arguments.dataDir); (void)snprintf(dnodeEpSet, TSDB_FILENAME_LEN*2, "%s/dnode/mnodeEpSet.json", arguments.dataDir);
modDnodeIpList(dnodeIpList); modDnodeEpSet(dnodeEpSet);
// 3. modify vnode config: config.json // 3. modify vnode config: config.json
char vnodeDir[TSDB_FILENAME_LEN*2] = {0}; char vnodeDir[TSDB_FILENAME_LEN*2] = {0};

2
src/kit/taosmigrate/taosmigrate.h
View File

@ -71,7 +71,7 @@ int tSystemShell(const char * cmd);
void taosMvFile(char* destFile, char *srcFile) ; void taosMvFile(char* destFile, char *srcFile) ;
void walModWalFile(char* walfile); void walModWalFile(char* walfile);
SdnodeIfo* getDnodeInfo(int32_t dnodeId); SdnodeIfo* getDnodeInfo(int32_t dnodeId);
void modDnodeIpList(char* dnodeIpList); void modDnodeEpSet(char* dnodeEpSet);
void modAllVnode(char *vnodeDir); void modAllVnode(char *vnodeDir);
#endif #endif

36
src/kit/taosmigrate/taosmigrateDnodeCfg.c
View File

@ -23,10 +23,10 @@
static SDMMnodeInfos tsDnodeIpInfos = {0}; static SDMMnodeInfos tsDnodeIpInfos = {0};
static bool dnodeReadMnodeInfos(char* dnodeIpList) { static bool dnodeReadMnodeInfos(char* dnodeEpSet) {
FILE *fp = fopen(dnodeIpList, "r"); FILE *fp = fopen(dnodeEpSet, "r");
if (!fp) { if (!fp) {
printf("failed to read mnodeIpList.json, file not exist\n"); printf("failed to read mnodeEpSet.json, file not exist\n");
return false; return false;
} }
@ -37,40 +37,40 @@ static bool dnodeReadMnodeInfos(char* dnodeIpList) {
if (len <= 0) { if (len <= 0) {
free(content); free(content);
fclose(fp); fclose(fp);
printf("failed to read mnodeIpList.json, content is null\n"); printf("failed to read mnodeEpSet.json, content is null\n");
return false; return false;
} }
content[len] = 0; content[len] = 0;
cJSON* root = cJSON_Parse(content); cJSON* root = cJSON_Parse(content);
if (root == NULL) { if (root == NULL) {
printf("failed to read mnodeIpList.json, invalid json format\n"); printf("failed to read mnodeEpSet.json, invalid json format\n");
goto PARSE_OVER; goto PARSE_OVER;
} }
cJSON* inUse = cJSON_GetObjectItem(root, "inUse"); cJSON* inUse = cJSON_GetObjectItem(root, "inUse");
if (!inUse || inUse->type != cJSON_Number) { if (!inUse || inUse->type != cJSON_Number) {
printf("failed to read mnodeIpList.json, inUse not found\n"); printf("failed to read mnodeEpSet.json, inUse not found\n");
goto PARSE_OVER; goto PARSE_OVER;
} }
tsDnodeIpInfos.inUse = inUse->valueint; tsDnodeIpInfos.inUse = inUse->valueint;
cJSON* nodeNum = cJSON_GetObjectItem(root, "nodeNum"); cJSON* nodeNum = cJSON_GetObjectItem(root, "nodeNum");
if (!nodeNum || nodeNum->type != cJSON_Number) { if (!nodeNum || nodeNum->type != cJSON_Number) {
printf("failed to read mnodeIpList.json, nodeNum not found\n"); printf("failed to read mnodeEpSet.json, nodeNum not found\n");
goto PARSE_OVER; goto PARSE_OVER;
} }
tsDnodeIpInfos.nodeNum = nodeNum->valueint; tsDnodeIpInfos.nodeNum = nodeNum->valueint;
cJSON* nodeInfos = cJSON_GetObjectItem(root, "nodeInfos"); cJSON* nodeInfos = cJSON_GetObjectItem(root, "nodeInfos");
if (!nodeInfos || nodeInfos->type != cJSON_Array) { if (!nodeInfos || nodeInfos->type != cJSON_Array) {
printf("failed to read mnodeIpList.json, nodeInfos not found\n"); printf("failed to read mnodeEpSet.json, nodeInfos not found\n");
goto PARSE_OVER; goto PARSE_OVER;
} }
int size = cJSON_GetArraySize(nodeInfos); int size = cJSON_GetArraySize(nodeInfos);
if (size != tsDnodeIpInfos.nodeNum) { if (size != tsDnodeIpInfos.nodeNum) {
printf("failed to read mnodeIpList.json, nodeInfos size not matched\n"); printf("failed to read mnodeEpSet.json, nodeInfos size not matched\n");
goto PARSE_OVER; goto PARSE_OVER;
} }
@ -80,14 +80,14 @@ static bool dnodeReadMnodeInfos(char* dnodeIpList) {
cJSON *nodeId = cJSON_GetObjectItem(nodeInfo, "nodeId"); cJSON *nodeId = cJSON_GetObjectItem(nodeInfo, "nodeId");
if (!nodeId || nodeId->type != cJSON_Number) { if (!nodeId || nodeId->type != cJSON_Number) {
printf("failed to read mnodeIpList.json, nodeId not found\n"); printf("failed to read mnodeEpSet.json, nodeId not found\n");
goto PARSE_OVER; goto PARSE_OVER;
} }
tsDnodeIpInfos.nodeInfos[i].nodeId = nodeId->valueint; tsDnodeIpInfos.nodeInfos[i].nodeId = nodeId->valueint;
cJSON *nodeEp = cJSON_GetObjectItem(nodeInfo, "nodeEp"); cJSON *nodeEp = cJSON_GetObjectItem(nodeInfo, "nodeEp");
if (!nodeEp || nodeEp->type != cJSON_String || nodeEp->valuestring == NULL) { if (!nodeEp || nodeEp->type != cJSON_String || nodeEp->valuestring == NULL) {
printf("failed to read mnodeIpList.json, nodeName not found\n"); printf("failed to read mnodeEpSet.json, nodeName not found\n");
goto PARSE_OVER; goto PARSE_OVER;
} }
strncpy(tsDnodeIpInfos.nodeInfos[i].nodeEp, nodeEp->valuestring, TSDB_EP_LEN); strncpy(tsDnodeIpInfos.nodeInfos[i].nodeEp, nodeEp->valuestring, TSDB_EP_LEN);
@ -102,7 +102,7 @@ static bool dnodeReadMnodeInfos(char* dnodeIpList) {
ret = true; ret = true;
//printf("read mnode iplist successed, numOfIps:%d inUse:%d\n", tsDnodeIpInfos.nodeNum, tsDnodeIpInfos.inUse); //printf("read mnode epSet successed, numOfEps:%d inUse:%d\n", tsDnodeIpInfos.nodeNum, tsDnodeIpInfos.inUse);
//for (int32_t i = 0; i < tsDnodeIpInfos.nodeNum; i++) { //for (int32_t i = 0; i < tsDnodeIpInfos.nodeNum; i++) {
// printf("mnode:%d, %s\n", tsDnodeIpInfos.nodeInfos[i].nodeId, tsDnodeIpInfos.nodeInfos[i].nodeEp); // printf("mnode:%d, %s\n", tsDnodeIpInfos.nodeInfos[i].nodeId, tsDnodeIpInfos.nodeInfos[i].nodeEp);
//} //}
@ -115,8 +115,8 @@ PARSE_OVER:
} }
static void dnodeSaveMnodeInfos(char* dnodeIpList) { static void dnodeSaveMnodeInfos(char* dnodeEpSet) {
FILE *fp = fopen(dnodeIpList, "w"); FILE *fp = fopen(dnodeEpSet, "w");
if (!fp) return; if (!fp) return;
int32_t len = 0; int32_t len = 0;
@ -143,13 +143,13 @@ static void dnodeSaveMnodeInfos(char* dnodeIpList) {
fclose(fp); fclose(fp);
free(content); free(content);
printf("mod mnode iplist successed\n"); printf("mod mnode epSet successed\n");
} }
void modDnodeIpList(char* dnodeIpList) void modDnodeEpSet(char* dnodeEpSet)
{ {
(void)dnodeReadMnodeInfos(dnodeIpList); (void)dnodeReadMnodeInfos(dnodeEpSet);
dnodeSaveMnodeInfos(dnodeIpList); dnodeSaveMnodeInfos(dnodeEpSet);
return; return;
} }

6
src/mnode/inc/mnodeDef.h
View File

@ -40,7 +40,7 @@ typedef struct SDnodeObj {
int32_t dnodeId; int32_t dnodeId;
int32_t openVnodes; int32_t openVnodes;
int64_t createdTime; int64_t createdTime;
int32_t totalVnodes; // from dnode status msg, config information int32_t resever0; // from dnode status msg, config information
int32_t customScore; // config by user int32_t customScore; // config by user
uint32_t lastAccess; uint32_t lastAccess;
uint16_t numOfCores; // from dnode status msg uint16_t numOfCores; // from dnode status msg
@ -50,7 +50,7 @@ typedef struct SDnodeObj {
int8_t alternativeRole; // from dnode status msg, 0-any, 1-mgmt, 2-dnode int8_t alternativeRole; // from dnode status msg, 0-any, 1-mgmt, 2-dnode
int8_t status; // set in balance function int8_t status; // set in balance function
int8_t isMgmt; int8_t isMgmt;
int8_t reserved0[14]; int8_t reserve1[14];
int8_t updateEnd[1]; int8_t updateEnd[1];
int32_t refCount; int32_t refCount;
uint32_t moduleStatus; uint32_t moduleStatus;
@ -61,7 +61,7 @@ typedef struct SDnodeObj {
int16_t cpuAvgUsage; // calc from sys.cpu int16_t cpuAvgUsage; // calc from sys.cpu
int16_t memoryAvgUsage; // calc from sys.mem int16_t memoryAvgUsage; // calc from sys.mem
int16_t bandwidthUsage; // calc from sys.band int16_t bandwidthUsage; // calc from sys.band
int8_t reserved1[2]; int8_t reserved2[2];
} SDnodeObj; } SDnodeObj;
typedef struct SMnodeObj { typedef struct SMnodeObj {

6
src/mnode/inc/mnodeMnode.h
View File

@ -42,12 +42,12 @@ void mnodeIncMnodeRef(struct SMnodeObj *pMnode);
void mnodeDecMnodeRef(struct SMnodeObj *pMnode); void mnodeDecMnodeRef(struct SMnodeObj *pMnode);
char * mnodeGetMnodeRoleStr(); char * mnodeGetMnodeRoleStr();
void mnodeGetMnodeIpSetForPeer(SRpcIpSet *ipSet); void mnodeGetMnodeEpSetForPeer(SRpcEpSet *epSet);
void mnodeGetMnodeIpSetForShell(SRpcIpSet *ipSet); void mnodeGetMnodeEpSetForShell(SRpcEpSet *epSet);
char* mnodeGetMnodeMasterEp(); char* mnodeGetMnodeMasterEp();
void mnodeGetMnodeInfos(void *mnodes); void mnodeGetMnodeInfos(void *mnodes);
void mnodeUpdateMnodeIpSet(); void mnodeUpdateMnodeEpSet();
#ifdef __cplusplus #ifdef __cplusplus
} }

6
src/mnode/inc/mnodeVgroup.h
View File

@ -44,12 +44,12 @@ int32_t mnodeGetAvailableVgroup(struct SMnodeMsg *pMsg, SVgObj **pVgroup, int32_
void mnodeAddTableIntoVgroup(SVgObj *pVgroup, SChildTableObj *pTable); void mnodeAddTableIntoVgroup(SVgObj *pVgroup, SChildTableObj *pTable);
void mnodeRemoveTableFromVgroup(SVgObj *pVgroup, SChildTableObj *pTable); void mnodeRemoveTableFromVgroup(SVgObj *pVgroup, SChildTableObj *pTable);
void mnodeSendDropVnodeMsg(int32_t vgId, SRpcIpSet *ipSet, void *ahandle); void mnodeSendDropVnodeMsg(int32_t vgId, SRpcEpSet *epSet, void *ahandle);
void mnodeSendCreateVgroupMsg(SVgObj *pVgroup, void *ahandle); void mnodeSendCreateVgroupMsg(SVgObj *pVgroup, void *ahandle);
void mnodeSendAlterVgroupMsg(SVgObj *pVgroup); void mnodeSendAlterVgroupMsg(SVgObj *pVgroup);
SRpcIpSet mnodeGetIpSetFromVgroup(SVgObj *pVgroup); SRpcEpSet mnodeGetEpSetFromVgroup(SVgObj *pVgroup);
SRpcIpSet mnodeGetIpSetFromIp(char *ep); SRpcEpSet mnodeGetEpSetFromIp(char *ep);
#ifdef __cplusplus #ifdef __cplusplus
} }

4
src/mnode/src/mnodeBalance.c
View File

@ -39,11 +39,11 @@ int32_t balanceAllocVnodes(SVgObj *pVgroup) {
pIter = mnodeGetNextDnode(pIter, &pDnode); pIter = mnodeGetNextDnode(pIter, &pDnode);
if (pDnode == NULL) break; if (pDnode == NULL) break;
if (pDnode->totalVnodes > 0 && pDnode->openVnodes < pDnode->totalVnodes) { if (pDnode->numOfCores > 0 && pDnode->openVnodes < TSDB_MAX_VNODES) {
float openVnodes = pDnode->openVnodes; float openVnodes = pDnode->openVnodes;
if (pDnode->isMgmt) openVnodes += tsMnodeEqualVnodeNum; if (pDnode->isMgmt) openVnodes += tsMnodeEqualVnodeNum;
float usage = openVnodes / pDnode->totalVnodes; float usage = openVnodes / pDnode->numOfCores;
if (usage <= vnodeUsage) { if (usage <= vnodeUsage) {
pSelDnode = pDnode; pSelDnode = pDnode;
vnodeUsage = usage; vnodeUsage = usage;

40
src/mnode/src/mnodeDnode.c
View File

@ -289,14 +289,14 @@ static int32_t mnodeProcessCfgDnodeMsg(SMnodeMsg *pMsg) {
} }
} }
SRpcIpSet ipSet = mnodeGetIpSetFromIp(pCmCfgDnode->ep); SRpcEpSet epSet = mnodeGetEpSetFromIp(pCmCfgDnode->ep);
if (dnodeId != 0) { if (dnodeId != 0) {
SDnodeObj *pDnode = mnodeGetDnode(dnodeId); SDnodeObj *pDnode = mnodeGetDnode(dnodeId);
if (pDnode == NULL) { if (pDnode == NULL) {
mError("failed to cfg dnode, invalid dnodeId:%d", dnodeId); mError("failed to cfg dnode, invalid dnodeId:%d", dnodeId);
return TSDB_CODE_MND_DNODE_NOT_EXIST; return TSDB_CODE_MND_DNODE_NOT_EXIST;
} }
ipSet = mnodeGetIpSetFromIp(pDnode->dnodeEp); epSet = mnodeGetEpSetFromIp(pDnode->dnodeEp);
mnodeDecDnodeRef(pDnode); mnodeDecDnodeRef(pDnode);
} }
@ -313,7 +313,7 @@ static int32_t mnodeProcessCfgDnodeMsg(SMnodeMsg *pMsg) {
}; };
mInfo("dnode:%s, is configured by %s", pCmCfgDnode->ep, pMsg->pUser->user); mInfo("dnode:%s, is configured by %s", pCmCfgDnode->ep, pMsg->pUser->user);
dnodeSendMsgToDnode(&ipSet, &rpcMdCfgDnodeMsg); dnodeSendMsgToDnode(&epSet, &rpcMdCfgDnodeMsg);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -345,7 +345,6 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
pStatus->moduleStatus = htonl(pStatus->moduleStatus); pStatus->moduleStatus = htonl(pStatus->moduleStatus);
pStatus->lastReboot = htonl(pStatus->lastReboot); pStatus->lastReboot = htonl(pStatus->lastReboot);
pStatus->numOfCores = htons(pStatus->numOfCores); pStatus->numOfCores = htons(pStatus->numOfCores);
pStatus->numOfTotalVnodes = htons(pStatus->numOfTotalVnodes);
uint32_t version = htonl(pStatus->version); uint32_t version = htonl(pStatus->version);
if (version != tsVersion) { if (version != tsVersion) {
@ -372,7 +371,6 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
pDnode->numOfCores = pStatus->numOfCores; pDnode->numOfCores = pStatus->numOfCores;
pDnode->diskAvailable = pStatus->diskAvailable; pDnode->diskAvailable = pStatus->diskAvailable;
pDnode->alternativeRole = pStatus->alternativeRole; pDnode->alternativeRole = pStatus->alternativeRole;
pDnode->totalVnodes = pStatus->numOfTotalVnodes;
pDnode->moduleStatus = pStatus->moduleStatus; pDnode->moduleStatus = pStatus->moduleStatus;
if (pStatus->dnodeId == 0) { if (pStatus->dnodeId == 0) {
@ -401,9 +399,9 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
SVgObj *pVgroup = mnodeGetVgroup(pVload->vgId); SVgObj *pVgroup = mnodeGetVgroup(pVload->vgId);
if (pVgroup == NULL) { if (pVgroup == NULL) {
SRpcIpSet ipSet = mnodeGetIpSetFromIp(pDnode->dnodeEp); SRpcEpSet epSet = mnodeGetEpSetFromIp(pDnode->dnodeEp);
mInfo("dnode:%d, vgId:%d not exist in mnode, drop it", pDnode->dnodeId, pVload->vgId); mInfo("dnode:%d, vgId:%d not exist in mnode, drop it", pDnode->dnodeId, pVload->vgId);
mnodeSendDropVnodeMsg(pVload->vgId, &ipSet, NULL); mnodeSendDropVnodeMsg(pVload->vgId, &epSet, NULL);
} else { } else {
mnodeUpdateVgroupStatus(pVgroup, pDnode, pVload); mnodeUpdateVgroupStatus(pVgroup, pDnode, pVload);
pAccess->vgId = htonl(pVload->vgId); pAccess->vgId = htonl(pVload->vgId);
@ -452,17 +450,29 @@ static int32_t mnodeCreateDnode(char *ep, SMnodeMsg *pMsg) {
return grantCode; return grantCode;
} }
char dnodeEp[TSDB_EP_LEN] = {0};
tstrncpy(dnodeEp, ep, TSDB_EP_LEN);
strtrim(dnodeEp);
char *temp = strchr(dnodeEp, ':');
if (!temp) {
int len = strlen(dnodeEp);
if (dnodeEp[len - 1] == ';') dnodeEp[len - 1] = 0;
len = strlen(dnodeEp);
snprintf(dnodeEp + len, TSDB_EP_LEN - len, ":%d", tsServerPort);
}
ep = dnodeEp;
SDnodeObj *pDnode = mnodeGetDnodeByEp(ep); SDnodeObj *pDnode = mnodeGetDnodeByEp(ep);
if (pDnode != NULL) { if (pDnode != NULL) {
mnodeDecDnodeRef(pDnode); mnodeDecDnodeRef(pDnode);
mError("dnode:%d is alredy exist, %s:%d", pDnode->dnodeId, pDnode->dnodeFqdn, pDnode->dnodePort); mError("dnode:%d is already exist, %s:%d", pDnode->dnodeId, pDnode->dnodeFqdn, pDnode->dnodePort);
return TSDB_CODE_MND_DNODE_ALREADY_EXIST; return TSDB_CODE_MND_DNODE_ALREADY_EXIST;
} }
pDnode = (SDnodeObj *) calloc(1, sizeof(SDnodeObj)); pDnode = (SDnodeObj *) calloc(1, sizeof(SDnodeObj));
pDnode->createdTime = taosGetTimestampMs(); pDnode->createdTime = taosGetTimestampMs();
pDnode->status = TAOS_DN_STATUS_OFFLINE; pDnode->status = TAOS_DN_STATUS_OFFLINE;
pDnode->totalVnodes = TSDB_INVALID_VNODE_NUM;
tstrncpy(pDnode->dnodeEp, ep, TSDB_EP_LEN); tstrncpy(pDnode->dnodeEp, ep, TSDB_EP_LEN);
taosGetFqdnPortFromEp(ep, pDnode->dnodeFqdn, &pDnode->dnodePort); taosGetFqdnPortFromEp(ep, pDnode->dnodeFqdn, &pDnode->dnodePort);
@ -506,10 +516,14 @@ int32_t mnodeDropDnode(SDnodeObj *pDnode, void *pMsg) {
static int32_t mnodeDropDnodeByEp(char *ep, SMnodeMsg *pMsg) { static int32_t mnodeDropDnodeByEp(char *ep, SMnodeMsg *pMsg) {
SDnodeObj *pDnode = mnodeGetDnodeByEp(ep); SDnodeObj *pDnode = mnodeGetDnodeByEp(ep);
if (pDnode == NULL) {
int32_t dnodeId = (int32_t)strtol(ep, NULL, 10);
pDnode = mnodeGetDnode(dnodeId);
if (pDnode == NULL) { if (pDnode == NULL) {
mError("dnode:%s, is not exist", ep); mError("dnode:%s, is not exist", ep);
return TSDB_CODE_MND_DNODE_NOT_EXIST; return TSDB_CODE_MND_DNODE_NOT_EXIST;
} }
}
if (strcmp(pDnode->dnodeEp, mnodeGetMnodeMasterEp()) == 0) { if (strcmp(pDnode->dnodeEp, mnodeGetMnodeMasterEp()) == 0) {
mError("dnode:%d, can't drop dnode:%s which is master", pDnode->dnodeId, ep); mError("dnode:%d, can't drop dnode:%s which is master", pDnode->dnodeId, ep);
@ -575,13 +589,13 @@ static int32_t mnodeGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC
pShow->bytes[cols] = 2; pShow->bytes[cols] = 2;
pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
strcpy(pSchema[cols].name, "open_vnodes"); strcpy(pSchema[cols].name, "vnodes");
pSchema[cols].bytes = htons(pShow->bytes[cols]); pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++; cols++;
pShow->bytes[cols] = 2; pShow->bytes[cols] = 2;
pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
strcpy(pSchema[cols].name, "total_vnodes"); strcpy(pSchema[cols].name, "cores");
pSchema[cols].bytes = htons(pShow->bytes[cols]); pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++; cols++;
@ -593,7 +607,7 @@ static int32_t mnodeGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC
pShow->bytes[cols] = 6 + VARSTR_HEADER_SIZE; pShow->bytes[cols] = 6 + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY; pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "alternativeRole"); strcpy(pSchema[cols].name, "role");
pSchema[cols].bytes = htons(pShow->bytes[cols]); pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++; cols++;
@ -645,7 +659,7 @@ static int32_t mnodeRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, vo
cols++; cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int16_t *)pWrite = pDnode->totalVnodes; *(int16_t *)pWrite = pDnode->numOfCores;
cols++; cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;

48
src/mnode/src/mnodeMnode.c
View File

@ -35,8 +35,8 @@
static void * tsMnodeSdb = NULL; static void * tsMnodeSdb = NULL;
static int32_t tsMnodeUpdateSize = 0; static int32_t tsMnodeUpdateSize = 0;
static SRpcIpSet tsMnodeIpSetForShell; static SRpcEpSet tsMnodeEpSetForShell;
static SRpcIpSet tsMnodeIpSetForPeer; static SRpcEpSet tsMnodeEpSetForPeer;
static SDMMnodeInfos tsMnodeInfos; static SDMMnodeInfos tsMnodeInfos;
static int32_t mnodeGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mnodeGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
static int32_t mnodeRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn); static int32_t mnodeRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn);
@ -123,7 +123,7 @@ static int32_t mnodeMnodeActionRestored() {
sdbFreeIter(pIter); sdbFreeIter(pIter);
} }
mnodeUpdateMnodeIpSet(); mnodeUpdateMnodeEpSet();
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -204,13 +204,13 @@ char *mnodeGetMnodeRoleStr(int32_t role) {
} }
} }
void mnodeUpdateMnodeIpSet() { void mnodeUpdateMnodeEpSet() {
mInfo("update mnodes ipset, numOfIps:%d ", mnodeGetMnodesNum()); mInfo("update mnodes epSet, numOfEps:%d ", mnodeGetMnodesNum());
mnodeMnodeWrLock(); mnodeMnodeWrLock();
memset(&tsMnodeIpSetForShell, 0, sizeof(SRpcIpSet)); memset(&tsMnodeEpSetForShell, 0, sizeof(SRpcEpSet));
memset(&tsMnodeIpSetForPeer, 0, sizeof(SRpcIpSet)); memset(&tsMnodeEpSetForPeer, 0, sizeof(SRpcEpSet));
memset(&tsMnodeInfos, 0, sizeof(SDMMnodeInfos)); memset(&tsMnodeInfos, 0, sizeof(SDMMnodeInfos));
int32_t index = 0; int32_t index = 0;
@ -222,20 +222,20 @@ void mnodeUpdateMnodeIpSet() {
SDnodeObj *pDnode = mnodeGetDnode(pMnode->mnodeId); SDnodeObj *pDnode = mnodeGetDnode(pMnode->mnodeId);
if (pDnode != NULL) { if (pDnode != NULL) {
strcpy(tsMnodeIpSetForShell.fqdn[index], pDnode->dnodeFqdn); strcpy(tsMnodeEpSetForShell.fqdn[index], pDnode->dnodeFqdn);
tsMnodeIpSetForShell.port[index] = htons(pDnode->dnodePort); tsMnodeEpSetForShell.port[index] = htons(pDnode->dnodePort);
mDebug("mnode:%d, for shell fqdn:%s %d", pDnode->dnodeId, tsMnodeIpSetForShell.fqdn[index], htons(tsMnodeIpSetForShell.port[index])); mDebug("mnode:%d, for shell fqdn:%s %d", pDnode->dnodeId, tsMnodeEpSetForShell.fqdn[index], htons(tsMnodeEpSetForShell.port[index]));
strcpy(tsMnodeIpSetForPeer.fqdn[index], pDnode->dnodeFqdn); strcpy(tsMnodeEpSetForPeer.fqdn[index], pDnode->dnodeFqdn);
tsMnodeIpSetForPeer.port[index] = htons(pDnode->dnodePort + TSDB_PORT_DNODEDNODE); tsMnodeEpSetForPeer.port[index] = htons(pDnode->dnodePort + TSDB_PORT_DNODEDNODE);
mDebug("mnode:%d, for peer fqdn:%s %d", pDnode->dnodeId, tsMnodeIpSetForPeer.fqdn[index], htons(tsMnodeIpSetForPeer.port[index])); mDebug("mnode:%d, for peer fqdn:%s %d", pDnode->dnodeId, tsMnodeEpSetForPeer.fqdn[index], htons(tsMnodeEpSetForPeer.port[index]));
tsMnodeInfos.nodeInfos[index].nodeId = htonl(pMnode->mnodeId); tsMnodeInfos.nodeInfos[index].nodeId = htonl(pMnode->mnodeId);
strcpy(tsMnodeInfos.nodeInfos[index].nodeEp, pDnode->dnodeEp); strcpy(tsMnodeInfos.nodeInfos[index].nodeEp, pDnode->dnodeEp);
if (pMnode->role == TAOS_SYNC_ROLE_MASTER) { if (pMnode->role == TAOS_SYNC_ROLE_MASTER) {
tsMnodeIpSetForShell.inUse = index; tsMnodeEpSetForShell.inUse = index;
tsMnodeIpSetForPeer.inUse = index; tsMnodeEpSetForPeer.inUse = index;
tsMnodeInfos.inUse = index; tsMnodeInfos.inUse = index;
} }
@ -248,23 +248,23 @@ void mnodeUpdateMnodeIpSet() {
} }
tsMnodeInfos.nodeNum = index; tsMnodeInfos.nodeNum = index;
tsMnodeIpSetForShell.numOfIps = index; tsMnodeEpSetForShell.numOfEps = index;
tsMnodeIpSetForPeer.numOfIps = index; tsMnodeEpSetForPeer.numOfEps = index;
sdbFreeIter(pIter); sdbFreeIter(pIter);
mnodeMnodeUnLock(); mnodeMnodeUnLock();
} }
void mnodeGetMnodeIpSetForPeer(SRpcIpSet *ipSet) { void mnodeGetMnodeEpSetForPeer(SRpcEpSet *epSet) {
mnodeMnodeRdLock(); mnodeMnodeRdLock();
*ipSet = tsMnodeIpSetForPeer; *epSet = tsMnodeEpSetForPeer;
mnodeMnodeUnLock(); mnodeMnodeUnLock();
} }
void mnodeGetMnodeIpSetForShell(SRpcIpSet *ipSet) { void mnodeGetMnodeEpSetForShell(SRpcEpSet *epSet) {
mnodeMnodeRdLock(); mnodeMnodeRdLock();
*ipSet = tsMnodeIpSetForShell; *epSet = tsMnodeEpSetForShell;
mnodeMnodeUnLock(); mnodeMnodeUnLock();
} }
@ -295,7 +295,7 @@ int32_t mnodeAddMnode(int32_t dnodeId) {
code = TSDB_CODE_MND_SDB_ERROR; code = TSDB_CODE_MND_SDB_ERROR;
} }
mnodeUpdateMnodeIpSet(); mnodeUpdateMnodeEpSet();
return code; return code;
} }
@ -308,7 +308,7 @@ void mnodeDropMnodeLocal(int32_t dnodeId) {
mnodeDecMnodeRef(pMnode); mnodeDecMnodeRef(pMnode);
} }
mnodeUpdateMnodeIpSet(); mnodeUpdateMnodeEpSet();
} }
int32_t mnodeDropMnode(int32_t dnodeId) { int32_t mnodeDropMnode(int32_t dnodeId) {
@ -330,7 +330,7 @@ int32_t mnodeDropMnode(int32_t dnodeId) {
sdbDecRef(tsMnodeSdb, pMnode); sdbDecRef(tsMnodeSdb, pMnode);
mnodeUpdateMnodeIpSet(); mnodeUpdateMnodeEpSet();
return code; return code;
} }

14
src/mnode/src/mnodePeer.c
View File

@ -53,14 +53,14 @@ int32_t mnodeProcessPeerReq(SMnodeMsg *pMsg) {
if (!sdbIsMaster()) { if (!sdbIsMaster()) {
SMnodeRsp *rpcRsp = &pMsg->rpcRsp; SMnodeRsp *rpcRsp = &pMsg->rpcRsp;
SRpcIpSet *ipSet = rpcMallocCont(sizeof(SRpcIpSet)); SRpcEpSet *epSet = rpcMallocCont(sizeof(SRpcEpSet));
mnodeGetMnodeIpSetForPeer(ipSet); mnodeGetMnodeEpSetForPeer(epSet);
rpcRsp->rsp = ipSet; rpcRsp->rsp = epSet;
rpcRsp->len = sizeof(SRpcIpSet); rpcRsp->len = sizeof(SRpcEpSet);
mDebug("%p, msg:%s in mpeer queue, will be redireced inUse:%d", pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType], ipSet->inUse); mDebug("%p, msg:%s in mpeer queue, will be redireced inUse:%d", pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType], epSet->inUse);
for (int32_t i = 0; i < ipSet->numOfIps; ++i) { for (int32_t i = 0; i < epSet->numOfEps; ++i) {
mDebug("mnode index:%d ip:%s:%d", i, ipSet->fqdn[i], htons(ipSet->port[i])); mDebug("mnode index:%d ep:%s:%d", i, epSet->fqdn[i], htons(epSet->port[i]));
} }
return TSDB_CODE_RPC_REDIRECT; return TSDB_CODE_RPC_REDIRECT;

14
src/mnode/src/mnodeRead.c
View File

@ -49,14 +49,14 @@ int32_t mnodeProcessRead(SMnodeMsg *pMsg) {
if (!sdbIsMaster()) { if (!sdbIsMaster()) {
SMnodeRsp *rpcRsp = &pMsg->rpcRsp; SMnodeRsp *rpcRsp = &pMsg->rpcRsp;
SRpcIpSet *ipSet = rpcMallocCont(sizeof(SRpcIpSet)); SRpcEpSet *epSet = rpcMallocCont(sizeof(SRpcEpSet));
mnodeGetMnodeIpSetForShell(ipSet); mnodeGetMnodeEpSetForShell(epSet);
rpcRsp->rsp = ipSet; rpcRsp->rsp = epSet;
rpcRsp->len = sizeof(SRpcIpSet); rpcRsp->len = sizeof(SRpcEpSet);
mDebug("%p, msg:%s in mread queue, will be redireced, inUse:%d", pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType], ipSet->inUse); mDebug("%p, msg:%s in mread queue, will be redireced, inUse:%d", pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType], epSet->inUse);
for (int32_t i = 0; i < ipSet->numOfIps; ++i) { for (int32_t i = 0; i < epSet->numOfEps; ++i) {
mDebug("mnode index:%d ip:%s:%d", i, ipSet->fqdn[i], htons(ipSet->port[i])); mDebug("mnode index:%d ep:%s:%d", i, epSet->fqdn[i], htons(epSet->port[i]));
} }
return TSDB_CODE_RPC_REDIRECT; return TSDB_CODE_RPC_REDIRECT;

2
src/mnode/src/mnodeSdb.c
View File

@ -219,7 +219,7 @@ void sdbUpdateMnodeRoles() {
} }
} }
mnodeUpdateMnodeIpSet(); mnodeUpdateMnodeEpSet();
} }
static uint32_t sdbGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int32_t *size, uint64_t *fversion) { static uint32_t sdbGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int32_t *size, uint64_t *fversion) {

4
src/mnode/src/mnodeShow.c
View File

@ -270,7 +270,7 @@ static int32_t mnodeProcessHeartBeatMsg(SMnodeMsg *pMsg) {
pHBRsp->onlineDnodes = htonl(mnodeGetOnlinDnodesNum()); pHBRsp->onlineDnodes = htonl(mnodeGetOnlinDnodesNum());
pHBRsp->totalDnodes = htonl(mnodeGetDnodesNum()); pHBRsp->totalDnodes = htonl(mnodeGetDnodesNum());
mnodeGetMnodeIpSetForShell(&pHBRsp->ipList); mnodeGetMnodeEpSetForShell(&pHBRsp->epSet);
pMsg->rpcRsp.rsp = pHBRsp; pMsg->rpcRsp.rsp = pHBRsp;
pMsg->rpcRsp.len = sizeof(SCMHeartBeatRsp); pMsg->rpcRsp.len = sizeof(SCMHeartBeatRsp);
@ -335,7 +335,7 @@ static int32_t mnodeProcessConnectMsg(SMnodeMsg *pMsg) {
pConnectRsp->writeAuth = pUser->writeAuth; pConnectRsp->writeAuth = pUser->writeAuth;
pConnectRsp->superAuth = pUser->superAuth; pConnectRsp->superAuth = pUser->superAuth;
mnodeGetMnodeIpSetForShell(&pConnectRsp->ipList); mnodeGetMnodeEpSetForShell(&pConnectRsp->epSet);
connect_over: connect_over:
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {

28
src/mnode/src/mnodeTable.c
View File

@ -910,9 +910,9 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) {
mInfo("app:%p:%p, stable:%s, send drop stable msg to vgId:%d", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, mInfo("app:%p:%p, stable:%s, send drop stable msg to vgId:%d", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId,
pVgroup->vgId); pVgroup->vgId);
SRpcIpSet ipSet = mnodeGetIpSetFromVgroup(pVgroup); SRpcEpSet epSet = mnodeGetEpSetFromVgroup(pVgroup);
SRpcMsg rpcMsg = {.pCont = pDrop, .contLen = sizeof(SMDDropSTableMsg), .msgType = TSDB_MSG_TYPE_MD_DROP_STABLE}; SRpcMsg rpcMsg = {.pCont = pDrop, .contLen = sizeof(SMDDropSTableMsg), .msgType = TSDB_MSG_TYPE_MD_DROP_STABLE};
dnodeSendMsgToDnode(&ipSet, &rpcMsg); dnodeSendMsgToDnode(&epSet, &rpcMsg);
mnodeDecVgroupRef(pVgroup); mnodeDecVgroupRef(pVgroup);
} }
taosHashDestroyIter(pIter); taosHashDestroyIter(pIter);
@ -1484,10 +1484,10 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) {
SDnodeObj *pDnode = pVgroup->vnodeGid[vn].pDnode; SDnodeObj *pDnode = pVgroup->vnodeGid[vn].pDnode;
if (pDnode == NULL) break; if (pDnode == NULL) break;
tstrncpy(pVgroupInfo->vgroups[vgSize].ipAddr[vn].fqdn, pDnode->dnodeFqdn, TSDB_FQDN_LEN); tstrncpy(pVgroupInfo->vgroups[vgSize].epAddr[vn].fqdn, pDnode->dnodeFqdn, TSDB_FQDN_LEN);
pVgroupInfo->vgroups[vgSize].ipAddr[vn].port = htons(pDnode->dnodePort); pVgroupInfo->vgroups[vgSize].epAddr[vn].port = htons(pDnode->dnodePort);
pVgroupInfo->vgroups[vgSize].numOfIps++; pVgroupInfo->vgroups[vgSize].numOfEps++;
} }
vgSize++; vgSize++;
@ -1615,7 +1615,7 @@ static int32_t mnodeDoCreateChildTableCb(SMnodeMsg *pMsg, int32_t code) {
return terrno; return terrno;
} }
SRpcIpSet ipSet = mnodeGetIpSetFromVgroup(pMsg->pVgroup); SRpcEpSet epSet = mnodeGetEpSetFromVgroup(pMsg->pVgroup);
SRpcMsg rpcMsg = { SRpcMsg rpcMsg = {
.ahandle = pMsg, .ahandle = pMsg,
.pCont = pMDCreate, .pCont = pMDCreate,
@ -1624,7 +1624,7 @@ static int32_t mnodeDoCreateChildTableCb(SMnodeMsg *pMsg, int32_t code) {
.msgType = TSDB_MSG_TYPE_MD_CREATE_TABLE .msgType = TSDB_MSG_TYPE_MD_CREATE_TABLE
}; };
dnodeSendMsgToDnode(&ipSet, &rpcMsg); dnodeSendMsgToDnode(&epSet, &rpcMsg);
return TSDB_CODE_MND_ACTION_IN_PROGRESS; return TSDB_CODE_MND_ACTION_IN_PROGRESS;
} }
@ -1788,7 +1788,7 @@ static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg, bool needReturn) {
pDrop->sid = htonl(pTable->sid); pDrop->sid = htonl(pTable->sid);
pDrop->uid = htobe64(pTable->uid); pDrop->uid = htobe64(pTable->uid);
SRpcIpSet ipSet = mnodeGetIpSetFromVgroup(pMsg->pVgroup); SRpcEpSet epSet = mnodeGetEpSetFromVgroup(pMsg->pVgroup);
mInfo("app:%p:%p, table:%s, send drop ctable msg, vgId:%d sid:%d uid:%" PRIu64, pMsg->rpcMsg.ahandle, pMsg, mInfo("app:%p:%p, table:%s, send drop ctable msg, vgId:%d sid:%d uid:%" PRIu64, pMsg->rpcMsg.ahandle, pMsg,
pDrop->tableId, pTable->vgId, pTable->sid, pTable->uid); pDrop->tableId, pTable->vgId, pTable->sid, pTable->uid);
@ -1803,7 +1803,7 @@ static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg, bool needReturn) {
if (!needReturn) rpcMsg.ahandle = NULL; if (!needReturn) rpcMsg.ahandle = NULL;
dnodeSendMsgToDnode(&ipSet, &rpcMsg); dnodeSendMsgToDnode(&epSet, &rpcMsg);
return TSDB_CODE_MND_ACTION_IN_PROGRESS; return TSDB_CODE_MND_ACTION_IN_PROGRESS;
} }
@ -1842,7 +1842,7 @@ static int32_t mnodeAlterNormalTableColumnCb(SMnodeMsg *pMsg, int32_t code) {
} }
} }
SRpcIpSet ipSet = mnodeGetIpSetFromVgroup(pMsg->pVgroup); SRpcEpSet epSet = mnodeGetEpSetFromVgroup(pMsg->pVgroup);
SRpcMsg rpcMsg = { SRpcMsg rpcMsg = {
.ahandle = pMsg, .ahandle = pMsg,
.pCont = pMDCreate, .pCont = pMDCreate,
@ -1854,7 +1854,7 @@ static int32_t mnodeAlterNormalTableColumnCb(SMnodeMsg *pMsg, int32_t code) {
mDebug("app:%p:%p, ctable %s, send alter column msg to vgId:%d", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, mDebug("app:%p:%p, ctable %s, send alter column msg to vgId:%d", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId,
pMsg->pVgroup->vgId); pMsg->pVgroup->vgId);
dnodeSendMsgToDnode(&ipSet, &rpcMsg); dnodeSendMsgToDnode(&epSet, &rpcMsg);
return TSDB_CODE_MND_ACTION_IN_PROGRESS; return TSDB_CODE_MND_ACTION_IN_PROGRESS;
} }
@ -1996,9 +1996,9 @@ static int32_t mnodeDoGetChildTableMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta) {
for (int32_t i = 0; i < pMsg->pVgroup->numOfVnodes; ++i) { for (int32_t i = 0; i < pMsg->pVgroup->numOfVnodes; ++i) {
SDnodeObj *pDnode = mnodeGetDnode(pMsg->pVgroup->vnodeGid[i].dnodeId); SDnodeObj *pDnode = mnodeGetDnode(pMsg->pVgroup->vnodeGid[i].dnodeId);
if (pDnode == NULL) break; if (pDnode == NULL) break;
strcpy(pMeta->vgroup.ipAddr[i].fqdn, pDnode->dnodeFqdn); strcpy(pMeta->vgroup.epAddr[i].fqdn, pDnode->dnodeFqdn);
pMeta->vgroup.ipAddr[i].port = htons(pDnode->dnodePort + TSDB_PORT_DNODESHELL); pMeta->vgroup.epAddr[i].port = htons(pDnode->dnodePort + TSDB_PORT_DNODESHELL);
pMeta->vgroup.numOfIps++; pMeta->vgroup.numOfEps++;
mnodeDecDnodeRef(pDnode); mnodeDecDnodeRef(pDnode);
} }
pMeta->vgroup.vgId = htonl(pMsg->pVgroup->vgId); pMeta->vgroup.vgId = htonl(pMsg->pVgroup->vgId);

67
src/mnode/src/mnodeVgroup.c
View File

@ -317,9 +317,9 @@ void mnodeUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *pDnode, SVnodeLoad *pVl
} }
if (!dnodeExist) { if (!dnodeExist) {
SRpcIpSet ipSet = mnodeGetIpSetFromIp(pDnode->dnodeEp); SRpcEpSet epSet = mnodeGetEpSetFromIp(pDnode->dnodeEp);
mError("vgId:%d, dnode:%d not exist in mnode, drop it", pVload->vgId, pDnode->dnodeId); mError("vgId:%d, dnode:%d not exist in mnode, drop it", pVload->vgId, pDnode->dnodeId);
mnodeSendDropVnodeMsg(pVload->vgId, &ipSet, NULL); mnodeSendDropVnodeMsg(pVload->vgId, &epSet, NULL);
return; return;
} }
@ -585,9 +585,9 @@ static int32_t mnodeGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p
pSchema[cols].bytes = htons(pShow->bytes[cols]); pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++; cols++;
pShow->bytes[cols] = 4; pShow->bytes[cols] = 12 + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_INT; pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "poolSize"); strcpy(pSchema[cols].name, "status");
pSchema[cols].bytes = htons(pShow->bytes[cols]); pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++; cols++;
@ -689,7 +689,8 @@ static int32_t mnodeRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, v
cols++; cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int32_t *)pWrite = taosIdPoolMaxSize(pVgroup->idPool); char* status = vgroupStatus[pVgroup->status];
STR_TO_VARSTR(pWrite, status);
cols++; cols++;
int32_t onlineVnodes = 0; int32_t onlineVnodes = 0;
@ -808,29 +809,29 @@ static SMDCreateVnodeMsg *mnodeBuildVnodeMsg(SVgObj *pVgroup) {
return pVnode; return pVnode;
} }
SRpcIpSet mnodeGetIpSetFromVgroup(SVgObj *pVgroup) { SRpcEpSet mnodeGetEpSetFromVgroup(SVgObj *pVgroup) {
SRpcIpSet ipSet = { SRpcEpSet epSet = {
.numOfIps = pVgroup->numOfVnodes, .numOfEps = pVgroup->numOfVnodes,
.inUse = 0, .inUse = 0,
}; };
for (int i = 0; i < pVgroup->numOfVnodes; ++i) { for (int i = 0; i < pVgroup->numOfVnodes; ++i) {
strcpy(ipSet.fqdn[i], pVgroup->vnodeGid[i].pDnode->dnodeFqdn); strcpy(epSet.fqdn[i], pVgroup->vnodeGid[i].pDnode->dnodeFqdn);
ipSet.port[i] = pVgroup->vnodeGid[i].pDnode->dnodePort + TSDB_PORT_DNODEDNODE; epSet.port[i] = pVgroup->vnodeGid[i].pDnode->dnodePort + TSDB_PORT_DNODEDNODE;
} }
return ipSet; return epSet;
} }
SRpcIpSet mnodeGetIpSetFromIp(char *ep) { SRpcEpSet mnodeGetEpSetFromIp(char *ep) {
SRpcIpSet ipSet; SRpcEpSet epSet;
ipSet.numOfIps = 1; epSet.numOfEps = 1;
ipSet.inUse = 0; epSet.inUse = 0;
taosGetFqdnPortFromEp(ep, ipSet.fqdn[0], &ipSet.port[0]); taosGetFqdnPortFromEp(ep, epSet.fqdn[0], &epSet.port[0]);
ipSet.port[0] += TSDB_PORT_DNODEDNODE; epSet.port[0] += TSDB_PORT_DNODEDNODE;
return ipSet; return epSet;
} }
static void mnodeSendAlterVnodeMsg(SVgObj *pVgroup, SRpcIpSet *ipSet) { static void mnodeSendAlterVnodeMsg(SVgObj *pVgroup, SRpcEpSet *epSet) {
SMDAlterVnodeMsg *pAlter = mnodeBuildVnodeMsg(pVgroup); SMDAlterVnodeMsg *pAlter = mnodeBuildVnodeMsg(pVgroup);
SRpcMsg rpcMsg = { SRpcMsg rpcMsg = {
.ahandle = NULL, .ahandle = NULL,
@ -839,21 +840,21 @@ static void mnodeSendAlterVnodeMsg(SVgObj *pVgroup, SRpcIpSet *ipSet) {
.code = 0, .code = 0,
.msgType = TSDB_MSG_TYPE_MD_ALTER_VNODE .msgType = TSDB_MSG_TYPE_MD_ALTER_VNODE
}; };
dnodeSendMsgToDnode(ipSet, &rpcMsg); dnodeSendMsgToDnode(epSet, &rpcMsg);
} }
void mnodeSendAlterVgroupMsg(SVgObj *pVgroup) { void mnodeSendAlterVgroupMsg(SVgObj *pVgroup) {
mDebug("vgId:%d, send alter all vnodes msg, numOfVnodes:%d db:%s", pVgroup->vgId, pVgroup->numOfVnodes, mDebug("vgId:%d, send alter all vnodes msg, numOfVnodes:%d db:%s", pVgroup->vgId, pVgroup->numOfVnodes,
pVgroup->dbName); pVgroup->dbName);
for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
SRpcIpSet ipSet = mnodeGetIpSetFromIp(pVgroup->vnodeGid[i].pDnode->dnodeEp); SRpcEpSet epSet = mnodeGetEpSetFromIp(pVgroup->vnodeGid[i].pDnode->dnodeEp);
mDebug("vgId:%d, index:%d, send alter vnode msg to dnode %s", pVgroup->vgId, i, mDebug("vgId:%d, index:%d, send alter vnode msg to dnode %s", pVgroup->vgId, i,
pVgroup->vnodeGid[i].pDnode->dnodeEp); pVgroup->vnodeGid[i].pDnode->dnodeEp);
mnodeSendAlterVnodeMsg(pVgroup, &ipSet); mnodeSendAlterVnodeMsg(pVgroup, &epSet);
} }
} }
static void mnodeSendCreateVnodeMsg(SVgObj *pVgroup, SRpcIpSet *ipSet, void *ahandle) { static void mnodeSendCreateVnodeMsg(SVgObj *pVgroup, SRpcEpSet *epSet, void *ahandle) {
SMDCreateVnodeMsg *pCreate = mnodeBuildVnodeMsg(pVgroup); SMDCreateVnodeMsg *pCreate = mnodeBuildVnodeMsg(pVgroup);
SRpcMsg rpcMsg = { SRpcMsg rpcMsg = {
.ahandle = ahandle, .ahandle = ahandle,
@ -862,17 +863,17 @@ static void mnodeSendCreateVnodeMsg(SVgObj *pVgroup, SRpcIpSet *ipSet, void *aha
.code = 0, .code = 0,
.msgType = TSDB_MSG_TYPE_MD_CREATE_VNODE .msgType = TSDB_MSG_TYPE_MD_CREATE_VNODE
}; };
dnodeSendMsgToDnode(ipSet, &rpcMsg); dnodeSendMsgToDnode(epSet, &rpcMsg);
} }
void mnodeSendCreateVgroupMsg(SVgObj *pVgroup, void *ahandle) { void mnodeSendCreateVgroupMsg(SVgObj *pVgroup, void *ahandle) {
mDebug("vgId:%d, send create all vnodes msg, numOfVnodes:%d db:%s", pVgroup->vgId, pVgroup->numOfVnodes, mDebug("vgId:%d, send create all vnodes msg, numOfVnodes:%d db:%s", pVgroup->vgId, pVgroup->numOfVnodes,
pVgroup->dbName); pVgroup->dbName);
for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
SRpcIpSet ipSet = mnodeGetIpSetFromIp(pVgroup->vnodeGid[i].pDnode->dnodeEp); SRpcEpSet epSet = mnodeGetEpSetFromIp(pVgroup->vnodeGid[i].pDnode->dnodeEp);
mDebug("vgId:%d, index:%d, send create vnode msg to dnode %s, ahandle:%p", pVgroup->vgId, mDebug("vgId:%d, index:%d, send create vnode msg to dnode %s, ahandle:%p", pVgroup->vgId,
i, pVgroup->vnodeGid[i].pDnode->dnodeEp, ahandle); i, pVgroup->vnodeGid[i].pDnode->dnodeEp, ahandle);
mnodeSendCreateVnodeMsg(pVgroup, &ipSet, ahandle); mnodeSendCreateVnodeMsg(pVgroup, &epSet, ahandle);
} }
} }
@ -925,7 +926,7 @@ static SMDDropVnodeMsg *mnodeBuildDropVnodeMsg(int32_t vgId) {
return pDrop; return pDrop;
} }
void mnodeSendDropVnodeMsg(int32_t vgId, SRpcIpSet *ipSet, void *ahandle) { void mnodeSendDropVnodeMsg(int32_t vgId, SRpcEpSet *epSet, void *ahandle) {
SMDDropVnodeMsg *pDrop = mnodeBuildDropVnodeMsg(vgId); SMDDropVnodeMsg *pDrop = mnodeBuildDropVnodeMsg(vgId);
SRpcMsg rpcMsg = { SRpcMsg rpcMsg = {
.ahandle = ahandle, .ahandle = ahandle,
@ -934,16 +935,16 @@ void mnodeSendDropVnodeMsg(int32_t vgId, SRpcIpSet *ipSet, void *ahandle) {
.code = 0, .code = 0,
.msgType = TSDB_MSG_TYPE_MD_DROP_VNODE .msgType = TSDB_MSG_TYPE_MD_DROP_VNODE
}; };
dnodeSendMsgToDnode(ipSet, &rpcMsg); dnodeSendMsgToDnode(epSet, &rpcMsg);
} }
static void mnodeSendDropVgroupMsg(SVgObj *pVgroup, void *ahandle) { static void mnodeSendDropVgroupMsg(SVgObj *pVgroup, void *ahandle) {
pVgroup->status = TAOS_VG_STATUS_DROPPING; // deleting pVgroup->status = TAOS_VG_STATUS_DROPPING; // deleting
mDebug("vgId:%d, send drop all vnodes msg, ahandle:%p", pVgroup->vgId, ahandle); mDebug("vgId:%d, send drop all vnodes msg, ahandle:%p", pVgroup->vgId, ahandle);
for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
SRpcIpSet ipSet = mnodeGetIpSetFromIp(pVgroup->vnodeGid[i].pDnode->dnodeEp); SRpcEpSet epSet = mnodeGetEpSetFromIp(pVgroup->vnodeGid[i].pDnode->dnodeEp);
mDebug("vgId:%d, send drop vnode msg to dnode:%d, ahandle:%p", pVgroup->vgId, pVgroup->vnodeGid[i].dnodeId, ahandle); mDebug("vgId:%d, send drop vnode msg to dnode:%d, ahandle:%p", pVgroup->vgId, pVgroup->vnodeGid[i].dnodeId, ahandle);
mnodeSendDropVnodeMsg(pVgroup->vgId, &ipSet, ahandle); mnodeSendDropVnodeMsg(pVgroup->vgId, &epSet, ahandle);
} }
} }
@ -997,8 +998,8 @@ static int32_t mnodeProcessVnodeCfgMsg(SMnodeMsg *pMsg) {
} }
mDebug("vgId:%d, send create vnode msg to dnode %s for vnode cfg msg", pVgroup->vgId, pDnode->dnodeEp); mDebug("vgId:%d, send create vnode msg to dnode %s for vnode cfg msg", pVgroup->vgId, pDnode->dnodeEp);
SRpcIpSet ipSet = mnodeGetIpSetFromIp(pDnode->dnodeEp); SRpcEpSet epSet = mnodeGetEpSetFromIp(pDnode->dnodeEp);
mnodeSendCreateVnodeMsg(pVgroup, &ipSet, NULL); mnodeSendCreateVnodeMsg(pVgroup, &epSet, NULL);
mnodeDecDnodeRef(pDnode); mnodeDecDnodeRef(pDnode);
mnodeDecVgroupRef(pVgroup); mnodeDecVgroupRef(pVgroup);

16
src/mnode/src/mnodeWrite.c
View File

@ -49,16 +49,16 @@ int32_t mnodeProcessWrite(SMnodeMsg *pMsg) {
if (!sdbIsMaster()) { if (!sdbIsMaster()) {
SMnodeRsp *rpcRsp = &pMsg->rpcRsp; SMnodeRsp *rpcRsp = &pMsg->rpcRsp;
SRpcIpSet *ipSet = rpcMallocCont(sizeof(SRpcIpSet)); SRpcEpSet *epSet = rpcMallocCont(sizeof(SRpcEpSet));
mnodeGetMnodeIpSetForShell(ipSet); mnodeGetMnodeEpSetForShell(epSet);
rpcRsp->rsp = ipSet; rpcRsp->rsp = epSet;
rpcRsp->len = sizeof(SRpcIpSet); rpcRsp->len = sizeof(SRpcEpSet);
mDebug("app:%p:%p, msg:%s will be redireced inUse:%d", pMsg->rpcMsg.ahandle, pMsg, taosMsg[pMsg->rpcMsg.msgType], mDebug("app:%p:%p, msg:%s will be redireced inUse:%d", pMsg->rpcMsg.ahandle, pMsg, taosMsg[pMsg->rpcMsg.msgType],
ipSet->inUse); epSet->inUse);
for (int32_t i = 0; i < ipSet->numOfIps; ++i) { for (int32_t i = 0; i < epSet->numOfEps; ++i) {
mDebug("app:%p:%p, mnode index:%d ip:%s:%d", pMsg->rpcMsg.ahandle, pMsg, i, ipSet->fqdn[i], mDebug("app:%p:%p, mnode index:%d ep:%s:%d", pMsg->rpcMsg.ahandle, pMsg, i, epSet->fqdn[i],
htons(ipSet->port[i])); htons(epSet->port[i]));
} }
return TSDB_CODE_RPC_REDIRECT; return TSDB_CODE_RPC_REDIRECT;

4
src/plugins/monitor/src/monitorMain.c
View File

@ -76,6 +76,8 @@ static void monitorInitDatabase();
static void monitorInitDatabaseCb(void *param, TAOS_RES *result, int32_t code); static void monitorInitDatabaseCb(void *param, TAOS_RES *result, int32_t code);
static void monitorStartTimer(); static void monitorStartTimer();
static void monitorSaveSystemInfo(); static void monitorSaveSystemInfo();
extern int32_t (*monitorStartSystemFp)();
extern void (*monitorStopSystemFp)();
static void monitorCheckDiskUsage(void *para, void *unused) { static void monitorCheckDiskUsage(void *para, void *unused) {
taosGetDisk(); taosGetDisk();
@ -85,6 +87,8 @@ static void monitorCheckDiskUsage(void *para, void *unused) {
int32_t monitorInitSystem() { int32_t monitorInitSystem() {
taos_init(); taos_init();
taosTmrReset(monitorCheckDiskUsage, CHECK_INTERVAL, NULL, tscTmr, &tsMonitorConn.diskTimer); taosTmrReset(monitorCheckDiskUsage, CHECK_INTERVAL, NULL, tscTmr, &tsMonitorConn.diskTimer);
monitorStartSystemFp = monitorStartSystem;
monitorStopSystemFp = monitorStopSystem;
return 0; return 0;
} }

1
src/query/inc/qExecutor.h
View File

@ -121,7 +121,6 @@ typedef struct SQueryCostInfo {
uint32_t loadBlockStatis; uint32_t loadBlockStatis;
uint32_t discardBlocks; uint32_t discardBlocks;
uint64_t elapsedTime; uint64_t elapsedTime;
uint64_t ioTime;
uint64_t computTime; uint64_t computTime;
} SQueryCostInfo; } SQueryCostInfo;

23
src/query/inc/qUtil.h
View File

@ -23,7 +23,7 @@ void copyTimeWindowResBuf(SQueryRuntimeEnv* pRuntimeEnv, SWindowResult* dst, con
int32_t initWindowResInfo(SWindowResInfo* pWindowResInfo, SQueryRuntimeEnv* pRuntimeEnv, int32_t size, int32_t initWindowResInfo(SWindowResInfo* pWindowResInfo, SQueryRuntimeEnv* pRuntimeEnv, int32_t size,
int32_t threshold, int16_t type); int32_t threshold, int16_t type);
void cleanupTimeWindowInfo(SWindowResInfo* pWindowResInfo, int32_t numOfCols); void cleanupTimeWindowInfo(SWindowResInfo* pWindowResInfo);
void resetTimeWindowInfo(SQueryRuntimeEnv* pRuntimeEnv, SWindowResInfo* pWindowResInfo); void resetTimeWindowInfo(SQueryRuntimeEnv* pRuntimeEnv, SWindowResInfo* pWindowResInfo);
void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num); void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num);
@ -32,14 +32,29 @@ int32_t numOfClosedTimeWindow(SWindowResInfo* pWindowResInfo);
void closeTimeWindow(SWindowResInfo* pWindowResInfo, int32_t slot); void closeTimeWindow(SWindowResInfo* pWindowResInfo, int32_t slot);
void closeAllTimeWindow(SWindowResInfo* pWindowResInfo); void closeAllTimeWindow(SWindowResInfo* pWindowResInfo);
void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_t order); void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_t order);
SWindowResult *getWindowResult(SWindowResInfo *pWindowResInfo, int32_t slot);
static FORCE_INLINE SWindowResult *getWindowResult(SWindowResInfo *pWindowResInfo, int32_t slot) {
assert(pWindowResInfo != NULL && slot >= 0 && slot < pWindowResInfo->size);
return &pWindowResInfo->pResult[slot];
}
#define curTimeWindow(_winres) ((_winres)->curIndex) #define curTimeWindow(_winres) ((_winres)->curIndex)
#define GET_ROW_PARAM_FOR_MULTIOUTPUT(_q, tbq, sq) (((tbq) && (!sq))? (_q)->pSelectExpr[1].base.arg->argValue.i64:1)
bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot); bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot);
int32_t createQueryResultInfo(SQuery *pQuery, SWindowResult *pResultRow, bool isSTableQuery, SPosInfo *posInfo, size_t interBufSize); int32_t createQueryResultInfo(SQuery *pQuery, SWindowResult *pResultRow, bool isSTableQuery, size_t interBufSize);
char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int32_t columnIndex, SWindowResult *pResult); static FORCE_INLINE char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int32_t columnIndex, SWindowResult *pResult) {
assert(pResult != NULL && pRuntimeEnv != NULL);
SQuery *pQuery = pRuntimeEnv->pQuery;
tFilePage *page = GET_RES_BUF_PAGE_BY_ID(pRuntimeEnv->pResultBuf, pResult->pos.pageId);
int32_t realRowId = pResult->pos.rowId * GET_ROW_PARAM_FOR_MULTIOUTPUT(pQuery, pRuntimeEnv->topBotQuery, pRuntimeEnv->stableQuery);
return ((char *)page->data) + pRuntimeEnv->offset[columnIndex] * pRuntimeEnv->numOfRowsPerPage +
pQuery->pSelectExpr[columnIndex].bytes * realRowId;
}
__filter_func_t *getRangeFilterFuncArray(int32_t type); __filter_func_t *getRangeFilterFuncArray(int32_t type);
__filter_func_t *getValueFilterFuncArray(int32_t type); __filter_func_t *getValueFilterFuncArray(int32_t type);

14
src/query/inc/qresultBuf.h
View File

@ -22,12 +22,9 @@ extern "C" {
#include "os.h" #include "os.h"
#include "qextbuffer.h" #include "qextbuffer.h"
#include "hash.h"
typedef struct SIDList { typedef struct SArray* SIDList;
uint32_t alloc;
int32_t size;
int32_t* pData;
} SIDList;
typedef struct SDiskbasedResultBuf { typedef struct SDiskbasedResultBuf {
int32_t numOfRowsPerPage; int32_t numOfRowsPerPage;
@ -39,9 +36,8 @@ typedef struct SDiskbasedResultBuf {
char* pBuf; // mmap buffer pointer char* pBuf; // mmap buffer pointer
char* path; // file path char* path; // file path
uint32_t numOfAllocGroupIds; // number of allocated id list SHashObj* idsTable; // id hash table
void* idsTable; // id hash table SIDList list; // for each id, there is a page id list
SIDList* list; // for each id, there is a page id list
} SDiskbasedResultBuf; } SDiskbasedResultBuf;
#define DEFAULT_INTERN_BUF_PAGE_SIZE (8192L*5) #define DEFAULT_INTERN_BUF_PAGE_SIZE (8192L*5)
@ -112,7 +108,7 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf, void* handle);
* @param pList * @param pList
* @return * @return
*/ */
int32_t getLastPageId(SIDList *pList); int32_t getLastPageId(SIDList pList);
#ifdef __cplusplus #ifdef __cplusplus
} }

5
src/query/inc/tsqlfunction.h
View File

@ -132,13 +132,10 @@ typedef struct SQLPreAggVal {
typedef struct SInterpInfoDetail { typedef struct SInterpInfoDetail {
TSKEY ts; // interp specified timestamp TSKEY ts; // interp specified timestamp
int8_t hasResult;
int8_t type; int8_t type;
int8_t primaryCol; int8_t primaryCol;
} SInterpInfoDetail; } SInterpInfoDetail;
typedef struct SInterpInfo { SInterpInfoDetail *pInterpDetail; } SInterpInfo;
typedef struct SResultInfo { typedef struct SResultInfo {
int8_t hasResult; // result generated, not NULL value int8_t hasResult; // result generated, not NULL value
bool initialized; // output buffer has been initialized bool initialized; // output buffer has been initialized
@ -146,7 +143,7 @@ typedef struct SResultInfo {
bool superTableQ; // is super table query bool superTableQ; // is super table query
int32_t numOfRes; // num of output result in current buffer int32_t numOfRes; // num of output result in current buffer
int32_t bufLen; // buffer size int32_t bufLen; // buffer size
void * interResultBuf; // output result buffer void* interResultBuf; // output result buffer
} SResultInfo; } SResultInfo;
struct SQLFunctionCtx; struct SQLFunctionCtx;

424
src/query/src/qExecutor.c
View File

@ -26,7 +26,6 @@
#include "query.h" #include "query.h"
#include "queryLog.h" #include "queryLog.h"
#include "qast.h" #include "qast.h"
#include "tfile.h"
#include "tlosertree.h" #include "tlosertree.h"
#include "tscompression.h" #include "tscompression.h"
#include "ttime.h" #include "ttime.h"
@ -144,7 +143,7 @@ static void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData,
SDataStatis *pStatis, void *param, int32_t colIndex); SDataStatis *pStatis, void *param, int32_t colIndex);
static void initCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv); static void initCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv);
static void destroyTableQueryInfo(STableQueryInfo *pTableQueryInfo, int32_t numOfCols); static void destroyTableQueryInfo(STableQueryInfo *pTableQueryInfo);
static void resetCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv); static void resetCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv);
static bool hasMainOutput(SQuery *pQuery); static bool hasMainOutput(SQuery *pQuery);
static void buildTagQueryResult(SQInfo *pQInfo); static void buildTagQueryResult(SQInfo *pQInfo);
@ -361,17 +360,17 @@ static bool hasTagValOutput(SQuery* pQuery) {
* @return * @return
*/ */
static bool hasNullValue(SColIndex* pColIndex, SDataStatis *pStatis, SDataStatis **pColStatis) { static bool hasNullValue(SColIndex* pColIndex, SDataStatis *pStatis, SDataStatis **pColStatis) {
if (TSDB_COL_IS_TAG(pColIndex->flag) || pColIndex->colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { if (pStatis != NULL && !TSDB_COL_IS_TAG(pColIndex->flag)) {
return false;
}
if (pStatis != NULL) {
*pColStatis = &pStatis[pColIndex->colIndex]; *pColStatis = &pStatis[pColIndex->colIndex];
assert((*pColStatis)->colId == pColIndex->colId); assert((*pColStatis)->colId == pColIndex->colId);
} else { } else {
*pColStatis = NULL; *pColStatis = NULL;
} }
if (TSDB_COL_IS_TAG(pColIndex->flag) || pColIndex->colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) {
return false;
}
if ((*pColStatis) != NULL && (*pColStatis)->numOfNull == 0) { if ((*pColStatis) != NULL && (*pColStatis)->numOfNull == 0) {
return false; return false;
} }
@ -387,31 +386,33 @@ static SWindowResult *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWin
if (p1 != NULL) { if (p1 != NULL) {
pWindowResInfo->curIndex = *p1; pWindowResInfo->curIndex = *p1;
} else { } else {
if (masterscan) { // more than the capacity, reallocate the resources if (!masterscan) { // not master scan, do not add new timewindow
if (pWindowResInfo->size >= pWindowResInfo->capacity) { return NULL;
int64_t newCap = pWindowResInfo->capacity * 2; }
// more than the capacity, reallocate the resources
if (pWindowResInfo->size >= pWindowResInfo->capacity) {
int64_t newCap = pWindowResInfo->capacity * 1.5;
char *t = realloc(pWindowResInfo->pResult, newCap * sizeof(SWindowResult)); char *t = realloc(pWindowResInfo->pResult, newCap * sizeof(SWindowResult));
if (t != NULL) { if (t != NULL) {
pWindowResInfo->pResult = (SWindowResult *)t; pWindowResInfo->pResult = (SWindowResult *)t;
memset(&pWindowResInfo->pResult[pWindowResInfo->capacity], 0, sizeof(SWindowResult) * pWindowResInfo->capacity);
int32_t inc = newCap - pWindowResInfo->capacity;
memset(&pWindowResInfo->pResult[pWindowResInfo->capacity], 0, sizeof(SWindowResult) * inc);
} else { } else {
// todo // todo
} }
for (int32_t i = pWindowResInfo->capacity; i < newCap; ++i) { for (int32_t i = pWindowResInfo->capacity; i < newCap; ++i) {
SPosInfo pos = {-1, -1}; createQueryResultInfo(pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, pRuntimeEnv->interBufSize);
createQueryResultInfo(pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, &pos, pRuntimeEnv->interBufSize);
} }
pWindowResInfo->capacity = newCap; pWindowResInfo->capacity = newCap;
} }
// add a new result set for a new group // add a new result set for a new group
pWindowResInfo->curIndex = pWindowResInfo->size++; pWindowResInfo->curIndex = pWindowResInfo->size++;
taosHashPut(pWindowResInfo->hashList, pData, bytes, (char *)&pWindowResInfo->curIndex, sizeof(int32_t)); taosHashPut(pWindowResInfo->hashList, pData, bytes, (char *)&pWindowResInfo->curIndex, sizeof(int32_t));
} else {
return NULL;
}
} }
return getWindowResult(pWindowResInfo, pWindowResInfo->curIndex); return getWindowResult(pWindowResInfo, pWindowResInfo->curIndex);
@ -470,10 +471,10 @@ static int32_t addNewWindowResultBuf(SWindowResult *pWindowRes, SDiskbasedResult
int32_t pageId = -1; int32_t pageId = -1;
SIDList list = getDataBufPagesIdList(pResultBuf, sid); SIDList list = getDataBufPagesIdList(pResultBuf, sid);
if (list.size == 0) { if (taosArrayGetSize(list) == 0) {
pData = getNewDataBuf(pResultBuf, sid, &pageId); pData = getNewDataBuf(pResultBuf, sid, &pageId);
} else { } else {
pageId = getLastPageId(&list); pageId = getLastPageId(list);
pData = GET_RES_BUF_PAGE_BY_ID(pResultBuf, pageId); pData = GET_RES_BUF_PAGE_BY_ID(pResultBuf, pageId);
if (pData->num >= numOfRowsPerPage) { if (pData->num >= numOfRowsPerPage) {
@ -511,10 +512,11 @@ static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowRes
} }
*newWind = true; *newWind = true;
// not assign result buffer yet, add new result buffer // not assign result buffer yet, add new result buffer
if (pWindowRes->pos.pageId == -1) { if (pWindowRes->pos.pageId == -1) {
int32_t ret = addNewWindowResultBuf(pWindowRes, pResultBuf, sid, pRuntimeEnv->numOfRowsPerPage); int32_t ret = addNewWindowResultBuf(pWindowRes, pResultBuf, sid, pRuntimeEnv->numOfRowsPerPage);
if (ret != 0) { if (ret != TSDB_CODE_SUCCESS) {
return -1; return -1;
} }
} }
@ -531,7 +533,7 @@ static SWindowStatus *getTimeWindowResStatus(SWindowResInfo *pWindowResInfo, int
return &pWindowResInfo->pResult[slot].status; return &pWindowResInfo->pResult[slot].status;
} }
static int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, int16_t pos, static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, int16_t pos,
int16_t order, int64_t *pData) { int16_t order, int64_t *pData) {
int32_t forwardStep = 0; int32_t forwardStep = 0;
@ -647,13 +649,9 @@ static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlo
if (QUERY_IS_ASC_QUERY(pQuery)) { if (QUERY_IS_ASC_QUERY(pQuery)) {
if (ekey < pDataBlockInfo->window.ekey) { if (ekey < pDataBlockInfo->window.ekey) {
num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn); num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn);
if (num == 0) { // no qualified data in current block, do not update the lastKey value if (updateLastKey) { // update the last key
assert(ekey < pPrimaryColumn[startPos]);
} else {
if (updateLastKey) {
item->lastKey = pPrimaryColumn[startPos + (num - 1)] + step; item->lastKey = pPrimaryColumn[startPos + (num - 1)] + step;
} }
}
} else { } else {
num = pDataBlockInfo->rows - startPos; num = pDataBlockInfo->rows - startPos;
if (updateLastKey) { if (updateLastKey) {
@ -663,13 +661,9 @@ static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlo
} else { // desc } else { // desc
if (ekey > pDataBlockInfo->window.skey) { if (ekey > pDataBlockInfo->window.skey) {
num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn); num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn);
if (num == 0) { // no qualified data in current block, do not update the lastKey value if (updateLastKey) { // update the last key
assert(ekey > pPrimaryColumn[startPos]);
} else {
if (updateLastKey) {
item->lastKey = pPrimaryColumn[startPos - (num - 1)] + step; item->lastKey = pPrimaryColumn[startPos - (num - 1)] + step;
} }
}
} else { } else {
num = startPos + 1; num = startPos + 1;
if (updateLastKey) { if (updateLastKey) {
@ -912,13 +906,20 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *
} }
int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order);
if (QUERY_IS_INTERVAL_QUERY(pQuery) && tsCols != NULL) { if (QUERY_IS_INTERVAL_QUERY(pQuery)/* && tsCols != NULL*/) {
TSKEY ts = TSKEY_INITIAL_VAL;
if (tsCols == NULL) {
ts = QUERY_IS_ASC_QUERY(pQuery)? pDataBlockInfo->window.skey:pDataBlockInfo->window.ekey;
} else {
int32_t offset = GET_COL_DATA_POS(pQuery, 0, step); int32_t offset = GET_COL_DATA_POS(pQuery, 0, step);
TSKEY ts = tsCols[offset]; ts = tsCols[offset];
}
bool hasTimeWindow = false; bool hasTimeWindow = false;
STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery); STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery);
if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo->tid, &win, masterScan, &hasTimeWindow) != TSDB_CODE_SUCCESS) { if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo->tid, &win, masterScan, &hasTimeWindow) !=
TSDB_CODE_SUCCESS) {
tfree(sasArray); tfree(sasArray);
return; return;
} }
@ -946,7 +947,8 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *
// null data, failed to allocate more memory buffer // null data, failed to allocate more memory buffer
hasTimeWindow = false; hasTimeWindow = false;
if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo->tid, &nextWin, masterScan, &hasTimeWindow) != TSDB_CODE_SUCCESS) { if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo->tid, &nextWin, masterScan,
&hasTimeWindow) != TSDB_CODE_SUCCESS) {
break; break;
} }
@ -957,7 +959,7 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *
TSKEY ekey = reviseWindowEkey(pQuery, &nextWin); TSKEY ekey = reviseWindowEkey(pQuery, &nextWin);
forwardStep = getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, tsCols, startPos, ekey, searchFn, true); forwardStep = getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, tsCols, startPos, ekey, searchFn, true);
SWindowStatus* pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo)); SWindowStatus *pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo));
doBlockwiseApplyFunctions(pRuntimeEnv, pStatus, &nextWin, startPos, forwardStep, tsCols, pDataBlockInfo->rows); doBlockwiseApplyFunctions(pRuntimeEnv, pStatus, &nextWin, startPos, forwardStep, tsCols, pDataBlockInfo->rows);
} }
@ -1478,7 +1480,9 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order
qDebug("QInfo:%p setup runtime env", GET_QINFO_ADDR(pRuntimeEnv)); qDebug("QInfo:%p setup runtime env", GET_QINFO_ADDR(pRuntimeEnv));
SQuery *pQuery = pRuntimeEnv->pQuery; SQuery *pQuery = pRuntimeEnv->pQuery;
pRuntimeEnv->resultInfo = calloc(pQuery->numOfOutput, sizeof(SResultInfo)); size_t size = pRuntimeEnv->interBufSize + pQuery->numOfOutput * sizeof(SResultInfo);
pRuntimeEnv->resultInfo = calloc(1, size);
pRuntimeEnv->pCtx = (SQLFunctionCtx *)calloc(pQuery->numOfOutput, sizeof(SQLFunctionCtx)); pRuntimeEnv->pCtx = (SQLFunctionCtx *)calloc(pQuery->numOfOutput, sizeof(SQLFunctionCtx));
if (pRuntimeEnv->resultInfo == NULL || pRuntimeEnv->pCtx == NULL) { if (pRuntimeEnv->resultInfo == NULL || pRuntimeEnv->pCtx == NULL) {
@ -1549,7 +1553,7 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order
} }
} }
char* buf = calloc(1, pRuntimeEnv->interBufSize); char* buf = (char*) pRuntimeEnv->resultInfo + sizeof(SResultInfo) * pQuery->numOfOutput;
// set the intermediate result output buffer // set the intermediate result output buffer
setWindowResultInfo(pRuntimeEnv->resultInfo, pQuery, pRuntimeEnv->stableQuery, buf); setWindowResultInfo(pRuntimeEnv->resultInfo, pQuery, pRuntimeEnv->stableQuery, buf);
@ -1578,7 +1582,7 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) {
SQInfo* pQInfo = (SQInfo*) GET_QINFO_ADDR(pRuntimeEnv); SQInfo* pQInfo = (SQInfo*) GET_QINFO_ADDR(pRuntimeEnv);
qDebug("QInfo:%p teardown runtime env", pQInfo); qDebug("QInfo:%p teardown runtime env", pQInfo);
cleanupTimeWindowInfo(&pRuntimeEnv->windowResInfo, pQuery->numOfOutput); cleanupTimeWindowInfo(&pRuntimeEnv->windowResInfo);
if (pRuntimeEnv->pCtx != NULL) { if (pRuntimeEnv->pCtx != NULL) {
for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
@ -1592,7 +1596,6 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) {
tfree(pCtx->tagInfo.pTagCtxList); tfree(pCtx->tagInfo.pTagCtxList);
} }
tfree(pRuntimeEnv->resultInfo[0].interResultBuf);
tfree(pRuntimeEnv->resultInfo); tfree(pRuntimeEnv->resultInfo);
tfree(pRuntimeEnv->pCtx); tfree(pRuntimeEnv->pCtx);
} }
@ -1608,7 +1611,7 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) {
#define IS_QUERY_KILLED(_q) ((_q)->code == TSDB_CODE_TSC_QUERY_CANCELLED) #define IS_QUERY_KILLED(_q) ((_q)->code == TSDB_CODE_TSC_QUERY_CANCELLED)
static void setQueryKilled(SQInfo *pQInfo) { pQInfo->code = TSDB_CODE_TSC_QUERY_CANCELLED; } static void setQueryKilled(SQInfo *pQInfo) { pQInfo->code = TSDB_CODE_TSC_QUERY_CANCELLED;}
static bool isFixedOutputQuery(SQueryRuntimeEnv* pRuntimeEnv) { static bool isFixedOutputQuery(SQueryRuntimeEnv* pRuntimeEnv) {
SQuery* pQuery = pRuntimeEnv->pQuery; SQuery* pQuery = pRuntimeEnv->pQuery;
@ -1912,24 +1915,11 @@ static int32_t getInitialPageNum(SQInfo *pQInfo) {
return num; return num;
} }
#define GET_ROW_PARAM_FOR_MULTIOUTPUT(_q, tbq, sq) (((tbq) && (!sq))? (_q)->pSelectExpr[1].base.arg->argValue.i64:1)
static FORCE_INLINE int32_t getNumOfRowsInResultPage(SQuery *pQuery, bool topBotQuery, bool isSTableQuery) { static FORCE_INLINE int32_t getNumOfRowsInResultPage(SQuery *pQuery, bool topBotQuery, bool isSTableQuery) {
int32_t rowSize = pQuery->rowSize * GET_ROW_PARAM_FOR_MULTIOUTPUT(pQuery, topBotQuery, isSTableQuery); int32_t rowSize = pQuery->rowSize * GET_ROW_PARAM_FOR_MULTIOUTPUT(pQuery, topBotQuery, isSTableQuery);
return (DEFAULT_INTERN_BUF_PAGE_SIZE - sizeof(tFilePage)) / rowSize; return (DEFAULT_INTERN_BUF_PAGE_SIZE - sizeof(tFilePage)) / rowSize;
} }
char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int32_t columnIndex, SWindowResult *pResult) {
assert(pResult != NULL && pRuntimeEnv != NULL);
SQuery *pQuery = pRuntimeEnv->pQuery;
tFilePage *page = GET_RES_BUF_PAGE_BY_ID(pRuntimeEnv->pResultBuf, pResult->pos.pageId);
int32_t realRowId = pResult->pos.rowId * GET_ROW_PARAM_FOR_MULTIOUTPUT(pQuery, pRuntimeEnv->topBotQuery, pRuntimeEnv->stableQuery);
return ((char *)page->data) + pRuntimeEnv->offset[columnIndex] * pRuntimeEnv->numOfRowsPerPage +
pQuery->pSelectExpr[columnIndex].bytes * realRowId;
}
#define IS_PREFILTER_TYPE(_t) ((_t) != TSDB_DATA_TYPE_BINARY && (_t) != TSDB_DATA_TYPE_NCHAR) #define IS_PREFILTER_TYPE(_t) ((_t) != TSDB_DATA_TYPE_BINARY && (_t) != TSDB_DATA_TYPE_NCHAR)
static bool needToLoadDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SDataStatis *pDataStatis, SQLFunctionCtx *pCtx, static bool needToLoadDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SDataStatis *pDataStatis, SQLFunctionCtx *pCtx,
@ -1997,23 +1987,80 @@ static bool needToLoadDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SDataStatis *pDat
return false; return false;
} }
#define PT_IN_WINDOW(_p, _w) ((_p) > (_w).skey && (_p) < (_w).ekey)
static bool overlapWithTimeWindow(SQuery* pQuery, SDataBlockInfo* pBlockInfo) {
STimeWindow w = {0};
TSKEY sk = MIN(pQuery->window.skey, pQuery->window.ekey);
TSKEY ek = MAX(pQuery->window.skey, pQuery->window.ekey);
if (QUERY_IS_ASC_QUERY(pQuery)) {
getAlignQueryTimeWindow(pQuery, pBlockInfo->window.skey, sk, ek, &w);
if (PT_IN_WINDOW(w.ekey, pBlockInfo->window)) {
return true;
}
while(1) {
GET_NEXT_TIMEWINDOW(pQuery, &w);
if (w.skey > pBlockInfo->window.skey) {
break;
}
if (PT_IN_WINDOW(w.skey, pBlockInfo->window) || PT_IN_WINDOW(w.ekey, pBlockInfo->window)) {
return true;
}
}
} else {
getAlignQueryTimeWindow(pQuery, pBlockInfo->window.ekey, sk, ek, &w);
if (PT_IN_WINDOW(w.skey, pBlockInfo->window)) {
return true;
}
while(1) {
GET_NEXT_TIMEWINDOW(pQuery, &w);
if (w.ekey < pBlockInfo->window.skey) {
break;
}
if (PT_IN_WINDOW(w.skey, pBlockInfo->window) || PT_IN_WINDOW(w.ekey, pBlockInfo->window)) {
return true;
}
}
}
return false;
}
int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, void* pQueryHandle, SDataBlockInfo* pBlockInfo, SDataStatis **pStatis, SArray** pDataBlock) { int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, void* pQueryHandle, SDataBlockInfo* pBlockInfo, SDataStatis **pStatis, SArray** pDataBlock) {
SQuery *pQuery = pRuntimeEnv->pQuery; SQuery *pQuery = pRuntimeEnv->pQuery;
uint32_t status = 0; uint32_t status = 0;
if (pQuery->numOfFilterCols > 0) { if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf > 0) {
status = BLK_DATA_ALL_NEEDED; status = BLK_DATA_ALL_NEEDED;
} else { // check if this data block is required to load } else { // check if this data block is required to load
// Calculate all time windows that are overlapping or contain current data block.
// If current data block is contained by all possible time window, loading current
// data block is not needed.
if (QUERY_IS_INTERVAL_QUERY(pQuery) && overlapWithTimeWindow(pQuery, pBlockInfo)) {
status = BLK_DATA_ALL_NEEDED;
}
if (status != BLK_DATA_ALL_NEEDED) {
for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
SSqlFuncMsg* pSqlFunc = &pQuery->pSelectExpr[i].base; SSqlFuncMsg* pSqlFunc = &pQuery->pSelectExpr[i].base;
int32_t functionId = pSqlFunc->functionId; int32_t functionId = pSqlFunc->functionId;
int32_t colId = pSqlFunc->colInfo.colId; int32_t colId = pSqlFunc->colInfo.colId;
status |= aAggs[functionId].dataReqFunc(&pRuntimeEnv->pCtx[i], pBlockInfo->window.skey, pBlockInfo->window.ekey, colId);
}
if (pRuntimeEnv->pTSBuf > 0 || QUERY_IS_INTERVAL_QUERY(pQuery)) { status |= aAggs[functionId].dataReqFunc(&pRuntimeEnv->pCtx[i], pBlockInfo->window.skey, pBlockInfo->window.ekey, colId);
status |= BLK_DATA_ALL_NEEDED; if ((status & BLK_DATA_ALL_NEEDED) == BLK_DATA_ALL_NEEDED) {
break;
}
}
} }
} }
@ -2189,6 +2236,27 @@ static void ensureOutputBuffer(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pB
} }
} }
static void doSetInitialTimewindow(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pBlockInfo) {
SQuery* pQuery = pRuntimeEnv->pQuery;
if (QUERY_IS_INTERVAL_QUERY(pQuery) && pRuntimeEnv->windowResInfo.prevSKey == TSKEY_INITIAL_VAL) {
STimeWindow w = TSWINDOW_INITIALIZER;
SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo;
if (QUERY_IS_ASC_QUERY(pQuery)) {
getAlignQueryTimeWindow(pQuery, pBlockInfo->window.skey, pBlockInfo->window.skey, pQuery->window.ekey, &w);
pWindowResInfo->startTime = w.skey;
pWindowResInfo->prevSKey = w.skey;
} else {
// the start position of the first time window in the endpoint that spreads beyond the queried last timestamp
getAlignQueryTimeWindow(pQuery, pBlockInfo->window.ekey, pQuery->window.ekey, pBlockInfo->window.ekey, &w);
pWindowResInfo->startTime = pQuery->window.skey;
pWindowResInfo->prevSKey = w.skey;
}
}
}
static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) {
SQuery *pQuery = pRuntimeEnv->pQuery; SQuery *pQuery = pRuntimeEnv->pQuery;
STableQueryInfo* pTableQueryInfo = pQuery->current; STableQueryInfo* pTableQueryInfo = pQuery->current;
@ -2216,24 +2284,7 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) {
} }
tsdbRetrieveDataBlockInfo(pQueryHandle, &blockInfo); tsdbRetrieveDataBlockInfo(pQueryHandle, &blockInfo);
doSetInitialTimewindow(pRuntimeEnv, &blockInfo);
// todo extract methods
if (QUERY_IS_INTERVAL_QUERY(pQuery) && pRuntimeEnv->windowResInfo.prevSKey == TSKEY_INITIAL_VAL) {
STimeWindow w = TSWINDOW_INITIALIZER;
SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo;
if (QUERY_IS_ASC_QUERY(pQuery)) {
getAlignQueryTimeWindow(pQuery, blockInfo.window.skey, blockInfo.window.skey, pQuery->window.ekey, &w);
pWindowResInfo->startTime = w.skey;
pWindowResInfo->prevSKey = w.skey;
} else {
// the start position of the first time window in the endpoint that spreads beyond the queried last timestamp
getAlignQueryTimeWindow(pQuery, blockInfo.window.ekey, pQuery->window.ekey, blockInfo.window.ekey, &w);
pWindowResInfo->startTime = pQuery->window.skey;
pWindowResInfo->prevSKey = w.skey;
}
}
// in case of prj/diff query, ensure the output buffer is sufficient to accommodate the results of current block // in case of prj/diff query, ensure the output buffer is sufficient to accommodate the results of current block
ensureOutputBuffer(pRuntimeEnv, &blockInfo); ensureOutputBuffer(pRuntimeEnv, &blockInfo);
@ -2267,7 +2318,6 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) {
if (QUERY_IS_INTERVAL_QUERY(pQuery) && IS_MASTER_SCAN(pRuntimeEnv)) { if (QUERY_IS_INTERVAL_QUERY(pQuery) && IS_MASTER_SCAN(pRuntimeEnv)) {
if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) {
closeAllTimeWindow(&pRuntimeEnv->windowResInfo); closeAllTimeWindow(&pRuntimeEnv->windowResInfo);
// removeRedundantWindow(&pRuntimeEnv->windowResInfo, pTableQueryInfo->lastKey - step, step);
pRuntimeEnv->windowResInfo.curIndex = pRuntimeEnv->windowResInfo.size - 1; // point to the last time window pRuntimeEnv->windowResInfo.curIndex = pRuntimeEnv->windowResInfo.size - 1; // point to the last time window
} else { } else {
assert(Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)); assert(Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL));
@ -2490,11 +2540,7 @@ void UNUSED_FUNC displayInterResult(tFilePage **pdata, SQueryRuntimeEnv* pRuntim
switch (pQuery->pSelectExpr[i].type) { switch (pQuery->pSelectExpr[i].type) {
case TSDB_DATA_TYPE_BINARY: { case TSDB_DATA_TYPE_BINARY: {
// int32_t colIndex = pQuery->pSelectExpr[i].base.colInfo.colIndex;
int32_t type = pQuery->pSelectExpr[i].type; int32_t type = pQuery->pSelectExpr[i].type;
// } else {
// type = pMeterObj->schema[colIndex].type;
// }
printBinaryData(pQuery->pSelectExpr[i].base.functionId, pdata[i]->data + pQuery->pSelectExpr[i].bytes * j, printBinaryData(pQuery->pSelectExpr[i].base.functionId, pdata[i]->data + pQuery->pSelectExpr[i].bytes * j,
type); type);
break; break;
@ -2617,16 +2663,19 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) {
SIDList list = getDataBufPagesIdList(pResultBuf, pQInfo->offset + id); SIDList list = getDataBufPagesIdList(pResultBuf, pQInfo->offset + id);
int32_t total = 0; int32_t total = 0;
for (int32_t i = 0; i < list.size; ++i) { int32_t size = taosArrayGetSize(list);
tFilePage *pData = GET_RES_BUF_PAGE_BY_ID(pResultBuf, list.pData[i]); for (int32_t i = 0; i < size; ++i) {
int32_t* pgId = taosArrayGet(list, i);
tFilePage *pData = GET_RES_BUF_PAGE_BY_ID(pResultBuf, *pgId);
total += pData->num; total += pData->num;
} }
int32_t rows = total; int32_t rows = total;
int32_t offset = 0; int32_t offset = 0;
for (int32_t num = 0; num < list.size; ++num) { for (int32_t j = 0; j < size; ++j) {
tFilePage *pData = GET_RES_BUF_PAGE_BY_ID(pResultBuf, list.pData[num]); int32_t* pgId = taosArrayGet(list, j);
tFilePage *pData = GET_RES_BUF_PAGE_BY_ID(pResultBuf, *pgId);
for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes;
@ -2692,7 +2741,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) {
STableQueryInfo *item = taosArrayGetP(pGroup, i); STableQueryInfo *item = taosArrayGetP(pGroup, i);
SIDList list = getDataBufPagesIdList(pRuntimeEnv->pResultBuf, TSDB_TABLEID(item->pTable)->tid); SIDList list = getDataBufPagesIdList(pRuntimeEnv->pResultBuf, TSDB_TABLEID(item->pTable)->tid);
if (list.size > 0 && item->windowResInfo.size > 0) { if (taosArrayGetSize(list) > 0 && item->windowResInfo.size > 0) {
pTableList[numOfTables] = item; pTableList[numOfTables] = item;
numOfTables += 1; numOfTables += 1;
} }
@ -2960,19 +3009,18 @@ void switchCtxOrder(SQueryRuntimeEnv *pRuntimeEnv) {
} }
} }
int32_t createQueryResultInfo(SQuery *pQuery, SWindowResult *pResultRow, bool isSTableQuery, SPosInfo *posInfo, size_t interBufSize) { int32_t createQueryResultInfo(SQuery *pQuery, SWindowResult *pResultRow, bool isSTableQuery, size_t interBufSize) {
int32_t numOfCols = pQuery->numOfOutput; int32_t numOfCols = pQuery->numOfOutput;
pResultRow->resultInfo = calloc((size_t)numOfCols, sizeof(SResultInfo)); size_t size = numOfCols * sizeof(SResultInfo) + interBufSize;
pResultRow->resultInfo = calloc(1, size);
if (pResultRow->resultInfo == NULL) { if (pResultRow->resultInfo == NULL) {
return TSDB_CODE_QRY_OUT_OF_MEMORY; return TSDB_CODE_QRY_OUT_OF_MEMORY;
} }
pResultRow->pos = *posInfo;
char* buf = calloc(1, interBufSize); pResultRow->pos = (SPosInfo) {-1, -1};
if (buf == NULL) {
return TSDB_CODE_QRY_OUT_OF_MEMORY; char* buf = (char*) pResultRow->resultInfo + numOfCols * sizeof(SResultInfo);
}
// set the intermediate result output buffer // set the intermediate result output buffer
setWindowResultInfo(pResultRow->resultInfo, pQuery, isSTableQuery, buf); setWindowResultInfo(pResultRow->resultInfo, pQuery, isSTableQuery, buf);
@ -3178,6 +3226,13 @@ static void setEnvBeforeReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatusI
SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY);
SWITCH_ORDER(pQuery->order.order); SWITCH_ORDER(pQuery->order.order);
if (QUERY_IS_ASC_QUERY(pQuery)) {
assert(pQuery->window.skey <= pQuery->window.ekey);
} else {
assert(pQuery->window.skey >= pQuery->window.ekey);
}
SET_REVERSE_SCAN_FLAG(pRuntimeEnv); SET_REVERSE_SCAN_FLAG(pRuntimeEnv);
STsdbQueryCond cond = { STsdbQueryCond cond = {
@ -3217,8 +3272,7 @@ static void clearEnvAfterReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatus
SET_MASTER_SCAN_FLAG(pRuntimeEnv); SET_MASTER_SCAN_FLAG(pRuntimeEnv);
// update the pQuery->window.skey and pQuery->window.ekey to limit the scan scope of sliding query // update the pQuery->window.skey and pQuery->window.ekey to limit the scan scope of sliding query during reverse scan
// during reverse scan
pTableQueryInfo->lastKey = pStatus->lastKey; pTableQueryInfo->lastKey = pStatus->lastKey;
pQuery->status = pStatus->status; pQuery->status = pStatus->status;
@ -3244,7 +3298,12 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) {
if (pRuntimeEnv->scanFlag == MASTER_SCAN) { if (pRuntimeEnv->scanFlag == MASTER_SCAN) {
qstatus.status = pQuery->status; qstatus.status = pQuery->status;
// do nothing if no data blocks are found qualified during scan
if (qstatus.lastKey != pTableQueryInfo->lastKey) {
qstatus.curWindow.ekey = pTableQueryInfo->lastKey - step; qstatus.curWindow.ekey = pTableQueryInfo->lastKey - step;
}
qstatus.lastKey = pTableQueryInfo->lastKey; qstatus.lastKey = pTableQueryInfo->lastKey;
} }
@ -3373,12 +3432,12 @@ static STableQueryInfo *createTableQueryInfo(SQueryRuntimeEnv *pRuntimeEnv, void
return pTableQueryInfo; return pTableQueryInfo;
} }
void destroyTableQueryInfo(STableQueryInfo *pTableQueryInfo, int32_t numOfCols) { void destroyTableQueryInfo(STableQueryInfo *pTableQueryInfo) {
if (pTableQueryInfo == NULL) { if (pTableQueryInfo == NULL) {
return; return;
} }
cleanupTimeWindowInfo(&pTableQueryInfo->windowResInfo, numOfCols); cleanupTimeWindowInfo(&pTableQueryInfo->windowResInfo);
} }
#define CHECK_QUERY_TIME_RANGE(_q, _tableInfo) \ #define CHECK_QUERY_TIME_RANGE(_q, _tableInfo) \
@ -3837,45 +3896,11 @@ int32_t doFillGapsInResults(SQueryRuntimeEnv* pRuntimeEnv, tFilePage **pDst, int
static void queryCostStatis(SQInfo *pQInfo) { static void queryCostStatis(SQInfo *pQInfo) {
SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv;
SQueryCostInfo *pSummary = &pRuntimeEnv->summary; SQueryCostInfo *pSummary = &pRuntimeEnv->summary;
// if (pRuntimeEnv->pResultBuf == NULL) {
//// pSummary->tmpBufferInDisk = 0;
// } else {
//// pSummary->tmpBufferInDisk = getResBufSize(pRuntimeEnv->pResultBuf);
// }
//
// qDebug("QInfo:%p cost: comp blocks:%d, size:%d Bytes, elapsed time:%.2f ms", pQInfo, pSummary->readCompInfo,
// pSummary->totalCompInfoSize, pSummary->loadCompInfoUs / 1000.0);
//
// qDebug("QInfo:%p cost: field info: %d, size:%d Bytes, avg size:%.2f Bytes, elapsed time:%.2f ms", pQInfo,
// pSummary->readField, pSummary->totalFieldSize, (double)pSummary->totalFieldSize / pSummary->readField,
// pSummary->loadFieldUs / 1000.0);
//
// qDebug(
// "QInfo:%p cost: file blocks:%d, size:%d Bytes, elapsed time:%.2f ms, skipped:%d, in-memory gen null:%d Bytes",
// pQInfo, pSummary->readDiskBlocks, pSummary->totalBlockSize, pSummary->loadBlocksUs / 1000.0,
// pSummary->skippedFileBlocks, pSummary->totalGenData);
qDebug("QInfo:%p :cost summary: elapsed time:%"PRId64" us, io time:%"PRId64" us, total blocks:%d, load block statis:%d," qDebug("QInfo:%p :cost summary: elapsed time:%"PRId64" us, total blocks:%d, load block statis:%d,"
" load data block:%d, total rows:%"PRId64 ", check rows:%"PRId64, " load data block:%d, total rows:%"PRId64 ", check rows:%"PRId64,
pQInfo, pSummary->elapsedTime, pSummary->ioTime, pSummary->totalBlocks, pSummary->loadBlockStatis, pQInfo, pSummary->elapsedTime, pSummary->totalBlocks, pSummary->loadBlockStatis,
pSummary->loadBlocks, pSummary->totalRows, pSummary->totalCheckedRows); pSummary->loadBlocks, pSummary->totalRows, pSummary->totalCheckedRows);
// qDebug("QInfo:%p cost: temp file:%d Bytes", pQInfo, pSummary->tmpBufferInDisk);
//
// qDebug("QInfo:%p cost: file:%d, table:%d", pQInfo, pSummary->numOfFiles, pSummary->numOfTables);
// qDebug("QInfo:%p cost: seek ops:%d", pQInfo, pSummary->numOfSeek);
//
// double total = pSummary->fileTimeUs + pSummary->cacheTimeUs;
// double io = pSummary->loadCompInfoUs + pSummary->loadBlocksUs + pSummary->loadFieldUs;
// double computing = total - io;
//
// qDebug(
// "QInfo:%p cost: total elapsed time:%.2f ms, file:%.2f ms(%.2f%), cache:%.2f ms(%.2f%). io:%.2f ms(%.2f%),"
// "comput:%.2fms(%.2f%)",
// pQInfo, total / 1000.0, pSummary->fileTimeUs / 1000.0, pSummary->fileTimeUs * 100 / total,
// pSummary->cacheTimeUs / 1000.0, pSummary->cacheTimeUs * 100 / total, io / 1000.0, io * 100 / total,
// computing / 1000.0, computing * 100 / total);
} }
static void updateOffsetVal(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBlockInfo) { static void updateOffsetVal(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBlockInfo) {
@ -4189,14 +4214,14 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo
pRuntimeEnv->numOfRowsPerPage = getNumOfRowsInResultPage(pQuery, pRuntimeEnv->topBotQuery, isSTableQuery); pRuntimeEnv->numOfRowsPerPage = getNumOfRowsInResultPage(pQuery, pRuntimeEnv->topBotQuery, isSTableQuery);
if (isSTableQuery) { if (isSTableQuery && !onlyQueryTags(pRuntimeEnv->pQuery)) {
int32_t rows = getInitialPageNum(pQInfo); int32_t rows = getInitialPageNum(pQInfo);
code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rows, pQuery->rowSize, pQInfo); code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rows, pQuery->rowSize, pQInfo);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return code; return code;
} }
if (pQuery->intervalTime == 0) { if (!QUERY_IS_INTERVAL_QUERY(pQuery)) {
int16_t type = TSDB_DATA_TYPE_NULL; int16_t type = TSDB_DATA_TYPE_NULL;
if (pRuntimeEnv->groupbyNormalCol) { // group by columns not tags; if (pRuntimeEnv->groupbyNormalCol) { // group by columns not tags;
@ -4317,7 +4342,6 @@ static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) {
SDataStatis *pStatis = NULL; SDataStatis *pStatis = NULL;
SArray *pDataBlock = NULL; SArray *pDataBlock = NULL;
if (loadDataBlockOnDemand(pRuntimeEnv, pQueryHandle, &blockInfo, &pStatis, &pDataBlock) == BLK_DATA_DISCARD) { if (loadDataBlockOnDemand(pRuntimeEnv, pQueryHandle, &blockInfo, &pStatis, &pDataBlock) == BLK_DATA_DISCARD) {
pQuery->current->lastKey = QUERY_IS_ASC_QUERY(pQuery)? blockInfo.window.ekey + step:blockInfo.window.skey + step; pQuery->current->lastKey = QUERY_IS_ASC_QUERY(pQuery)? blockInfo.window.ekey + step:blockInfo.window.skey + step;
continue; continue;
@ -4603,7 +4627,6 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
// TODO handle the limit offset problem // TODO handle the limit offset problem
if (pQuery->numOfFilterCols == 0 && pQuery->limit.offset > 0) { if (pQuery->numOfFilterCols == 0 && pQuery->limit.offset > 0) {
// skipBlocks(pRuntimeEnv);
if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) {
pQInfo->tableIndex++; pQInfo->tableIndex++;
continue; continue;
@ -4740,8 +4763,6 @@ static void doRestoreContext(SQInfo *pQInfo) {
static void doCloseAllTimeWindowAfterScan(SQInfo *pQInfo) { static void doCloseAllTimeWindowAfterScan(SQInfo *pQInfo) {
SQuery *pQuery = pQInfo->runtimeEnv.pQuery; SQuery *pQuery = pQInfo->runtimeEnv.pQuery;
// int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order);
if (QUERY_IS_INTERVAL_QUERY(pQuery)) { if (QUERY_IS_INTERVAL_QUERY(pQuery)) {
size_t numOfGroup = GET_NUM_OF_TABLEGROUP(pQInfo); size_t numOfGroup = GET_NUM_OF_TABLEGROUP(pQInfo);
for (int32_t i = 0; i < numOfGroup; ++i) { for (int32_t i = 0; i < numOfGroup; ++i) {
@ -4751,7 +4772,6 @@ static void doCloseAllTimeWindowAfterScan(SQInfo *pQInfo) {
for (int32_t j = 0; j < num; ++j) { for (int32_t j = 0; j < num; ++j) {
STableQueryInfo* item = taosArrayGetP(group, j); STableQueryInfo* item = taosArrayGetP(group, j);
closeAllTimeWindow(&item->windowResInfo); closeAllTimeWindow(&item->windowResInfo);
// removeRedundantWindow(&item->windowResInfo, item->lastKey - step, step);
} }
} }
} else { // close results for group result } else { // close results for group result
@ -4844,7 +4864,7 @@ static void tableFixedOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo)
SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv;
SQuery *pQuery = pRuntimeEnv->pQuery; SQuery *pQuery = pRuntimeEnv->pQuery;
if (!isTopBottomQuery(pQuery) && pQuery->limit.offset > 0) { // no need to execute, since the output will be ignore. if (!pRuntimeEnv->topBotQuery && pQuery->limit.offset > 0) { // no need to execute, since the output will be ignore.
return; return;
} }
@ -5684,6 +5704,20 @@ static int compareTableIdInfo(const void* a, const void* b) {
static void freeQInfo(SQInfo *pQInfo); static void freeQInfo(SQInfo *pQInfo);
static void calResultBufSize(SQuery* pQuery) {
const int32_t RESULT_MSG_MIN_SIZE = 1024 * (1024 + 512); // bytes
const int32_t RESULT_MSG_MIN_ROWS = 8192;
const float RESULT_THRESHOLD_RATIO = 0.85;
int32_t numOfRes = RESULT_MSG_MIN_SIZE / pQuery->rowSize;
if (numOfRes < RESULT_MSG_MIN_ROWS) {
numOfRes = RESULT_MSG_MIN_ROWS;
}
pQuery->rec.capacity = numOfRes;
pQuery->rec.threshold = numOfRes * RESULT_THRESHOLD_RATIO;
}
static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList, SSqlGroupbyExpr *pGroupbyExpr, SExprInfo *pExprs, static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList, SSqlGroupbyExpr *pGroupbyExpr, SExprInfo *pExprs,
STableGroupInfo *pTableGroupInfo, SColumnInfo* pTagCols) { STableGroupInfo *pTableGroupInfo, SColumnInfo* pTagCols) {
int16_t numOfCols = pQueryMsg->numOfCols; int16_t numOfCols = pQueryMsg->numOfCols;
@ -5718,7 +5752,6 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList,
pQuery->numOfTags = pQueryMsg->numOfTags; pQuery->numOfTags = pQueryMsg->numOfTags;
pQuery->tagColList = pTagCols; pQuery->tagColList = pTagCols;
// todo do not allocate ??
pQuery->colList = calloc(numOfCols, sizeof(SSingleColumnFilterInfo)); pQuery->colList = calloc(numOfCols, sizeof(SSingleColumnFilterInfo));
if (pQuery->colList == NULL) { if (pQuery->colList == NULL) {
goto _cleanup; goto _cleanup;
@ -5748,9 +5781,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList,
goto _cleanup; goto _cleanup;
} }
// set the output buffer capacity calResultBufSize(pQuery);
pQuery->rec.capacity = 4096;
pQuery->rec.threshold = 4000;
for (int32_t col = 0; col < pQuery->numOfOutput; ++col) { for (int32_t col = 0; col < pQuery->numOfOutput; ++col) {
assert(pExprs[col].interBytes >= pExprs[col].bytes); assert(pExprs[col].interBytes >= pExprs[col].bytes);
@ -5799,7 +5830,6 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList,
if (p1 == NULL) { if (p1 == NULL) {
goto _cleanup; goto _cleanup;
} }
taosArrayPush(pQInfo->tableqinfoGroupInfo.pGroupList, &p1);
for(int32_t j = 0; j < s; ++j) { for(int32_t j = 0; j < s; ++j) {
void* pTable = taosArrayGetP(pa, j); void* pTable = taosArrayGetP(pa, j);
@ -5822,6 +5852,8 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList,
taosHashPut(pQInfo->tableqinfoGroupInfo.map, &id->tid, sizeof(id->tid), &item, POINTER_BYTES); taosHashPut(pQInfo->tableqinfoGroupInfo.map, &id->tid, sizeof(id->tid), &item, POINTER_BYTES);
index += 1; index += 1;
} }
taosArrayPush(pQInfo->tableqinfoGroupInfo.pGroupList, &p1);
} }
pQInfo->arrTableIdInfo = taosArrayInit(tableIndex, sizeof(STableIdInfo)); pQInfo->arrTableIdInfo = taosArrayInit(tableIndex, sizeof(STableIdInfo));
@ -5841,7 +5873,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList,
return pQInfo; return pQInfo;
_cleanup_qinfo: _cleanup_qinfo:
tsdbDestoryTableGroup(pTableGroupInfo); tsdbDestroyTableGroup(pTableGroupInfo);
_cleanup_query: _cleanup_query:
taosArrayDestroy(pGroupbyExpr->columnInfo); taosArrayDestroy(pGroupbyExpr->columnInfo);
@ -5981,9 +6013,7 @@ static void freeQInfo(SQInfo *pQInfo) {
size_t num = taosArrayGetSize(p); size_t num = taosArrayGetSize(p);
for(int32_t j = 0; j < num; ++j) { for(int32_t j = 0; j < num; ++j) {
STableQueryInfo* item = taosArrayGetP(p, j); STableQueryInfo* item = taosArrayGetP(p, j);
if (item != NULL) { destroyTableQueryInfo(item);
destroyTableQueryInfo(item, pQuery->numOfOutput);
}
} }
taosArrayDestroy(p); taosArrayDestroy(p);
@ -5993,7 +6023,7 @@ static void freeQInfo(SQInfo *pQInfo) {
tfree(pQInfo->pBuf); tfree(pQInfo->pBuf);
taosArrayDestroy(pQInfo->tableqinfoGroupInfo.pGroupList); taosArrayDestroy(pQInfo->tableqinfoGroupInfo.pGroupList);
taosHashCleanup(pQInfo->tableqinfoGroupInfo.map); taosHashCleanup(pQInfo->tableqinfoGroupInfo.map);
tsdbDestoryTableGroup(&pQInfo->tableGroupInfo); tsdbDestroyTableGroup(&pQInfo->tableGroupInfo);
taosArrayDestroy(pQInfo->arrTableIdInfo); taosArrayDestroy(pQInfo->arrTableIdInfo);
if (pQuery->pGroupbyExpr != NULL) { if (pQuery->pGroupbyExpr != NULL) {
@ -6013,7 +6043,6 @@ static void freeQInfo(SQInfo *pQInfo) {
} }
tfree(pQuery->sdata); tfree(pQuery->sdata);
tfree(pQuery); tfree(pQuery);
qDebug("QInfo:%p QInfo is freed", pQInfo); qDebug("QInfo:%p QInfo is freed", pQInfo);
@ -6168,9 +6197,7 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, vo
code = tsdbQuerySTableByTagCond(tsdb, id->uid, tagCond, pQueryMsg->tagCondLen, pQueryMsg->tagNameRelType, tbnameCond, &tableGroupInfo, pGroupColIndex, code = tsdbQuerySTableByTagCond(tsdb, id->uid, tagCond, pQueryMsg->tagCondLen, pQueryMsg->tagNameRelType, tbnameCond, &tableGroupInfo, pGroupColIndex,
numOfGroupByCols); numOfGroupByCols);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
if (code == TSDB_CODE_QRY_EXCEED_TAGS_LIMIT) {
qError("qmsg:%p failed to QueryStable, reason: %s", pQueryMsg, tstrerror(code)); qError("qmsg:%p failed to QueryStable, reason: %s", pQueryMsg, tstrerror(code));
}
goto _over; goto _over;
} }
} else { } else {
@ -6269,7 +6296,7 @@ void qTableQuery(qinfo_t qinfo) {
SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv;
if (onlyQueryTags(pQInfo->runtimeEnv.pQuery)) { if (onlyQueryTags(pQInfo->runtimeEnv.pQuery)) {
assert(pQInfo->runtimeEnv.pQueryHandle == NULL); assert(pQInfo->runtimeEnv.pQueryHandle == NULL);
buildTagQueryResult(pQInfo); // todo support the limit/offset buildTagQueryResult(pQInfo);
} else if (pQInfo->runtimeEnv.stableQuery) { } else if (pQInfo->runtimeEnv.stableQuery) {
stableQueryImpl(pQInfo); stableQueryImpl(pQInfo);
} else { } else {
@ -6330,7 +6357,6 @@ bool qHasMoreResultsToRetrieve(qinfo_t qinfo) {
} }
if (ret) { if (ret) {
// T_REF_INC(pQInfo);
qDebug("QInfo:%p has more results waits for client retrieve", pQInfo); qDebug("QInfo:%p has more results waits for client retrieve", pQInfo);
} }
@ -6391,6 +6417,22 @@ int32_t qKillQuery(qinfo_t qinfo) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static void doSetTagValueToResultBuf(char* output, const char* val, int16_t type, int16_t bytes) {
if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
if (val == NULL) {
setVardataNull(output, type);
} else {
memcpy(output, val, varDataTLen(val));
}
} else {
if (val == NULL) {
setNull(output, type, bytes);
} else { // todo here stop will cause client crash
memcpy(output, val, bytes);
}
}
}
static void buildTagQueryResult(SQInfo* pQInfo) { static void buildTagQueryResult(SQInfo* pQInfo) {
SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv;
SQuery * pQuery = pRuntimeEnv->pQuery; SQuery * pQuery = pRuntimeEnv->pQuery;
@ -6447,25 +6489,11 @@ static void buildTagQueryResult(SQInfo* pQInfo) {
output += sizeof(pQInfo->vgId); output += sizeof(pQInfo->vgId);
if (pExprInfo->base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { if (pExprInfo->base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) {
char *data = tsdbGetTableName(item->pTable); char* data = tsdbGetTableName(item->pTable);
memcpy(output, data, varDataTLen(data)); memcpy(output, data, varDataTLen(data));
} else { } else {
char *val = tsdbGetTableTagVal(item->pTable, pExprInfo->base.colInfo.colId, type, bytes); char* data = tsdbGetTableTagVal(item->pTable, pExprInfo->base.colInfo.colId, type, bytes);
doSetTagValueToResultBuf(output, data, type, bytes);
// todo refactor
if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
if (val == NULL) {
setVardataNull(output, type);
} else {
memcpy(output, val, varDataTLen(val));
}
} else {
if (val == NULL) {
setNull(output, type, bytes);
} else { // todo here stop will cause client crash
memcpy(output, val, bytes);
}
}
} }
count += 1; count += 1;
@ -6482,38 +6510,44 @@ static void buildTagQueryResult(SQInfo* pQInfo) {
} else { // return only the tags|table name etc. } else { // return only the tags|table name etc.
count = 0; count = 0;
SSchema tbnameSchema = tGetTableNameColumnSchema(); SSchema tbnameSchema = tGetTableNameColumnSchema();
while(pQInfo->tableIndex < num && count < pQuery->rec.capacity) {
int32_t maxNumOfTables = pQuery->rec.capacity;
if (pQuery->limit.limit >= 0 && pQuery->limit.limit < pQuery->rec.capacity) {
maxNumOfTables = pQuery->limit.limit;
}
while(pQInfo->tableIndex < num && count < maxNumOfTables) {
int32_t i = pQInfo->tableIndex++; int32_t i = pQInfo->tableIndex++;
// discard current result due to offset
if (pQuery->limit.offset > 0) {
pQuery->limit.offset -= 1;
continue;
}
SExprInfo* pExprInfo = pQuery->pSelectExpr; SExprInfo* pExprInfo = pQuery->pSelectExpr;
STableQueryInfo* item = taosArrayGetP(pa, i); STableQueryInfo* item = taosArrayGetP(pa, i);
char *data = NULL, *dst = NULL;
int16_t type = 0, bytes = 0;
for(int32_t j = 0; j < pQuery->numOfOutput; ++j) { for(int32_t j = 0; j < pQuery->numOfOutput; ++j) {
if (pExprInfo[j].base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { if (pExprInfo[j].base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) {
char* data = tsdbGetTableName(item->pTable); bytes = tbnameSchema.bytes;
char* dst = pQuery->sdata[j]->data + count * tbnameSchema.bytes; type = tbnameSchema.type;
memcpy(dst, data, varDataTLen(data));
} else {// todo refactor
int16_t type = pExprInfo[j].type;
int16_t bytes = pExprInfo[j].bytes;
char* data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.colInfo.colId, type, bytes); data = tsdbGetTableName(item->pTable);
char* dst = pQuery->sdata[j]->data + count * pExprInfo[j].bytes; dst = pQuery->sdata[j]->data + count * tbnameSchema.bytes;
} else {
type = pExprInfo[j].type;
bytes = pExprInfo[j].bytes;
data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.colInfo.colId, type, bytes);
dst = pQuery->sdata[j]->data + count * pExprInfo[j].bytes;
if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
if (data == NULL) {
setVardataNull(dst, type);
} else {
memcpy(dst, data, varDataTLen(data));
}
} else {
if (data == NULL) {
setNull(dst, type, bytes);
} else {
memcpy(dst, data, pExprInfo[j].bytes);
}
}
} }
doSetTagValueToResultBuf(dst, data, type, bytes);
} }
count += 1; count += 1;
} }

17
src/query/src/qUtil.c
View File

@ -53,9 +53,9 @@ int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, SQueryRuntimeEnv *pRun
if (pWindowResInfo->pResult == NULL) { if (pWindowResInfo->pResult == NULL) {
return TSDB_CODE_QRY_OUT_OF_MEMORY; return TSDB_CODE_QRY_OUT_OF_MEMORY;
} }
for (int32_t i = 0; i < pWindowResInfo->capacity; ++i) { for (int32_t i = 0; i < pWindowResInfo->capacity; ++i) {
SPosInfo posInfo = {-1, -1}; int32_t code = createQueryResultInfo(pRuntimeEnv->pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, pRuntimeEnv->interBufSize);
int32_t code = createQueryResultInfo(pRuntimeEnv->pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, &posInfo, pRuntimeEnv->interBufSize);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return code; return code;
} }
@ -64,16 +64,15 @@ int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, SQueryRuntimeEnv *pRun
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
void destroyTimeWindowRes(SWindowResult *pWindowRes, int32_t nOutputCols) { void destroyTimeWindowRes(SWindowResult *pWindowRes) {
if (pWindowRes == NULL) { if (pWindowRes == NULL) {
return; return;
} }
free(pWindowRes->resultInfo[0].interResultBuf);
free(pWindowRes->resultInfo); free(pWindowRes->resultInfo);
} }
void cleanupTimeWindowInfo(SWindowResInfo *pWindowResInfo, int32_t numOfCols) { void cleanupTimeWindowInfo(SWindowResInfo *pWindowResInfo) {
if (pWindowResInfo == NULL) { if (pWindowResInfo == NULL) {
return; return;
} }
@ -84,8 +83,7 @@ void cleanupTimeWindowInfo(SWindowResInfo *pWindowResInfo, int32_t numOfCols) {
if (pWindowResInfo->pResult != NULL) { if (pWindowResInfo->pResult != NULL) {
for (int32_t i = 0; i < pWindowResInfo->capacity; ++i) { for (int32_t i = 0; i < pWindowResInfo->capacity; ++i) {
SWindowResult *pResult = &pWindowResInfo->pResult[i]; destroyTimeWindowRes(&pWindowResInfo->pResult[i]);
destroyTimeWindowRes(pResult, numOfCols);
} }
} }
@ -225,11 +223,6 @@ void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_
} }
} }
SWindowResult *getWindowResult(SWindowResInfo *pWindowResInfo, int32_t slot) {
assert(pWindowResInfo != NULL && slot >= 0 && slot < pWindowResInfo->size);
return &pWindowResInfo->pResult[slot];
}
bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot) { bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot) {
return (getWindowResult(pWindowResInfo, slot)->status.closed == true); return (getWindowResult(pWindowResInfo, slot)->status.closed == true);
} }

7
src/query/src/qast.c
View File

@ -1064,10 +1064,9 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) {
if (*e == TS_PATH_DELIMITER[0]) { if (*e == TS_PATH_DELIMITER[0]) {
cond = e + 1; cond = e + 1;
} else if (*e == ',') { } else if (*e == ',') {
size_t len = e - cond + VARSTR_HEADER_SIZE; size_t len = e - cond;
char* p = exception_malloc(len); char* p = exception_malloc(len + VARSTR_HEADER_SIZE);
varDataSetLen(p, len - VARSTR_HEADER_SIZE); STR_WITH_SIZE_TO_VARSTR(p, cond, len);
memcpy(varDataVal(p), cond, len);
cond += len; cond += len;
taosArrayPush(pVal->arr, &p); taosArrayPush(pVal->arr, &p);
} }

91
src/query/src/qresultBuf.c
View File

@ -2,7 +2,6 @@
#include "hash.h" #include "hash.h"
#include "qextbuffer.h" #include "qextbuffer.h"
#include "taoserror.h" #include "taoserror.h"
#include "tsqlfunction.h"
#include "queryLog.h" #include "queryLog.h"
int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t size, int32_t rowSize, void* handle) { int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t size, int32_t rowSize, void* handle) {
@ -20,35 +19,31 @@ int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t si
// init id hash table // init id hash table
pResBuf->idsTable = taosHashInit(size, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false); pResBuf->idsTable = taosHashInit(size, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false);
pResBuf->list = calloc(size, sizeof(SIDList)); pResBuf->list = taosArrayInit(size, POINTER_BYTES);
pResBuf->numOfAllocGroupIds = size;
char path[4096] = {0}; char path[4096] = {0};
getTmpfilePath("tsdb_q_buf", path); getTmpfilePath("tsdb_qbuf", path);
pResBuf->path = strdup(path); pResBuf->path = strdup(path);
pResBuf->fd = open(pResBuf->path, O_CREAT | O_RDWR, 0666); pResBuf->fd = open(pResBuf->path, O_CREAT | O_RDWR, 0666);
memset(path, 0, tListLen(path));
if (!FD_VALID(pResBuf->fd)) { if (!FD_VALID(pResBuf->fd)) {
qError("failed to create tmp file: %s on disk. %s", pResBuf->path, strerror(errno)); qError("failed to create tmp file: %s on disk. %s", pResBuf->path, strerror(errno));
return TSDB_CODE_QRY_NO_DISKSPACE; return TAOS_SYSTEM_ERROR(errno);
} }
int32_t ret = ftruncate(pResBuf->fd, pResBuf->numOfPages * DEFAULT_INTERN_BUF_PAGE_SIZE); int32_t ret = ftruncate(pResBuf->fd, pResBuf->numOfPages * DEFAULT_INTERN_BUF_PAGE_SIZE);
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
qError("failed to create tmp file: %s on disk. %s", pResBuf->path, strerror(errno)); qError("failed to create tmp file: %s on disk. %s", pResBuf->path, strerror(errno));
return TSDB_CODE_QRY_NO_DISKSPACE; return TAOS_SYSTEM_ERROR(errno);
} }
pResBuf->pBuf = mmap(NULL, pResBuf->totalBufSize, PROT_READ | PROT_WRITE, MAP_SHARED, pResBuf->fd, 0); pResBuf->pBuf = mmap(NULL, pResBuf->totalBufSize, PROT_READ | PROT_WRITE, MAP_SHARED, pResBuf->fd, 0);
if (pResBuf->pBuf == MAP_FAILED) { if (pResBuf->pBuf == MAP_FAILED) {
qError("QInfo:%p failed to map temp file: %s. %s", handle, pResBuf->path, strerror(errno)); qError("QInfo:%p failed to map temp file: %s. %s", handle, pResBuf->path, strerror(errno));
return TSDB_CODE_QRY_OUT_OF_MEMORY; // todo change error code return TAOS_SYSTEM_ERROR(errno);
} }
qDebug("QInfo:%p create tmp file for output result, %s, %" PRId64 "bytes", handle, pResBuf->path, qDebug("QInfo:%p create tmp file for output result:%s, %" PRId64 "bytes", handle, pResBuf->path,
pResBuf->totalBufSize); pResBuf->totalBufSize);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -86,11 +81,11 @@ static int32_t extendDiskFileSize(SDiskbasedResultBuf* pResultBuf, int32_t numOf
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static bool noMoreAvailablePages(SDiskbasedResultBuf* pResultBuf) { static FORCE_INLINE bool noMoreAvailablePages(SDiskbasedResultBuf* pResultBuf) {
return (pResultBuf->allocateId == pResultBuf->numOfPages - 1); return (pResultBuf->allocateId == pResultBuf->numOfPages - 1);
} }
static int32_t getGroupIndex(SDiskbasedResultBuf* pResultBuf, int32_t groupId) { static FORCE_INLINE int32_t getGroupIndex(SDiskbasedResultBuf* pResultBuf, int32_t groupId) {
assert(pResultBuf != NULL); assert(pResultBuf != NULL);
char* p = taosHashGet(pResultBuf->idsTable, (const char*)&groupId, sizeof(int32_t)); char* p = taosHashGet(pResultBuf->idsTable, (const char*)&groupId, sizeof(int32_t));
@ -99,61 +94,30 @@ static int32_t getGroupIndex(SDiskbasedResultBuf* pResultBuf, int32_t groupId) {
} }
int32_t slot = GET_INT32_VAL(p); int32_t slot = GET_INT32_VAL(p);
assert(slot >= 0 && slot < pResultBuf->numOfAllocGroupIds); assert(slot >= 0 && slot < taosHashGetSize(pResultBuf->idsTable));
return slot; return slot;
} }
static int32_t addNewGroupId(SDiskbasedResultBuf* pResultBuf, int32_t groupId) { static int32_t addNewGroupId(SDiskbasedResultBuf* pResultBuf, int32_t groupId) {
int32_t num = getNumOfResultBufGroupId(pResultBuf); // the num is the newest allocated group id slot int32_t num = getNumOfResultBufGroupId(pResultBuf); // the num is the newest allocated group id slot
if (pResultBuf->numOfAllocGroupIds <= num) {
size_t n = pResultBuf->numOfAllocGroupIds << 1u;
SIDList* p = (SIDList*)realloc(pResultBuf->list, sizeof(SIDList) * n);
assert(p != NULL);
memset(&p[pResultBuf->numOfAllocGroupIds], 0, sizeof(SIDList) * pResultBuf->numOfAllocGroupIds);
pResultBuf->list = p;
pResultBuf->numOfAllocGroupIds = n;
}
taosHashPut(pResultBuf->idsTable, (const char*)&groupId, sizeof(int32_t), &num, sizeof(int32_t)); taosHashPut(pResultBuf->idsTable, (const char*)&groupId, sizeof(int32_t), &num, sizeof(int32_t));
SArray* pa = taosArrayInit(1, sizeof(int32_t));
taosArrayPush(pResultBuf->list, &pa);
assert(taosArrayGetSize(pResultBuf->list) == taosHashGetSize(pResultBuf->idsTable));
return num; return num;
} }
static int32_t doRegisterId(SIDList* pList, int32_t id) {
if (pList->size >= pList->alloc) {
int32_t s = 0;
if (pList->alloc == 0) {
s = 4;
assert(pList->pData == NULL);
} else {
s = pList->alloc << 1u;
}
int32_t* c = realloc(pList->pData, s * sizeof(int32_t));
assert(c);
memset(&c[pList->alloc], 0, sizeof(int32_t) * pList->alloc);
pList->pData = c;
pList->alloc = s;
}
pList->pData[pList->size++] = id;
return 0;
}
static void registerPageId(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t pageId) { static void registerPageId(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t pageId) {
int32_t slot = getGroupIndex(pResultBuf, groupId); int32_t slot = getGroupIndex(pResultBuf, groupId);
if (slot < 0) { if (slot < 0) {
slot = addNewGroupId(pResultBuf, groupId); slot = addNewGroupId(pResultBuf, groupId);
} }
SIDList* pList = &pResultBuf->list[slot]; SIDList pList = taosArrayGetP(pResultBuf->list, slot);
doRegisterId(pList, pageId); taosArrayPush(pList, &pageId);
} }
tFilePage* getNewDataBuf(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t* pageId) { tFilePage* getNewDataBuf(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t* pageId) {
@ -178,12 +142,11 @@ tFilePage* getNewDataBuf(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32
int32_t getNumOfRowsPerPage(SDiskbasedResultBuf* pResultBuf) { return pResultBuf->numOfRowsPerPage; } int32_t getNumOfRowsPerPage(SDiskbasedResultBuf* pResultBuf) { return pResultBuf->numOfRowsPerPage; }
SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId) { SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId) {
SIDList list = {0};
int32_t slot = getGroupIndex(pResultBuf, groupId); int32_t slot = getGroupIndex(pResultBuf, groupId);
if (slot < 0) { if (slot < 0) {
return list; return taosArrayInit(1, sizeof(int32_t));
} else { } else {
return pResultBuf->list[slot]; return taosArrayGetP(pResultBuf->list, slot);
} }
} }
@ -202,22 +165,20 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf, void* handle) {
tfree(pResultBuf->path); tfree(pResultBuf->path);
for (int32_t i = 0; i < pResultBuf->numOfAllocGroupIds; ++i) { size_t size = taosArrayGetSize(pResultBuf->list);
SIDList* pList = &pResultBuf->list[i]; for (int32_t i = 0; i < size; ++i) {
tfree(pList->pData); SArray* pa = taosArrayGetP(pResultBuf->list, i);
taosArrayDestroy(pa);
} }
tfree(pResultBuf->list); taosArrayDestroy(pResultBuf->list);
taosHashCleanup(pResultBuf->idsTable); taosHashCleanup(pResultBuf->idsTable);
tfree(pResultBuf); tfree(pResultBuf);
} }
int32_t getLastPageId(SIDList *pList) { int32_t getLastPageId(SIDList pList) {
if (pList == NULL || pList->size <= 0) { size_t size = taosArrayGetSize(pList);
return -1; return *(int32_t*) taosArrayGet(pList, size - 1);
}
return pList->pData[pList->size - 1];
} }

60
src/rpc/src/rpcMain.c
View File

@ -55,7 +55,7 @@ typedef struct {
char secret[TSDB_KEY_LEN]; // secret for the link char secret[TSDB_KEY_LEN]; // secret for the link
char ckey[TSDB_KEY_LEN]; // ciphering key char ckey[TSDB_KEY_LEN]; // ciphering key
void (*cfp)(SRpcMsg *, SRpcIpSet *); void (*cfp)(SRpcMsg *, SRpcEpSet *);
int (*afp)(char *user, char *spi, char *encrypt, char *secret, char *ckey); int (*afp)(char *user, char *spi, char *encrypt, char *secret, char *ckey);
int32_t refCount; int32_t refCount;
@ -71,7 +71,7 @@ typedef struct {
typedef struct { typedef struct {
SRpcInfo *pRpc; // associated SRpcInfo SRpcInfo *pRpc; // associated SRpcInfo
SRpcIpSet ipSet; // ip list provided by app SRpcEpSet epSet; // ip list provided by app
void *ahandle; // handle provided by app void *ahandle; // handle provided by app
void *signature; // for validation void *signature; // for validation
struct SRpcConn *pConn; // pConn allocated struct SRpcConn *pConn; // pConn allocated
@ -80,12 +80,12 @@ typedef struct {
int32_t contLen; // content length int32_t contLen; // content length
int32_t code; // error code int32_t code; // error code
int16_t numOfTry; // number of try for different servers int16_t numOfTry; // number of try for different servers
int8_t oldInUse; // server IP inUse passed by app int8_t oldInUse; // server EP inUse passed by app
int8_t redirect; // flag to indicate redirect int8_t redirect; // flag to indicate redirect
int8_t connType; // connection type int8_t connType; // connection type
SRpcMsg *pRsp; // for synchronous API SRpcMsg *pRsp; // for synchronous API
tsem_t *pSem; // for synchronous API tsem_t *pSem; // for synchronous API
SRpcIpSet *pSet; // for synchronous API SRpcEpSet *pSet; // for synchronous API
char msg[0]; // RpcHead starts from here char msg[0]; // RpcHead starts from here
} SRpcReqContext; } SRpcReqContext;
@ -355,7 +355,7 @@ void *rpcReallocCont(void *ptr, int contLen) {
return start + sizeof(SRpcReqContext) + sizeof(SRpcHead); return start + sizeof(SRpcReqContext) + sizeof(SRpcHead);
} }
void rpcSendRequest(void *shandle, const SRpcIpSet *pIpSet, SRpcMsg *pMsg) { void rpcSendRequest(void *shandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg) {
SRpcInfo *pRpc = (SRpcInfo *)shandle; SRpcInfo *pRpc = (SRpcInfo *)shandle;
SRpcReqContext *pContext; SRpcReqContext *pContext;
@ -364,11 +364,11 @@ void rpcSendRequest(void *shandle, const SRpcIpSet *pIpSet, SRpcMsg *pMsg) {
pContext->ahandle = pMsg->ahandle; pContext->ahandle = pMsg->ahandle;
pContext->signature = pContext; pContext->signature = pContext;
pContext->pRpc = (SRpcInfo *)shandle; pContext->pRpc = (SRpcInfo *)shandle;
pContext->ipSet = *pIpSet; pContext->epSet = *pEpSet;
pContext->contLen = contLen; pContext->contLen = contLen;
pContext->pCont = pMsg->pCont; pContext->pCont = pMsg->pCont;
pContext->msgType = pMsg->msgType; pContext->msgType = pMsg->msgType;
pContext->oldInUse = pIpSet->inUse; pContext->oldInUse = pEpSet->inUse;
pContext->connType = RPC_CONN_UDPC; pContext->connType = RPC_CONN_UDPC;
if (contLen > tsRpcMaxUdpSize) pContext->connType = RPC_CONN_TCPC; if (contLen > tsRpcMaxUdpSize) pContext->connType = RPC_CONN_TCPC;
@ -458,15 +458,15 @@ void rpcSendResponse(const SRpcMsg *pRsp) {
return; return;
} }
void rpcSendRedirectRsp(void *thandle, const SRpcIpSet *pIpSet) { void rpcSendRedirectRsp(void *thandle, const SRpcEpSet *pEpSet) {
SRpcMsg rpcMsg; SRpcMsg rpcMsg;
memset(&rpcMsg, 0, sizeof(rpcMsg)); memset(&rpcMsg, 0, sizeof(rpcMsg));
rpcMsg.contLen = sizeof(SRpcIpSet); rpcMsg.contLen = sizeof(SRpcEpSet);
rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen);
if (rpcMsg.pCont == NULL) return; if (rpcMsg.pCont == NULL) return;
memcpy(rpcMsg.pCont, pIpSet, sizeof(SRpcIpSet)); memcpy(rpcMsg.pCont, pEpSet, sizeof(SRpcEpSet));
rpcMsg.code = TSDB_CODE_RPC_REDIRECT; rpcMsg.code = TSDB_CODE_RPC_REDIRECT;
rpcMsg.handle = thandle; rpcMsg.handle = thandle;
@ -488,7 +488,7 @@ int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo) {
return 0; return 0;
} }
void rpcSendRecv(void *shandle, SRpcIpSet *pIpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { void rpcSendRecv(void *shandle, SRpcEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
SRpcReqContext *pContext; SRpcReqContext *pContext;
pContext = (SRpcReqContext *) (pMsg->pCont-sizeof(SRpcHead)-sizeof(SRpcReqContext)); pContext = (SRpcReqContext *) (pMsg->pCont-sizeof(SRpcHead)-sizeof(SRpcReqContext));
@ -498,9 +498,9 @@ void rpcSendRecv(void *shandle, SRpcIpSet *pIpSet, SRpcMsg *pMsg, SRpcMsg *pRsp)
tsem_init(&sem, 0, 0); tsem_init(&sem, 0, 0);
pContext->pSem = &sem; pContext->pSem = &sem;
pContext->pRsp = pRsp; pContext->pRsp = pRsp;
pContext->pSet = pIpSet; pContext->pSet = pEpSet;
rpcSendRequest(shandle, pIpSet, pMsg); rpcSendRequest(shandle, pEpSet, pMsg);
tsem_wait(&sem); tsem_wait(&sem);
tsem_destroy(&sem); tsem_destroy(&sem);
@ -755,11 +755,11 @@ static SRpcConn *rpcGetConnObj(SRpcInfo *pRpc, int sid, SRecvInfo *pRecv) {
static SRpcConn *rpcSetupConnToServer(SRpcReqContext *pContext) { static SRpcConn *rpcSetupConnToServer(SRpcReqContext *pContext) {
SRpcConn *pConn; SRpcConn *pConn;
SRpcInfo *pRpc = pContext->pRpc; SRpcInfo *pRpc = pContext->pRpc;
SRpcIpSet *pIpSet = &pContext->ipSet; SRpcEpSet *pEpSet = &pContext->epSet;
pConn = rpcGetConnFromCache(pRpc->pCache, pIpSet->fqdn[pIpSet->inUse], pIpSet->port[pIpSet->inUse], pContext->connType); pConn = rpcGetConnFromCache(pRpc->pCache, pEpSet->fqdn[pEpSet->inUse], pEpSet->port[pEpSet->inUse], pContext->connType);
if ( pConn == NULL || pConn->user[0] == 0) { if ( pConn == NULL || pConn->user[0] == 0) {
pConn = rpcOpenConn(pRpc, pIpSet->fqdn[pIpSet->inUse], pIpSet->port[pIpSet->inUse], pContext->connType); pConn = rpcOpenConn(pRpc, pEpSet->fqdn[pEpSet->inUse], pEpSet->port[pEpSet->inUse], pContext->connType);
} }
if (pConn) { if (pConn) {
@ -1020,16 +1020,16 @@ static void rpcNotifyClient(SRpcReqContext *pContext, SRpcMsg *pMsg) {
pContext->pConn = NULL; pContext->pConn = NULL;
if (pContext->pRsp) { if (pContext->pRsp) {
// for synchronous API // for synchronous API
memcpy(pContext->pSet, &pContext->ipSet, sizeof(SRpcIpSet)); memcpy(pContext->pSet, &pContext->epSet, sizeof(SRpcEpSet));
memcpy(pContext->pRsp, pMsg, sizeof(SRpcMsg)); memcpy(pContext->pRsp, pMsg, sizeof(SRpcMsg));
tsem_post(pContext->pSem); tsem_post(pContext->pSem);
} else { } else {
// for asynchronous API // for asynchronous API
SRpcIpSet *pIpSet = NULL; SRpcEpSet *pEpSet = NULL;
if (pContext->ipSet.inUse != pContext->oldInUse || pContext->redirect) if (pContext->epSet.inUse != pContext->oldInUse || pContext->redirect)
pIpSet = &pContext->ipSet; pEpSet = &pContext->epSet;
(*pRpc->cfp)(pMsg, pIpSet); (*pRpc->cfp)(pMsg, pEpSet);
} }
// free the request message // free the request message
@ -1070,9 +1070,9 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead) {
pConn->pContext = NULL; pConn->pContext = NULL;
pConn->pReqMsg = NULL; pConn->pReqMsg = NULL;
// for UDP, port may be changed by server, the port in ipSet shall be used for cache // for UDP, port may be changed by server, the port in epSet shall be used for cache
if (pHead->code != TSDB_CODE_RPC_TOO_SLOW) { if (pHead->code != TSDB_CODE_RPC_TOO_SLOW) {
rpcAddConnIntoCache(pRpc->pCache, pConn, pConn->peerFqdn, pContext->ipSet.port[pContext->ipSet.inUse], pConn->connType); rpcAddConnIntoCache(pRpc->pCache, pConn, pConn->peerFqdn, pContext->epSet.port[pContext->epSet.inUse], pConn->connType);
} else { } else {
rpcCloseConn(pConn); rpcCloseConn(pConn);
} }
@ -1087,10 +1087,10 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead) {
if (pHead->code == TSDB_CODE_RPC_REDIRECT) { if (pHead->code == TSDB_CODE_RPC_REDIRECT) {
pContext->numOfTry = 0; pContext->numOfTry = 0;
memcpy(&pContext->ipSet, pHead->content, sizeof(pContext->ipSet)); memcpy(&pContext->epSet, pHead->content, sizeof(pContext->epSet));
tDebug("%s, redirect is received, numOfIps:%d", pConn->info, pContext->ipSet.numOfIps); tDebug("%s, redirect is received, numOfEps:%d", pConn->info, pContext->epSet.numOfEps);
for (int i=0; i<pContext->ipSet.numOfIps; ++i) for (int i=0; i<pContext->epSet.numOfEps; ++i)
pContext->ipSet.port[i] = htons(pContext->ipSet.port[i]); pContext->epSet.port[i] = htons(pContext->epSet.port[i]);
rpcSendReqToServer(pRpc, pContext); rpcSendReqToServer(pRpc, pContext);
rpcFreeCont(rpcMsg.pCont); rpcFreeCont(rpcMsg.pCont);
} else if (pHead->code == TSDB_CODE_RPC_NOT_READY) { } else if (pHead->code == TSDB_CODE_RPC_NOT_READY) {
@ -1269,7 +1269,7 @@ static void rpcProcessConnError(void *param, void *id) {
tDebug("%s %p, connection error happens", pRpc->label, pContext->ahandle); tDebug("%s %p, connection error happens", pRpc->label, pContext->ahandle);
if (pContext->numOfTry >= pContext->ipSet.numOfIps) { if (pContext->numOfTry >= pContext->epSet.numOfEps) {
rpcMsg.msgType = pContext->msgType+1; rpcMsg.msgType = pContext->msgType+1;
rpcMsg.ahandle = pContext->ahandle; rpcMsg.ahandle = pContext->ahandle;
rpcMsg.code = pContext->code; rpcMsg.code = pContext->code;
@ -1279,8 +1279,8 @@ static void rpcProcessConnError(void *param, void *id) {
rpcNotifyClient(pContext, &rpcMsg); rpcNotifyClient(pContext, &rpcMsg);
} else { } else {
// move to next IP // move to next IP
pContext->ipSet.inUse++; pContext->epSet.inUse++;
pContext->ipSet.inUse = pContext->ipSet.inUse % pContext->ipSet.numOfIps; pContext->epSet.inUse = pContext->epSet.inUse % pContext->epSet.numOfEps;
rpcSendReqToServer(pRpc, pContext); rpcSendReqToServer(pRpc, pContext);
} }
} }

2
src/rpc/src/rpcTcp.c
View File

@ -374,7 +374,7 @@ int taosSendTcpData(uint32_t ip, uint16_t port, void *data, int len, void *chand
if (chandle == NULL) return -1; if (chandle == NULL) return -1;
return (int)send(pFdObj->fd, data, (size_t)len, 0); return taosWriteMsg(pFdObj->fd, data, len);
} }
static void taosReportBrokenLink(SFdObj *pFdObj) { static void taosReportBrokenLink(SFdObj *pFdObj) {

30
src/rpc/test/rclient.c
View File

@ -22,7 +22,7 @@
typedef struct { typedef struct {
int index; int index;
SRpcIpSet ipSet; SRpcEpSet epSet;
int num; int num;
int numOfReqs; int numOfReqs;
int msgSize; int msgSize;
@ -32,11 +32,11 @@ typedef struct {
void *pRpc; void *pRpc;
} SInfo; } SInfo;
static void processResponse(SRpcMsg *pMsg, SRpcIpSet *pIpSet) { static void processResponse(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
SInfo *pInfo = (SInfo *)pMsg->ahandle; SInfo *pInfo = (SInfo *)pMsg->ahandle;
tDebug("thread:%d, response is received, type:%d contLen:%d code:0x%x", pInfo->index, pMsg->msgType, pMsg->contLen, pMsg->code); tDebug("thread:%d, response is received, type:%d contLen:%d code:0x%x", pInfo->index, pMsg->msgType, pMsg->contLen, pMsg->code);
if (pIpSet) pInfo->ipSet = *pIpSet; if (pEpSet) pInfo->epSet = *pEpSet;
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);
sem_post(&pInfo->rspSem); sem_post(&pInfo->rspSem);
@ -57,7 +57,7 @@ static void *sendRequest(void *param) {
rpcMsg.ahandle = pInfo; rpcMsg.ahandle = pInfo;
rpcMsg.msgType = 1; rpcMsg.msgType = 1;
tDebug("thread:%d, send request, contLen:%d num:%d", pInfo->index, pInfo->msgSize, pInfo->num); tDebug("thread:%d, send request, contLen:%d num:%d", pInfo->index, pInfo->msgSize, pInfo->num);
rpcSendRequest(pInfo->pRpc, &pInfo->ipSet, &rpcMsg); rpcSendRequest(pInfo->pRpc, &pInfo->epSet, &rpcMsg);
if ( pInfo->num % 20000 == 0 ) if ( pInfo->num % 20000 == 0 )
tInfo("thread:%d, %d requests have been sent", pInfo->index, pInfo->num); tInfo("thread:%d, %d requests have been sent", pInfo->index, pInfo->num);
sem_wait(&pInfo->rspSem); sem_wait(&pInfo->rspSem);
@ -71,7 +71,7 @@ static void *sendRequest(void *param) {
int main(int argc, char *argv[]) { int main(int argc, char *argv[]) {
SRpcInit rpcInit; SRpcInit rpcInit;
SRpcIpSet ipSet; SRpcEpSet epSet;
int msgSize = 128; int msgSize = 128;
int numOfReqs = 0; int numOfReqs = 0;
int appThreads = 1; int appThreads = 1;
@ -82,12 +82,12 @@ int main(int argc, char *argv[]) {
pthread_attr_t thattr; pthread_attr_t thattr;
// server info // server info
ipSet.numOfIps = 1; epSet.numOfEps = 1;
ipSet.inUse = 0; epSet.inUse = 0;
ipSet.port[0] = 7000; epSet.port[0] = 7000;
ipSet.port[1] = 7000; epSet.port[1] = 7000;
strcpy(ipSet.fqdn[0], serverIp); strcpy(epSet.fqdn[0], serverIp);
strcpy(ipSet.fqdn[1], "192.168.0.1"); strcpy(epSet.fqdn[1], "192.168.0.1");
// client info // client info
memset(&rpcInit, 0, sizeof(rpcInit)); memset(&rpcInit, 0, sizeof(rpcInit));
@ -105,9 +105,9 @@ int main(int argc, char *argv[]) {
for (int i=1; i<argc; ++i) { for (int i=1; i<argc; ++i) {
if (strcmp(argv[i], "-p")==0 && i < argc-1) { if (strcmp(argv[i], "-p")==0 && i < argc-1) {
ipSet.port[0] = atoi(argv[++i]); epSet.port[0] = atoi(argv[++i]);
} else if (strcmp(argv[i], "-i") ==0 && i < argc-1) { } else if (strcmp(argv[i], "-i") ==0 && i < argc-1) {
tstrncpy(ipSet.fqdn[0], argv[++i], sizeof(ipSet.fqdn[0])); tstrncpy(epSet.fqdn[0], argv[++i], sizeof(epSet.fqdn[0]));
} else if (strcmp(argv[i], "-t")==0 && i < argc-1) { } else if (strcmp(argv[i], "-t")==0 && i < argc-1) {
rpcInit.numOfThreads = atoi(argv[++i]); rpcInit.numOfThreads = atoi(argv[++i]);
} else if (strcmp(argv[i], "-m")==0 && i < argc-1) { } else if (strcmp(argv[i], "-m")==0 && i < argc-1) {
@ -131,7 +131,7 @@ int main(int argc, char *argv[]) {
} else { } else {
printf("\nusage: %s [options] \n", argv[0]); printf("\nusage: %s [options] \n", argv[0]);
printf(" [-i ip]: first server IP address, default is:%s\n", serverIp); printf(" [-i ip]: first server IP address, default is:%s\n", serverIp);
printf(" [-p port]: server port number, default is:%d\n", ipSet.port[0]); printf(" [-p port]: server port number, default is:%d\n", epSet.port[0]);
printf(" [-t threads]: number of rpc threads, default is:%d\n", rpcInit.numOfThreads); printf(" [-t threads]: number of rpc threads, default is:%d\n", rpcInit.numOfThreads);
printf(" [-s sessions]: number of rpc sessions, default is:%d\n", rpcInit.sessions); printf(" [-s sessions]: number of rpc sessions, default is:%d\n", rpcInit.sessions);
printf(" [-m msgSize]: message body size, default is:%d\n", msgSize); printf(" [-m msgSize]: message body size, default is:%d\n", msgSize);
@ -168,7 +168,7 @@ int main(int argc, char *argv[]) {
for (int i=0; i<appThreads; ++i) { for (int i=0; i<appThreads; ++i) {
pInfo->index = i; pInfo->index = i;
pInfo->ipSet = ipSet; pInfo->epSet = epSet;
pInfo->numOfReqs = numOfReqs; pInfo->numOfReqs = numOfReqs;
pInfo->msgSize = msgSize; pInfo->msgSize = msgSize;
sem_init(&pInfo->rspSem, 0, 0); sem_init(&pInfo->rspSem, 0, 0);

26
src/rpc/test/rsclient.c
View File

@ -23,7 +23,7 @@
typedef struct { typedef struct {
int index; int index;
SRpcIpSet ipSet; SRpcEpSet epSet;
int num; int num;
int numOfReqs; int numOfReqs;
int msgSize; int msgSize;
@ -51,7 +51,7 @@ static void *sendRequest(void *param) {
rpcMsg.msgType = 1; rpcMsg.msgType = 1;
tDebug("thread:%d, send request, contLen:%d num:%d", pInfo->index, pInfo->msgSize, pInfo->num); tDebug("thread:%d, send request, contLen:%d num:%d", pInfo->index, pInfo->msgSize, pInfo->num);
rpcSendRecv(pInfo->pRpc, &pInfo->ipSet, &rpcMsg, &rspMsg); rpcSendRecv(pInfo->pRpc, &pInfo->epSet, &rpcMsg, &rspMsg);
// handle response // handle response
if (rspMsg.code != 0) terror++; if (rspMsg.code != 0) terror++;
@ -72,7 +72,7 @@ static void *sendRequest(void *param) {
int main(int argc, char *argv[]) { int main(int argc, char *argv[]) {
SRpcInit rpcInit; SRpcInit rpcInit;
SRpcIpSet ipSet; SRpcEpSet epSet;
int msgSize = 128; int msgSize = 128;
int numOfReqs = 0; int numOfReqs = 0;
int appThreads = 1; int appThreads = 1;
@ -83,12 +83,12 @@ int main(int argc, char *argv[]) {
pthread_attr_t thattr; pthread_attr_t thattr;
// server info // server info
ipSet.numOfIps = 1; epSet.numOfEps = 1;
ipSet.inUse = 0; epSet.inUse = 0;
ipSet.port[0] = 7000; epSet.port[0] = 7000;
ipSet.port[1] = 7000; epSet.port[1] = 7000;
strcpy(ipSet.fqdn[0], serverIp); strcpy(epSet.fqdn[0], serverIp);
strcpy(ipSet.fqdn[1], "192.168.0.1"); strcpy(epSet.fqdn[1], "192.168.0.1");
// client info // client info
memset(&rpcInit, 0, sizeof(rpcInit)); memset(&rpcInit, 0, sizeof(rpcInit));
@ -106,9 +106,9 @@ int main(int argc, char *argv[]) {
for (int i=1; i<argc; ++i) { for (int i=1; i<argc; ++i) {
if (strcmp(argv[i], "-p")==0 && i < argc-1) { if (strcmp(argv[i], "-p")==0 && i < argc-1) {
ipSet.port[0] = atoi(argv[++i]); epSet.port[0] = atoi(argv[++i]);
} else if (strcmp(argv[i], "-i") ==0 && i < argc-1) { } else if (strcmp(argv[i], "-i") ==0 && i < argc-1) {
tstrncpy(ipSet.fqdn[0], argv[++i], sizeof(ipSet.fqdn[0])); tstrncpy(epSet.fqdn[0], argv[++i], sizeof(epSet.fqdn[0]));
} else if (strcmp(argv[i], "-t")==0 && i < argc-1) { } else if (strcmp(argv[i], "-t")==0 && i < argc-1) {
rpcInit.numOfThreads = atoi(argv[++i]); rpcInit.numOfThreads = atoi(argv[++i]);
} else if (strcmp(argv[i], "-m")==0 && i < argc-1) { } else if (strcmp(argv[i], "-m")==0 && i < argc-1) {
@ -132,7 +132,7 @@ int main(int argc, char *argv[]) {
} else { } else {
printf("\nusage: %s [options] \n", argv[0]); printf("\nusage: %s [options] \n", argv[0]);
printf(" [-i ip]: first server IP address, default is:%s\n", serverIp); printf(" [-i ip]: first server IP address, default is:%s\n", serverIp);
printf(" [-p port]: server port number, default is:%d\n", ipSet.port[0]); printf(" [-p port]: server port number, default is:%d\n", epSet.port[0]);
printf(" [-t threads]: number of rpc threads, default is:%d\n", rpcInit.numOfThreads); printf(" [-t threads]: number of rpc threads, default is:%d\n", rpcInit.numOfThreads);
printf(" [-s sessions]: number of rpc sessions, default is:%d\n", rpcInit.sessions); printf(" [-s sessions]: number of rpc sessions, default is:%d\n", rpcInit.sessions);
printf(" [-m msgSize]: message body size, default is:%d\n", msgSize); printf(" [-m msgSize]: message body size, default is:%d\n", msgSize);
@ -168,7 +168,7 @@ int main(int argc, char *argv[]) {
for (int i=0; i<appThreads; ++i) { for (int i=0; i<appThreads; ++i) {
pInfo->index = i; pInfo->index = i;
pInfo->ipSet = ipSet; pInfo->epSet = epSet;
pInfo->numOfReqs = numOfReqs; pInfo->numOfReqs = numOfReqs;
pInfo->msgSize = msgSize; pInfo->msgSize = msgSize;
sem_init(&pInfo->rspSem, 0, 0); sem_init(&pInfo->rspSem, 0, 0);

2
src/rpc/test/rserver.c
View File

@ -103,7 +103,7 @@ int retrieveAuthInfo(char *meterId, char *spi, char *encrypt, char *secret, char
return ret; return ret;
} }
void processRequestMsg(SRpcMsg *pMsg, SRpcIpSet *pIpSet) { void processRequestMsg(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
SRpcMsg *pTemp; SRpcMsg *pTemp;
pTemp = taosAllocateQitem(sizeof(SRpcMsg)); pTemp = taosAllocateQitem(sizeof(SRpcMsg));

3
src/tsdb/src/tsdbMain.c
View File

@ -148,7 +148,7 @@ void tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit) {
STsdbRepo *pRepo = (STsdbRepo *)repo; STsdbRepo *pRepo = (STsdbRepo *)repo;
int vgId = REPO_ID(pRepo); int vgId = REPO_ID(pRepo);
tsdbStopStream(repo); tsdbStopStream(pRepo);
if (toCommit) { if (toCommit) {
tsdbAsyncCommit(pRepo); tsdbAsyncCommit(pRepo);
@ -1126,6 +1126,7 @@ static void tsdbStartStream(STsdbRepo *pRepo) {
} }
} }
static void tsdbStopStream(STsdbRepo *pRepo) { static void tsdbStopStream(STsdbRepo *pRepo) {
STsdbMeta *pMeta = pRepo->tsdbMeta; STsdbMeta *pMeta = pRepo->tsdbMeta;

7
src/tsdb/src/tsdbMemTable.c
View File

@ -172,7 +172,7 @@ void *tsdbAllocBytes(STsdbRepo *pRepo, int bytes) {
STsdbBufBlock *pBufBlock = tsdbGetCurrBufBlock(pRepo); STsdbBufBlock *pBufBlock = tsdbGetCurrBufBlock(pRepo);
if (pBufBlock != NULL && pBufBlock->remain < bytes) { if (pBufBlock != NULL && pBufBlock->remain < bytes) {
if (listNEles(pRepo->mem->bufBlockList) >= pCfg->totalBlocks / 2) { // need to commit mem if (listNEles(pRepo->mem->bufBlockList) >= pCfg->totalBlocks / 3) { // need to commit mem
if (tsdbAsyncCommit(pRepo) < 0) return NULL; if (tsdbAsyncCommit(pRepo) < 0) return NULL;
} else { } else {
if (tsdbLockRepo(pRepo) < 0) return NULL; if (tsdbLockRepo(pRepo) < 0) return NULL;
@ -204,6 +204,9 @@ void *tsdbAllocBytes(STsdbRepo *pRepo, int bytes) {
pBufBlock->offset += bytes; pBufBlock->offset += bytes;
pBufBlock->remain -= bytes; pBufBlock->remain -= bytes;
tsdbTrace("vgId:%d allocate %d bytes from buffer block, nBlocks %d offset %d remain %d", REPO_ID(pRepo), bytes,
listNEles(pRepo->mem->bufBlockList), pBufBlock->offset, pBufBlock->remain);
return ptr; return ptr;
} }
@ -324,6 +327,8 @@ static void tsdbFreeBytes(STsdbRepo *pRepo, void *ptr, int bytes) {
pBufBlock->offset -= bytes; pBufBlock->offset -= bytes;
pBufBlock->remain += bytes; pBufBlock->remain += bytes;
ASSERT(ptr == POINTER_SHIFT(pBufBlock->data, pBufBlock->offset)); ASSERT(ptr == POINTER_SHIFT(pBufBlock->data, pBufBlock->offset));
tsdbTrace("vgId:%d return %d bytes to buffer block, nBlocks %d offset %d remain %d", REPO_ID(pRepo), bytes,
listNEles(pRepo->mem->bufBlockList), pBufBlock->offset, pBufBlock->remain);
} }
static SMemTable* tsdbNewMemTable(STsdbCfg* pCfg) { static SMemTable* tsdbNewMemTable(STsdbCfg* pCfg) {

2
src/tsdb/src/tsdbMeta.c
View File

@ -592,7 +592,7 @@ void tsdbUpdateTableSchema(STsdbRepo *pRepo, STable *pTable, STSchema *pSchema,
int tlen = tsdbGetTableEncodeSize(TSDB_UPDATE_META, pCTable); int tlen = tsdbGetTableEncodeSize(TSDB_UPDATE_META, pCTable);
void *buf = tsdbAllocBytes(pRepo, tlen); void *buf = tsdbAllocBytes(pRepo, tlen);
ASSERT(buf != NULL); ASSERT(buf != NULL);
tsdbInsertTableAct(pRepo, TSDB_UPDATE_META, buf, pTable); tsdbInsertTableAct(pRepo, TSDB_UPDATE_META, buf, pCTable);
} }
} }

29
src/tsdb/src/tsdbRWHelper.c
View File

@ -589,20 +589,25 @@ void tsdbGetDataStatis(SRWHelper *pHelper, SDataStatis *pStatis, int numOfCols)
int tsdbLoadBlockDataCols(SRWHelper *pHelper, SCompBlock *pCompBlock, SCompInfo *pCompInfo, int16_t *colIds, int numOfColIds) { int tsdbLoadBlockDataCols(SRWHelper *pHelper, SCompBlock *pCompBlock, SCompInfo *pCompInfo, int16_t *colIds, int numOfColIds) {
ASSERT(pCompBlock->numOfSubBlocks >= 1); // Must be super block ASSERT(pCompBlock->numOfSubBlocks >= 1); // Must be super block
SCompBlock *pTCompBlock = pCompBlock;
int numOfSubBlocks = pCompBlock->numOfSubBlocks; int numOfSubBlocks = pCompBlock->numOfSubBlocks;
if (numOfSubBlocks > 1) if (numOfSubBlocks > 1)
pCompBlock = (SCompBlock *)POINTER_SHIFT((pCompInfo == NULL) ? pHelper->pCompInfo : pCompInfo, pCompBlock->offset); pTCompBlock = (SCompBlock *)POINTER_SHIFT((pCompInfo == NULL) ? pHelper->pCompInfo : pCompInfo, pCompBlock->offset);
tdResetDataCols(pHelper->pDataCols[0]); tdResetDataCols(pHelper->pDataCols[0]);
if (tsdbLoadBlockDataColsImpl(pHelper, pCompBlock, pHelper->pDataCols[0], colIds, numOfColIds) < 0) goto _err; if (tsdbLoadBlockDataColsImpl(pHelper, pTCompBlock, pHelper->pDataCols[0], colIds, numOfColIds) < 0) goto _err;
for (int i = 1; i < numOfSubBlocks; i++) { for (int i = 1; i < numOfSubBlocks; i++) {
tdResetDataCols(pHelper->pDataCols[1]); tdResetDataCols(pHelper->pDataCols[1]);
pCompBlock++; pTCompBlock++;
if (tsdbLoadBlockDataColsImpl(pHelper, pCompBlock, pHelper->pDataCols[1], colIds, numOfColIds) < 0) goto _err; if (tsdbLoadBlockDataColsImpl(pHelper, pTCompBlock, pHelper->pDataCols[1], colIds, numOfColIds) < 0) goto _err;
if (tdMergeDataCols(pHelper->pDataCols[0], pHelper->pDataCols[1], pHelper->pDataCols[1]->numOfRows) < 0) goto _err; if (tdMergeDataCols(pHelper->pDataCols[0], pHelper->pDataCols[1], pHelper->pDataCols[1]->numOfRows) < 0) goto _err;
} }
ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows &&
dataColsKeyFirst(pHelper->pDataCols[0]) == pCompBlock->keyFirst &&
dataColsKeyLast(pHelper->pDataCols[0]) == pCompBlock->keyLast);
return 0; return 0;
_err: _err:
@ -610,19 +615,25 @@ _err:
} }
int tsdbLoadBlockData(SRWHelper *pHelper, SCompBlock *pCompBlock, SCompInfo *pCompInfo) { int tsdbLoadBlockData(SRWHelper *pHelper, SCompBlock *pCompBlock, SCompInfo *pCompInfo) {
SCompBlock *pTCompBlock = pCompBlock;
int numOfSubBlock = pCompBlock->numOfSubBlocks; int numOfSubBlock = pCompBlock->numOfSubBlocks;
if (numOfSubBlock > 1) if (numOfSubBlock > 1)
pCompBlock = (SCompBlock *)POINTER_SHIFT((pCompInfo == NULL) ? pHelper->pCompInfo : pCompInfo, pCompBlock->offset); pTCompBlock = (SCompBlock *)POINTER_SHIFT((pCompInfo == NULL) ? pHelper->pCompInfo : pCompInfo, pCompBlock->offset);
tdResetDataCols(pHelper->pDataCols[0]); tdResetDataCols(pHelper->pDataCols[0]);
if (tsdbLoadBlockDataImpl(pHelper, pCompBlock, pHelper->pDataCols[0]) < 0) goto _err; if (tsdbLoadBlockDataImpl(pHelper, pTCompBlock, pHelper->pDataCols[0]) < 0) goto _err;
for (int i = 1; i < numOfSubBlock; i++) { for (int i = 1; i < numOfSubBlock; i++) {
tdResetDataCols(pHelper->pDataCols[1]); tdResetDataCols(pHelper->pDataCols[1]);
pCompBlock++; pTCompBlock++;
if (tsdbLoadBlockDataImpl(pHelper, pCompBlock, pHelper->pDataCols[1]) < 0) goto _err; if (tsdbLoadBlockDataImpl(pHelper, pTCompBlock, pHelper->pDataCols[1]) < 0) goto _err;
if (tdMergeDataCols(pHelper->pDataCols[0], pHelper->pDataCols[1], pHelper->pDataCols[1]->numOfRows) < 0) goto _err; if (tdMergeDataCols(pHelper->pDataCols[0], pHelper->pDataCols[1], pHelper->pDataCols[1]->numOfRows) < 0) goto _err;
} }
ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows &&
dataColsKeyFirst(pHelper->pDataCols[0]) == pCompBlock->keyFirst &&
dataColsKeyLast(pHelper->pDataCols[0]) == pCompBlock->keyLast);
return 0; return 0;
_err: _err:
@ -1227,7 +1238,6 @@ static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDa
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
goto _err; goto _err;
} }
ASSERT(pCompData->numOfCols == pCompBlock->numOfCols);
int32_t tsize = TSDB_GET_COMPCOL_LEN(pCompBlock->numOfCols); int32_t tsize = TSDB_GET_COMPCOL_LEN(pCompBlock->numOfCols);
if (!taosCheckChecksumWhole((uint8_t *)pCompData, tsize)) { if (!taosCheckChecksumWhole((uint8_t *)pCompData, tsize)) {
@ -1236,6 +1246,7 @@ static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDa
terrno = TSDB_CODE_TDB_FILE_CORRUPTED; terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
goto _err; goto _err;
} }
ASSERT(pCompData->numOfCols == pCompBlock->numOfCols);
pDataCols->numOfRows = pCompBlock->numOfRows; pDataCols->numOfRows = pCompBlock->numOfRows;

149
src/tsdb/src/tsdbRead.c
View File

@ -90,6 +90,12 @@ typedef struct SBlockOrderSupporter {
int32_t* numOfBlocksPerTable; int32_t* numOfBlocksPerTable;
} SBlockOrderSupporter; } SBlockOrderSupporter;
typedef struct SIOCostSummary {
int64_t blockLoadTime;
int64_t statisInfoLoadTime;
int64_t checkForNextTime;
} SIOCostSummary;
typedef struct STsdbQueryHandle { typedef struct STsdbQueryHandle {
STsdbRepo* pTsdb; STsdbRepo* pTsdb;
SQueryFilePos cur; // current position SQueryFilePos cur; // current position
@ -101,7 +107,7 @@ typedef struct STsdbQueryHandle {
bool locateStart; bool locateStart;
int32_t outputCapacity; int32_t outputCapacity;
int32_t realNumOfRows; int32_t realNumOfRows;
SArray* pTableCheckInfo; //SArray<STableCheckInfo> SArray* pTableCheckInfo; // SArray<STableCheckInfo>
int32_t activeIndex; int32_t activeIndex;
bool checkFiles; // check file stage bool checkFiles; // check file stage
void* qinfo; // query info handle, for debug purpose void* qinfo; // query info handle, for debug purpose
@ -116,6 +122,8 @@ typedef struct STsdbQueryHandle {
SArray* defaultLoadColumn;// default load column SArray* defaultLoadColumn;// default load column
SDataBlockLoadInfo dataBlockLoadInfo; /* record current block load information */ SDataBlockLoadInfo dataBlockLoadInfo; /* record current block load information */
SLoadCompBlockInfo compBlockLoadInfo; /* record current compblock information in SQuery */ SLoadCompBlockInfo compBlockLoadInfo; /* record current compblock information in SQuery */
SIOCostSummary cost;
} STsdbQueryHandle; } STsdbQueryHandle;
static void changeQueryHandleForLastrowQuery(TsdbQueryHandleT pqHandle); static void changeQueryHandleForLastrowQuery(TsdbQueryHandleT pqHandle);
@ -183,6 +191,7 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab
free(pQueryHandle); free(pQueryHandle);
return NULL; return NULL;
} }
tsdbTakeMemSnapshot(pQueryHandle->pTsdb, &pQueryHandle->mem, &pQueryHandle->imem); tsdbTakeMemSnapshot(pQueryHandle->pTsdb, &pQueryHandle->mem, &pQueryHandle->imem);
size_t sizeOfGroup = taosArrayGetSize(groupList->pGroupList); size_t sizeOfGroup = taosArrayGetSize(groupList->pGroupList);
@ -340,6 +349,11 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh
return true; return true;
} }
static void destroyTableMemIterator(STableCheckInfo* pCheckInfo) {
tSkipListDestroyIter(pCheckInfo->iter);
tSkipListDestroyIter(pCheckInfo->iiter);
}
SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo) { SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo) {
SDataRow rmem = NULL, rimem = NULL; SDataRow rmem = NULL, rimem = NULL;
if (pCheckInfo->iter) { if (pCheckInfo->iter) {
@ -384,7 +398,7 @@ SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo) {
return NULL; return NULL;
} }
bool moveToNextRow(STableCheckInfo* pCheckInfo) { static bool moveToNextRow(STableCheckInfo* pCheckInfo) {
bool hasNext = false; bool hasNext = false;
if (pCheckInfo->chosen == 0) { if (pCheckInfo->chosen == 0) {
if (pCheckInfo->iter != NULL) { if (pCheckInfo->iter != NULL) {
@ -507,7 +521,7 @@ static int32_t binarySearchForBlock(SCompBlock* pBlock, int32_t numOfBlocks, TSK
return midSlot; return midSlot;
} }
static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlocks, int32_t type) { static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlocks) {
SFileGroup* fileGroup = pQueryHandle->pFileGroup; SFileGroup* fileGroup = pQueryHandle->pFileGroup;
assert(fileGroup->files[TSDB_FILE_TYPE_HEAD].fname > 0); assert(fileGroup->files[TSDB_FILE_TYPE_HEAD].fname > 0);
@ -524,13 +538,15 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo
for (int32_t i = 0; i < numOfTables; ++i) { for (int32_t i = 0; i < numOfTables; ++i) {
STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i);
pCheckInfo->numOfBlocks = 0;
SCompIdx* compIndex = &pQueryHandle->rhelper.pCompIdx[pCheckInfo->tableId.tid]; SCompIdx* compIndex = &pQueryHandle->rhelper.pCompIdx[pCheckInfo->tableId.tid];
if (compIndex->len == 0 || compIndex->numOfBlocks == 0 ||
compIndex->uid != pCheckInfo->tableId.uid) { // no data block in this file, try next file // no data block in this file, try next file
pCheckInfo->numOfBlocks = 0; if (compIndex->len == 0 || compIndex->numOfBlocks == 0 || compIndex->uid != pCheckInfo->tableId.uid) {
continue; // no data blocks in the file belongs to pCheckInfo->pTable continue; // no data blocks in the file belongs to pCheckInfo->pTable
} else { }
if (pCheckInfo->compSize < compIndex->len) { if (pCheckInfo->compSize < compIndex->len) {
assert(compIndex->len > 0); assert(compIndex->len > 0);
@ -546,8 +562,16 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo
tsdbLoadCompInfo(&(pQueryHandle->rhelper), (void *)(pCheckInfo->pCompInfo)); tsdbLoadCompInfo(&(pQueryHandle->rhelper), (void *)(pCheckInfo->pCompInfo));
SCompInfo* pCompInfo = pCheckInfo->pCompInfo; SCompInfo* pCompInfo = pCheckInfo->pCompInfo;
TSKEY s = MIN(pCheckInfo->lastKey, pQueryHandle->window.ekey); TSKEY s = TSKEY_INITIAL_VAL, e = TSKEY_INITIAL_VAL;
TSKEY e = MAX(pCheckInfo->lastKey, pQueryHandle->window.ekey);
if (ASCENDING_TRAVERSE(pQueryHandle->order)) {
assert(pCheckInfo->lastKey <= pQueryHandle->window.ekey && pQueryHandle->window.skey <= pQueryHandle->window.ekey);
} else {
assert(pCheckInfo->lastKey >= pQueryHandle->window.ekey && pQueryHandle->window.skey >= pQueryHandle->window.ekey);
}
s = MIN(pCheckInfo->lastKey, pQueryHandle->window.ekey);
e = MAX(pCheckInfo->lastKey, pQueryHandle->window.ekey);
// discard the unqualified data block based on the query time window // discard the unqualified data block based on the query time window
int32_t start = binarySearchForBlock(pCompInfo->blocks, compIndex->numOfBlocks, s, TSDB_ORDER_ASC); int32_t start = binarySearchForBlock(pCompInfo->blocks, compIndex->numOfBlocks, s, TSDB_ORDER_ASC);
@ -570,7 +594,6 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo
(*numOfBlocks) += pCheckInfo->numOfBlocks; (*numOfBlocks) += pCheckInfo->numOfBlocks;
} }
}
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -583,22 +606,13 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo
.uid = (_checkInfo)->tableId.uid}) .uid = (_checkInfo)->tableId.uid})
static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo) { static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo) {
STsdbRepo *pRepo = pQueryHandle->pTsdb; STsdbRepo *pRepo = pQueryHandle->pTsdb;
// TODO refactor
SCompData* data = calloc(1, sizeof(SCompData) + sizeof(SCompCol) * pBlock->numOfCols);
data->numOfCols = pBlock->numOfCols;
data->uid = pCheckInfo->pTableObj->tableId.uid;
bool blockLoaded = false; bool blockLoaded = false;
int64_t st = taosGetTimestampUs(); int64_t st = taosGetTimestampUs();
if (pCheckInfo->pDataCols == NULL) { if (pCheckInfo->pDataCols == NULL) {
STsdbMeta* pMeta = tsdbGetMeta(pRepo); STsdbMeta* pMeta = tsdbGetMeta(pRepo);
// TODO
pCheckInfo->pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pRepo->config.maxRowsPerFileBlock); pCheckInfo->pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pRepo->config.maxRowsPerFileBlock);
} }
@ -607,7 +621,9 @@ static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlo
tdInitDataCols(pQueryHandle->rhelper.pDataCols[0], pSchema); tdInitDataCols(pQueryHandle->rhelper.pDataCols[0], pSchema);
tdInitDataCols(pQueryHandle->rhelper.pDataCols[1], pSchema); tdInitDataCols(pQueryHandle->rhelper.pDataCols[1], pSchema);
if (tsdbLoadBlockData(&(pQueryHandle->rhelper), pBlock, pCheckInfo->pCompInfo) == 0) { int16_t* colIds = pQueryHandle->defaultLoadColumn->pData;
int32_t ret = tsdbLoadBlockDataCols(&(pQueryHandle->rhelper), pBlock, pCheckInfo->pCompInfo, colIds, QH_GET_NUM_OF_COLS(pQueryHandle));
if (ret == TSDB_CODE_SUCCESS) {
SDataBlockLoadInfo* pBlockLoadInfo = &pQueryHandle->dataBlockLoadInfo; SDataBlockLoadInfo* pBlockLoadInfo = &pQueryHandle->dataBlockLoadInfo;
pBlockLoadInfo->fileGroup = pQueryHandle->pFileGroup; pBlockLoadInfo->fileGroup = pQueryHandle->pFileGroup;
@ -621,10 +637,10 @@ static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlo
assert(pCols->numOfRows != 0 && pCols->numOfRows <= pBlock->numOfRows); assert(pCols->numOfRows != 0 && pCols->numOfRows <= pBlock->numOfRows);
pBlock->numOfRows = pCols->numOfRows; pBlock->numOfRows = pCols->numOfRows;
tfree(data);
int64_t et = taosGetTimestampUs() - st; int64_t elapsedTime = (taosGetTimestampUs() - st);
tsdbDebug("%p load file block into buffer, elapsed time:%"PRId64 " us", pQueryHandle, et); pQueryHandle->cost.blockLoadTime += elapsedTime;
tsdbDebug("%p load file block into buffer, elapsed time:%"PRId64 " us", pQueryHandle, elapsedTime);
return blockLoaded; return blockLoaded;
} }
@ -1367,7 +1383,6 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
// todo opt for only one table case
static int32_t getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle, bool* exists) { static int32_t getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle, bool* exists) {
pQueryHandle->numOfBlocks = 0; pQueryHandle->numOfBlocks = 0;
SQueryFilePos* cur = &pQueryHandle->cur; SQueryFilePos* cur = &pQueryHandle->cur;
@ -1378,8 +1393,7 @@ static int32_t getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle, bool* ex
int32_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); int32_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
while ((pQueryHandle->pFileGroup = tsdbGetFileGroupNext(&pQueryHandle->fileIter)) != NULL) { while ((pQueryHandle->pFileGroup = tsdbGetFileGroupNext(&pQueryHandle->fileIter)) != NULL) {
int32_t type = ASCENDING_TRAVERSE(pQueryHandle->order)? QUERY_RANGE_GREATER_EQUAL:QUERY_RANGE_LESS_EQUAL; if ((code = getFileCompInfo(pQueryHandle, &numOfBlocks)) != TSDB_CODE_SUCCESS) {
if ((code = getFileCompInfo(pQueryHandle, &numOfBlocks, type)) != TSDB_CODE_SUCCESS) {
break; break;
} }
@ -1488,6 +1502,9 @@ static bool doHasDataInBuffer(STsdbQueryHandle* pQueryHandle) {
bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle; STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle;
int64_t stime = taosGetTimestampUs();
int64_t elapsedTime = stime;
size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
assert(numOfTables > 0); assert(numOfTables > 0);
@ -1514,7 +1531,7 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
pQueryHandle->cur.win = (STimeWindow){pQueryHandle->window.skey, pQueryHandle->window.skey}; pQueryHandle->cur.win = (STimeWindow){pQueryHandle->window.skey, pQueryHandle->window.skey};
pQueryHandle->window = pQueryHandle->cur.win; pQueryHandle->window = pQueryHandle->cur.win;
pQueryHandle->cur.rows = 1; pQueryHandle->cur.rows = 1;
pQueryHandle->type = TSDB_QUERY_TYPE_EXTERNAL; pQueryHandle->type = TSDB_QUERY_TYPE_ALL;
return true; return true;
} else { } else {
STsdbQueryHandle* pSecQueryHandle = calloc(1, sizeof(STsdbQueryHandle)); STsdbQueryHandle* pSecQueryHandle = calloc(1, sizeof(STsdbQueryHandle));
@ -1532,6 +1549,7 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
free(pSecQueryHandle); free(pSecQueryHandle);
return false; return false;
} }
tsdbTakeMemSnapshot(pSecQueryHandle->pTsdb, &pSecQueryHandle->mem, &pSecQueryHandle->imem); tsdbTakeMemSnapshot(pSecQueryHandle->pTsdb, &pSecQueryHandle->mem, &pSecQueryHandle->imem);
// allocate buffer in order to load data blocks from file // allocate buffer in order to load data blocks from file
@ -1547,7 +1565,6 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
colInfo.info = pCol->info; colInfo.info = pCol->info;
colInfo.pData = calloc(1, EXTRA_BYTES + pQueryHandle->outputCapacity * pCol->info.bytes); colInfo.pData = calloc(1, EXTRA_BYTES + pQueryHandle->outputCapacity * pCol->info.bytes);
taosArrayPush(pSecQueryHandle->pColumns, &colInfo); taosArrayPush(pSecQueryHandle->pColumns, &colInfo);
pSecQueryHandle->statis[i].colId = colInfo.info.colId;
} }
size_t si = taosArrayGetSize(pQueryHandle->pTableCheckInfo); size_t si = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
@ -1557,7 +1574,6 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
for (int32_t j = 0; j < si; ++j) { for (int32_t j = 0; j < si; ++j) {
STableCheckInfo* pCheckInfo = (STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, j); STableCheckInfo* pCheckInfo = (STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, j);
STableCheckInfo info = { STableCheckInfo info = {
.lastKey = pSecQueryHandle->window.skey, .lastKey = pSecQueryHandle->window.skey,
.tableId = pCheckInfo->tableId, .tableId = pCheckInfo->tableId,
@ -1569,12 +1585,13 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
tsdbInitDataBlockLoadInfo(&pSecQueryHandle->dataBlockLoadInfo); tsdbInitDataBlockLoadInfo(&pSecQueryHandle->dataBlockLoadInfo);
tsdbInitCompBlockLoadInfo(&pSecQueryHandle->compBlockLoadInfo); tsdbInitCompBlockLoadInfo(&pSecQueryHandle->compBlockLoadInfo);
pSecQueryHandle->defaultLoadColumn = taosArrayClone(pQueryHandle->defaultLoadColumn);
bool ret = tsdbNextDataBlock((void*) pSecQueryHandle); bool ret = tsdbNextDataBlock((void*) pSecQueryHandle);
assert(ret); assert(ret);
/*SDataBlockInfo* pBlockInfo =*/ tsdbRetrieveDataBlockInfo((void*) pSecQueryHandle, &blockInfo); tsdbRetrieveDataBlockInfo((void*) pSecQueryHandle, &blockInfo);
/*SArray *pDataBlock = */tsdbRetrieveDataBlock((void*) pSecQueryHandle, pSecQueryHandle->defaultLoadColumn); tsdbRetrieveDataBlock((void*) pSecQueryHandle, pSecQueryHandle->defaultLoadColumn);
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i); SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i);
@ -1588,14 +1605,25 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
SColumnInfoData* pTSCol = taosArrayGet(pQueryHandle->pColumns, 0); SColumnInfoData* pTSCol = taosArrayGet(pQueryHandle->pColumns, 0);
// it is ascending order
pQueryHandle->cur.win = (STimeWindow){((TSKEY*)pTSCol->pData)[0], ((TSKEY*)pTSCol->pData)[1]}; pQueryHandle->cur.win = (STimeWindow){((TSKEY*)pTSCol->pData)[0], ((TSKEY*)pTSCol->pData)[1]};
pQueryHandle->window = pQueryHandle->cur.win; pQueryHandle->window = pQueryHandle->cur.win;
pQueryHandle->cur.rows = 2; pQueryHandle->cur.rows = 2;
pQueryHandle->cur.mixBlock = true;
pQueryHandle->order = TSDB_ORDER_DESC;
int32_t step = -1;// one step for ascending order traverse
for (int32_t j = 0; j < si; ++j) {
STableCheckInfo* pCheckInfo = (STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, j);
pCheckInfo->lastKey = pQueryHandle->cur.win.ekey + step;
}
tsdbCleanupQueryHandle(pSecQueryHandle); tsdbCleanupQueryHandle(pSecQueryHandle);
} }
//disable it after retrieve data
pQueryHandle->type = TSDB_QUERY_TYPE_EXTERNAL; pQueryHandle->type = TSDB_QUERY_TYPE_EXTERNAL;
pQueryHandle->checkFiles = false;
return true; return true;
} }
@ -1607,6 +1635,8 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
} }
if (exists) { if (exists) {
elapsedTime = taosGetTimestampUs() - stime;
pQueryHandle->cost.checkForNextTime += elapsedTime;
return exists; return exists;
} }
@ -1617,6 +1647,9 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
// TODO: opt by consider the scan order // TODO: opt by consider the scan order
bool ret = doHasDataInBuffer(pQueryHandle); bool ret = doHasDataInBuffer(pQueryHandle);
terrno = TSDB_CODE_SUCCESS; terrno = TSDB_CODE_SUCCESS;
elapsedTime = taosGetTimestampUs() - stime;
pQueryHandle->cost.checkForNextTime += elapsedTime;
return ret; return ret;
} }
@ -1794,41 +1827,44 @@ void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT* pQueryHandle, SDataBlockInfo* p
int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataStatis** pBlockStatis) { int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataStatis** pBlockStatis) {
STsdbQueryHandle* pHandle = (STsdbQueryHandle*) pQueryHandle; STsdbQueryHandle* pHandle = (STsdbQueryHandle*) pQueryHandle;
SQueryFilePos* cur = &pHandle->cur; SQueryFilePos* c = &pHandle->cur;
if (cur->mixBlock) { if (c->mixBlock) {
*pBlockStatis = NULL; *pBlockStatis = NULL;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
assert((cur->slot >= 0 && cur->slot < pHandle->numOfBlocks) || STableBlockInfo* pBlockInfo = &pHandle->pDataBlockInfo[c->slot];
((cur->slot == pHandle->numOfBlocks) && (cur->slot == 0))); assert((c->slot >= 0 && c->slot < pHandle->numOfBlocks) || ((c->slot == pHandle->numOfBlocks) && (c->slot == 0)));
STableBlockInfo* pBlockInfo = &pHandle->pDataBlockInfo[cur->slot]; // file block with sub-blocks has no statistics data
// file block with subblocks has no statistics data
if (pBlockInfo->compBlock->numOfSubBlocks > 1) { if (pBlockInfo->compBlock->numOfSubBlocks > 1) {
*pBlockStatis = NULL; *pBlockStatis = NULL;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int64_t stime = taosGetTimestampUs();
tsdbLoadCompData(&pHandle->rhelper, pBlockInfo->compBlock, NULL); tsdbLoadCompData(&pHandle->rhelper, pBlockInfo->compBlock, NULL);
// todo opt perf int16_t* colIds = pHandle->defaultLoadColumn->pData;
size_t numOfCols = QH_GET_NUM_OF_COLS(pHandle);
for(int32_t i = 0; i < numOfCols; ++i) {
SDataStatis* st = &pHandle->statis[i];
int32_t colId = st->colId;
memset(st, 0, sizeof(SDataStatis)); size_t numOfCols = QH_GET_NUM_OF_COLS(pHandle);
st->colId = colId; memset(pHandle->statis, 0, numOfCols * sizeof(SDataStatis));
for(int32_t i = 0; i < numOfCols; ++i) {
pHandle->statis[i].colId = colIds[i];
} }
tsdbGetDataStatis(&pHandle->rhelper, pHandle->statis, numOfCols); tsdbGetDataStatis(&pHandle->rhelper, pHandle->statis, numOfCols);
*pBlockStatis = pHandle->statis; // always load the first primary timestamp column data
SDataStatis* pPrimaryColStatis = &pHandle->statis[0];
assert(pPrimaryColStatis->colId == PRIMARYKEY_TIMESTAMP_COL_INDEX);
pPrimaryColStatis->numOfNull = 0;
pPrimaryColStatis->min = pBlockInfo->compBlock->keyFirst;
pPrimaryColStatis->max = pBlockInfo->compBlock->keyLast;
//update the number of NULL data rows //update the number of NULL data rows
for(int32_t i = 0; i < numOfCols; ++i) { for(int32_t i = 1; i < numOfCols; ++i) {
if (pHandle->statis[i].numOfNull == -1) { // set the column data are all NULL if (pHandle->statis[i].numOfNull == -1) { // set the column data are all NULL
pHandle->statis[i].numOfNull = pBlockInfo->compBlock->numOfRows; pHandle->statis[i].numOfNull = pBlockInfo->compBlock->numOfRows;
} }
@ -1841,6 +1877,10 @@ int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataSta
} }
} }
int64_t elapsed = taosGetTimestampUs() - stime;
pHandle->cost.statisInfoLoadTime += elapsed;
*pBlockStatis = pHandle->statis;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -1893,8 +1933,6 @@ SArray* tsdbRetrieveDataBlock(TsdbQueryHandleT* pQueryHandle, SArray* pIdList) {
} }
} }
SArray* tsdbRetrieveDataRow(TsdbQueryHandleT* pQueryHandle, SArray* pIdList, SQueryRowCond* pCond) { return NULL; }
static int32_t getAllTableList(STable* pSuperTable, SArray* list) { static int32_t getAllTableList(STable* pSuperTable, SArray* list) {
SSkipListIterator* iter = tSkipListCreateIter(pSuperTable->pIndex); SSkipListIterator* iter = tSkipListCreateIter(pSuperTable->pIndex);
while (tSkipListIterNext(iter)) { while (tSkipListIterNext(iter)) {
@ -1923,6 +1961,7 @@ static void destroyHelper(void* param) {
free(param); free(param);
} }
#define TAG_INVALID_COLUMN_INDEX -2
static int32_t getTagColumnIndex(STSchema* pTSchema, SSchema* pSchema) { static int32_t getTagColumnIndex(STSchema* pTSchema, SSchema* pSchema) {
// filter on table name(TBNAME) // filter on table name(TBNAME)
if (strcasecmp(pSchema->name, TSQL_TBNAME_L) == 0) { if (strcasecmp(pSchema->name, TSQL_TBNAME_L) == 0) {
@ -1954,9 +1993,8 @@ void filterPrepare(void* expr, void* param) {
tVariant* pCond = pExpr->_node.pRight->pVal; tVariant* pCond = pExpr->_node.pRight->pVal;
SSchema* pSchema = pExpr->_node.pLeft->pSchema; SSchema* pSchema = pExpr->_node.pLeft->pSchema;
// todo : if current super table does not change schema yet, this function may fail to get correct schema, test case
int32_t index = getTagColumnIndex(pTSSchema, pSchema); int32_t index = getTagColumnIndex(pTSSchema, pSchema);
assert((index >= 0 && i < TSDB_MAX_TAGS) || (index == TSDB_TBNAME_COLUMN_INDEX)); assert((index >= 0 && i < TSDB_MAX_TAGS) || (index == TSDB_TBNAME_COLUMN_INDEX) || index == TAG_INVALID_COLUMN_INDEX);
pInfo->sch = *pSchema; pInfo->sch = *pSchema;
pInfo->colIndex = index; pInfo->colIndex = index;
@ -2334,7 +2372,7 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) {
size_t size = taosArrayGetSize(pQueryHandle->pTableCheckInfo); size_t size = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
for (int32_t i = 0; i < size; ++i) { for (int32_t i = 0; i < size; ++i) {
STableCheckInfo* pTableCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); STableCheckInfo* pTableCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i);
tSkipListDestroyIter(pTableCheckInfo->iter); destroyTableMemIterator(pTableCheckInfo);
if (pTableCheckInfo->pDataCols != NULL) { if (pTableCheckInfo->pDataCols != NULL) {
tfree(pTableCheckInfo->pDataCols->buf); tfree(pTableCheckInfo->pDataCols->buf);
@ -2362,10 +2400,15 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) {
tsdbUnRefMemTable(pQueryHandle->pTsdb, pQueryHandle->imem); tsdbUnRefMemTable(pQueryHandle->pTsdb, pQueryHandle->imem);
tsdbDestroyHelper(&pQueryHandle->rhelper); tsdbDestroyHelper(&pQueryHandle->rhelper);
SIOCostSummary* pCost = &pQueryHandle->cost;
tsdbDebug(":io-cost summary: statis-info:%"PRId64"us, datablock:%" PRId64"us, check data:%"PRId64"us, %p",
pCost->statisInfoLoadTime, pCost->blockLoadTime, pCost->checkForNextTime, pQueryHandle->qinfo);
tfree(pQueryHandle); tfree(pQueryHandle);
} }
void tsdbDestoryTableGroup(STableGroupInfo *pGroupList) { void tsdbDestroyTableGroup(STableGroupInfo *pGroupList) {
assert(pGroupList != NULL); assert(pGroupList != NULL);
size_t numOfGroup = taosArrayGetSize(pGroupList->pGroupList); size_t numOfGroup = taosArrayGetSize(pGroupList->pGroupList);

15
src/tsdb/tests/tsdbTests.cpp
View File

@ -39,7 +39,9 @@ static int insertData(SInsertInfo *pInfo) {
pBlock->uid = pInfo->uid; pBlock->uid = pInfo->uid;
pBlock->tid = pInfo->tid; pBlock->tid = pInfo->tid;
pBlock->sversion = pInfo->sversion; pBlock->sversion = pInfo->sversion;
pBlock->len = 0; pBlock->dataLen = 0;
pBlock->schemaLen = 0;
pBlock->numOfRows = 0;
for (int i = 0; i < pInfo->rowsPerSubmit; i++) { for (int i = 0; i < pInfo->rowsPerSubmit; i++) {
// start_time += 1000; // start_time += 1000;
if (pInfo->isAscend) { if (pInfo->isAscend) {
@ -47,7 +49,7 @@ static int insertData(SInsertInfo *pInfo) {
} else { } else {
start_time -= pInfo->interval; start_time -= pInfo->interval;
} }
SDataRow row = (SDataRow)(pBlock->data + pBlock->len); SDataRow row = (SDataRow)(pBlock->data + pBlock->dataLen);
tdInitDataRow(row, pInfo->pSchema); tdInitDataRow(row, pInfo->pSchema);
for (int j = 0; j < schemaNCols(pInfo->pSchema); j++) { for (int j = 0; j < schemaNCols(pInfo->pSchema); j++) {
@ -59,13 +61,15 @@ static int insertData(SInsertInfo *pInfo) {
tdAppendColVal(row, (void *)(&val), pTCol->type, pTCol->bytes, pTCol->offset); tdAppendColVal(row, (void *)(&val), pTCol->type, pTCol->bytes, pTCol->offset);
} }
} }
pBlock->len += dataRowLen(row); pBlock->dataLen += dataRowLen(row);
pBlock->numOfRows++;
} }
pMsg->length = pMsg->length + sizeof(SSubmitBlk) + pBlock->len; pMsg->length = sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + pBlock->dataLen;
pMsg->numOfBlocks = 1; pMsg->numOfBlocks = 1;
pBlock->len = htonl(pBlock->len); pBlock->dataLen = htonl(pBlock->dataLen);
pBlock->numOfRows = htonl(pBlock->numOfRows); pBlock->numOfRows = htonl(pBlock->numOfRows);
pBlock->schemaLen = htonl(pBlock->schemaLen);
pBlock->uid = htobe64(pBlock->uid); pBlock->uid = htobe64(pBlock->uid);
pBlock->tid = htonl(pBlock->tid); pBlock->tid = htonl(pBlock->tid);
@ -74,7 +78,6 @@ static int insertData(SInsertInfo *pInfo) {
pMsg->length = htonl(pMsg->length); pMsg->length = htonl(pMsg->length);
pMsg->numOfBlocks = htonl(pMsg->numOfBlocks); pMsg->numOfBlocks = htonl(pMsg->numOfBlocks);
pMsg->compressed = htonl(pMsg->numOfBlocks);
if (tsdbInsertData(pInfo->pRepo, pMsg, NULL) < 0) { if (tsdbInsertData(pInfo->pRepo, pMsg, NULL) < 0) {
tfree(pMsg); tfree(pMsg);

46
src/util/src/tcache.c
View File

@ -225,7 +225,7 @@ static void doCleanupDataCache(SCacheObj *pCacheObj);
* refresh cache to remove data in both hash list and trash, if any nodes' refcount == 0, every pCacheObj->refreshTime * refresh cache to remove data in both hash list and trash, if any nodes' refcount == 0, every pCacheObj->refreshTime
* @param handle Cache object handle * @param handle Cache object handle
*/ */
static void* taosCacheTimedRefresh(void *pCacheObj); static void* taosCacheTimedRefresh(void *handle);
SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool extendLifespan, __cache_free_fn_t fn, const char* cacheName) { SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool extendLifespan, __cache_free_fn_t fn, const char* cacheName) {
if (refreshTimeInSeconds <= 0) { if (refreshTimeInSeconds <= 0) {
@ -455,51 +455,11 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) {
__cache_unlock(pCacheObj); __cache_unlock(pCacheObj);
} else { } else {
uDebug("cache:%s, key:%p, %p is released, refcnt:%d", pCacheObj->name, pNode->key, pNode->data, T_REF_VAL_GET(pNode) - 1);
__cache_wr_lock(pCacheObj);
// NOTE: once refcount is decrease, pNode may be freed by other thread immediately. // NOTE: once refcount is decrease, pNode may be freed by other thread immediately.
int32_t ref = T_REF_DEC(pNode); int32_t ref = T_REF_DEC(pNode);
uDebug("cache:%s, key:%p, %p is released, refcnt:%d, in trashcan:%d", pCacheObj->name, pNode->key, pNode->data, ref,
if (inTrashCan && (ref == 0)) { inTrashCan);
// Remove it if the ref count is 0.
// The ref count does not need to load and check again after lock acquired, since ref count can not be increased when
// the node is in trashcan.
assert(pNode->pTNodeHeader->pData == pNode);
taosRemoveFromTrashCan(pCacheObj, pNode->pTNodeHeader);
} }
__cache_unlock(pCacheObj);
}
// else {
// if (_remove) { // not in trash can, but need to remove it
// __cache_wr_lock(pCacheObj);
//
// /*
// * If not referenced by other users. Otherwise move this node to trashcan wait for all users
// * releasing this resources.
// *
// * NOTE: previous ref is 0, and current ref is still 0, remove it. If previous is not 0, there is another thread
// * that tries to do the same thing.
// */
// if (ref == 0) {
// if (T_REF_VAL_GET(pNode) == 0) {
// taosCacheReleaseNode(pCacheObj, pNode);
// } else {
// taosCacheMoveToTrash(pCacheObj, pNode);
// }
// } else if (ref > 0) {
// if (!pNode->inTrashCan) {
// assert(pNode->pTNodeHeader == NULL);
// taosCacheMoveToTrash(pCacheObj, pNode);
// }
// }
//
// __cache_unlock(pCacheObj);
// }
// }
} }
void taosCacheEmpty(SCacheObj *pCacheObj) { void taosCacheEmpty(SCacheObj *pCacheObj) {

16
src/vnode/src/vnodeMain.c
View File

@ -259,7 +259,6 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
appH.cqH = pVnode->cq; appH.cqH = pVnode->cq;
appH.cqCreateFunc = cqCreate; appH.cqCreateFunc = cqCreate;
appH.cqDropFunc = cqDrop; appH.cqDropFunc = cqDrop;
appH.configFunc = dnodeSendCfgTableToRecv;
sprintf(temp, "%s/tsdb", rootDir); sprintf(temp, "%s/tsdb", rootDir);
pVnode->tsdb = tsdbOpenRepo(temp, &appH); pVnode->tsdb = tsdbOpenRepo(temp, &appH);
if (pVnode->tsdb == NULL) { if (pVnode->tsdb == NULL) {
@ -341,6 +340,13 @@ void vnodeRelease(void *pVnodeRaw) {
tsdbCloseRepo(pVnode->tsdb, 1); tsdbCloseRepo(pVnode->tsdb, 1);
pVnode->tsdb = NULL; pVnode->tsdb = NULL;
// stop continuous query
if (pVnode->cq) {
void *cq = pVnode->cq;
pVnode->cq = NULL;
cqClose(cq);
}
if (pVnode->wal) if (pVnode->wal)
walClose(pVnode->wal); walClose(pVnode->wal);
pVnode->wal = NULL; pVnode->wal = NULL;
@ -512,13 +518,6 @@ static void vnodeCleanUp(SVnodeObj *pVnode) {
syncStop(sync); syncStop(sync);
} }
// stop continuous query
if (pVnode->cq) {
void *cq = pVnode->cq;
pVnode->cq = NULL;
cqClose(cq);
}
vTrace("vgId:%d, vnode will cleanup, refCount:%d", pVnode->vgId, pVnode->refCount); vTrace("vgId:%d, vnode will cleanup, refCount:%d", pVnode->vgId, pVnode->refCount);
// release local resources only after cutting off outside connections // release local resources only after cutting off outside connections
@ -588,7 +587,6 @@ static int vnodeResetTsdb(SVnodeObj *pVnode)
appH.cqH = pVnode->cq; appH.cqH = pVnode->cq;
appH.cqCreateFunc = cqCreate; appH.cqCreateFunc = cqCreate;
appH.cqDropFunc = cqDrop; appH.cqDropFunc = cqDrop;
appH.configFunc = dnodeSendCfgTableToRecv;
pVnode->tsdb = tsdbOpenRepo(rootDir, &appH); pVnode->tsdb = tsdbOpenRepo(rootDir, &appH);
pVnode->status = TAOS_VN_STATUS_READY; pVnode->status = TAOS_VN_STATUS_READY;

1
tests/comparisonTest/tdengine/q5.txt Normal file
View File

@ -0,0 +1 @@
select * from db.devices;

75
tests/comparisonTest/tdengine/tdengineTest.c
View File

@ -13,8 +13,9 @@ typedef struct {
char sql[256]; char sql[256];
char dataDir[256]; char dataDir[256];
int filesNum; int filesNum;
int writeClients; int clients;
int rowsPerRequest; int rowsPerRequest;
int write;
} ProArgs; } ProArgs;
typedef struct { typedef struct {
@ -41,7 +42,7 @@ int main(int argc, char *argv[]) {
statis.totalRows = 0; statis.totalRows = 0;
parseArg(argc, argv); parseArg(argc, argv);
if (arguments.writeClients > 0) { if (arguments.write) {
writeData(); writeData();
} else { } else {
readData(); readData();
@ -52,7 +53,7 @@ void parseArg(int argc, char *argv[]) {
strcpy(arguments.sql, "./sqlCmd.txt"); strcpy(arguments.sql, "./sqlCmd.txt");
strcpy(arguments.dataDir, "./testdata"); strcpy(arguments.dataDir, "./testdata");
arguments.filesNum = 2; arguments.filesNum = 2;
arguments.writeClients = 0; arguments.clients = 1;
arguments.rowsPerRequest = 100; arguments.rowsPerRequest = 100;
for (int i = 1; i < argc; ++i) { for (int i = 1; i < argc; ++i) {
@ -83,12 +84,12 @@ void parseArg(int argc, char *argv[]) {
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
} }
else if (strcmp(argv[i], "-writeClients") == 0) { else if (strcmp(argv[i], "-clients") == 0) {
if (i < argc - 1) { if (i < argc - 1) {
arguments.writeClients = atoi(argv[++i]); arguments.clients = atoi(argv[++i]);
} }
else { else {
fprintf(stderr, "'-writeClients' requires a parameter, default:%d\n", arguments.writeClients); fprintf(stderr, "'-clients' requires a parameter, default:%d\n", arguments.clients);
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
} }
@ -101,6 +102,9 @@ void parseArg(int argc, char *argv[]) {
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
} }
else if (strcmp(argv[i], "-w") == 0) {
arguments.write = 1;
}
} }
} }
@ -215,7 +219,7 @@ void writeDataImp(void *param) {
void writeData() { void writeData() {
printf("write data\n"); printf("write data\n");
printf("---- writeClients: %d\n", arguments.writeClients); printf("---- clients: %d\n", arguments.clients);
printf("---- dataDir: %s\n", arguments.dataDir); printf("---- dataDir: %s\n", arguments.dataDir);
printf("---- numOfFiles: %d\n", arguments.filesNum); printf("---- numOfFiles: %d\n", arguments.filesNum);
printf("---- rowsPerRequest: %d\n", arguments.rowsPerRequest); printf("---- rowsPerRequest: %d\n", arguments.rowsPerRequest);
@ -243,12 +247,12 @@ void writeData() {
int64_t st = getTimeStampMs(); int64_t st = getTimeStampMs();
int a = arguments.filesNum / arguments.writeClients; int a = arguments.filesNum / arguments.clients;
int b = arguments.filesNum % arguments.writeClients; int b = arguments.filesNum % arguments.clients;
int last = 0; int last = 0;
ThreadObj *threads = calloc((size_t)arguments.writeClients, sizeof(ThreadObj)); ThreadObj *threads = calloc((size_t)arguments.clients, sizeof(ThreadObj));
for (int i = 0; i < arguments.writeClients; ++i) { for (int i = 0; i < arguments.clients; ++i) {
ThreadObj *pthread = threads + i; ThreadObj *pthread = threads + i;
pthread_attr_t thattr; pthread_attr_t thattr;
pthread->threadId = i + 1; pthread->threadId = i + 1;
@ -264,7 +268,7 @@ void writeData() {
pthread_create(&pthread->pid, &thattr, (void *(*)(void *))writeDataImp, pthread); pthread_create(&pthread->pid, &thattr, (void *(*)(void *))writeDataImp, pthread);
} }
for (int i = 0; i < arguments.writeClients; i++) { for (int i = 0; i < arguments.clients; i++) {
pthread_join(threads[i].pid, NULL); pthread_join(threads[i].pid, NULL);
} }
@ -272,17 +276,15 @@ void writeData() {
float seconds = (float)elapsed / 1000; float seconds = (float)elapsed / 1000;
float rs = (float)statis.totalRows / seconds; float rs = (float)statis.totalRows / seconds;
free(threads);
printf("---- Spent %f seconds to insert %ld records, speed: %f Rows/Second\n", seconds, statis.totalRows, rs); printf("---- Spent %f seconds to insert %ld records, speed: %f Rows/Second\n", seconds, statis.totalRows, rs);
} }
void readData() { void readDataImp(void *param)
printf("read data\n"); {
printf("---- sql: %s\n", arguments.sql); ThreadObj *pThread = (ThreadObj *)param;
printf("Thread %d\n", pThread->threadId);
void *taos = taos_connect("127.0.0.1", "root", "taosdata", NULL, 0);
if (taos == NULL)
taos_error(taos);
FILE *fp = fopen(arguments.sql, "r"); FILE *fp = fopen(arguments.sql, "r");
if (fp == NULL) { if (fp == NULL) {
printf("failed to open file %s\n", arguments.sql); printf("failed to open file %s\n", arguments.sql);
@ -290,6 +292,10 @@ void readData() {
} }
printf("open file %s success\n", arguments.sql); printf("open file %s success\n", arguments.sql);
void *taos = taos_connect("127.0.0.1", "root", "taosdata", NULL, 0);
if (taos == NULL)
taos_error(taos);
char *line = NULL; char *line = NULL;
size_t len = 0; size_t len = 0;
while (!feof(fp)) { while (!feof(fp)) {
@ -325,9 +331,36 @@ void readData() {
int64_t elapsed = getTimeStampMs() - st; int64_t elapsed = getTimeStampMs() - st;
float seconds = (float)elapsed / 1000; float seconds = (float)elapsed / 1000;
printf("---- Spent %f seconds to query: %s", seconds, line); printf("---- Spent %f seconds to retrieve %d records, Thread:%d query: %s\n", seconds, rows, pThread->threadId, line);
} }
fclose(fp); fclose(fp);
} }
void readData() {
printf("read data\n");
printf("---- sql: %s\n", arguments.sql);
printf("---- clients: %d\n", arguments.clients);
void *taos = taos_connect("127.0.0.1", "root", "taosdata", NULL, 0);
if (taos == NULL)
taos_error(taos);
ThreadObj *threads = calloc((size_t)arguments.clients, sizeof(ThreadObj));
for (int i = 0; i < arguments.clients; ++i) {
ThreadObj *pthread = threads + i;
pthread_attr_t thattr;
pthread->threadId = i + 1;
pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
pthread_create(&pthread->pid, &thattr, (void *(*)(void *))readDataImp, pthread);
}
for (int i = 0; i < arguments.clients; i++) {
pthread_join(threads[i].pid, NULL);
}
free(threads);
}

127
tests/perftest-scripts/tdengineTestQ1Loop.sh Executable file
View File

@ -0,0 +1,127 @@
#!/bin/bash
DATA_DIR=/mnt/root/testdata
NUM_LOOP=5
function printTo {
if $verbose ; then
echo $1
fi
}
TDTESTQ1OUT=tdengineTestQ1.out
function runTest {
totalG0=0
totalG10=0
totalG20=0
totalG30=0
totalG40=0
totalG50=0
totalG60=0
totalG70=0
totalG80=0
totalG90=0
for i in `seq 1 $NUM_LOOP`; do
printTo "loop i:$i, $TDTEST_DIR/tdengineTest \
-sql q1.txt"
restartTaosd
$TDTEST_DIR/tdengineTest \
-sql $TDTEST_DIR/q1.txt > $TDTESTQ1OUT
G0=`grep "devgroup=0" $TDTESTQ1OUT| awk '{print $3}'`
totalG0=`echo "scale=4; $totalG0 + $G0" | bc`
printTo "i: $i, G0: $G0, totalG0:$totalG0"
G10=`grep "devgroup=10" $TDTESTQ1OUT| awk '{print $3}'`
totalG10=`echo "scale=4; $totalG10 + $G10" | bc`
printTo "i: $i, G10: $G10, totalG10:$totalG10"
G20=`grep "devgroup=20" $TDTESTQ1OUT| awk '{print $3}'`
totalG20=`echo "scale=4; $totalG20 + $G20" | bc`
G30=`grep "devgroup=30" $TDTESTQ1OUT| awk '{print $3}'`
totalG30=`echo "scale=4; $totalG30 + $G30" | bc`
G40=`grep "devgroup=40" $TDTESTQ1OUT| awk '{print $3}'`
totalG40=`echo "scale=4; $totalG40 + $G40" | bc`
G50=`grep "devgroup=50" $TDTESTQ1OUT| awk '{print $3}'`
totalG50=`echo "scale=4; $totalG50 + $G50" | bc`
G60=`grep "devgroup=60" $TDTESTQ1OUT| awk '{print $3}'`
totalG60=`echo "scale=4; $totalG60 + $G60" | bc`
G70=`grep "devgroup=70" $TDTESTQ1OUT| awk '{print $3}'`
totalG70=`echo "scale=4; $totalG70 + $G70" | bc`
G80=`grep "devgroup=80" $TDTESTQ1OUT| awk '{print $3}'`
totalG80=`echo "scale=4; $totalG80 + $G80" | bc`
G90=`grep "devgroup=90" $TDTESTQ1OUT| awk '{print $3}'`
totalG90=`echo "scale=4; $totalG90 + $G90" | bc`
printTo "i: $i, G90: $G90, totalG90:$totalG90"
done
avgG0=`echo "scale=4; x = $totalG0 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgG10=`echo "scale=4; x = $totalG10 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgG20=`echo "scale=4; x = $totalG20 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgG30=`echo "scale=4; x = $totalG30 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgG40=`echo "scale=4; x = $totalG40 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgG50=`echo "scale=4; x = $totalG50 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgG60=`echo "scale=4; x = $totalG60 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgG70=`echo "scale=4; x = $totalG70 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgG80=`echo "scale=4; x = $totalG80 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgG90=`echo "scale=4; x = $totalG90 / $NUM_LOOP; if(x<1) print 0; x" | bc`
echo "Latency, G-0, G-10, G-20, G-30, G-40, G-50, G-60, G-70, G-80, G-90"
echo "TDengine, $avgG0, $avgG10, $avgG20, $avgG30, $avgG40, $avgG50, $avgG60, $avgG70, $avgG80, $avgG90"
}
function restartTaosd {
printTo "Stop taosd"
systemctl stop taosd
PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
while [ -n "$PID" ]
do
pkill -TERM -x taosd
sleep 1
PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
done
printTo "Start taosd"
$TAOSD_DIR/taosd > /dev/null 2>&1 &
sleep 10
}
################ Main ################
master=false
develop=true
verbose=false
for arg in "$@"
do
case $arg in
-v)
verbose=true
;;
master)
master=true
develop=false
;;
develop)
master=false
develop=true
;;
*)
;;
esac
done
if $master ; then
echo "Test master branch.."
cp /mnt/root/cfg/master/taos.cfg /etc/taos/taos.cfg
WORK_DIR=/mnt/root/TDengine.master
else
echo "Test develop branch.."
cp /mnt/root/cfg/10billion/taos.cfg /etc/taos/taos.cfg
WORK_DIR=/mnt/root/TDengine
fi
TAOSD_DIR=$WORK_DIR/debug/build/bin
TDTEST_DIR=$WORK_DIR/tests/comparisonTest/tdengine
runTest
echo "Test done!"

348
tests/perftest-scripts/tdengineTestQ2Loop.sh Executable file
View File

@ -0,0 +1,348 @@
#!/bin/bash
DATA_DIR=/mnt/root/testdata
NUM_LOOP=5
function printTo {
if $verbose ; then
echo $1
fi
}
TDTESTQ2OUT=tdengineTestQ2.out
function runTest {
totalCount10=0
totalCount20=0
totalCount30=0
totalCount40=0
totalCount50=0
totalCount60=0
totalCount70=0
totalCount80=0
totalCount90=0
totalCount100=0
totalAvg10=0
totalAvg20=0
totalAvg30=0
totalAvg40=0
totalAvg50=0
totalAvg60=0
totalAvg70=0
totalAvg80=0
totalAvg90=0
totalAvg100=0
totalSum10=0
totalSum20=0
totalSum30=0
totalSum40=0
totalSum50=0
totalSum60=0
totalSum70=0
totalSum80=0
totalSum90=0
totalSum100=0
totalMax10=0
totalMax20=0
totalMax30=0
totalMax40=0
totalMax50=0
totalMax60=0
totalMax70=0
totalMax80=0
totalMax90=0
totalMax100=0
totalMin10=0
totalMin20=0
totalMin30=0
totalMin40=0
totalMin50=0
totalMin60=0
totalMin70=0
totalMin80=0
totalMin90=0
totalMin100=0
totalSpread10=0
totalSpread20=0
totalSpread30=0
totalSpread40=0
totalSpread50=0
totalSpread60=0
totalSpread70=0
totalSpread80=0
totalSpread90=0
totalSpread100=0
for i in `seq 1 $NUM_LOOP`; do
printTo "loop i:$i, $TDTEST_DIR/tdengineTest \
-sql q2.txt"
restartTaosd
$TDTEST_DIR/tdengineTest \
-sql $TDTEST_DIR/q2.txt > $TDTESTQ2OUT
Count10=`cat $TDTESTQ2OUT | grep "count" | grep "devgroup<10;" | awk '{print $3}'`
totalCount10=`echo "scale=4; $totalCount10 + $Count10" | bc`
Count20=`cat $TDTESTQ2OUT | grep "count" | grep "devgroup<20;" | awk '{print $3}'`
totalCount20=`echo "scale=4; $totalCount20 + $Count20" | bc`
Count30=`cat $TDTESTQ2OUT | grep count | grep "devgroup<30;" | awk '{print $3}'`
totalCount30=`echo "scale=4; $totalCount30 + $Count30" | bc`
Count40=`cat $TDTESTQ2OUT | grep count | grep "devgroup<40;" | awk '{print $3}'`
totalCount40=`echo "scale=4; $totalCount40 + $Count40" | bc`
Count50=`cat $TDTESTQ2OUT | grep count | grep "devgroup<50;" | awk '{print $3}'`
totalCount50=`echo "scale=4; $totalCount50 + $Count50" | bc`
Count60=`cat $TDTESTQ2OUT | grep count | grep "devgroup<60;" | awk '{print $3}'`
totalCount60=`echo "scale=4; $totalCount60 + $Count60" | bc`
Count70=`cat $TDTESTQ2OUT | grep count | grep "devgroup<70;" | awk '{print $3}'`
totalCount70=`echo "scale=4; $totalCount70 + $Count70" | bc`
Count80=`cat $TDTESTQ2OUT | grep count | grep "devgroup<80;" | awk '{print $3}'`
totalCount80=`echo "scale=4; $totalCount80 + $Count80" | bc`
Count90=`cat $TDTESTQ2OUT | grep count | grep "devgroup<90;" | awk '{print $3}'`
totalCount90=`echo "scale=4; $totalCount90 + $Count90" | bc`
Count100=`cat $TDTESTQ2OUT | grep count | grep "db.devices;" | awk '{print $3}'`
totalCount100=`echo "scale=4; $totalCount100 + $Count100" | bc`
Avg10=`cat $TDTESTQ2OUT | grep "avg" | grep "devgroup<10;" | awk '{print $3}'`
totalAvg10=`echo "scale=4; $totalAvg10 + $Avg10" | bc`
Avg20=`cat $TDTESTQ2OUT | grep "avg" | grep "devgroup<20;" | awk '{print $3}'`
totalAvg20=`echo "scale=4; $totalAvg20 + $Avg20" | bc`
Avg30=`cat $TDTESTQ2OUT | grep avg | grep "devgroup<30;" | awk '{print $3}'`
totalAvg30=`echo "scale=4; $totalAvg30 + $Avg30" | bc`
Avg40=`cat $TDTESTQ2OUT | grep avg | grep "devgroup<40;" | awk '{print $3}'`
totalAvg40=`echo "scale=4; $totalAvg40 + $Avg40" | bc`
Avg50=`cat $TDTESTQ2OUT | grep avg | grep "devgroup<50;" | awk '{print $3}'`
totalAvg50=`echo "scale=4; $totalAvg50 + $Avg50" | bc`
Avg60=`cat $TDTESTQ2OUT | grep avg | grep "devgroup<60;" | awk '{print $3}'`
totalAvg60=`echo "scale=4; $totalAvg60 + $Avg60" | bc`
Avg70=`cat $TDTESTQ2OUT | grep avg | grep "devgroup<70;" | awk '{print $3}'`
totalAvg70=`echo "scale=4; $totalAvg70 + $Avg70" | bc`
Avg80=`cat $TDTESTQ2OUT | grep avg | grep "devgroup<80;" | awk '{print $3}'`
totalAvg80=`echo "scale=4; $totalAvg80 + $Avg80" | bc`
Avg90=`cat $TDTESTQ2OUT | grep avg | grep "devgroup<90;" | awk '{print $3}'`
totalAvg90=`echo "scale=4; $totalAvg90 + $Avg90" | bc`
Avg100=`cat $TDTESTQ2OUT | grep avg | grep "db.devices;" | awk '{print $3}'`
totalAvg100=`echo "scale=4; $totalAvg100 + $Avg100" | bc`
Sum10=`cat $TDTESTQ2OUT | grep "sum" | grep "devgroup<10;" | awk '{print $3}'`
totalSum10=`echo "scale=4; $totalSum10 + $Sum10" | bc`
Sum20=`cat $TDTESTQ2OUT | grep "sum" | grep "devgroup<20;" | awk '{print $3}'`
totalSum20=`echo "scale=4; $totalSum20 + $Sum20" | bc`
Sum30=`cat $TDTESTQ2OUT | grep sum | grep "devgroup<30;" | awk '{print $3}'`
totalSum30=`echo "scale=4; $totalSum30 + $Sum30" | bc`
Sum40=`cat $TDTESTQ2OUT | grep sum | grep "devgroup<40;" | awk '{print $3}'`
totalSum40=`echo "scale=4; $totalSum40 + $Sum40" | bc`
Sum50=`cat $TDTESTQ2OUT | grep sum | grep "devgroup<50;" | awk '{print $3}'`
totalSum50=`echo "scale=4; $totalSum50 + $Sum50" | bc`
Sum60=`cat $TDTESTQ2OUT | grep sum | grep "devgroup<60;" | awk '{print $3}'`
totalSum60=`echo "scale=4; $totalSum60 + $Sum60" | bc`
Sum70=`cat $TDTESTQ2OUT | grep sum | grep "devgroup<70;" | awk '{print $3}'`
totalSum70=`echo "scale=4; $totalSum70 + $Sum70" | bc`
Sum80=`cat $TDTESTQ2OUT | grep sum | grep "devgroup<80;" | awk '{print $3}'`
totalSum80=`echo "scale=4; $totalSum80 + $Sum80" | bc`
Sum90=`cat $TDTESTQ2OUT | grep sum | grep "devgroup<90;" | awk '{print $3}'`
totalSum90=`echo "scale=4; $totalSum90 + $Sum90" | bc`
Sum100=`cat $TDTESTQ2OUT | grep sum | grep "db.devices;" | awk '{print $3}'`
totalSum100=`echo "scale=4; $totalSum100 + $Sum100" | bc`
Max10=`cat $TDTESTQ2OUT | grep "max" | grep "devgroup<10;" | awk '{print $3}'`
totalMax10=`echo "scale=4; $totalMax10 + $Max10" | bc`
Max20=`cat $TDTESTQ2OUT | grep "max" | grep "devgroup<20;" | awk '{print $3}'`
totalMax20=`echo "scale=4; $totalMax20 + $Max20" | bc`
Max30=`cat $TDTESTQ2OUT | grep max | grep "devgroup<30;" | awk '{print $3}'`
totalMax30=`echo "scale=4; $totalMax30 + $Max30" | bc`
Max40=`cat $TDTESTQ2OUT | grep max | grep "devgroup<40;" | awk '{print $3}'`
totalMax40=`echo "scale=4; $totalMax40 + $Max40" | bc`
Max50=`cat $TDTESTQ2OUT | grep max | grep "devgroup<50;" | awk '{print $3}'`
totalMax50=`echo "scale=4; $totalMax50 + $Max50" | bc`
Max60=`cat $TDTESTQ2OUT | grep max | grep "devgroup<60;" | awk '{print $3}'`
totalMax60=`echo "scale=4; $totalMax60 + $Max60" | bc`
Max70=`cat $TDTESTQ2OUT | grep max | grep "devgroup<70;" | awk '{print $3}'`
totalMax70=`echo "scale=4; $totalMax70 + $Max70" | bc`
Max80=`cat $TDTESTQ2OUT | grep max | grep "devgroup<80;" | awk '{print $3}'`
totalMax80=`echo "scale=4; $totalMax80 + $Max80" | bc`
Max90=`cat $TDTESTQ2OUT | grep max | grep "devgroup<90;" | awk '{print $3}'`
totalMax90=`echo "scale=4; $totalMax90 + $Max90" | bc`
Max100=`cat $TDTESTQ2OUT | grep max | grep "db.devices;" | awk '{print $3}'`
totalMax100=`echo "scale=4; $totalMax100 + $Max100" | bc`
Min10=`cat $TDTESTQ2OUT | grep "min" | grep "devgroup<10;" | awk '{print $3}'`
totalMin10=`echo "scale=4; $totalMin10 + $Min10" | bc`
Min20=`cat $TDTESTQ2OUT | grep "min" | grep "devgroup<20;" | awk '{print $3}'`
totalMin20=`echo "scale=4; $totalMin20 + $Min20" | bc`
Min30=`cat $TDTESTQ2OUT | grep min | grep "devgroup<30;" | awk '{print $3}'`
totalMin30=`echo "scale=4; $totalMin30 + $Min30" | bc`
Min40=`cat $TDTESTQ2OUT | grep min | grep "devgroup<40;" | awk '{print $3}'`
totalMin40=`echo "scale=4; $totalMin40 + $Min40" | bc`
Min50=`cat $TDTESTQ2OUT | grep min | grep "devgroup<50;" | awk '{print $3}'`
totalMin50=`echo "scale=4; $totalMin50 + $Min50" | bc`
Min60=`cat $TDTESTQ2OUT | grep min | grep "devgroup<60;" | awk '{print $3}'`
totalMin60=`echo "scale=4; $totalMin60 + $Min60" | bc`
Min70=`cat $TDTESTQ2OUT | grep min | grep "devgroup<70;" | awk '{print $3}'`
totalMin70=`echo "scale=4; $totalMin70 + $Min70" | bc`
Min80=`cat $TDTESTQ2OUT | grep min | grep "devgroup<80;" | awk '{print $3}'`
totalMin80=`echo "scale=4; $totalMin80 + $Min80" | bc`
Min90=`cat $TDTESTQ2OUT | grep min | grep "devgroup<90;" | awk '{print $3}'`
totalMin90=`echo "scale=4; $totalMin90 + $Min90" | bc`
Min100=`cat $TDTESTQ2OUT | grep min | grep "db.devices;" | awk '{print $3}'`
totalMin100=`echo "scale=4; $totalMin100 + $Min100" | bc`
Spread10=`cat $TDTESTQ2OUT | grep "spread" | grep "devgroup<10;" | awk '{print $3}'`
totalSpread10=`echo "scale=4; $totalSpread10 + $Spread10" | bc`
Spread20=`cat $TDTESTQ2OUT | grep "spread" | grep "devgroup<20;" | awk '{print $3}'`
totalSpread20=`echo "scale=4; $totalSpread20 + $Spread20" | bc`
Spread30=`cat $TDTESTQ2OUT | grep spread | grep "devgroup<30;" | awk '{print $3}'`
totalSpread30=`echo "scale=4; $totalSpread30 + $Spread30" | bc`
Spread40=`cat $TDTESTQ2OUT | grep spread | grep "devgroup<40;" | awk '{print $3}'`
totalSpread40=`echo "scale=4; $totalSpread40 + $Spread40" | bc`
Spread50=`cat $TDTESTQ2OUT | grep spread | grep "devgroup<50;" | awk '{print $3}'`
totalSpread50=`echo "scale=4; $totalSpread50 + $Spread50" | bc`
Spread60=`cat $TDTESTQ2OUT | grep spread | grep "devgroup<60;" | awk '{print $3}'`
totalSpread60=`echo "scale=4; $totalSpread60 + $Spread60" | bc`
Spread70=`cat $TDTESTQ2OUT | grep spread | grep "devgroup<70;" | awk '{print $3}'`
totalSpread70=`echo "scale=4; $totalSpread70 + $Spread70" | bc`
Spread80=`cat $TDTESTQ2OUT | grep spread | grep "devgroup<80;" | awk '{print $3}'`
totalSpread80=`echo "scale=4; $totalSpread80 + $Spread80" | bc`
Spread90=`cat $TDTESTQ2OUT | grep spread | grep "devgroup<90;" | awk '{print $3}'`
totalSpread90=`echo "scale=4; $totalSpread90 + $Spread90" | bc`
Spread100=`cat $TDTESTQ2OUT | grep spread | grep "db.devices;" | awk '{print $3}'`
totalSpread100=`echo "scale=4; $totalSpread100 + $Spread100" | bc`
done
avgCount10=`echo "scale=4; x = $totalCount10 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgCount20=`echo "scale=4; x = $totalCount20 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgCount30=`echo "scale=4; x = $totalCount30 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgCount40=`echo "scale=4; x = $totalCount40 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgCount50=`echo "scale=4; x = $totalCount50 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgCount60=`echo "scale=4; x = $totalCount60 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgCount70=`echo "scale=4; x = $totalCount70 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgCount80=`echo "scale=4; x = $totalCount80 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgCount90=`echo "scale=4; x = $totalCount90 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgCount100=`echo "scale=4; x = $totalCount100 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgAvg10=`echo "scale=4; x = $totalAvg10 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgAvg20=`echo "scale=4; x = $totalAvg20 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgAvg30=`echo "scale=4; x = $totalAvg30 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgAvg40=`echo "scale=4; x = $totalAvg40 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgAvg50=`echo "scale=4; x = $totalAvg50 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgAvg60=`echo "scale=4; x = $totalAvg60 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgAvg70=`echo "scale=4; x = $totalAvg70 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgAvg80=`echo "scale=4; x = $totalAvg80 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgAvg90=`echo "scale=4; x = $totalAvg90 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgAvg100=`echo "scale=4; x = $totalAvg100 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgSum10=`echo "scale=4; x = $totalSum10 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgSum20=`echo "scale=4; x = $totalSum20 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgSum30=`echo "scale=4; x = $totalSum30 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgSum40=`echo "scale=4; x = $totalSum40 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgSum50=`echo "scale=4; x = $totalSum50 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgSum60=`echo "scale=4; x = $totalSum60 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgSum70=`echo "scale=4; x = $totalSum70 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgSum80=`echo "scale=4; x = $totalSum80 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgSum90=`echo "scale=4; x = $totalSum90 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgSum100=`echo "scale=4; x = $totalSum100 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgMax10=`echo "scale=4; x = $totalMax10 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgMax20=`echo "scale=4; x = $totalMax20 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgMax30=`echo "scale=4; x = $totalMax30 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgMax40=`echo "scale=4; x = $totalMax40 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgMax50=`echo "scale=4; x = $totalMax50 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgMax60=`echo "scale=4; x = $totalMax60 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgMax70=`echo "scale=4; x = $totalMax70 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgMax80=`echo "scale=4; x = $totalMax80 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgMax90=`echo "scale=4; x = $totalMax90 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgMax100=`echo "scale=4; x = $totalMax100 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgMin10=`echo "scale=4; x = $totalMin10 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgMin20=`echo "scale=4; x = $totalMin20 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgMin30=`echo "scale=4; x = $totalMin30 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgMin40=`echo "scale=4; x = $totalMin40 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgMin50=`echo "scale=4; x = $totalMin50 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgMin60=`echo "scale=4; x = $totalMin60 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgMin70=`echo "scale=4; x = $totalMin70 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgMin80=`echo "scale=4; x = $totalMin80 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgMin90=`echo "scale=4; x = $totalMin90 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgMin100=`echo "scale=4; x = $totalMin100 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgSpread10=`echo "scale=4; x = $totalSpread10 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgSpread20=`echo "scale=4; x = $totalSpread20 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgSpread30=`echo "scale=4; x = $totalSpread30 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgSpread40=`echo "scale=4; x = $totalSpread40 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgSpread50=`echo "scale=4; x = $totalSpread50 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgSpread60=`echo "scale=4; x = $totalSpread60 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgSpread70=`echo "scale=4; x = $totalSpread70 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgSpread80=`echo "scale=4; x = $totalSpread80 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgSpread90=`echo "scale=4; x = $totalSpread90 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgSpread100=`echo "scale=4; x = $totalSpread100 / $NUM_LOOP; if(x<1) print 0; x" | bc`
echo "Latency, 10%, 20%, 30%, 40%, 50%, 60%, 70%, 80%, 90%, 100%"
echo "Count, $avgCount10, $avgCount20, $avgCount30, $avgCount40, $avgCount50, $avgCount60, $avgCount70, $avgCount80, $avgCount90, $avgCount100"
echo "Avg, $avgAvg10, $avgAvg20, $avgAvg30, $avgAvg40, $avgAvg50, $avgAvg60, $avgAvg70, $avgAvg80, $avgAvg90, $avgAvg100"
echo "Sum, $avgSum10, $avgSum20, $avgSum30, $avgSum40, $avgSum50, $avgSum60, $avgSum70, $avgSum80, $avgSum90, $avgSum100"
echo "Max, $avgMax10, $avgMax20, $avgMax30, $avgMax40, $avgMax50, $avgMax60, $avgMax70, $avgMax80, $avgMax90, $avgMax100"
echo "Min, $avgMin10, $avgMin20, $avgMin30, $avgMin40, $avgMin50, $avgMin60, $avgMin70, $avgMin80, $avgMin90, $avgMin100"
echo "Spread, $avgSpread10, $avgSpread20, $avgSpread30, $avgSpread40, $avgSpread50, $avgSpread60, $avgSpread70, $avgSpread80, $avgSpread90, $avgSpread100"
}
function restartTaosd {
printTo "Stop taosd"
systemctl stop taosd
PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
while [ -n "$PID" ]
do
pkill -TERM -x taosd
sleep 1
PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
done
printTo "Start taosd"
$TAOSD_DIR/taosd > /dev/null 2>&1 &
sleep 10
}
################ Main ################
master=false
develop=true
verbose=false
for arg in "$@"
do
case $arg in
-v)
verbose=true
;;
master)
master=true
develop=false
;;
develop)
master=false
develop=true
;;
*)
;;
esac
done
if $master ; then
echo "Test master branch.."
cp /mnt/root/cfg/master/taos.cfg /etc/taos/taos.cfg
WORK_DIR=/mnt/root/TDengine.master
else
echo "Test develop branch.."
cp /mnt/root/cfg/10billion/taos.cfg /etc/taos/taos.cfg
WORK_DIR=/mnt/root/TDengine
fi
TAOSD_DIR=$WORK_DIR/debug/build/bin
TDTEST_DIR=$WORK_DIR/tests/comparisonTest/tdengine
runTest
echo "Test done!"

124
tests/perftest-scripts/tdengineTestQ3Loop.sh Executable file
View File

@ -0,0 +1,124 @@
#!/bin/bash
DATA_DIR=/mnt/root/testdata
NUM_LOOP=5
function printTo {
if $verbose ; then
echo $1
fi
}
TDTESTQ3OUT=tdengineTestQ3.out
function runTest {
totalG10=0
totalG20=0
totalG30=0
totalG40=0
totalG50=0
totalG60=0
totalG70=0
totalG80=0
totalG90=0
totalG100=0
for i in `seq 1 $NUM_LOOP`; do
printTo "loop i:$i, $TDTEST_DIR/tdengineTest \
-sql q3.txt"
restartTaosd
$TDTEST_DIR/tdengineTest \
-sql $TDTEST_DIR/q3.txt > $TDTESTQ3OUT
G10=`grep "devgroup<10" $TDTESTQ3OUT| awk '{print $3}'`
totalG10=`echo "scale=4; $totalG10 + $G10" | bc`
G20=`grep "devgroup<20" $TDTESTQ3OUT| awk '{print $3}'`
totalG20=`echo "scale=4; $totalG20 + $G20" | bc`
G30=`grep "devgroup<30" $TDTESTQ3OUT| awk '{print $3}'`
totalG30=`echo "scale=4; $totalG30 + $G30" | bc`
G40=`grep "devgroup<40" $TDTESTQ3OUT| awk '{print $3}'`
totalG40=`echo "scale=4; $totalG40 + $G40" | bc`
G50=`grep "devgroup<50" $TDTESTQ3OUT| awk '{print $3}'`
totalG50=`echo "scale=4; $totalG50 + $G50" | bc`
G60=`grep "devgroup<60" $TDTESTQ3OUT| awk '{print $3}'`
totalG60=`echo "scale=4; $totalG60 + $G60" | bc`
G70=`grep "devgroup<70" $TDTESTQ3OUT| awk '{print $3}'`
totalG70=`echo "scale=4; $totalG70 + $G70" | bc`
G80=`grep "devgroup<80" $TDTESTQ3OUT| awk '{print $3}'`
totalG80=`echo "scale=4; $totalG80 + $G80" | bc`
G90=`grep "devgroup<90" $TDTESTQ3OUT| awk '{print $3}'`
totalG90=`echo "scale=4; $totalG90 + $G90" | bc`
G100=`grep "db.devices group by devgroup;" $TDTESTQ3OUT| awk '{print $3}'`
totalG100=`echo "scale=4; $totalG100 + $G100" | bc`
done
avgG10=`echo "scale=4; x = $totalG10 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgG20=`echo "scale=4; x = $totalG20 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgG30=`echo "scale=4; x = $totalG30 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgG40=`echo "scale=4; x = $totalG40 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgG50=`echo "scale=4; x = $totalG50 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgG60=`echo "scale=4; x = $totalG60 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgG70=`echo "scale=4; x = $totalG70 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgG80=`echo "scale=4; x = $totalG80 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgG90=`echo "scale=4; x = $totalG90 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgG100=`echo "scale=4; x = $totalG100 / $NUM_LOOP; if(x<1) print 0; x" | bc`
echo "Latency, 10%, 20%, 30%, 40%, 50%, 60%, 70%, 80%, 90%, 100%"
echo "TDengine, $avgG10, $avgG20, $avgG30, $avgG40, $avgG50, $avgG60, $avgG70, $avgG80, $avgG90, $avgG100"
}
function restartTaosd {
printTo "Stop taosd"
systemctl stop taosd
PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
while [ -n "$PID" ]
do
pkill -TERM -x taosd
sleep 1
PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
done
printTo "Start taosd"
$TAOSD_DIR/taosd > /dev/null 2>&1 &
sleep 10
}
################ Main ################
master=false
develop=true
verbose=false
for arg in "$@"
do
case $arg in
-v)
verbose=true
;;
master)
master=true
develop=false
;;
develop)
master=false
develop=true
;;
*)
;;
esac
done
if $master ; then
echo "Test master branch.."
cp /mnt/root/cfg/master/taos.cfg /etc/taos/taos.cfg
WORK_DIR=/mnt/root/TDengine.master
else
echo "Test develop branch.."
cp /mnt/root/cfg/10billion/taos.cfg /etc/taos/taos.cfg
WORK_DIR=/mnt/root/TDengine
fi
TAOSD_DIR=$WORK_DIR/debug/build/bin
TDTEST_DIR=$WORK_DIR/tests/comparisonTest/tdengine
runTest
echo "Test done!"

124
tests/perftest-scripts/tdengineTestQ4Loop.sh Executable file
View File

@ -0,0 +1,124 @@
#!/bin/bash
DATA_DIR=/mnt/root/testdata
NUM_LOOP=5
function printTo {
if $verbose ; then
echo $1
fi
}
TDTESTQ4OUT=tdengineTestQ4.out
function runTest {
totalG10=0
totalG20=0
totalG30=0
totalG40=0
totalG50=0
totalG60=0
totalG70=0
totalG80=0
totalG90=0
totalG100=0
for i in `seq 1 $NUM_LOOP`; do
printTo "loop i:$i, $TDTEST_DIR/tdengineTest \
-sql q4.txt"
restartTaosd
$TDTEST_DIR/tdengineTest \
-sql $TDTEST_DIR/q4.txt > $TDTESTQ4OUT
G10=`grep "devgroup<10" $TDTESTQ4OUT| awk '{print $3}'`
totalG10=`echo "scale=4; $totalG10 + $G10" | bc`
G20=`grep "devgroup<20" $TDTESTQ4OUT| awk '{print $3}'`
totalG20=`echo "scale=4; $totalG20 + $G20" | bc`
G30=`grep "devgroup<30" $TDTESTQ4OUT| awk '{print $3}'`
totalG30=`echo "scale=4; $totalG30 + $G30" | bc`
G40=`grep "devgroup<40" $TDTESTQ4OUT| awk '{print $3}'`
totalG40=`echo "scale=4; $totalG40 + $G40" | bc`
G50=`grep "devgroup<50" $TDTESTQ4OUT| awk '{print $3}'`
totalG50=`echo "scale=4; $totalG50 + $G50" | bc`
G60=`grep "devgroup<60" $TDTESTQ4OUT| awk '{print $3}'`
totalG60=`echo "scale=4; $totalG60 + $G60" | bc`
G70=`grep "devgroup<70" $TDTESTQ4OUT| awk '{print $3}'`
totalG70=`echo "scale=4; $totalG70 + $G70" | bc`
G80=`grep "devgroup<80" $TDTESTQ4OUT| awk '{print $3}'`
totalG80=`echo "scale=4; $totalG80 + $G80" | bc`
G90=`grep "devgroup<90" $TDTESTQ4OUT| awk '{print $3}'`
totalG90=`echo "scale=4; $totalG90 + $G90" | bc`
G100=`grep "db.devices interval" $TDTESTQ4OUT| awk '{print $3}'`
totalG100=`echo "scale=4; $totalG100 + $G100" | bc`
done
avgG10=`echo "scale=4; x = $totalG10 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgG20=`echo "scale=4; x = $totalG20 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgG30=`echo "scale=4; x = $totalG30 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgG40=`echo "scale=4; x = $totalG40 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgG50=`echo "scale=4; x = $totalG50 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgG60=`echo "scale=4; x = $totalG60 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgG70=`echo "scale=4; x = $totalG70 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgG80=`echo "scale=4; x = $totalG80 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgG90=`echo "scale=4; x = $totalG90 / $NUM_LOOP; if(x<1) print 0; x" | bc`
avgG100=`echo "scale=4; x = $totalG100 / $NUM_LOOP; if(x<1) print 0; x" | bc`
echo "Latency, 10%, 20%, 30%, 40%, 50%, 60%, 70%, 80%, 90%, 100%"
echo "TDengine, $avgG10, $avgG20, $avgG30, $avgG40, $avgG50, $avgG60, $avgG70, $avgG80, $avgG90, $avgG100"
}
function restartTaosd {
printTo "Stop taosd"
systemctl stop taosd
PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
while [ -n "$PID" ]
do
pkill -TERM -x taosd
sleep 1
PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
done
printTo "Start taosd"
$TAOSD_DIR/taosd > /dev/null 2>&1 &
sleep 10
}
################ Main ################
master=false
develop=true
verbose=false
for arg in "$@"
do
case $arg in
-v)
verbose=true
;;
master)
master=true
develop=false
;;
develop)
master=false
develop=true
;;
*)
;;
esac
done
if $master ; then
echo "Test master branch.."
cp /mnt/root/cfg/master/taos.cfg /etc/taos/taos.cfg
WORK_DIR=/mnt/root/TDengine.master
else
echo "Test develop branch.."
cp /mnt/root/cfg/10billion/taos.cfg /etc/taos/taos.cfg
WORK_DIR=/mnt/root/TDengine
fi
TAOSD_DIR=$WORK_DIR/debug/build/bin
TDTEST_DIR=$WORK_DIR/tests/comparisonTest/tdengine
runTest
echo "Test done!"

107
tests/perftest-scripts/tdengineTestQ5Loop.sh Executable file
View File

@ -0,0 +1,107 @@
#!/bin/bash
DATA_DIR=/mnt/root/testdata
NUM_LOOP=5
function printTo {
if $verbose ; then
echo $1
fi
}
TDTESTQ5OUT=tdengineTestQ5.out
function runTest {
totalThroughput=0
for i in `seq 1 $NUM_LOOP`; do
for c in `seq 1 $clients`; do
records[$c]=0
spentTime[$c]=0
throughput[$c]=0
done
printTo "loop i:$i, $TDTEST_DIR/tdengineTest \
-clients $clients -sql q5.txt"
restartTaosd
beginMS=`date +%s%3N`
$TDTEST_DIR/tdengineTest \
-clients $clients -sql $TDTEST_DIR/q5.txt > $TDTESTQ5OUT
endMS=`date +%s%3N`
totalRecords=0
for c in `seq 1 $clients`; do
records[$c]=`grep Thread:$c $TDTESTQ5OUT | awk '{print $7}'`
totalRecords=`echo "$totalRecords + ${records[$c]}"|bc`
done
spending=`echo "scale=4; x = ($endMS - $beginMS)/1000; if (x<1) print 0; x"|bc`
throughput=`echo "scale=4; x= $totalRecords / $spending; if (x<1) print 0; x" | bc`
printTo "spending: $spending sec, throughput: $throughput"
totalThroughput=`echo "scale=4; x = $totalThroughput + $throughput; if(x<1) print 0; x"|bc`
done
avgThrougput=`echo "scale=4; x = $totalThroughput / $NUM_LOOP; if (x<1) print 0; x"|bc`
echo "avg Throughput: $avgThrougput"
}
function restartTaosd {
printTo "Stop taosd"
systemctl stop taosd
PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
while [ -n "$PID" ]
do
pkill -TERM -x taosd
sleep 1
PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
done
printTo "Start taosd"
$TAOSD_DIR/taosd > /dev/null 2>&1 &
sleep 10
}
################ Main ################
master=false
develop=true
verbose=false
clients=1
while : ; do
case $1 in
-v)
verbose=true
shift ;;
master)
master=true
develop=false
shift ;;
develop)
master=false
develop=true
shift ;;
-c)
clients=$2
shift 2;;
*)
break ;;
esac
done
if $master ; then
printTo "Test master branch.."
cp /mnt/root/cfg/master/taos.cfg /etc/taos/taos.cfg
WORK_DIR=/mnt/root/TDengine.master
else
printTo "Test develop branch.."
cp /mnt/root/cfg/10billion/taos.cfg /etc/taos/taos.cfg
WORK_DIR=/mnt/root/TDengine
fi
TAOSD_DIR=$WORK_DIR/debug/build/bin
TDTEST_DIR=$WORK_DIR/tests/comparisonTest/tdengine
runTest
printTo "Test done!"

116
tests/perftest-scripts/tdengineTestWriteLoop.sh Executable file
View File

@ -0,0 +1,116 @@
#!/bin/bash
DATA_DIR=/mnt/root/testdata
NUM_LOOP=5
NUM_OF_FILES=100
rowsPerRequest=(1 100 500 1000 2000)
function printTo {
if $verbose ; then
echo $1
fi
}
function runTest {
printf "R/R, "
for c in `seq 1 $clients`; do
if [ "$c" == "1" ]; then
printf "$c client, "
else
printf "$c clients, "
fi
done
printf "\n"
for r in ${rowsPerRequest[@]}; do
printf "$r, "
for c in `seq 1 $clients`; do
totalRPR=0
for i in `seq 1 $NUM_LOOP`; do
restartTaosd
$TAOSD_DIR/taos -s "drop database db" > /dev/null 2>&1
printTo "loop i:$i, $TDTEST_DIR/tdengineTest \
-dataDir $DATA_DIR \
-numOfFiles $NUM_OF_FILES \
-w -clients $c \
-rowsPerRequest $r"
RPR=`$TDTEST_DIR/tdengineTest \
-dataDir $DATA_DIR \
-numOfFiles 1 \
-w -clients $c \
-rowsPerRequest $r \
| grep speed | awk '{print $(NF-1)}'`
totalRPR=`echo "scale=4; $totalRPR + $RPR" | bc`
printTo "rows:$r, clients:$c, i:$i RPR:$RPR"
done
avgRPR=`echo "scale=4; $totalRPR / $NUM_LOOP" | bc`
printf "$avgRPR, "
done
printf "\n"
done
}
function restartTaosd {
printTo "Stop taosd"
systemctl stop taosd
PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
while [ -n "$PID" ]
do
pkill -TERM -x taosd
sleep 1
PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
done
printTo "Start taosd"
$TAOSD_DIR/taosd > /dev/null 2>&1 &
sleep 10
}
################ Main ################
master=false
develop=true
verbose=false
clients=1
while : ; do
case $1 in
-v)
verbose=true
shift ;;
master)
master=true
develop=false
shift ;;
develop)
master=false
develop=true
shift ;;
-c)
clients=$2
shift 2;;
*)
break ;;
esac
done
if $master ; then
echo "Test master branch.."
cp /mnt/root/cfg/master/taos.cfg /etc/taos/taos.cfg
WORK_DIR=/mnt/root/TDengine.master
else
echo "Test develop branch.."
cp /mnt/root/cfg/10billion/taos.cfg /etc/taos/taos.cfg
WORK_DIR=/mnt/root/TDengine
fi
TAOSD_DIR=$WORK_DIR/debug/build/bin
TDTEST_DIR=$WORK_DIR/tests/comparisonTest/tdengine
runTest
echo "Test done!"

2
tests/pytest/tag_lite/add.py
View File

@ -100,6 +100,8 @@ class TDTestCase:
# TSIM: sql alter table $mt add tag tgcol4 int # TSIM: sql alter table $mt add tag tgcol4 int
tdLog.info('alter table %s add tag tgcol4 int' % (mt)) tdLog.info('alter table %s add tag tgcol4 int' % (mt))
tdSql.execute('alter table %s add tag tgcol4 int' % (mt)) tdSql.execute('alter table %s add tag tgcol4 int' % (mt))
tdLog.info('select * from %s where tgcol4=6' % (mt))
tdSql.query('select * from %s where tgcol4=6' % (mt))
# TSIM: sql reset query cache # TSIM: sql reset query cache
tdLog.info('reset query cache') tdLog.info('reset query cache')
tdSql.execute('reset query cache') tdSql.execute('reset query cache')

1
tests/pytest/util/dnodes-no-random-fail.py
View File

@ -178,7 +178,6 @@ class TDDnode:
self.cfg("walLevel", "2") self.cfg("walLevel", "2")
self.cfg("fsync", "1000") self.cfg("fsync", "1000")
self.cfg("statusInterval", "1") self.cfg("statusInterval", "1")
self.cfg("numOfTotalVnodes", "64")
self.cfg("numOfMnodes", "3") self.cfg("numOfMnodes", "3")
self.cfg("numOfThreadsPerCore", "2.0") self.cfg("numOfThreadsPerCore", "2.0")
self.cfg("monitor", "0") self.cfg("monitor", "0")

1
tests/pytest/util/dnodes-random-fail.py
View File

@ -178,7 +178,6 @@ class TDDnode:
self.cfg("walLevel", "2") self.cfg("walLevel", "2")
self.cfg("fsync", "1000") self.cfg("fsync", "1000")
self.cfg("statusInterval", "1") self.cfg("statusInterval", "1")
self.cfg("numOfTotalVnodes", "64")
self.cfg("numOfMnodes", "3") self.cfg("numOfMnodes", "3")
self.cfg("numOfThreadsPerCore", "2.0") self.cfg("numOfThreadsPerCore", "2.0")
self.cfg("monitor", "0") self.cfg("monitor", "0")

1
tests/pytest/util/dnodes.py
View File

@ -180,7 +180,6 @@ class TDDnode:
self.cfg("walLevel", "2") self.cfg("walLevel", "2")
self.cfg("fsync", "1000") self.cfg("fsync", "1000")
self.cfg("statusInterval", "1") self.cfg("statusInterval", "1")
self.cfg("numOfTotalVnodes", "64")
self.cfg("numOfMnodes", "3") self.cfg("numOfMnodes", "3")
self.cfg("numOfThreadsPerCore", "2.0") self.cfg("numOfThreadsPerCore", "2.0")
self.cfg("monitor", "0") self.cfg("monitor", "0")

4
tests/script/general/alter/cached_schema_after_alter.sim
View File

@ -68,7 +68,7 @@ endi
if $data01 != 1 then if $data01 != 1 then
return -1 return -1
endi endi
if $data02 != NULL then if $data02 != 1 then
return -1 return -1
endi endi
@ -80,7 +80,7 @@ endi
if $data01 != 1 then if $data01 != 1 then
return -1 return -1
endi endi
if $data02 != NULL then if $data02 != 1 then
return -1 return -1
endi endi

1
tests/script/general/db/alter_option.sim
View File

@ -1,7 +1,6 @@
system sh/stop_dnodes.sh system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode1 -i 1
system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/cfg.sh -n dnode1 -c wallevel -v 0
system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4
system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 1000 system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 1000
system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode1 -s start

4
tests/script/general/db/basic.sim
View File

@ -99,9 +99,7 @@ $db = $dbPrefix . $i
$tb = $tbPrefix . $i $tb = $tbPrefix . $i
sql create database $db sql create database $db
sql use $db sql use $db
sql create table $tb (ts timestamp, speed int) -x step6 sql create table $tb (ts timestamp, speed int)
return -1
step6:
print =============== step7 print =============== step7
$i = 0 $i = 0

1
tests/script/general/db/delete.sim
View File

@ -1,7 +1,6 @@
system sh/stop_dnodes.sh system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode1 -i 1
system sh/cfg.sh -n dnode1 -c wallevel -v 2 system sh/cfg.sh -n dnode1 -c wallevel -v 2
system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 10
system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 10 system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 10
system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 1000 system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 1000

5
tests/script/general/db/delete_reuse1.sim
View File

@ -20,11 +20,6 @@ system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4
system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4
system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4
system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4
system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 4
system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4
system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4
print ========= start dnodes print ========= start dnodes
system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode1 -s start
sleep 3000 sleep 3000

5
tests/script/general/db/delete_reuse2.sim
View File

@ -20,11 +20,6 @@ system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4
system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4
system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4
system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4
system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 4
system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4
system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4
print ========= start dnodes print ========= start dnodes
sleep 2000 sleep 2000
system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode1 -s start

7
tests/script/general/db/delete_reusevnode.sim
View File

@ -2,10 +2,6 @@ system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode1 -i 1
system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 10
system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 10
system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 10
print ========= start dnodes print ========= start dnodes
system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode1 -s start
sleep 3000 sleep 3000
@ -44,9 +40,6 @@ system sh/stop_dnodes.sh
sleep 3000 sleep 3000
system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode1 -i 1
system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 10
system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 10
system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 10
print ========= start dnodes print ========= start dnodes
system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode1 -s start
sql connect sql connect

3
tests/script/general/db/delete_reusevnode2.sim
View File

@ -1,8 +1,5 @@
system sh/stop_dnodes.sh system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode1 -i 1
system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 10
system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 10
system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 10
print ========= start dnodes print ========= start dnodes
system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode1 -s start

5
tests/script/general/db/delete_writing1.sim
View File

@ -20,11 +20,6 @@ system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4
system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4
system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4
system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4
system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 4
system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4
system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4
print ========= start dnodes print ========= start dnodes
system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode1 -s start
sleep 3000 sleep 3000

1
tests/script/general/db/delete_writing2.sim
View File

@ -2,7 +2,6 @@ system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode1 -i 1
system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/cfg.sh -n dnode1 -c wallevel -v 0
system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4
print ========= start dnodes print ========= start dnodes
system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode1 -s start

1
tests/script/general/db/len.sim
View File

@ -3,7 +3,6 @@ system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode1 -i 1
system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/cfg.sh -n dnode1 -c walLevel -v 0
system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4
system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 2000 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 2000
system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode1 -s start

1
tests/script/general/db/repeat.sim
View File

@ -1,7 +1,6 @@
system sh/stop_dnodes.sh system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode1 -i 1
system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/cfg.sh -n dnode1 -c wallevel -v 0
system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4
system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode1 -s start

1
tests/script/general/db/vnodes.sim
View File

@ -8,7 +8,6 @@ system sh/deploy.sh -n dnode1 -i 1
system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/cfg.sh -n dnode1 -c walLevel -v 0
system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v $maxTables system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v $maxTables
system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v $totalVnodes system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v $totalVnodes
system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v $totalVnodes
system sh/cfg.sh -n dnode1 -c maxVnodeConnections -v 100000 system sh/cfg.sh -n dnode1 -c maxVnodeConnections -v 100000
system sh/cfg.sh -n dnode1 -c maxMeterConnections -v 100000 system sh/cfg.sh -n dnode1 -c maxMeterConnections -v 100000
system sh/cfg.sh -n dnode1 -c maxShellConns -v 100000 system sh/cfg.sh -n dnode1 -c maxShellConns -v 100000

1
tests/script/general/parser/bug.sim
View File

@ -20,7 +20,6 @@ system sh/cfg.sh -n dnode1 -c maxVnodeConnections -v 30000
system sh/cfg.sh -n dnode1 -c maxMgmtConnections -v 30000 system sh/cfg.sh -n dnode1 -c maxMgmtConnections -v 30000
system sh/cfg.sh -n dnode1 -c maxMeterConnections -v 30000 system sh/cfg.sh -n dnode1 -c maxMeterConnections -v 30000
system sh/cfg.sh -n dnode1 -c maxShellConns -v 30000 system sh/cfg.sh -n dnode1 -c maxShellConns -v 30000
system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 60
system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode1 -s start
sql connect sql connect

11
tests/script/general/parser/first_last_query.sim
View File

@ -126,3 +126,14 @@ endi
if $data01 != 0 then if $data01 != 0 then
return -1 return -1
endi endi
#add check for out of range first/last query
sql select first(ts),last(ts) from first_tb4 where ts>'2018-9-18 1:40:01';
if $row != 0 then
return -1
endi
sql select first(ts),last(ts) from first_tb4 where ts<'2018-9-17 8:50:0';
if $row != 0 then
return -1
endi

15
tests/script/general/parser/null_char.sim
View File

@ -105,6 +105,21 @@ if $data03 != 1 then
return -1 return -1
endi endi
sql select tag1 from st2 limit 20 offset 1
if $rows != 0 then
return -1
endi
sql select tag1 from st2 limit 10 offset 2
if $rows != 0 then
return -1
endi
sql select tag1 from st2 limit 0 offset 0
if $rows != 0 then
return -1
endi
sql create table st3 using mt2 tags (NULL, 'ABC', 103, 'FALSE') sql create table st3 using mt2 tags (NULL, 'ABC', 103, 'FALSE')
sql select tag1, tag2, tag3, tag5 from st3 sql select tag1, tag2, tag3, tag5 from st3
if $rows != 1 then if $rows != 1 then

1
tests/script/general/parser/select_across_vnodes.sim
View File

@ -3,7 +3,6 @@ system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode1 -i 1
system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/cfg.sh -n dnode1 -c walLevel -v 0
system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 5 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 5
system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 8
system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode1 -s start
sleep 3000 sleep 3000
sql connect sql connect

1
tests/script/general/parser/select_from_cache_disk.sim
View File

@ -3,7 +3,6 @@ system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode1 -i 1
system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/cfg.sh -n dnode1 -c walLevel -v 0
system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 2 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 2
system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 8
system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode1 -s start
sleep 3000 sleep 3000
sql connect sql connect

1
tests/script/general/parser/slimit.sim
View File

@ -3,7 +3,6 @@ system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode1 -i 1
system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/cfg.sh -n dnode1 -c walLevel -v 0
system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 8
system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode1 -s start
sleep 3000 sleep 3000
sql connect sql connect

1
tests/script/general/parser/slimit1.sim
View File

@ -3,7 +3,6 @@ system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode1 -i 1
system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/cfg.sh -n dnode1 -c walLevel -v 0
system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 2 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 2
system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 10
system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode1 -s start
sleep 3000 sleep 3000
sql connect sql connect

1
tests/script/general/parser/slimit_alter_tags.sim
View File

@ -3,7 +3,6 @@ system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode1 -i 1
system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/cfg.sh -n dnode1 -c walLevel -v 0
system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 2 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 2
system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 10
system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode1 -s start
sleep 3000 sleep 3000
sql connect sql connect

6
tests/script/general/parser/testSuite.sim
View File

@ -91,13 +91,11 @@ run general/parser/select_with_tags.sim
sleep 2000 sleep 2000
run general/parser/groupby.sim run general/parser/groupby.sim
sleep 2000 sleep 2000
run general/parser/tags_filter.sim
sleep 2000
run general/parser/union.sim run general/parser/union.sim
sleep 2000 sleep 2000
run general/parser/sliding.sim run general/parser/sliding.sim
sleep 2000
run general/parser/fill_us.sim
sleep 2000
run general/parser/tags_filter.sim
#sleep 2000 #sleep 2000
#run general/parser/repeatStream.sim #run general/parser/repeatStream.sim

1
tests/script/general/parser/timestamp.sim
View File

@ -2,7 +2,6 @@ system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode1 -i 1
system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/cfg.sh -n dnode1 -c walLevel -v 0
system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 8
system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode1 -s start
sleep 3000 sleep 3000
sql connect sql connect

160
tests/script/general/parser/union.sim
View File

@ -1,10 +1,10 @@
system sh/stop_dnodes.sh #system sh/stop_dnodes.sh
#
system sh/deploy.sh -n dnode1 -i 1 #system sh/deploy.sh -n dnode1 -i 1
system sh/cfg.sh -n dnode1 -c walLevel -v 0 #system sh/cfg.sh -n dnode1 -c walLevel -v 0
system sh/cfg.sh -n dnode1 -c debugFlag -v 135 #system sh/cfg.sh -n dnode1 -c debugFlag -v 135
system sh/cfg.sh -n dnode1 -c rpcDebugFlag -v 135 #system sh/cfg.sh -n dnode1 -c rpcDebugFlag -v 135
system sh/exec.sh -n dnode1 -s start #system sh/exec.sh -n dnode1 -s start
sleep 1000 sleep 1000
sql connect sql connect
@ -24,77 +24,77 @@ $mt = $mtPrefix . $i
$j = 1 $j = 1
$mt1 = $mtPrefix . $j $mt1 = $mtPrefix . $j
#
sql drop database if exits $db -x step1 #sql drop database if exits $db -x step1
step1: #step1:
sql create database if not exists $db maxtables 4 #sql create database if not exists $db maxtables 4
sql use $db sql use $db
sql create table $mt (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int) #sql create table $mt (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int)
#
$i = 0 #$i = 0
$t = 1578203484000 #$t = 1578203484000
#
while $i < $tbNum #while $i < $tbNum
$tb = $tbPrefix . $i # $tb = $tbPrefix . $i
sql create table $tb using $mt tags( $i ) # sql create table $tb using $mt tags( $i )
#
$x = 0 # $x = 0
while $x < $rowNum # while $x < $rowNum
$ms = $x * 1000 # $ms = $x * 1000
$ms = $ms * 60 # $ms = $ms * 60
#
$c = $x / 100 # $c = $x / 100
$c = $c * 100 # $c = $c * 100
$c = $x - $c # $c = $x - $c
$binary = 'binary . $c # $binary = 'binary . $c
$binary = $binary . ' # $binary = $binary . '
$nchar = 'nchar . $c # $nchar = 'nchar . $c
$nchar = $nchar . ' # $nchar = $nchar . '
#
$t1 = $t + $ms # $t1 = $t + $ms
sql insert into $tb values ($t1 , $c , $c , $c , $c , $c , $c , $c , $binary , $nchar ) # sql insert into $tb values ($t1 , $c , $c , $c , $c , $c , $c , $c , $binary , $nchar )
$x = $x + 1 # $x = $x + 1
endw # endw
#
$i = $i + 1 # $i = $i + 1
endw #endw
#
sql create table $mt1 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int) #sql create table $mt1 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int)
#
$j = 0 #$j = 0
$t = 1578203484000 #$t = 1578203484000
$rowNum = 1000 #$rowNum = 1000
$tbNum = 5 #$tbNum = 5
$i = 0 #$i = 0
#
while $i < $tbNum #while $i < $tbNum
$tb1 = $tbPrefix1 . $j # $tb1 = $tbPrefix1 . $j
sql create table $tb1 using $mt1 tags( $i ) # sql create table $tb1 using $mt1 tags( $i )
#
$x = 0 # $x = 0
while $x < $rowNum # while $x < $rowNum
$ms = $x * 1000 # $ms = $x * 1000
$ms = $ms * 60 # $ms = $ms * 60
#
$c = $x / 100 # $c = $x / 100
$c = $c * 100 # $c = $c * 100
$c = $x - $c # $c = $x - $c
$binary = 'binary . $c # $binary = 'binary . $c
$binary = $binary . ' # $binary = $binary . '
$nchar = 'nchar . $c # $nchar = 'nchar . $c
$nchar = $nchar . ' # $nchar = $nchar . '
#
$t1 = $t + $ms # $t1 = $t + $ms
sql insert into $tb1 values ($t1 , $c , $c , $c , $c , $c , $c , $c , $binary , $nchar ) # sql insert into $tb1 values ($t1 , $c , $c , $c , $c , $c , $c , $c , $binary , $nchar )
$x = $x + 1 # $x = $x + 1
endw # endw
#
$i = $i + 1 # $i = $i + 1
$j = $j + 1 # $j = $j + 1
endw #endw
#
print sleep 1sec. #print sleep 1sec.
sleep 1000 #sleep 1000
$i = 1 $i = 1
$tb = $tbPrefix . $i $tb = $tbPrefix . $i
@ -222,7 +222,7 @@ endi
print ===========================================tags union print ===========================================tags union
# two super table tag union, limit is not active during retrieve tags query # two super table tag union, limit is not active during retrieve tags query
sql select t1 from union_mt0 union all select t1 from union_mt0 limit 1 sql select t1 from union_mt0 union all select t1 from union_mt0
if $rows != 20 then if $rows != 20 then
return -1 return -1
endi endi
@ -235,6 +235,10 @@ if $data90 != 9 then
return -1 return -1
endi endi
#sql select t1 from union_mt0 union all select t1 from union_mt0 limit 1
#if $row != 11 then
# return -1
#endi
#========================================== two super table join subclause #========================================== two super table join subclause
print ================two super table join subclause print ================two super table join subclause
sql select avg(union_mt0.c1) as c from union_mt0 interval(1h) limit 10 union all select union_mt1.ts, union_mt1.c1/1.0 as c from union_mt0, union_mt1 where union_mt1.ts=union_mt0.ts and union_mt1.t1=union_mt0.t1 limit 5; sql select avg(union_mt0.c1) as c from union_mt0 interval(1h) limit 10 union all select union_mt1.ts, union_mt1.c1/1.0 as c from union_mt0, union_mt1 where union_mt1.ts=union_mt0.ts and union_mt1.t1=union_mt0.t1 limit 5;

1
tests/script/general/stable/disk.sim
View File

@ -3,7 +3,6 @@ system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode1 -i 1
system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/cfg.sh -n dnode1 -c walLevel -v 0
system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4
system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode1 -s start

4
tests/script/general/stable/dnode3.sim
View File

@ -8,10 +8,6 @@ system sh/cfg.sh -n dnode1 -c walLevel -v 0
system sh/cfg.sh -n dnode2 -c walLevel -v 0 system sh/cfg.sh -n dnode2 -c walLevel -v 0
system sh/cfg.sh -n dnode3 -c walLevel -v 0 system sh/cfg.sh -n dnode3 -c walLevel -v 0
system sh/cfg.sh -n dnode4 -c walLevel -v 0 system sh/cfg.sh -n dnode4 -c walLevel -v 0
system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4
system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 4
system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4
system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4
system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4

Some files were not shown because too many files have changed in this diff Show More