From 4224343001f93ffc2d84e04a8ee4712ec22ccae9 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 27 Dec 2021 01:14:20 -0800 Subject: [PATCH 01/21] add test cases --- source/dnode/mgmt/impl/src/dndDnode.c | 20 ++++++++------------ source/dnode/mgmt/impl/test/sut/src/base.cpp | 3 ++- tests/script/general/db/basic1.sim | 7 +++++++ tests/script/general/table/basic1.sim | 9 +++++++++ tests/script/jenkins/basic.txt | 2 ++ 5 files changed, 28 insertions(+), 13 deletions(-) diff --git a/source/dnode/mgmt/impl/src/dndDnode.c b/source/dnode/mgmt/impl/src/dndDnode.c index c67e55f048..ca0552b8ad 100644 --- a/source/dnode/mgmt/impl/src/dndDnode.c +++ b/source/dnode/mgmt/impl/src/dndDnode.c @@ -17,6 +17,7 @@ #include "dndDnode.h" #include "dndTransport.h" #include "dndVnodes.h" +#include "dndMnode.h" static int32_t dndInitMgmtWorker(SDnode *pDnode); static void dndCleanupMgmtWorker(SDnode *pDnode); @@ -28,10 +29,10 @@ static int32_t dndReadDnodes(SDnode *pDnode); static int32_t dndWriteDnodes(SDnode *pDnode); static void *dnodeThreadRoutine(void *param); -static void dndProcessConfigDnodeReq(SDnode *pDnode, SRpcMsg *pMsg); -static void dndProcessStatusRsp(SDnode *pDnode, SRpcMsg *pMsg); -static void dndProcessAuthRsp(SDnode *pDnode, SRpcMsg *pMsg); -static void dndProcessGrantRsp(SDnode *pDnode, SRpcMsg *pMsg); +static int32_t dndProcessConfigDnodeReq(SDnode *pDnode, SRpcMsg *pMsg); +static void dndProcessStatusRsp(SDnode *pDnode, SRpcMsg *pMsg); +static void dndProcessAuthRsp(SDnode *pDnode, SRpcMsg *pMsg); +static void dndProcessGrantRsp(SDnode *pDnode, SRpcMsg *pMsg); int32_t dndGetDnodeId(SDnode *pDnode) { SDnodeMgmt *pMgmt = &pDnode->dmgmt; @@ -458,13 +459,11 @@ static void dndProcessAuthRsp(SDnode *pDnode, SRpcMsg *pMsg) { assert(1); } static void dndProcessGrantRsp(SDnode *pDnode, SRpcMsg *pMsg) { assert(1); } -static void dndProcessConfigDnodeReq(SDnode *pDnode, SRpcMsg *pMsg) { +static int32_t dndProcessConfigDnodeReq(SDnode *pDnode, SRpcMsg *pMsg) { dError("config msg is received, but not supported yet"); SCfgDnodeMsg *pCfg = pMsg->pCont; - int32_t code = TSDB_CODE_OPS_NOT_SUPPORT; - SRpcMsg rspMsg = {.handle = pMsg->handle, .pCont = NULL, .contLen = 0, .code = code}; - rpcSendResponse(&rspMsg); + return TSDB_CODE_OPS_NOT_SUPPORT; } void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pMsg) { @@ -646,11 +645,8 @@ static void dndProcessMgmtQueue(SDnode *pDnode, SRpcMsg *pMsg) { case TDMT_DND_DROP_MNODE: code = dndProcessDropMnodeReq(pDnode, pMsg); break; - case TDMT_DND_NETWORK_TEST: - dndProcessStartupReq(pDnode, pMsg); - break; case TDMT_DND_CONFIG_DNODE: - dndProcessConfigDnodeReq(pDnode, pMsg); + code = dndProcessConfigDnodeReq(pDnode, pMsg); break; case TDMT_MND_STATUS_RSP: dndProcessStatusRsp(pDnode, pMsg); diff --git a/source/dnode/mgmt/impl/test/sut/src/base.cpp b/source/dnode/mgmt/impl/test/sut/src/base.cpp index f3fe8cfa86..98371e9893 100644 --- a/source/dnode/mgmt/impl/test/sut/src/base.cpp +++ b/source/dnode/mgmt/impl/test/sut/src/base.cpp @@ -31,6 +31,7 @@ void Testbase::InitLog(const char* path) { tsdbDebugFlag = 0; cqDebugFlag = 0; tscEmbeddedInUtil = 1; + tsAsyncLog = 0; taosRemoveDir(path); taosMkDir(path); @@ -47,7 +48,7 @@ void Testbase::Init(const char* path, int16_t port) { char firstEp[TSDB_EP_LEN] = {0}; snprintf(firstEp, TSDB_EP_LEN, "%s:%u", fqdn, port); - InitLog("/tmp/tdlog"); + InitLog("/tmp/td"); server.Start(path, fqdn, port, firstEp); client.Init("root", "taosdata", fqdn, port); taosMsleep(1100); diff --git a/tests/script/general/db/basic1.sim b/tests/script/general/db/basic1.sim index 666a9dfd9e..618b1377b8 100644 --- a/tests/script/general/db/basic1.sim +++ b/tests/script/general/db/basic1.sim @@ -58,4 +58,11 @@ if $data03 != 0 then return -1 endi +print =============== show vgroups +sql use d4 + +if $rows == 0 then + return -1 +endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/table/basic1.sim b/tests/script/general/table/basic1.sim index ded8d79a3f..04e44f7308 100644 --- a/tests/script/general/table/basic1.sim +++ b/tests/script/general/table/basic1.sim @@ -23,6 +23,15 @@ endi print $data00 $data01 $data02 +sql create table st2 (ts timestamp, i float) tags (j bigint) +sql show stables +if $rows != 2 then + return -1 +endi + +print $data00 $data01 $data02 +print $data00 $data11 $data12 + return print =============== create child table diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 1cc15f731d..0ef43aaced 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -7,5 +7,7 @@ # ---- db ./test.sh -f general/db/basic1.sim +# ---- table +./test.sh -f general/table/basic1.sim #======================b1-end=============== From 08e32652b78e71f1c5d9dc4fcc727e1ed3e9c25f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 27 Dec 2021 01:59:18 -0800 Subject: [PATCH 02/21] removde http testcases --- tests/script/general/http/autocreate.sim | 33 - tests/script/general/http/bug.go | 344 ---------- tests/script/general/http/chunked.sim | 37 -- tests/script/general/http/grafana.sim | 182 ----- tests/script/general/http/grafana_bug.sim | 271 -------- tests/script/general/http/gzip.sim | 39 -- tests/script/general/http/http.json | 23 - tests/script/general/http/httpTestNew.go | 247 ------- tests/script/general/http/httpTest_cloud.go | 387 ----------- tests/script/general/http/httpTest_private.go | 387 ----------- tests/script/general/http/prepare.sim | 54 -- tests/script/general/http/restful.sim | 82 --- tests/script/general/http/restful_dbname.sim | 124 ---- tests/script/general/http/restful_full.sim | 240 ------- tests/script/general/http/restful_insert.sim | 53 -- tests/script/general/http/restful_limit.sim | 46 -- tests/script/general/http/taos.json | 23 - tests/script/general/http/taos_cloud.json | 21 - tests/script/general/http/telegraf.json | 7 - tests/script/general/http/telegraf.req | 621 ------------------ tests/script/general/http/telegraf.sim | 292 -------- tests/script/general/http/telegrafTest.go | 238 ------- tests/script/general/http/telegraf_help.txt | 24 - tests/script/general/http/testSuite.sim | 10 - 24 files changed, 3785 deletions(-) delete mode 100644 tests/script/general/http/autocreate.sim delete mode 100644 tests/script/general/http/bug.go delete mode 100644 tests/script/general/http/chunked.sim delete mode 100644 tests/script/general/http/grafana.sim delete mode 100644 tests/script/general/http/grafana_bug.sim delete mode 100644 tests/script/general/http/gzip.sim delete mode 100644 tests/script/general/http/http.json delete mode 100644 tests/script/general/http/httpTestNew.go delete mode 100644 tests/script/general/http/httpTest_cloud.go delete mode 100644 tests/script/general/http/httpTest_private.go delete mode 100644 tests/script/general/http/prepare.sim delete mode 100644 tests/script/general/http/restful.sim delete mode 100644 tests/script/general/http/restful_dbname.sim delete mode 100644 tests/script/general/http/restful_full.sim delete mode 100644 tests/script/general/http/restful_insert.sim delete mode 100644 tests/script/general/http/restful_limit.sim delete mode 100644 tests/script/general/http/taos.json delete mode 100644 tests/script/general/http/taos_cloud.json delete mode 100644 tests/script/general/http/telegraf.json delete mode 100644 tests/script/general/http/telegraf.req delete mode 100644 tests/script/general/http/telegraf.sim delete mode 100644 tests/script/general/http/telegrafTest.go delete mode 100644 tests/script/general/http/telegraf_help.txt delete mode 100644 tests/script/general/http/testSuite.sim diff --git a/tests/script/general/http/autocreate.sim b/tests/script/general/http/autocreate.sim deleted file mode 100644 index 39af990b50..0000000000 --- a/tests/script/general/http/autocreate.sim +++ /dev/null @@ -1,33 +0,0 @@ -system sh/stop_dnodes.sh -sleep 2000 -system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c wallevel -v 0 -system sh/cfg.sh -n dnode1 -c http -v 1 -system sh/cfg.sh -n dnode1 -c httpEnableRecordSql -v 1 -system sh/exec.sh -n dnode1 -s start - -sleep 2000 -sql connect - -print ============================ dnode1 start - -print =============== step1 - prepare data -sql create database db -sql use db -sql create table if not exists db.win_cpu(ts timestamp,f_percent_dpc_time double,f_percent_idle_time double,f_percent_interrupt_time double,f_percent_privileged_time double,f_percent_processor_time double,f_percent_user_time double) tags(t_host binary(32),t_instance binary(32),t_objectname binary(32)); - -print =============== step2 - auto create - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'import into db.win_cpu_windows_1_processor using db.win_cpu tags('windows','1','Processor') values(1564641722000,0.000000,95.598305,0.000000,0.000000,0.000000,0.000000);' 127.0.0.1:7111/rest/sql -print curl 127.0.0.1:7111/rest/sql -----> $system_content -#if $system_content != @{"status":"succ","head":["ts","i"],"data":[["2017-12-25 21:28:41.022",1],["2017-12-25 21:28:42.022",2],["2017-12-25 21:28:43.022",3],["2017-12-25 21:28:44.022",4],["2017-12-25 21:28:45.022",5],["2017-12-25 21:28:46.022",6],["2017-12-25 21:28:47.022",7],["2017-12-25 21:28:48.022",8],["2017-12-25 21:28:49.022",9],["2017-12-25 21:28:50.022",10],["2017-12-25 21:28:51.022",11]],"rows":11}@ then -# return -1 -#endi - -sql select * from db.win_cpu_windows_1_processor -print rows: $rows -if $rows != 1 then - return -1 -endi - -#system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/http/bug.go b/tests/script/general/http/bug.go deleted file mode 100644 index 9455a48c56..0000000000 --- a/tests/script/general/http/bug.go +++ /dev/null @@ -1,344 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package main - -import ( - "bytes" - "encoding/json" - "fmt" - "io/ioutil" - "math/rand" - "net/http" - "os" - "sync" - "sync/atomic" - "time" - "flag" -) - -var ( - token string - url string - config Config - request int64 - period int64 - errorNum int64 -) - -type Config struct { - HostIp string `json:"hostIp"` - ConnNum int `json:"connNum"` - InsertModel string `json:"insertModel"` - WaitTime int `json:"waitTime"` - TableDesc string `json:"tableDesc"` - TablePrefix string `json:"tablePrefix"` - TablePerConn int `json:"tablePerConn"` - TableCreate bool `json:"tableCreate"` - TableStart int `json:"tableStart"` - DbName string `json:"dbName"` - DbReplica int `json:"dbReplica"` - DbKeep int `json:"dbKeep"` - DbDays int `json:"dbDays"` - MetricsName string `json:"metricsName"` - TagNum int `json:"tagNum"` - DataNum int `json:"dataNum"` - DataBegin int64 `json:"dataBegin"` - DataInterval int `json:"dataInterval"` - DataBatch int `json:"dataBatch"` - DataInsert bool `json:"dataInsert"` - DataRandom bool `json:"dataRandom"` -} - -type TokenResult struct { - Status string `json:"status"` - Code int `json:"code"` - Desc string `json:"desc"` -} - -type JsonResult struct { - Status string `json:"status"` - Code int `json:"code"` -} - -func readFile(filename string) { - file, err := os.Open(filename) - if err != nil { - println("taos_cloud.json not found") - panic(err) - } - defer file.Close() - - dec := json.NewDecoder(file) - err = dec.Decode(&config) - if err != nil { - println("taos_cloud.json parse error") - panic(err) - } - - if config.TagNum <= 0 { - config.TagNum = 1 - } - request = 0 - period = 0 - errorNum = 0 - - fmt.Println("================config parameters======================") - fmt.Println("HostIp:", config.HostIp) - fmt.Println("connNum:", config.ConnNum) - fmt.Println("insertModel:", config.InsertModel) - fmt.Println("waitTime:", config.WaitTime) - fmt.Println("tableDesc:", config.TableDesc) - fmt.Println("tablePrefix:", config.TablePrefix) - fmt.Println("tablePerConn:", config.TablePerConn) - fmt.Println("tableCreate:", config.TableCreate) - fmt.Println("tableStart:", config.TableStart) - fmt.Println("dbName:", config.DbName) - fmt.Println("dbReplica:", config.DbReplica) - fmt.Println("dbKeep:", config.DbKeep) - fmt.Println("dbDays:", config.DbDays) - fmt.Println("metricsName:", config.MetricsName) - fmt.Println("tagNum:", config.TagNum) - fmt.Println("dataNum:", config.DataNum) - fmt.Println("dataBegin:", config.DataBegin) - fmt.Println("dataInterval:", config.DataInterval) - fmt.Println("dataBatch:", config.DataBatch) - fmt.Println("dataInsert:", config.DataInsert) - fmt.Println("dataRandom:", config.DataRandom) - - fmt.Println("================http token=============================") - token, err = getToken() - url = fmt.Sprintf("http://%s:%d/rest/sql", config.HostIp, 6020) - - fmt.Println("httpToken:", token) - fmt.Println("httpUrl:", url) - - if err != nil { - panic(err) - } -} - -func getToken() (string, error) { - resp, err := http.Get(fmt.Sprintf("http://%s:%d/rest/login/%s/%s", config.HostIp, 6020, "root", "taosdata")) - if err != nil { - return "", err - } - - defer resp.Body.Close() - - var tokenResult TokenResult - - data, err := ioutil.ReadAll(resp.Body) - - if err != nil { - return "", err - } - - err = json.Unmarshal(data, &tokenResult) - if err != nil { - return "", err - } - - if tokenResult.Status != "succ" { - fmt.Println("get http token failed") - fmt.Println(tokenResult) - return "", err - } - - return tokenResult.Desc, nil -} - -func exec(client *http.Client, sql string) { - for times := 0; times < 10; times++ { - - req, err1 := http.NewRequest("POST", url, bytes.NewReader([]byte(sql))) - if err1 != nil { - continue - } - req.Header.Add("Authorization", "Taosd "+token) - - begin := time.Now() - resp, err := client.Do(req) - - if err != nil { - continue - } - - data, err := ioutil.ReadAll(resp.Body) - if err != nil { - resp.Body.Close() - continue - } - - spend := (time.Since(begin).Nanoseconds()) - - var jsonResult JsonResult - err = json.Unmarshal(data, &jsonResult) - if err != nil { - resp.Body.Close() - continue - } - - if jsonResult.Status != "succ" { - resp.Body.Close() - continue - } - atomic.AddInt64(&request, 1) - if (request < 103) { - return - } - - atomic.AddInt64(&period, spend) - if request%5000 == 0 && request != 0 { - requestAvg := float64(period) / float64(1000000) / float64(request) - qps := float64(1000) / float64(requestAvg) * float64(config.ConnNum) - dps := qps * float64(config.DataBatch) - fmt.Println("====== req:", request, ", error:", errorNum, ", qps:", int64(qps), ", wait:", int64(requestAvg), "ms", ", data per second:", int64(dps)) - } - return - } - fmt.Println("xxxx>sql:", sql, ", retryTimes:", 10) - errorNum++ -} - -func insertTable(conn int) { - client := &http.Client{} - - tbStart := conn*config.TablePerConn + config.TableStart - tmStart := config.DataBegin - - - for j := 0; j < config.DataNum; j++ { - for i := 0; i < config.TablePerConn; i++ { - tmVal := int64(j)*int64(config.DataInterval) + tmStart + 1 - tbIndex := i + tbStart - - dataVal := j - if config.DataRandom { - dataVal = rand.Intn(1000) - } - - sql := fmt.Sprintf("import into %s.%s%d values(%d, %d)", config.DbName, config.TablePrefix, tbIndex, tmVal, dataVal) - exec(client, sql) - time.Sleep(time.Millisecond * time.Duration(10)) - } - } -} - -func insertLoop(conn int) { - client := &http.Client{} - - tbStart := conn*config.TablePerConn + config.TableStart - tmStart := config.DataBegin - - for j := 0; j < config.DataNum; j++ { - - for i := 0; i < config.TablePerConn; i++ { - tbIndex := i + tbStart - - tmVal := int64(j)*int64(config.DataInterval) + tmStart - - dataVal := j - if config.DataRandom { - dataVal = rand.Intn(1000) - } - - sql := fmt.Sprintf("insert into %s.%s%d values(%d, %d)", config.DbName, config.TablePrefix, tbIndex, tmVal, dataVal) - for k := 1; k < config.DataBatch; k++ { - tmVal := int64(j)*int64(config.DataInterval) + int64(k) + tmStart - - dataVal := j + k - if config.DataRandom { - dataVal = rand.Intn(1000) - } - sql += fmt.Sprintf("values(%d, %d)", tmVal, dataVal) - } - - j += (config.DataBatch - 1) - - exec(client, sql) - - if config.WaitTime != 0 { - time.Sleep(time.Millisecond * time.Duration(config.WaitTime)) - } - } - - } -} - -func insertTb(wg *sync.WaitGroup, conn int) { - defer wg.Done() - - if !config.DataInsert { - return - } - - if config.InsertModel == "insertTable" { - insertTable(conn) - } else { - insertLoop(conn) - } -} - -func selectData(wg *sync.WaitGroup, conn int) { - defer wg.Done() - - client := &http.Client{} - - - tbStart := conn*config.TablePerConn + config.TableStart - for j := 0; j < config.DataNum; j++ { - tbIndex := 0 + tbStart - sql := fmt.Sprintf("select max(i),min(i) from db.mt where tbname in ('%s%d'", config.TablePrefix, tbIndex) - for i := 1; i < 2000; i++ { - tbIndex := i + tbStart - sql += fmt.Sprintf(",'%s%d'", config.TablePrefix, tbIndex) - } - sql += ") group by orgno" - - //sql := fmt.Sprintf("select count(*) from db.mt") - //sql := fmt.Sprintf("select max(i),min(i) from db.mt", config.TablePrefix, tbIndex) - - exec(client, sql) - time.Sleep(time.Millisecond * time.Duration(10)) - } -} - -func main() { - filename := flag.String("config", "taos_cloud.json", "config file name") - - flag.Parse() - - readFile(*filename) - - fmt.Println("\n================http test start======================") - - var wg sync.WaitGroup - - fmt.Println("\n================select data ========================") - - - for i := 0; i < config.ConnNum; i++ { - wg.Add(1) - go insertTb(&wg, i) - } - for i := 0; i < config.ConnNum; i++ { - wg.Add(1) - go selectData(&wg, i) - } - wg.Wait() - - fmt.Println("\n================http test stop ======================") -} diff --git a/tests/script/general/http/chunked.sim b/tests/script/general/http/chunked.sim deleted file mode 100644 index c5855e5d29..0000000000 --- a/tests/script/general/http/chunked.sim +++ /dev/null @@ -1,37 +0,0 @@ -system sh/stop_dnodes.sh -sleep 2000 -system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c wallevel -v 0 -system sh/cfg.sh -n dnode1 -c http -v 1 -system sh/cfg.sh -n dnode1 -c maxSQLLength -v 340032 -system sh/exec.sh -n dnode1 -s start - -sleep 2000 -sql connect - -print ============================ dnode1 start - -print =============== step1 - prepare data -sql create database d1 -sql use d1 - -sql create table table_rest (ts timestamp, i int) -print sql length is 270KB -restful d1 table_rest 1591072800 10000 -restful d1 table_rest 1591172800 10000 -restful d1 table_rest 1591272800 10000 -restful d1 table_rest 1591372800 10000 -restful d1 table_rest 1591472800 10000 -restful d1 table_rest 1591572800 10000 -restful d1 table_rest 1591672800 10000 -restful d1 table_rest 1591772800 10000 -restful d1 table_rest 1591872800 10000 -restful d1 table_rest 1591972800 10000 - -sql select * from table_rest; -print rows: $rows -if $rows != 100000 then - return -1 -endi - -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/http/grafana.sim b/tests/script/general/http/grafana.sim deleted file mode 100644 index 414b859bd3..0000000000 --- a/tests/script/general/http/grafana.sim +++ /dev/null @@ -1,182 +0,0 @@ -system sh/stop_dnodes.sh -sleep 2000 -system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c http -v 1 -#system sh/cfg.sh -n dnode1 -c adminRowLimit -v 10 -system sh/cfg.sh -n dnode1 -c httpDebugFlag -v 135 -system sh/exec.sh -n dnode1 -s start - -sleep 2000 -sql connect - -print ============================ dnode1 start - -print =============== step1 - prepare data - -sql create database d1 -sql use d1 -sql create table table_gc (ts timestamp, db binary(10), tb binary(20), col binary(20)) -sql create table m1 (ts timestamp, v1 int, v2 float) -sql create table m2 (ts timestamp, v1 int, v2 float) - -sql insert into table_gc values('2017-12-25 21:28:41.022', 'd1', 'm1', 'v1') -sql insert into table_gc values('2017-12-25 21:28:42.022', 'd1', 'm1', 'v2') -sql insert into table_gc values('2017-12-25 21:28:43.022', 'd1', 'm2', 'v1') -sql insert into table_gc values('2017-12-25 21:28:44.022', 'd1', 'm2', 'v2') - -sql insert into m1 values(1514208523020, 1, 4.1) -sql insert into m1 values(1514208523021, 2, 5.1) -sql insert into m1 values(1514208523022, 3, 6.1) - -sql insert into m2 values(1514208523024, 3, 6.1) -sql insert into m2 values(1514208523025, 2, 5.1) -sql insert into m2 values(1514208523026, 1, 4.1) - -sql create table mt (ts timestamp, i int) tags(a int, b binary(10)) -sql create table t1 using mt tags (1, 'a') -sql create table t2 using mt tags (2, 'b') -sql create table t3 using mt tags (3, 'c') -sql insert into t1 values('2017-12-25 21:25:41', 1) -sql insert into t1 values('2017-12-25 21:26:41', 1) -sql insert into t1 values('2017-12-25 21:27:41', 1) -sql insert into t1 values('2017-12-25 21:28:41', 1) -sql insert into t1 values('2017-12-25 21:29:41', 1) -sql insert into t2 values('2017-12-25 21:25:41', 2) -sql insert into t2 values('2017-12-25 21:26:41', 2) -sql insert into t2 values('2017-12-25 21:27:41', 2) -sql insert into t2 values('2017-12-25 21:28:41', 2) -sql insert into t3 values('2017-12-25 21:25:41', 3) -sql insert into t3 values('2017-12-25 21:26:41', 3) -sql insert into t3 values('2017-12-25 21:27:41', 3) - -print =============== step2 - login - -system_content curl 127.0.0.1:7111/grafana/ -print 1-> $system_content -if $system_content != @{"status":"error","code":4357,"desc":"no auth info input"}@ then - return -1 -endi - -system_content curl 127.0.0.1:7111/grafana/xx -print 2-> $system_content -if $system_content != @{"status":"error","code":4357,"desc":"no auth info input"}@ then - return -1 -endi - -system_content curl 127.0.0.1:7111/grafana/login/xx/xx/ -print 3-> $system_content -if $system_content != @{"status":"error","code":849,"desc":"Invalid user"}@ then - return -1 -endi - -system_content curl 127.0.0.1:7111/grafana/root/1/123/1/1/3 -print 4-> $system_content -if $system_content != @{"status":"error","code":4357,"desc":"no auth info input"}@ then - return -1 -endi - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'show databases' 127.0.0.1:7111/grafana/login/1/root/1/ -print 5-> $system_content -if $system_content != @{"status":"error","code":849,"desc":"Invalid user"}@ then - return -1 -endi - -system_content curl -H 'Authorization: Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJ3d3cudGFvc2RhdGEuY29tIiwicGFzcyI6InRhb3NkYXRhIiwic3ViIjoicm9vdCJ9.xPv3b5odlR7YF8G_QWASjIRbMtA5v4ItToJ35fFgi' -d 'show databases' 127.0.0.1:7111/grafana/root/1/login -print 6-> $system_content -if $system_content != @{"status":"error","code":4387,"desc":"invalid format of Authorization"}@ then - return -1 -endi - -system_content curl -H 'Authorization: eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJ3d3cudGFvc2RhdGEuY29tIiwicGFzcyI6InRhb3NkYXRhIiwic3ViIjoicm9vdCJ9.xPv3b5odlR7YF8G_QWASjIRbMtA5v4ItToJ35fFgi' -d 'show databases' 127.0.0.1:7111/grafana/root/1/login -print 7-> $system_content -if $system_content != @{"status":"error","code":4387,"desc":"invalid format of Authorization"}@ then - return -1 -endi - -sleep 2000 -system_content curl 127.0.0.1:7111/grafana/login/root/taosdata -print 8-> $system_content -if $system_content != @{"status":"succ","code":0,"desc":"/KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04"}@ then - return -1 -endi - -print =============== step3 - heartbeat - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' 127.0.0.1:7111/grafana/d1/table_gc -print 9-> $system_content -if $system_content != @{"message":"Grafana server receive a quest from you!"}@ then - return -1 -endi - -print =============== step4 - search - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' 127.0.0.1:7111/grafana/heartbeat -print 10-> $system_content -if $system_content != @{"message":"Grafana server receive a quest from you!"}@ then - return -1 -endi - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' 127.0.0.1:7111/grafana/d1/table_invalid/search -print 11-> $system_content -if $system_content != @{"message":"Grafana server receive a quest from you!"}@ then - return -1 -endi - -system_content curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' 127.0.0.1:7111/grafana/d1/m1/search -print 12-> $system_content -if $system_content != @{"message":"Grafana server receive a quest from you!"}@ then - return -1 -endi - -print =============== step5 - query - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d '[{"refId":"A","alias":"taosd","sql":"select first(v1) from d1.m1 where ts > 1514208523020 and ts < 1514208523030 interval(1m)"},{"refId":"B","alias":"system","sql":"select first(v2) from d1.m1 where ts > 1514208523020 and ts < 1514208523030 interval(1m)"}]' 127.0.0.1:7111/grafana/query -print 13-> $system_content -if $system_content != @[{"refId":"A","target":"taosd","datapoints":[[2,1514208480000]]},{"refId":"B","target":"system","datapoints":[[5.10000,1514208480000]]}]@ then - return -1 -endi - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d '[{"refId":"A","alias":"","sql":"select first(v1) from d1.m1 where ts > 1514208523020 and ts < 1514208523030 interval(1m)"},{"refId":"B","alias":"","sql":"select first(v2) from d1.m1 where ts > 1514208523020 and ts < 1514208523030 interval(1m)"}]' 127.0.0.1:7111/grafana/query -print 14-> $system_content -if $system_content != @[{"refId":"A","target":"A","datapoints":[[2,1514208480000]]},{"refId":"B","target":"B","datapoints":[[5.10000,1514208480000]]}]@ then - return -1 -endi - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d '[{"refId":"A","alias":"","sql":"select count(v1) from d1.m1"},{"refId":"B","alias":"","sql":"select count(v2) from d1.m1"}]' 127.0.0.1:7111/grafana/query -print 15-> $system_content -if $system_content != @[{"refId":"A","target":"A","datapoints":[[3,"-"]]},{"refId":"B","target":"B","datapoints":[[3,"-"]]}]@ then - return -1 -endi - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d '[{"refId":"A","alias":"","sql":"select count(v1) from d1.m1"},{"refId":"B","alias":"","sql":"select count(v2) from d1.m1"}]' 127.0.0.1:7111/grafana/query -print 15-> $system_content -if $system_content != @[{"refId":"A","target":"A","datapoints":[[3,"-"]]},{"refId":"B","target":"B","datapoints":[[3,"-"]]}]@ then - return -1 -endi - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d '[{"refId":"A","alias":"","sql":"select count(i) from d1.mt group by a"},{"refId":"B","alias":"","sql":"select sum(i) from d1.mt group by b"}]' 127.0.0.1:7111/grafana/query -print 16-> $system_content -if $system_content != @[{"refId":"A","target":"{a:1,}","datapoints":[[5,"-"]]},{"refId":"A","target":"{a:2,}","datapoints":[[4,"-"]]},{"refId":"A","target":"{a:3,}","datapoints":[[3,"-"]]},{"refId":"B","target":"{b:a}","datapoints":[[5,"-"]]},{"refId":"B","target":"{b:b}","datapoints":[[8,"-"]]},{"refId":"B","target":"{b:c}","datapoints":[[9,"-"]]}]@ then - return -1 -endi - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d '[{"refId":"A","alias":"count","sql":"select count(i) from d1.mt group by a"},{"refId":"B","alias":"sum-","sql":"select sum(i) from d1.mt group by b"}]' 127.0.0.1:7111/grafana/query -print 17-> $system_content -if $system_content != @[{"refId":"A","target":"count{a:1,}","datapoints":[[5,"-"]]},{"refId":"A","target":"count{a:2,}","datapoints":[[4,"-"]]},{"refId":"A","target":"count{a:3,}","datapoints":[[3,"-"]]},{"refId":"B","target":"sum-{b:a}","datapoints":[[5,"-"]]},{"refId":"B","target":"sum-{b:b}","datapoints":[[8,"-"]]},{"refId":"B","target":"sum-{b:c}","datapoints":[[9,"-"]]}]@ then - return -1 -endi - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d '[{"refId":"A","alias":"count","sql":"select count(i) from d1.mt interval(1m) group by a "}]' 127.0.0.1:7111/grafana/query -print 18-> $system_content -if $system_content != @[{"refId":"A","target":"count{a:1,}","datapoints":[[1,1514208300000],[1,1514208360000],[1,1514208420000],[1,1514208480000],[1,1514208540000]]},{"refId":"A","target":"count{a:2,}","datapoints":[[1,1514208300000],[1,1514208360000],[1,1514208420000],[1,1514208480000]]},{"refId":"A","target":"count{a:3,}","datapoints":[[1,1514208300000],[1,1514208360000],[1,1514208420000]]}]@ then - return -1 -endi - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d '[{"refId":"A","alias":"","sql":"select sum(v2), count(v1) from d1.m1"},{"refId":"B","alias":"","sql":"select count(v2), sum(v2) from d1.m1"}]' 127.0.0.1:7111/grafana/query -print 19-> $system_content -if $system_content != @[{"refId":"A","target":"{count(v1):3}","datapoints":[[15.299999714,"-"]]},{"refId":"B","target":"{sum(v2):15.299999714}","datapoints":[[3,"-"]]}]@ then - return -1 -endi - -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/http/grafana_bug.sim b/tests/script/general/http/grafana_bug.sim deleted file mode 100644 index ed184e17c6..0000000000 --- a/tests/script/general/http/grafana_bug.sim +++ /dev/null @@ -1,271 +0,0 @@ -system sh/stop_dnodes.sh -sleep 2000 -system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c http -v 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -#system sh/cfg.sh -n dnode1 -c adminRowLimit -v 10 -system sh/cfg.sh -n dnode1 -c httpDebugFlag -v 135 -system sh/exec.sh -n dnode1 -s start - -sql connect -sleep 2000 - -print ============================ dnode1 start - -print =============== step0 - prepare data - -sql create database db -sql use db - -sql create table tb (ts timestamp, val int, val1 int, val2 int) -sql create table tb2 (ts timestamp, val int, val1 int, val2 int) -sql create table t2 (ts timestamp, val int) -sql create table tb3 (ts timestamp, val int, val1 int, val2 int) - -sql insert into tb values('2020-01-01 00:00:00.000', 1, 11, 21) -sql insert into tb values('2020-01-02 00:00:00.000', 1, 12, 22) -sql insert into tb values('2020-01-03 00:00:00.000', 2, 13, 23) -sql insert into tb values('2020-01-04 00:00:00.000', 2, 14, 24) - - -sql insert into tb2 values('2020-01-01 00:00:00.000', 21, 211, 221) -sql insert into tb2 values('2020-01-02 00:00:00.000', 21, 212, 222) -sql insert into tb2 values('2020-01-03 00:00:00.000', 22, 213, 223) -sql insert into tb2 values('2020-01-04 00:00:00.000', 22, 214, 224) - - -sql insert into tb3 values('2020-01-01 00:00:00.000', NULL, NULL, NULL) -sql insert into tb3 values('2020-01-02 00:00:00.000', NULL, NULL, NULL) -print =============== step1 - one query, 1 column, with timestamp - -system_content curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d '[ {"refId":"A","alias":"","sql":"select ts from db.tb where ts >= 1577980800000 "} ]' 127.0.0.1:7111/grafana/query -print step1-> $system_content -if $system_content != @[{"refId":"A","target":"A","datapoints":[[null,1577980800000],[null,1578067200000]]}]@ then - return -1 -endi - -print =============== step2 - one query, 2 column, with timestamp - -system_content curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d '[ {"refId":"A","alias":"","sql":"select ts,val from db.tb where ts >= 1577980800000 "} ]' 127.0.0.1:7111/grafana/query -print step2-> $system_content -if $system_content != @[{"refId":"A","target":"A","datapoints":[[2,1577980800000],[2,1578067200000]]}]@ then - return -1 -endi - -print =============== step3 - one query, 3 column, with timestamp - -system_content curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d '[ {"refId":"A","alias":"","sql":"select ts,val,val1 from db.tb where ts >= 1577980800000 "} ]' 127.0.0.1:7111/grafana/query -print step3.1-> $system_content -if $system_content != @[{"refId":"A","target":"{val1:13,}","datapoints":[[2,1577980800000]]},{"refId":"A","target":"{val1:14,}","datapoints":[[2,1578067200000]]}]@ then - return -1 -endi - -system_content curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d '[ {"refId":"A","alias":"","sql":"select ts,val,val1 from db.tb "} ]' 127.0.0.1:7111/grafana/query -print step3.2-> $system_content -if $system_content != @[{"refId":"A","target":"{val1:11,}","datapoints":[[1,1577808000000]]},{"refId":"A","target":"{val1:12,}","datapoints":[[1,1577894400000]]},{"refId":"A","target":"{val1:13,}","datapoints":[[2,1577980800000]]},{"refId":"A","target":"{val1:14,}","datapoints":[[2,1578067200000]]}]@ then - return -1 -endi - -system_content curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d '[ {"refId":"A","alias":"","sql":"select ts,val1,val from db.tb "} ]' 127.0.0.1:7111/grafana/query -print step3.3-> $system_content -if $system_content != @[{"refId":"A","target":"{val:1,}","datapoints":[[11,1577808000000],[12,1577894400000]]},{"refId":"A","target":"{val:2,}","datapoints":[[13,1577980800000],[14,1578067200000]]}]@ then - return -1 -endi - -print =============== step4 - one query, 4 column, with timestamp -system_content curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d '[ {"refId":"A","alias":"","sql":"select ts,val2,val1,val from db.tb "} ]' 127.0.0.1:7111/grafana/query -print step4.1-> $system_content -if $system_content != @[{"refId":"A","target":"{val1:11,, val:1,}","datapoints":[[21,1577808000000]]},{"refId":"A","target":"{val1:12,, val:1,}","datapoints":[[22,1577894400000]]},{"refId":"A","target":"{val1:13,, val:2,}","datapoints":[[23,1577980800000]]},{"refId":"A","target":"{val1:14,, val:2,}","datapoints":[[24,1578067200000]]}]@ then - return -1 -endi - -system_content curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d '[ {"refId":"A","alias":"","sql":"select ts,val,val1,val2 from db.tb "} ]' 127.0.0.1:7111/grafana/query -print step4.2-> $system_content -if $system_content != @[{"refId":"A","target":"{val1:11,, val2:21,}","datapoints":[[1,1577808000000]]},{"refId":"A","target":"{val1:12,, val2:22,}","datapoints":[[1,1577894400000]]},{"refId":"A","target":"{val1:13,, val2:23,}","datapoints":[[2,1577980800000]]},{"refId":"A","target":"{val1:14,, val2:24,}","datapoints":[[2,1578067200000]]}]@ then - return -1 -endi - -print =============== step5 - one query, 1 column, no timestamp -system_content curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d '[ {"refId":"A","alias":"","sql":"select val from db.tb where ts >= 1577980800000 "} ]' 127.0.0.1:7111/grafana/query -print step1-> $system_content -if $system_content != @[{"refId":"A","target":"A","datapoints":[[2,"-"],[2,"-"]]}]@ then - return -1 -endi - -print =============== step6 - one query, 2 column, no timestamp -system_content curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d '[ {"refId":"A","alias":"","sql":"select val1,val2 from db.tb where ts >= 1577980800000 "} ]' 127.0.0.1:7111/grafana/query -print step1-> $system_content -if $system_content != @[{"refId":"A","target":"{val2:23,}","datapoints":[[13,"-"]]},{"refId":"A","target":"{val2:24,}","datapoints":[[14,"-"]]}]@ then - return -1 -endi - -print =============== step7 - one query, 3 column, no timestamp -system_content curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d '[ {"refId":"A","alias":"","sql":"select val1,val2,val from db.tb where ts >= 1577980800000 "} ]' 127.0.0.1:7111/grafana/query -print step1-> $system_content -if $system_content != @[{"refId":"A","target":"{val2:23,, val:2,}","datapoints":[[13,"-"]]},{"refId":"A","target":"{val2:24,, val:2,}","datapoints":[[14,"-"]]}]@ then - return -1 -endi - -system_content curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d '[ {"refId":"A","alias":"","sql":"select val1,val2,val from db.tb"} ]' 127.0.0.1:7111/grafana/query -print step1-> $system_content -if $system_content != @[{"refId":"A","target":"{val2:21,, val:1,}","datapoints":[[11,"-"]]},{"refId":"A","target":"{val2:22,, val:1,}","datapoints":[[12,"-"]]},{"refId":"A","target":"{val2:23,, val:2,}","datapoints":[[13,"-"]]},{"refId":"A","target":"{val2:24,, val:2,}","datapoints":[[14,"-"]]}]@ then - return -1 -endi - -print =============== step8 - one query, no return -system_content curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d '[ {"refId":"A","alias":"","sql":"select val1,val2,val from db.tb where ts >= 1677980800000 "} ]' 127.0.0.1:7111/grafana/query -print step1-> $system_content -if $system_content != @[]@ then - return -1 -endi - -print =============== step9 - one query, insert sql -system_content curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d '[ {"refId":"A","alias":"","sql":"insert into db.t2 values(now, 1) "} ]' 127.0.0.1:7111/grafana/query -print step1-> $system_content -if $system_content != @[]@ then - return -1 -endi - -print =============== step10 - one query, error sql -system_content curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d '[ {"refId":"A","alias":"","sql":"select * from db.tt "} ]' 127.0.0.1:7111/grafana/query -print step1-> $system_content -if $system_content != @[]@ then - return -1 -endi - -print =============== step11 - two query, 1 column, with timestamp, 1 column, with timestamp - -system_content curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d '[ {"refId":"B","alias":"BB","sql":"select ts from db.tb2 where ts >= 1577980800000 "},{"refId":"A","alias":"","sql":"select ts from db.tb where ts >= 1577980800000 "} ]' 127.0.0.1:7111/grafana/query -print step1-> $system_content -if $system_content != @[{"refId":"B","target":"BB","datapoints":[[null,1577980800000],[null,1578067200000]]},{"refId":"A","target":"A","datapoints":[[null,1577980800000],[null,1578067200000]]}]@ then - return -1 -endi - -print =============== step12 - two query, 1 column, with timestamp, 2 column, with timestamp - -system_content curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d '[ {"refId":"B","alias":"BB","sql":"select ts from db.tb2 where ts >= 1577980800000 "},{"refId":"A","alias":"","sql":"select ts,val from db.tb where ts >= 1577980800000 "} ]' 127.0.0.1:7111/grafana/query -print step1-> $system_content -if $system_content != @[{"refId":"B","target":"BB","datapoints":[[null,1577980800000],[null,1578067200000]]},{"refId":"A","target":"A","datapoints":[[2,1577980800000],[2,1578067200000]]}]@ then - return -1 -endi - -print =============== step13 - two query, 1 column, with timestamp, 3 column, with timestamp -system_content curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d '[ {"refId":"B","alias":"BB","sql":"select ts from db.tb2 where ts >= 1577980800000 "},{"refId":"A","alias":"","sql":"select ts,val,val1 from db.tb where ts >= 1577980800000 "} ]' 127.0.0.1:7111/grafana/query -print step1-> $system_content -if $system_content != @[{"refId":"B","target":"BB","datapoints":[[null,1577980800000],[null,1578067200000]]},{"refId":"A","target":"{val1:13,}","datapoints":[[2,1577980800000]]},{"refId":"A","target":"{val1:14,}","datapoints":[[2,1578067200000]]}]@ then - return -1 -endi - -print =============== step14 - two query, 2 column, with timestamp, 2 column, with timestamp -system_content curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d '[ {"refId":"B","alias":"BB","sql":"select ts, val2 from db.tb2 where ts >= 1577980800000 "},{"refId":"A","alias":"AA","sql":"select ts,val from db.tb where ts >= 1577980800000 "} ]' 127.0.0.1:7111/grafana/query -print step1-> $system_content -if $system_content != @[{"refId":"B","target":"BB","datapoints":[[223,1577980800000],[224,1578067200000]]},{"refId":"A","target":"AA","datapoints":[[2,1577980800000],[2,1578067200000]]}]@ then - return -1 -endi - -print =============== step15 - two query, 2 column, with timestamp, 3 column, with timestamp -system_content curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d '[ {"refId":"B","alias":"BB","sql":"select ts, val2, val1 from db.tb2 where ts >= 1577980800000 "},{"refId":"A","alias":"AA","sql":"select ts,val from db.tb where ts >= 1577980800000 "} ]' 127.0.0.1:7111/grafana/query -print step1-> $system_content -if $system_content != @[{"refId":"B","target":"BB{val1:213,}","datapoints":[[223,1577980800000]]},{"refId":"B","target":"BB{val1:214,}","datapoints":[[224,1578067200000]]},{"refId":"A","target":"AA","datapoints":[[2,1577980800000],[2,1578067200000]]}]@ then - return -1 -endi - -print =============== step16 - two query, 3 column, with timestamp, 4 column, with timestamp -system_content curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d '[ {"refId":"B","alias":"BB","sql":"select ts, val, val1, val2, val1 from db.tb2 where ts >= 1577980800000 "},{"refId":"A","alias":"AA","sql":"select ts,val,val1 from db.tb where ts >= 1577980800000 "} ]' 127.0.0.1:7111/grafana/query -print step1-> $system_content -if $system_content != @[{"refId":"B","target":"BB{val1:213,, val2:223,, val1:213,}","datapoints":[[22,1577980800000]]},{"refId":"B","target":"BB{val1:214,, val2:224,, val1:214,}","datapoints":[[22,1578067200000]]},{"refId":"A","target":"AA{val1:13,}","datapoints":[[2,1577980800000]]},{"refId":"A","target":"AA{val1:14,}","datapoints":[[2,1578067200000]]}]@ then - return -1 -endi - -print =============== step17 - two query, 2 column, with timestamp, no return -system_content curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d '[ {"refId":"B","alias":"BB","sql":"select ts, val from db.tb2 where ts >= 1577980800000 "},{"refId":"A","alias":"AA","sql":"select ts,val from db.tb where ts >= 1677980800000 "} ]' 127.0.0.1:7111/grafana/query -print step1-> $system_content -if $system_content != @[{"refId":"B","target":"BB","datapoints":[[22,1577980800000],[22,1578067200000]]}]@ then - return -1 -endi - -print =============== step18 - two query, 2 column, with timestamp, invalid sql -system_content curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d '[ {"refId":"B","alias":"BB","sql":"select ts, val from db.tb2 where ts >= 1577980800000 "},{"refId":"A","alias":"AA","sql":"select ts,val from db.tb222 where ts >= 1677980800000 "} ]' 127.0.0.1:7111/grafana/query -print step1-> $system_content -if $system_content != @[{"refId":"B","target":"BB","datapoints":[[22,1577980800000],[22,1578067200000]]}]@ then - return -1 -endi - -print =============== step19 - two query, 2 column, with timestamp, insert sql -system_content curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d '[ {"refId":"B","alias":"BB","sql":"select ts, val from db.tb2 where ts >= 1577980800000 "},{"refId":"A","alias":"AA","sql":"insert into db.t2 values(now, 1)"} ]' 127.0.0.1:7111/grafana/query -print step1-> $system_content -if $system_content != @[{"refId":"B","target":"BB","datapoints":[[22,1577980800000],[22,1578067200000]]}]@ then - return -1 -endi - -print =============== step20 - two query, 1 column, no timestamp, 1 column, with timestamp -system_content curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d '[ {"refId":"B","alias":"BB","sql":"select val from db.tb2 where ts >= 1577980800000 "},{"refId":"A","alias":"AA","sql":"select ts from db.tb2 where ts >= 1577980800000 "} ]' 127.0.0.1:7111/grafana/query -print step1-> $system_content -if $system_content != @[{"refId":"B","target":"BB","datapoints":[[22,"-"],[22,"-"]]},{"refId":"A","target":"AA","datapoints":[[null,1577980800000],[null,1578067200000]]}]@ then - return -1 -endi - -print =============== step21 - two query, 1 column, no timestamp, 2 column, with timestamp -system_content curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d '[ {"refId":"B","alias":"BB","sql":"select val from db.tb2 where ts >= 1577980800000 "},{"refId":"A","alias":"AA","sql":"select ts,val2 from db.tb2 where ts >= 1577980800000 "} ]' 127.0.0.1:7111/grafana/query -print step1-> $system_content -if $system_content != @[{"refId":"B","target":"BB","datapoints":[[22,"-"],[22,"-"]]},{"refId":"A","target":"AA","datapoints":[[223,1577980800000],[224,1578067200000]]}]@ then - return -1 -endi - -print =============== step22 - two query, 1 column, no timestamp, 3 column, with timestamp -system_content curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d '[ {"refId":"B","alias":"BB","sql":"select val from db.tb2 where ts >= 1577980800000 "},{"refId":"A","alias":"AA","sql":"select ts,val1, val2 from db.tb2 where ts >= 1577980800000 "} ]' 127.0.0.1:7111/grafana/query -print step1-> $system_content -if $system_content != @[{"refId":"B","target":"BB","datapoints":[[22,"-"],[22,"-"]]},{"refId":"A","target":"AA{val2:223,}","datapoints":[[213,1577980800000]]},{"refId":"A","target":"AA{val2:224,}","datapoints":[[214,1578067200000]]}]@ then - return -1 -endi - -print =============== step23 - two query, 2 column, no timestamp, 1 column, no timestamp -system_content curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d '[ {"refId":"B","alias":"BB","sql":"select val1,val2 from db.tb2 where ts >= 1577980800000 "},{"refId":"A","alias":"AA","sql":"select val1 from db.tb2 where ts >= 1577980800000 "} ]' 127.0.0.1:7111/grafana/query -print step1-> $system_content -if $system_content != @[{"refId":"B","target":"BB{val2:223,}","datapoints":[[213,"-"]]},{"refId":"B","target":"BB{val2:224,}","datapoints":[[214,"-"]]},{"refId":"A","target":"AA","datapoints":[[213,"-"],[214,"-"]]}]@ then - return -1 -endi - -print =============== step24 - two query, 2 column, no timestamp, 2 column, no timestamp -system_content curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d '[ {"refId":"B","alias":"BB","sql":"select val1,val2 from db.tb2 where ts >= 1577980800000 "},{"refId":"A","alias":"AA","sql":"select val,val1 from db.tb2 where ts >= 1577980800000 "} ]' 127.0.0.1:7111/grafana/query -print step1-> $system_content -if $system_content != @[{"refId":"B","target":"BB{val2:223,}","datapoints":[[213,"-"]]},{"refId":"B","target":"BB{val2:224,}","datapoints":[[214,"-"]]},{"refId":"A","target":"AA{val1:213,}","datapoints":[[22,"-"]]},{"refId":"A","target":"AA{val1:214,}","datapoints":[[22,"-"]]}]@ then - return -1 -endi - -print =============== step25 - two query, 2 column, no timestamp, 3 column, no timestamp -system_content curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d '[ {"refId":"B","alias":"BB","sql":"select val1,val2 from db.tb2 where ts >= 1577980800000 "},{"refId":"A","alias":"AA","sql":"select val,val1,val2 from db.tb2 where ts >= 1577980800000 "} ]' 127.0.0.1:7111/grafana/query -print step1-> $system_content -if $system_content != @[{"refId":"B","target":"BB{val2:223,}","datapoints":[[213,"-"]]},{"refId":"B","target":"BB{val2:224,}","datapoints":[[214,"-"]]},{"refId":"A","target":"AA{val1:213,, val2:223,}","datapoints":[[22,"-"]]},{"refId":"A","target":"AA{val1:214,, val2:224,}","datapoints":[[22,"-"]]}]@ then - return -1 -endi - -print =============== step26 - 2 column, no timestamp, NULL -system_content curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d '[ {"refId":"A","alias":"","sql":"select * from db.tb3 "} ]' 127.0.0.1:7111/grafana/query -print step1-> $system_content -if $system_content != @[{"refId":"A","target":"{val1:nil, val2:nil}","datapoints":[[null,1577808000000],[null,1577894400000]]}]@ then - return -1 -endi - -sql create table tt (ts timestamp ,i int) tags(j binary(20),k binary(20)); -sql insert into t1 using tt tags('jnetworki','t1') values('2020-01-01 00:00:00.000',1)('2020-01-01 00:01:00.000',2)('2020-01-01 00:02:00.000',3)('2020-01-01 00:03:00.000',4)('2020-01-01 00:04:00.000',5); - -system_content curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d '[ {"refId":"A","alias":"","sql":"select max(i) from db.tt where j like \u0027%network%\u0027 and ts >= \u00272020-01-01 00:00:00.000\u0027 and ts < \u00272020-01-01 00:05:00.000\u0027 interval(5m) group by k "} ]' 127.0.0.1:7111/grafana/query -print step1-> $system_content -if $system_content != @[{"refId":"A","target":"{k:t1}","datapoints":[[5,1577808000000]]}]@ then - return -1 -endi - -system_content curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d '[ {"refId":"A","alias":"","sql":"select max(i) from db.tt where j like \u0027jnetwo%\u0027 and ts >= \u00272020-01-01 00:00:00.000\u0027 and ts < \u00272020-01-01 00:05:00.000\u0027 interval(5m) group by k "} ]' 127.0.0.1:7111/grafana/query -print step1-> $system_content -if $system_content != @[{"refId":"A","target":"{k:t1}","datapoints":[[5,1577808000000]]}]@ then - return -1 -endi - -system_content curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d '[ {"refId":"A","alias":"","sql":"select max(i) from db.tt where j like \u0027%networki\u0027 and ts >= \u00272020-01-01 00:00:00.000\u0027 and ts < \u00272020-01-01 00:05:00.000\u0027 interval(5m) group by k "} ]' 127.0.0.1:7111/grafana/query -print step1-> $system_content -if $system_content != @[{"refId":"A","target":"{k:t1}","datapoints":[[5,1577808000000]]}]@ then - return -1 -endi - -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/http/gzip.sim b/tests/script/general/http/gzip.sim deleted file mode 100644 index ce358d84a1..0000000000 --- a/tests/script/general/http/gzip.sim +++ /dev/null @@ -1,39 +0,0 @@ -system sh/stop_dnodes.sh -sleep 2000 -system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c wallevel -v 0 -system sh/cfg.sh -n dnode1 -c http -v 1 -system sh/cfg.sh -n dnode1 -c maxSQLLength -v 340032 -system sh/exec.sh -n dnode1 -s start - -sleep 2000 -sql connect - -print ============================ dnode1 start - -print =============== step1 - prepare data -sql create database d1 -sql use d1 - -sql create table table_rest (ts timestamp, i int) -print sql length is 270KB -restful d1 table_rest 1591072800 10000 gzip -restful d1 table_rest 1591172800 10000 gzip -restful d1 table_rest 1591272800 10000 gzip -restful d1 table_rest 1591372800 10000 gzip -restful d1 table_rest 1591472800 10000 gzip -restful d1 table_rest 1591572800 10000 gzip -restful d1 table_rest 1591672800 10000 gzip -restful d1 table_rest 1591772800 10000 gzip -restful d1 table_rest 1591872800 10000 gzip -restful d1 table_rest 1591972800 10000 gzip - -sql select * from table_rest; -print rows: $rows -if $rows != 100000 then - return -1 -endi - -system curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'select * from d1.table_rest' 127.0.0.1:7111/rest/sql --compressed - -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/http/http.json b/tests/script/general/http/http.json deleted file mode 100644 index 087f4eab5b..0000000000 --- a/tests/script/general/http/http.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "hostIp": "192.168.100.128", - "connNum": 1, - "insertModel": "insertTable", - "waitTime": 0, - "tableDesc": "ts timestamp i int", - "tablePrefix": "t", - "tablePerConn": 1, - "tableCreate": true, - "tableStart": 1, - "dbName": "db", - "dbReplica": 1, - "dbKeep": 3650, - "dbDays": 7, - "metricsName": "mt", - "tagNum": 10, - "dataNum": 100, - "dataBegin": 1485878400000, - "dataInterval": 1000, - "dataBatch": 1, - "dataInsert": true, - "dataRandom": false -} diff --git a/tests/script/general/http/httpTestNew.go b/tests/script/general/http/httpTestNew.go deleted file mode 100644 index 9ca7733d44..0000000000 --- a/tests/script/general/http/httpTestNew.go +++ /dev/null @@ -1,247 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package main - -import ( - "bytes" - "encoding/json" - "flag" - "fmt" - "io/ioutil" - "math/rand" - "net/http" - "os" - "sync" - "sync/atomic" - "time" -) - -var ( - token string - url string - config Config - request int64 - begin time.Time - errorNum int64 -) - -type Config struct { - HostIp string `json:"hostIp"` - TableNum int `json:"tableNum"` - DbName string `json:"dbName"` - MetricsName string `json:"metricsName"` - DataNum int `json:"dataNum"` - BatchNum int `json:"batchNum"` -} - -type TokenResult struct { - Status string `json:"status"` - Code int `json:"code"` - Desc string `json:"desc"` -} - -type JsonResult struct { - Status string `json:"status"` - Code int `json:"code"` -} - -func readFile(filename string) { - file, err := os.Open(filename) - if err != nil { - println("taos.json not found") - panic(err) - } - defer file.Close() - - dec := json.NewDecoder(file) - err = dec.Decode(&config) - if err != nil { - println("taos.json parse error") - panic(err) - } - - request = 0 - errorNum = 0 - - fmt.Println("================config parameters======================") - fmt.Println("HostIp:", config.HostIp) - fmt.Println("TableNum:", config.TableNum) - fmt.Println("dbName:", config.DbName) - fmt.Println("metricsName:", config.MetricsName) - fmt.Println("dataNum:", config.DataNum) - fmt.Println("batchNum:", config.BatchNum) - - fmt.Println("================http token=============================") - token, err = getToken() - url = fmt.Sprintf("http://%s:%d/rest/sql", config.HostIp, 6020) - - fmt.Println("httpToken:", token) - fmt.Println("httpUrl:", url) - - if err != nil { - panic(err) - } -} - -func getToken() (string, error) { - resp, err := http.Get(fmt.Sprintf("http://%s:%d/rest/login/%s/%s", config.HostIp, 6020, "root", "taosdata")) - if err != nil { - return "", err - } - - defer resp.Body.Close() - - var tokenResult TokenResult - - data, err := ioutil.ReadAll(resp.Body) - - if err != nil { - return "", err - } - - err = json.Unmarshal(data, &tokenResult) - if err != nil { - return "", err - } - - if tokenResult.Status != "succ" { - fmt.Println("get http token failed") - fmt.Println(tokenResult) - return "", err - } - - return tokenResult.Desc, nil -} - -func exec(client *http.Client, sql string) { - for reTryTimes := 0; reTryTimes < 10; reTryTimes++ { - - req, err1 := http.NewRequest("POST", url, bytes.NewReader([]byte(sql))) - if err1 != nil { - continue - } - req.Header.Add("Authorization", "Taosd "+token) - - resp, err := client.Do(req) - - if err != nil { - continue - } - - data, err := ioutil.ReadAll(resp.Body) - if err != nil { - resp.Body.Close() - continue - } - - var jsonResult JsonResult - err = json.Unmarshal(data, &jsonResult) - if err != nil { - resp.Body.Close() - continue - } - - if jsonResult.Status != "succ" { - resp.Body.Close() - continue - } - - atomic.AddInt64(&request, 1) - - if (request*int64(config.BatchNum))%100000 == 0 && request != 0 { - spend := time.Since(begin).Seconds() - if spend >= 1 && spend < 10000000 { - total := (request - errorNum - 2 - int64(config.TableNum)) * int64(config.BatchNum) - fmt.Printf("request:%d, error:%d, insert:%d, spend:%.2f seconds, dps:%.1f \n", request, errorNum, total, spend, float64(total)/float64(spend)) - } - } - - return - } - - //fmt.Println("exec failed, sql:", sql) - errorNum++ -} - -func createDb() { - fmt.Println("================create database =====================") - - client := &http.Client{} - sql := fmt.Sprintf("create database %s", config.DbName) - exec(client, sql) -} - -func createTb() { - fmt.Println("================create table ========================") - - client := &http.Client{} - sql := fmt.Sprintf("create table %s.%s(ts timestamp, f1 int, f2 int) tags (tb int)", config.DbName, config.MetricsName) - exec(client, sql) - - for i := 0; i < config.TableNum; i++ { - sql := fmt.Sprintf("create table %s.t%d using %s.%s tags(%d)", config.DbName, i, config.DbName, config.MetricsName, i) - exec(client, sql) - } -} - -func insertData(wg *sync.WaitGroup, tableIndex int) { - defer wg.Done() - - client := &http.Client{} - beginTime := int64(1519833600000) - - for i := 0; i < config.DataNum; i += config.BatchNum { - var sql bytes.Buffer - sql.WriteString(fmt.Sprintf("insert into %s.t%d values", config.DbName, tableIndex)) - - for j := 0; j < config.BatchNum; j++ { - sql.WriteString(fmt.Sprintf("(%d,%d,%d)", beginTime+int64(i)+int64(j), rand.Intn(1000), rand.Intn(1000))) - } - exec(client, sql.String()) - } -} - -func main() { - filename := flag.String("config", "http.json", "config file name") - - flag.Parse() - - readFile(*filename) - - fmt.Println("\n================http test start======================") - - createDb() - createTb() - - begin = time.Now() - - var wg sync.WaitGroup - - fmt.Println("================insert data ========================") - for i := 0; i < config.TableNum; i++ { - wg.Add(1) - go insertData(&wg, i) - } - - wg.Wait() - - fmt.Println("\n================http test stop ======================") - - spend := time.Since(begin).Seconds() - - total := (request - errorNum - 2 - int64(config.TableNum)) * int64(config.BatchNum) - fmt.Printf("request:%d, error:%d, insert:%d, spend:%.2f seconds, dps:%.1f \n", request, errorNum, total, spend, float64(total)/float64(spend)) -} diff --git a/tests/script/general/http/httpTest_cloud.go b/tests/script/general/http/httpTest_cloud.go deleted file mode 100644 index 1d849a245f..0000000000 --- a/tests/script/general/http/httpTest_cloud.go +++ /dev/null @@ -1,387 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package main - -import ( - "bytes" - "encoding/json" - "fmt" - "io/ioutil" - "math/rand" - "net/http" - "os" - "sync" - "sync/atomic" - "time" - "flag" -) - -var ( - token string - url string - config Config - request int64 - period int64 - errorNum int64 -) - -type Config struct { - HostIp string `json:"hostIp"` - ConnNum int `json:"connNum"` - InsertModel string `json:"insertModel"` - WaitTime int `json:"waitTime"` - TableDesc string `json:"tableDesc"` - TablePrefix string `json:"tablePrefix"` - TablePerConn int `json:"tablePerConn"` - TableCreate bool `json:"tableCreate"` - TableStart int `json:"tableStart"` - DbName string `json:"dbName"` - DbReplica int `json:"dbReplica"` - DbKeep int `json:"dbKeep"` - DbDays int `json:"dbDays"` - MetricsName string `json:"metricsName"` - TagNum int `json:"tagNum"` - DataNum int `json:"dataNum"` - DataBegin int64 `json:"dataBegin"` - DataInterval int `json:"dataInterval"` - DataBatch int `json:"dataBatch"` - DataInsert bool `json:"dataInsert"` - DataRandom bool `json:"dataRandom"` -} - -type TokenResult struct { - Status string `json:"status"` - Code int `json:"code"` - Desc string `json:"desc"` -} - -type JsonResult struct { - Status string `json:"status"` - Code int `json:"code"` -} - -func readFile(filename string) { - file, err := os.Open(filename) - if err != nil { - println("taos_cloud.json not found") - panic(err) - } - defer file.Close() - - dec := json.NewDecoder(file) - err = dec.Decode(&config) - if err != nil { - println("taos_cloud.json parse error") - panic(err) - } - - if config.TagNum <= 0 { - config.TagNum = 1 - } - request = 0 - period = 0 - errorNum = 0 - - fmt.Println("================config parameters======================") - fmt.Println("HostIp:", config.HostIp) - fmt.Println("connNum:", config.ConnNum) - fmt.Println("insertModel:", config.InsertModel) - fmt.Println("waitTime:", config.WaitTime) - fmt.Println("tableDesc:", config.TableDesc) - fmt.Println("tablePrefix:", config.TablePrefix) - fmt.Println("tablePerConn:", config.TablePerConn) - fmt.Println("tableCreate:", config.TableCreate) - fmt.Println("tableStart:", config.TableStart) - fmt.Println("dbName:", config.DbName) - fmt.Println("dbReplica:", config.DbReplica) - fmt.Println("dbKeep:", config.DbKeep) - fmt.Println("dbDays:", config.DbDays) - fmt.Println("metricsName:", config.MetricsName) - fmt.Println("tagNum:", config.TagNum) - fmt.Println("dataNum:", config.DataNum) - fmt.Println("dataBegin:", config.DataBegin) - fmt.Println("dataInterval:", config.DataInterval) - fmt.Println("dataBatch:", config.DataBatch) - fmt.Println("dataInsert:", config.DataInsert) - fmt.Println("dataRandom:", config.DataRandom) - - fmt.Println("================http token=============================") - token, err = getToken() - url = fmt.Sprintf("http://%s:%d/rest/sql", config.HostIp, 6020) - - fmt.Println("httpToken:", token) - fmt.Println("httpUrl:", url) - - if err != nil { - panic(err) - } -} - -func getToken() (string, error) { - resp, err := http.Get(fmt.Sprintf("http://%s:%d/rest/login/%s/%s", config.HostIp, 6020, "tiger", "tiger")) - if err != nil { - return "", err - } - - defer resp.Body.Close() - - var tokenResult TokenResult - - data, err := ioutil.ReadAll(resp.Body) - - if err != nil { - return "", err - } - - err = json.Unmarshal(data, &tokenResult) - if err != nil { - return "", err - } - - if tokenResult.Status != "succ" { - fmt.Println("get http token failed") - fmt.Println(tokenResult) - return "", err - } - - return tokenResult.Desc, nil -} - -func exec(client *http.Client, sql string) { - for times := 0; times < 10; times++ { - - req, err1 := http.NewRequest("POST", url, bytes.NewReader([]byte(sql))) - if err1 != nil { - continue - } - req.Header.Add("Authorization", "Taosd "+token) - - begin := time.Now() - resp, err := client.Do(req) - - if err != nil { - continue - } - - data, err := ioutil.ReadAll(resp.Body) - if err != nil { - resp.Body.Close() - continue - } - - spend := (time.Since(begin).Nanoseconds()) - - var jsonResult JsonResult - err = json.Unmarshal(data, &jsonResult) - if err != nil { - resp.Body.Close() - continue - } - - if jsonResult.Status != "succ" { - resp.Body.Close() - continue - } - atomic.AddInt64(&request, 1) - if (request < 103) { - return - } - - atomic.AddInt64(&period, spend) - if request%5000 == 0 && request != 0 { - requestAvg := float64(period) / float64(1000000) / float64(request) - qps := float64(1000) / float64(requestAvg) * float64(config.ConnNum) - dps := qps * float64(config.DataBatch) - fmt.Println("====== req:", request, ", error:", errorNum, ", qps:", int64(qps), ", wait:", int64(requestAvg), "ms", ", data per second:", int64(dps)) - } - return - } - fmt.Println("xxxx>sql:", sql, ", retryTimes:", 10) - errorNum++ -} - -func createDb() { - if !config.TableCreate { - return - } - - client := &http.Client{} - - fmt.Println("\n================create database =====================") - sql := fmt.Sprintf("create database %s keep %d", config.DbName, config.DbKeep) - exec(client, sql) -} - -func createTb() { - if !config.TableCreate { - return - } - client := &http.Client{} - - fmt.Println("\n================create table ========================") - sql := fmt.Sprintf("create table %s.%s(%s) tags (orgno int)", config.DbName, config.MetricsName, config.TableDesc) - exec(client, sql) - tbNum := config.TablePerConn*config.ConnNum + config.TableStart - for i := config.TableStart; i < tbNum; i++ { - sql := fmt.Sprintf("create table %s.%s%d using %s.%s tags(%d)", config.DbName, config.TablePrefix, i, config.DbName, config.MetricsName, i%config.TagNum+1) - exec(client, sql) - } - -} - -func insertTable(conn int) { - client := &http.Client{} - - tbStart := conn*config.TablePerConn + config.TableStart - tmStart := config.DataBegin - - for i := 0; i < config.TablePerConn; i++ { - tbIndex := i + tbStart - - for j := 0; j < config.DataNum; j++ { - tmVal := int64(j)*int64(config.DataInterval) + tmStart - - dataVal := j - if config.DataRandom { - dataVal = rand.Intn(1000) - } - - sql := fmt.Sprintf("insert into %s.%s%d values(%d, %d)", config.DbName, config.TablePrefix, tbIndex, tmVal, dataVal) - for k := 1; k < config.DataBatch; k++ { - tmVal := int64(j)*int64(config.DataInterval) + int64(k) + tmStart - - dataVal := j + k - if config.DataRandom { - dataVal = rand.Intn(1000) - } - - sql += fmt.Sprintf("(%d, %d)", tmVal, dataVal) - } - - j += (config.DataBatch - 1) - - exec(client, sql) - if config.WaitTime != 0 { - time.Sleep(time.Millisecond * time.Duration(config.WaitTime)) - } - } - } -} - -func insertLoop(conn int) { - client := &http.Client{} - - tbStart := conn*config.TablePerConn + config.TableStart - tmStart := config.DataBegin - - for j := 0; j < config.DataNum; j++ { - - for i := 0; i < config.TablePerConn; i++ { - tbIndex := i + tbStart - - tmVal := int64(j)*int64(config.DataInterval) + tmStart - - dataVal := j - if config.DataRandom { - dataVal = rand.Intn(1000) - } - - sql := fmt.Sprintf("insert into %s.%s%d values(%d, %d)", config.DbName, config.TablePrefix, tbIndex, tmVal, dataVal) - for k := 1; k < config.DataBatch; k++ { - tmVal := int64(j)*int64(config.DataInterval) + int64(k) + tmStart - - dataVal := j + k - if config.DataRandom { - dataVal = rand.Intn(1000) - } - sql += fmt.Sprintf("values(%d, %d)", tmVal, dataVal) - } - - j += (config.DataBatch - 1) - - exec(client, sql) - - if config.WaitTime != 0 { - time.Sleep(time.Millisecond * time.Duration(config.WaitTime)) - } - } - - } -} - -func insertTb(wg *sync.WaitGroup, conn int) { - defer wg.Done() - - if !config.DataInsert { - return - } - - if config.InsertModel == "insertTable" { - insertTable(conn) - } else { - insertLoop(conn) - } -} - -func selectData(wg *sync.WaitGroup, conn int) { - defer wg.Done() - - client := &http.Client{} - - for i := 0; i < config.DataNum; i++ { - exec(client, config.TableDesc) - } -} - -func main() { - filename := flag.String("config", "taos_cloud.json", "config file name") - - flag.Parse() - - readFile(*filename) - - fmt.Println("\n================http test start======================") - - var wg sync.WaitGroup - - if config.InsertModel == "selectData" { - fmt.Println("\n================select data ========================") - for i := 0; i < config.ConnNum; i++ { - wg.Add(1) - go selectData(&wg, i) - } - } else { - createDb() - createTb() - - if config.DataInsert { - fmt.Println("\n================insert data ========================") - } - - for i := 0; i < config.ConnNum; i++ { - wg.Add(1) - go insertTb(&wg, i) - } - } - - wg.Wait() - - fmt.Println("\n================http test stop ======================") - requestAvg := float64(period) / float64(1000000) / float64(request) - qps := float64(1000) / float64(requestAvg) * float64(config.ConnNum) - dps := qps * float64(config.DataBatch) - fmt.Println("====== req:", request, ", error:", errorNum, ", qps:", int64(qps), ", wait:", int64(requestAvg), "ms", ", data per second:", int64(dps)) -} diff --git a/tests/script/general/http/httpTest_private.go b/tests/script/general/http/httpTest_private.go deleted file mode 100644 index 66f19ec896..0000000000 --- a/tests/script/general/http/httpTest_private.go +++ /dev/null @@ -1,387 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package main - -import ( - "bytes" - "encoding/json" - "fmt" - "io/ioutil" - "math/rand" - "net/http" - "os" - "sync" - "sync/atomic" - "time" - "flag" -) - -var ( - token string - url string - config Config - request int64 - period int64 - errorNum int64 -) - -type Config struct { - HostIp string `json:"hostIp"` - ConnNum int `json:"connNum"` - InsertModel string `json:"insertModel"` - WaitTime int `json:"waitTime"` - TableDesc string `json:"tableDesc"` - TablePrefix string `json:"tablePrefix"` - TablePerConn int `json:"tablePerConn"` - TableCreate bool `json:"tableCreate"` - TableStart int `json:"tableStart"` - DbName string `json:"dbName"` - DbReplica int `json:"dbReplica"` - DbKeep int `json:"dbKeep"` - DbDays int `json:"dbDays"` - MetricsName string `json:"metricsName"` - TagNum int `json:"tagNum"` - DataNum int `json:"dataNum"` - DataBegin int64 `json:"dataBegin"` - DataInterval int `json:"dataInterval"` - DataBatch int `json:"dataBatch"` - DataInsert bool `json:"dataInsert"` - DataRandom bool `json:"dataRandom"` -} - -type TokenResult struct { - Status string `json:"status"` - Code int `json:"code"` - Desc string `json:"desc"` -} - -type JsonResult struct { - Status string `json:"status"` - Code int `json:"code"` -} - -func readFile(filename string) { - file, err := os.Open(filename) - if err != nil { - println("taos_cloud.json not found") - panic(err) - } - defer file.Close() - - dec := json.NewDecoder(file) - err = dec.Decode(&config) - if err != nil { - println("taos_cloud.json parse error") - panic(err) - } - - if config.TagNum <= 0 { - config.TagNum = 1 - } - request = 0 - period = 0 - errorNum = 0 - - fmt.Println("================config parameters======================") - fmt.Println("HostIp:", config.HostIp) - fmt.Println("connNum:", config.ConnNum) - fmt.Println("insertModel:", config.InsertModel) - fmt.Println("waitTime:", config.WaitTime) - fmt.Println("tableDesc:", config.TableDesc) - fmt.Println("tablePrefix:", config.TablePrefix) - fmt.Println("tablePerConn:", config.TablePerConn) - fmt.Println("tableCreate:", config.TableCreate) - fmt.Println("tableStart:", config.TableStart) - fmt.Println("dbName:", config.DbName) - fmt.Println("dbReplica:", config.DbReplica) - fmt.Println("dbKeep:", config.DbKeep) - fmt.Println("dbDays:", config.DbDays) - fmt.Println("metricsName:", config.MetricsName) - fmt.Println("tagNum:", config.TagNum) - fmt.Println("dataNum:", config.DataNum) - fmt.Println("dataBegin:", config.DataBegin) - fmt.Println("dataInterval:", config.DataInterval) - fmt.Println("dataBatch:", config.DataBatch) - fmt.Println("dataInsert:", config.DataInsert) - fmt.Println("dataRandom:", config.DataRandom) - - fmt.Println("================http token=============================") - token, err = getToken() - url = fmt.Sprintf("http://%s:%d/rest/sql", config.HostIp, 6020) - - fmt.Println("httpToken:", token) - fmt.Println("httpUrl:", url) - - if err != nil { - panic(err) - } -} - -func getToken() (string, error) { - resp, err := http.Get(fmt.Sprintf("http://%s:%d/rest/login/%s/%s", config.HostIp, 6020, "root", "taosdata")) - if err != nil { - return "", err - } - - defer resp.Body.Close() - - var tokenResult TokenResult - - data, err := ioutil.ReadAll(resp.Body) - - if err != nil { - return "", err - } - - err = json.Unmarshal(data, &tokenResult) - if err != nil { - return "", err - } - - if tokenResult.Status != "succ" { - fmt.Println("get http token failed") - fmt.Println(tokenResult) - return "", err - } - - return tokenResult.Desc, nil -} - -func exec(client *http.Client, sql string) { - for times := 0; times < 1; times++ { - - req, err1 := http.NewRequest("POST", url, bytes.NewReader([]byte(sql))) - if err1 != nil { - continue - } - req.Header.Add("Authorization", "Taosd "+token) - - begin := time.Now() - resp, err := client.Do(req) - - if err != nil { - continue - } - - data, err := ioutil.ReadAll(resp.Body) - if err != nil { - resp.Body.Close() - continue - } - - spend := (time.Since(begin).Nanoseconds()) - - var jsonResult JsonResult - err = json.Unmarshal(data, &jsonResult) - if err != nil { - resp.Body.Close() - continue - } - - if jsonResult.Status != "succ" { - resp.Body.Close() - continue - } - atomic.AddInt64(&request, 1) - if (request < 103) { - return - } - - atomic.AddInt64(&period, spend) - if request%5000 == 0 && request != 0 { - requestAvg := float64(period) / float64(1000000) / float64(request) - qps := float64(1000) / float64(requestAvg) * float64(config.ConnNum) - dps := qps * float64(config.DataBatch) - fmt.Println("====== req:", request, ", error:", errorNum, ", qps:", int64(qps), ", wait:", int64(requestAvg), "ms", ", data per second:", int64(dps)) - } - return - } - fmt.Println("xxxx>sql:", sql, ", retryTimes:", 10) - errorNum++ -} - -func createDb() { - if !config.TableCreate { - return - } - - client := &http.Client{} - - fmt.Println("\n================create database =====================") - sql := fmt.Sprintf("create database %s keep %d", config.DbName, config.DbKeep) - exec(client, sql) -} - -func createTb() { - if !config.TableCreate { - return - } - client := &http.Client{} - - fmt.Println("\n================create table ========================") - sql := fmt.Sprintf("create table %s.%s(%s) tags (orgno int)", config.DbName, config.MetricsName, config.TableDesc) - exec(client, sql) - tbNum := config.TablePerConn*config.ConnNum + config.TableStart - for i := config.TableStart; i < tbNum; i++ { - sql := fmt.Sprintf("create table %s.%s%d using %s.%s tags(%d)", config.DbName, config.TablePrefix, i, config.DbName, config.MetricsName, i%config.TagNum+1) - exec(client, sql) - } - -} - -func insertTable(conn int) { - client := &http.Client{} - - tbStart := conn*config.TablePerConn + config.TableStart - tmStart := config.DataBegin - - for j := 0; j < config.DataNum; j++ { - for i := 0; i < config.TablePerConn; i++ { - tbIndex := i + tbStart - - tmVal := int64(j)*int64(config.DataInterval) + tmStart - - dataVal := j - if config.DataRandom { - dataVal = rand.Intn(1000) - } - - sql := fmt.Sprintf("insert into %s.%s%d values(%d, %d)", config.DbName, config.TablePrefix, tbIndex, tmVal, dataVal) - for k := 1; k < config.DataBatch; k++ { - tmVal := int64(j)*int64(config.DataInterval) + int64(k) + tmStart - - dataVal := j + k - if config.DataRandom { - dataVal = rand.Intn(1000) - } - - sql += fmt.Sprintf("(%d, %d)", tmVal, dataVal) - } - - - exec(client, sql) - if config.WaitTime != 0 { - time.Sleep(time.Millisecond * time.Duration(config.WaitTime)) - } - } - j += (config.DataBatch - 1) - } -} - -func insertLoop(conn int) { - client := &http.Client{} - - tbStart := conn*config.TablePerConn + config.TableStart - tmStart := config.DataBegin - - for j := 0; j < config.DataNum; j++ { - - for i := 0; i < config.TablePerConn; i++ { - tbIndex := i + tbStart - - tmVal := int64(j)*int64(config.DataInterval) + tmStart - - dataVal := j - if config.DataRandom { - dataVal = rand.Intn(1000) - } - - sql := fmt.Sprintf("insert into %s.%s%d values(%d, %d)", config.DbName, config.TablePrefix, tbIndex, tmVal, dataVal) - for k := 1; k < config.DataBatch; k++ { - tmVal := int64(j)*int64(config.DataInterval) + int64(k) + tmStart - - dataVal := j + k - if config.DataRandom { - dataVal = rand.Intn(1000) - } - sql += fmt.Sprintf("values(%d, %d)", tmVal, dataVal) - } - - j += (config.DataBatch - 1) - - exec(client, sql) - - if config.WaitTime != 0 { - time.Sleep(time.Millisecond * time.Duration(config.WaitTime)) - } - } - - } -} - -func insertTb(wg *sync.WaitGroup, conn int) { - defer wg.Done() - - if !config.DataInsert { - return - } - - if config.InsertModel == "insertTable" { - insertTable(conn) - } else { - insertLoop(conn) - } -} - -func selectData(wg *sync.WaitGroup, conn int) { - defer wg.Done() - - client := &http.Client{} - - for i := 0; i < config.DataNum; i++ { - exec(client, config.TableDesc) - } -} - -func main() { - filename := flag.String("config", "taos_cloud.json", "config file name") - - flag.Parse() - - readFile(*filename) - - fmt.Println("\n================http test start======================") - - var wg sync.WaitGroup - - if config.InsertModel == "selectData" { - fmt.Println("\n================select data ========================") - for i := 0; i < config.ConnNum; i++ { - wg.Add(1) - go selectData(&wg, i) - } - } else { - createDb() - createTb() - - if config.DataInsert { - fmt.Println("\n================insert data ========================") - } - - for i := 0; i < config.ConnNum; i++ { - wg.Add(1) - go insertTb(&wg, i) - } - } - - wg.Wait() - - fmt.Println("\n================http test stop ======================") - requestAvg := float64(period) / float64(1000000) / float64(request) - qps := float64(1000) / float64(requestAvg) * float64(config.ConnNum) - dps := qps * float64(config.DataBatch) - fmt.Println("====== req:", request, ", error:", errorNum, ", qps:", int64(qps), ", wait:", int64(requestAvg), "ms", ", data per second:", int64(dps)) -} diff --git a/tests/script/general/http/prepare.sim b/tests/script/general/http/prepare.sim deleted file mode 100644 index 4bf6b61198..0000000000 --- a/tests/script/general/http/prepare.sim +++ /dev/null @@ -1,54 +0,0 @@ -system sh/stop_dnodes.sh -sleep 2000 -system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c http -v 1 -system sh/exec.sh -n dnode1 -s start - -sleep 2000 -sql connect - -print ============================ dnode1 start - -print =============== step1 - prepare data -sql create database d1 -sql use d1 - -sql create table t1 (ts timestamp, i int, b binary(100)) - -sql insert into t1 values('2017-12-25 21:28:41.022', 1, 11) -sql insert into t1 values('2017-12-25 21:28:42.022', 2, '22') -sql insert into t1 values('2017-12-25 21:28:43.022', 3, "33") -sql insert into t1 values('2017-12-25 21:28:44.022', 4, '44"') -sql insert into t1 values('2017-12-25 21:28:45.022', 5, "55'") -sql insert into t1 values('2017-12-25 21:28:46.022', 6, "66'6'") -sql insert into t1 values('2017-12-25 21:28:47.022', 7, '77"7"') -sql insert into t1 values('2017-12-25 21:28:48.022', 8, '88""88') -sql insert into t1 values('2017-12-25 21:28:49.022', 9, '99\99') -sql insert into t1 values('2017-12-25 21:28:51.022', 11, '11\\11') -sql insert into t1 values('2017-12-25 21:28:52.022', 12, '22\\11') -sql insert into t1 values('2017-12-25 21:28:53.022', 13, '33\\"33') -sql insert into t1 values('2017-12-25 21:28:54.022', 14, '44\\""44') - -sleep 4000 - -print =============== step2 - login - -system_content curl 127.0.0.1:7111/rest/login/root/taosdata -print curl 127.0.0.1:7111/rest/login/root/taosdata -----> $system_content - -if $system_content != @{"status":"succ","code":0,"desc":"/KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04"}@ then - return -1 -endi - -return - -print =============== step3 - query data - -system_content curl -H 'Authorization: Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJ3d3cudGFvc2RhdGEuY29tIiwia2V5IjoiYkVsOExjdkxVZDdhOHFkdE5abXRPTnJ5cDIwMW1hMDQiLCJzdWIiOiJyb290In0.k7CkgmpOJImIkLqZqzASlPmkdeEw7Wfk4XUrqGZX-LQ' -d 'select * from t1' 127.0.0.1:7111/rest/sql/d1 -print curl 127.0.0.1:7111/rest/sql/d1 -----> $system_content -if $system_content != @{"status":"succ","head":["ts","i","b"],"data":[["2017-12-25 21:28:54.022",14,"44\\\\\"\"44"],["2017-12-25 21:28:53.022",13,"33\\\\\"33"],["2017-12-25 21:28:52.022",12,"22\\\\11"],["2017-12-25 21:28:51.022",11,"11\\\\11"],["2017-12-25 21:28:49.022",9,"99\\99"],["2017-12-25 21:28:48.022",8,"88\"\"88"],["2017-12-25 21:28:47.022",7,"77\"7\""],["2017-12-25 21:28:46.022",6,"66'6'"],["2017-12-25 21:28:45.022",5,"55'"],["2017-12-25 21:28:44.022",4,"44\""],["2017-12-25 21:28:43.022",3,"33"],["2017-12-25 21:28:42.022",2,"22"],["2017-12-25 21:28:41.022",1,"11"]],"rows":13}@ then - return -1 -endi - -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/http/restful.sim b/tests/script/general/http/restful.sim deleted file mode 100644 index 6ebf5644e7..0000000000 --- a/tests/script/general/http/restful.sim +++ /dev/null @@ -1,82 +0,0 @@ -system sh/stop_dnodes.sh -system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c wallevel -v 0 -system sh/cfg.sh -n dnode1 -c http -v 1 -system sh/cfg.sh -n dnode1 -c httpEnableRecordSql -v 1 -system sh/exec.sh -n dnode1 -s start - -sql connect - -print ============================ dnode1 start - -print =============== step1 - prepare data -sql create database d1 -sql use d1 - -sql create table table_rest (ts timestamp, i int) - -sql insert into table_rest values('2017-12-25 21:28:41.022', 1) -sql insert into table_rest values('2017-12-25 21:28:42.022', 2) -sql insert into table_rest values('2017-12-25 21:28:43.022', 3) -sql insert into table_rest values('2017-12-25 21:28:44.022', 4) -sql insert into table_rest values('2017-12-25 21:28:45.022', 5) -sql insert into table_rest values('2017-12-25 21:28:46.022', 6) -sql insert into table_rest values('2017-12-25 21:28:47.022', 7) -sql insert into table_rest values('2017-12-25 21:28:48.022', 8) -sql insert into table_rest values('2017-12-25 21:28:49.022', 9) -sql insert into table_rest values('2017-12-25 21:28:50.022', 10) - -print =============== step2 - login - -system_content curl 127.0.0.1:7111/rest/login/root/taosdata -print curl 127.0.0.1:7111/rest/login/root/taosdata -----> $system_content - -if $system_content != {"status":"succ","code":0,"desc":"/KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04"} then - return -1 -endi - -print =============== step3 - query data - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'select * from d1.table_rest' 127.0.0.1:7111/rest/sql -print curl 127.0.0.1:7111/rest/sql -----> $system_content -if $system_content != @{"status":"succ","head":["ts","i"],"column_meta":[["ts",9,8],["i",4,4]],"data":[["2017-12-25 21:28:41.022",1],["2017-12-25 21:28:42.022",2],["2017-12-25 21:28:43.022",3],["2017-12-25 21:28:44.022",4],["2017-12-25 21:28:45.022",5],["2017-12-25 21:28:46.022",6],["2017-12-25 21:28:47.022",7],["2017-12-25 21:28:48.022",8],["2017-12-25 21:28:49.022",9],["2017-12-25 21:28:50.022",10]],"rows":10}@ then - return -1 -endi - -print =============== step4 - insert data -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d "insert into d1.table_rest values('2017-12-25 21:28:51.022', 11)" 127.0.0.1:7111/rest/sql -print curl 127.0.0.1:7111/rest/sql -----> $system_content -if $system_content != @{"status":"succ","head":["affected_rows"],"column_meta":[["affected_rows",4,4]],"data":[[1]],"rows":1}@ then - return -1 -endi - -print =============== step5 - query data - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'select * from d1.table_rest' 127.0.0.1:7111/rest/sql -print curl 127.0.0.1:7111/rest/sql -----> $system_content -if $system_content != @{"status":"succ","head":["ts","i"],"column_meta":[["ts",9,8],["i",4,4]],"data":[["2017-12-25 21:28:41.022",1],["2017-12-25 21:28:42.022",2],["2017-12-25 21:28:43.022",3],["2017-12-25 21:28:44.022",4],["2017-12-25 21:28:45.022",5],["2017-12-25 21:28:46.022",6],["2017-12-25 21:28:47.022",7],["2017-12-25 21:28:48.022",8],["2017-12-25 21:28:49.022",9],["2017-12-25 21:28:50.022",10],["2017-12-25 21:28:51.022",11]],"rows":11}@ then - return -1 -endi - -print =============== step6 - query no db data - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'show dnodes' 127.0.0.1:7111/rest/sql -print curl 127.0.0.1:7111/rest/sql -----> $system_content - -print =============== step7 - change password -sql create user u1 PASS 'abcd@1234' -sql create user u2 PASS 'abcd_1234' - -system_content curl 127.0.0.1:7111/rest/login/u1/abcd@1234 -print curl 127.0.0.1:7111/rest/login/u1/abcd@1234 -----> $system_content -if $system_content != @{"status":"succ","code":0,"desc":"jIlItaLFFIPa8qdtNZmtONryp201ma04SXX8PEJowKAB/46k1gwnPNryp201ma04"}@ then - return -1 -endi - -system_content curl 127.0.0.1:7111/rest/login/u2/aabcd_1234 -print curl 127.0.0.1:7111/rest/login/u2/abcd_1234 -----> $system_content -if $system_content != @{"status":"error","code":3,"desc":"Authentication failure"}@ then - return -1 -endi - -system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/http/restful_dbname.sim b/tests/script/general/http/restful_dbname.sim deleted file mode 100644 index 75cd198507..0000000000 --- a/tests/script/general/http/restful_dbname.sim +++ /dev/null @@ -1,124 +0,0 @@ -system sh/stop_dnodes.sh -sleep 2000 -system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c http -v 1 -system sh/cfg.sh -n dnode1 -c httpDbNameMandatory -v 1 -system sh/exec.sh -n dnode1 -s start - -sleep 2000 -sql connect -sql drop database if exists db - -print ============================ dnode1 start - -print =============== step1 - login - -system_content curl 127.0.0.1:7111/rest/login/root/taosdata -print curl 127.0.0.1:7111/rest/login/root/taosdata -----> $system_content - -if $system_content != @{"status":"succ","code":0,"desc":"/KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04"}@ then - return -1 -endi - -print =============== step2 - execute sql without db_name - -print curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'show databases' 127.0.0.1:7111/rest/sql - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'show databases' 127.0.0.1:7111/rest/sql - -if $system_content != @{"status":"error","code":4354,"desc":"invalid url format"}@ then - return -1 -endi - -print curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'create database if not exists db' 127.0.0.1:7111/rest/sql - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'create database if not exists db' 127.0.0.1:7111/rest/sql - -if $system_content != @{"status":"error","code":4354,"desc":"invalid url format"}@ then - return -1 -endi - -print curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'create table table_rest (ts timestamp, i int)' 127.0.0.1:7111/rest/sql - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'create table table_rest (ts timestamp, i int)' 127.0.0.1:7111/rest/sql - -if $system_content != @{"status":"error","code":4354,"desc":"invalid url format"}@ then - return -1 -endi - -print curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'insert into table_rest (now, 1)' 127.0.0.1:7111/rest/sql - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'insert into table_rest values (now, 1)' 127.0.0.1:7111/rest/sql - -if $system_content != @{"status":"error","code":4354,"desc":"invalid url format"}@ then - return -1 -endi - -print curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'select * from table_rest' 127.0.0.1:7111/rest/sql - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'select * from table_rest' 127.0.0.1:7111/rest/sql - -if $system_content != @{"status":"error","code":4354,"desc":"invalid url format"}@ then - return -1 -endi - -print curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'drop database if exists db' 127.0.0.1:7111/rest/sql - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'drop database if exists db' 127.0.0.1:7111/rest/sql - -if $system_content != @{"status":"error","code":4354,"desc":"invalid url format"}@ then - return -1 -endi - -print =============== step3 - execute sql with db_name - -print curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'show databases' 127.0.0.1:7111/rest/sql/databases - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'show databases' 127.0.0.1:7111/rest/sql/databases - -if $system_content != @{"status":"succ","head":["name","created_time","ntables","vgroups","replica","quorum","days","keep","cache(MB)","blocks","minrows","maxrows","wallevel","fsync","comp","cachelast","precision","update","status"],"column_meta":[["name",8,32],["created_time",9,8],["ntables",4,4],["vgroups",4,4],["replica",3,2],["quorum",3,2],["days",3,2],["keep",8,24],["cache(MB)",4,4],["blocks",4,4],["minrows",4,4],["maxrows",4,4],["wallevel",2,1],["fsync",4,4],["comp",2,1],["cachelast",2,1],["precision",8,3],["update",2,1],["status",8,10]],"data":[],"rows":0}@ then - return -1 -endi - -print curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'create database if not exists db' 127.0.0.1:7111/rest/sql/db - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'create database if not exists db' 127.0.0.1:7111/rest/sql/db - -if $system_content != @{"status":"succ","head":["affected_rows"],"column_meta":[["affected_rows",4,4]],"data":[[0]],"rows":1}@ then - return -1 -endi - -print curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'create table table_rest (ts timestamp, i int)' 127.0.0.1:7111/rest/sql/db - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'create table table_rest (ts timestamp, i int)' 127.0.0.1:7111/rest/sql/db - -if $system_content != @{"status":"succ","head":["affected_rows"],"column_meta":[["affected_rows",4,4]],"data":[[0]],"rows":1}@ then - return -1 -endi - -print curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'insert into table_rest (now, 1)' 127.0.0.1:7111/rest/sql/db - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'insert into table_rest values (1629904789233, 1)' 127.0.0.1:7111/rest/sql/db - -if $system_content != @{"status":"succ","head":["affected_rows"],"column_meta":[["affected_rows",4,4]],"data":[[1]],"rows":1}@ then - return -1 -endi - -print curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'select * from table_rest' 127.0.0.1:7111/rest/sql/db - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'select * from table_rest' 127.0.0.1:7111/rest/sql/db - -if $system_content != @{"status":"succ","head":["ts","i"],"column_meta":[["ts",9,8],["i",4,4]],"data":[["2021-08-25 23:19:49.233",1]],"rows":1}@ then - return -1 -endi - -print curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'drop database if exists db' 127.0.0.1:7111/rest/sql/db - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'drop database if exists db' 127.0.0.1:7111/rest/sql/db - -if $system_content != @{"status":"succ","head":["affected_rows"],"column_meta":[["affected_rows",4,4]],"data":[[0]],"rows":1}@ then - return -1 -endi - -system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/http/restful_full.sim b/tests/script/general/http/restful_full.sim deleted file mode 100644 index a5d3ac2b86..0000000000 --- a/tests/script/general/http/restful_full.sim +++ /dev/null @@ -1,240 +0,0 @@ -system sh/stop_dnodes.sh -sleep 2000 -system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c wallevel -v 0 -system sh/cfg.sh -n dnode1 -c http -v 1 -system sh/exec.sh -n dnode1 -s start - -#sleep 2000 -sql connect - -print ============================ dnode1 start - -print =============== step1 - login - -system_content curl 127.0.0.1:7111/rest/ -print 1-> $system_content -if $system_content != @{"status":"error","code":4357,"desc":"no auth info input"}@ then - print $system_content - return -1 -endi - -system_content curl 127.0.0.1:7111/rest/xx -print 2-> $system_content -if $system_content != @{"status":"error","code":4357,"desc":"no auth info input"}@ then - return -1 -endi - -system_content curl 127.0.0.1:7111/rest/login -print 3-> $system_content -if $system_content != @{"status":"error","code":4357,"desc":"no auth info input"}@ then - return -1 -endi - -#4 -system_content curl 127.0.0.1:7111/rest/login/root -print 4-> $system_content -if $system_content != @{"status":"error","code":4357,"desc":"no auth info input"}@ then - return -1 -endi - -system_content curl 127.0.0.1:7111/rest/login/root/123 -print 5-> $system_content -if $system_content != @{"status":"error","code":3,"desc":"Authentication failure"}@ then - return -1 -endi - -system_content curl 127.0.0.1:7111/rest/login/root/123/1/1/3 -print 6-> $system_content -if $system_content != @{"status":"error","code":3,"desc":"Authentication failure"}@ then - return -1 -endi - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'show databases' 127.0.0.1:7111/rest/login/root/1 -print 7-> $system_content -if $system_content != @{"status":"error","code":3,"desc":"Authentication failure"}@ then - return -1 -endi - -#8 -system_content curl -H 'Authorization: Beare eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9' -d 'show databases' 127.0.0.1:7111/rest/login/root/1 -print 8-> $system_content -if $system_content != @{"status":"error","code":4386,"desc":"invalid type of Authorization"}@ then - return -1 -endi - -system_content curl -H 'Authorization: Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJ3d3cudGFvc2RhdGEuY29tIiwicGFzcyI6InRhb3NkYXRhIiwic3ViIjoicm9vdCJ9.xPv3b5odlR7YF8G_QWASjIRbMtA5v4ItToJ35fFgi' -d 'show databases' 127.0.0.1:7111/rest/login/root/1 -print 8-> $system_content -if $system_content != @{"status":"error","code":4387,"desc":"invalid format of Authorization"}@ then - return -1 -endi - -system_content curl -H 'Authorization: eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJ3d3cudGFvc2RhdGEuY29tIiwicGFzcyI6InRhb3NkYXRhIiwic3ViIjoicm9vdCJ9.xPv3b5odlR7YF8G_QWASjIRbMtA5v4ItToJ35fFgi' -d 'show databases' 127.0.0.1:7111/rest/login/root/1 -print 9-> $system_content -if $system_content != @{"status":"error","code":4387,"desc":"invalid format of Authorization"}@ then - return -1 -endi - -sleep 2000 - -system_content curl 127.0.0.1:7111/rest/login/root/taosdata/ -print 10-> $system_content -if $system_content != @{"status":"succ","code":0,"desc":"/KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04"}@ then - return -1 -endi - -print =============== step2 - no db - -#11 -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'show databases' 127.0.0.1:7111/rest/sql -print 11-> $system_content -if $system_content != @{"status":"succ","head":["name","created_time","ntables","vgroups","replica","quorum","days","keep0,keep1,keep2","cache(MB)","blocks","minrows","maxrows","wallevel","fsync","comp","cachelast","precision","update","status"],"column_meta":[["name",8,32],["created_time",9,8],["ntables",4,4],["vgroups",4,4],["replica",3,2],["quorum",3,2],["days",3,2],["keep0,keep1,keep2",8,24],["cache(MB)",4,4],["blocks",4,4],["minrows",4,4],["maxrows",4,4],["wallevel",2,1],["fsync",4,4],["comp",2,1],["cachelast",2,1],["precision",8,3],["update",2,1],["status",8,10]],"data":[],"rows":0}@ then - return -1 -endi - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'create database d1' 127.0.0.1:7111/rest/sql -print 12-> $system_content -if $system_content != @{"status":"succ","head":["affected_rows"],"column_meta":[["affected_rows",4,4]],"data":[[0]],"rows":1}@ then - return -1 -endi - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'create database d1' 127.0.0.1:7111/rest/sql -print 13-> $system_content -if $system_content != @{"status":"error","code":897,"desc":"Database already exists"}@ then - return -1 -endi - -#14 -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d '' 127.0.0.1:7111/rest/sql -print 14-> $system_content -if $system_content != @{"status":"error","code":4359,"desc":"no sql input"}@ then - return -1 -endi - -#system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'use d1' 127.0.0.1:7111/rest/sql -#print 15-> $system_content -#if $system_content != @{"status":"error","code":5017,"desc":"no need to execute use db cmd"}@ then -#if $system_content != @{"status":"succ","head":["affected_rows"],"data":[[1]],"rows":1}@ then -# return -1 -#endi - -#system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' use d1' 127.0.0.1:7111/rest/sql -#print 16-> $system_content -#if $system_content != @{"status":"error","code":5017,"desc":"no need to execute use db cmd"}@ then -# return -1 -#endi - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' used1' 127.0.0.1:7111/rest/sql -print 17-> $system_content -if $system_content != @{"status":"error","code":534,"desc":"Syntax error in SQL"}@ then - return -1 -endi - -#18 -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' show tables;' 127.0.0.1:7111/rest/sql -print 18-> $system_content -if $system_content != @{"status":"error","code":896,"desc":"Database not specified or available"}@ then - return -1 -endi - -print =============== step3 - db - -#19 -#system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' show tables;' 127.0.0.1:7111/rest/sql/d4 -#print 19-> $system_content -#if $system_content != @{"status":"error","code":1000,"desc":"invalid DB"}@ then -# return -1 -#endi - -#system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' show tables;' 127.0.0.1:7111/rest/sql/d1 -#print 20-> $system_content -#if $system_content != @{"status":"succ","head":["name","created time","columns","metric"],"data":[],"rows":0}@ then -# return -1 -#endi - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' select * from d1.t1;' 127.0.0.1:7111/rest/sql -print 21-> $system_content -if $system_content != @{"status":"error","code":866,"desc":"Table does not exist"}@ then - return -1 -endi - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' create table d1.t1 (ts timestamp, speed int)' 127.0.0.1:7111/rest/sql -print 22-> $system_content -if $system_content != @{"status":"succ","head":["affected_rows"],"column_meta":[["affected_rows",4,4]],"data":[[0]],"rows":1}@ then - return -1 -endi - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' select * from d1.t1 ' 127.0.0.1:7111/rest/sql -print 23-> $system_content -if $system_content != @{"status":"succ","head":["ts","speed"],"column_meta":[["ts",9,8],["speed",4,4]],"data":[],"rows":0}@ then - return -1 -endi - -#24 -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d "insert into d1.t1 values('2017-12-25 21:28:41.022', 1)" 127.0.0.1:7111/rest/sql -print 24-> $system_content -if $system_content != @{"status":"succ","head":["affected_rows"],"column_meta":[["affected_rows",4,4]],"data":[[1]],"rows":1}@ then - return -1 -endi - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' select * from d1.t1 ' 127.0.0.1:7111/rest/sql -print 25-> $system_content -if $system_content != @{"status":"succ","head":["ts","speed"],"column_meta":[["ts",9,8],["speed",4,4]],"data":[["2017-12-25 21:28:41.022",1]],"rows":1}@ then - return -1 -endi - -#26 -print 25-> no print -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d "insert into d1.t1 values('2017-12-25 21:28:42.022', 2)" 127.0.0.1:7111/rest/sql - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d "insert into d1.t1 values('2017-12-25 21:28:43.022', 3)" 127.0.0.1:7111/rest/sql - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d "insert into d1.t1 values('2017-12-25 21:28:44.022', 4)" 127.0.0.1:7111/rest/sql - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d "insert into d1.t1 values('2017-12-25 21:28:45.022', 5)" 127.0.0.1:7111/rest/sql - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d "insert into d1.t1 values('2017-12-25 21:28:46.022', 6)" 127.0.0.1:7111/rest/sql - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d "insert into d1.t1 values('2017-12-25 21:28:47.022', 7)" 127.0.0.1:7111/rest/sql - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d "insert into d1.t1 values('2017-12-25 21:28:48.022', 8)" 127.0.0.1:7111/rest/sql - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d "insert into d1.t1 values('2017-12-25 21:28:49.022', 9)" 127.0.0.1:7111/rest/sql - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d "insert into d1.t1 values('2017-12-25 21:28:50.022', 10)" 127.0.0.1:7111/rest/sql - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d "insert into d1.t1 values('2017-12-25 21:28:51.022', 11)" 127.0.0.1:7111/rest/sql - -#27 -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' select * from d1.t1 ' 127.0.0.1:7111/rest/sql -print 27-> $system_content -if $system_content != @{"status":"succ","head":["ts","speed"],"column_meta":[["ts",9,8],["speed",4,4]],"data":[["2017-12-25 21:28:41.022",1],["2017-12-25 21:28:42.022",2],["2017-12-25 21:28:43.022",3],["2017-12-25 21:28:44.022",4],["2017-12-25 21:28:45.022",5],["2017-12-25 21:28:46.022",6],["2017-12-25 21:28:47.022",7],["2017-12-25 21:28:48.022",8],["2017-12-25 21:28:49.022",9],["2017-12-25 21:28:50.022",10],["2017-12-25 21:28:51.022",11]],"rows":11}@ then - return -1 -endi - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'create database d2' 127.0.0.1:7111/rest/sql -print 28-> $system_content -if $system_content != @{"status":"succ","head":["affected_rows"],"column_meta":[["affected_rows",4,4]],"data":[[0]],"rows":1}@ then - return -1 -endi - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' create table d2.t1 (ts timestamp, speed int)' 127.0.0.1:7111/rest/sql -print 29-> $system_content -if $system_content != @{"status":"succ","head":["affected_rows"],"column_meta":[["affected_rows",4,4]],"data":[[0]],"rows":1}@ then - return -1 -endi - -#30 -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d "insert into d2.t1 values('2017-12-25 21:28:41.022', 1)" 127.0.0.1:7111/rest/sql -print 30-> $system_content -if $system_content != @{"status":"succ","head":["affected_rows"],"column_meta":[["affected_rows",4,4]],"data":[[1]],"rows":1}@ then - return -1 -endi - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' select * from d2.t1 ' 127.0.0.1:7111/rest/sql -print 31-> $system_content -if $system_content != @{"status":"succ","head":["ts","speed"],"column_meta":[["ts",9,8],["speed",4,4]],"data":[["2017-12-25 21:28:41.022",1]],"rows":1}@ then - return -1 -endi - -system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/http/restful_insert.sim b/tests/script/general/http/restful_insert.sim deleted file mode 100644 index b77a1dd497..0000000000 --- a/tests/script/general/http/restful_insert.sim +++ /dev/null @@ -1,53 +0,0 @@ -system sh/stop_dnodes.sh -sleep 2000 -system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c http -v 1 -system sh/cfg.sh -n dnode1 -c httpEnableRecordSql -v 1 -system sh/exec.sh -n dnode1 -s start - -sleep 2000 -sql connect - -print ============================ dnode1 start - -print =============== step1 - prepare data -sql create database d1 -sql use d1 - -sql create table d1.table_rest0 (ts timestamp, i int) -sql create table d1.table_rest1 (ts timestamp, i int) -sql create table d1.table_rest2 (ts timestamp, i int) -sql create table d1.table_rest3 (ts timestamp, i int) -sql create table d1.table_rest4 (ts timestamp, i int) -sql create table d1.table_rest5 (ts timestamp, i int) -sql create table d1.table_rest6 (ts timestamp, i int) -sql create table d1.table_rest7 (ts timestamp, i int) -sql create table d1.table_rest8 (ts timestamp, i int) -sql create table d1.table_rest9 (ts timestamp, i int) - -print =============== step2 - login - -system_content curl 127.0.0.1:7111/rest/login/root/taosdata -print curl 127.0.0.1:7111/rest/login/root/taosdata -----> $system_content - -if $system_content != {"status":"succ","code":0,"desc":"/KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04"} then - return -1 -endi - -print =============== step3 - query data - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'insert into d1.table_rest1 values('now+1s', 1) d1.table_rest2 values('now+1s', 1) d1.table_rest3 values('now+1s', 1) d1.table_rest4 values('now+1s', 1) ' 127.0.0.1:7111/rest/sql -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'insert into d1.table_rest1 values('now+1s', 1) d1.table_rest2 values('now+1s', 1) d1.table_rest3 values('now+1s', 1) d1.table_rest4 values('now+1s', 1) ' 127.0.0.1:7111/rest/sql -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'insert into d1.table_rest1 values('now+1s', 1) d1.table_rest2 values('now+1s', 1) d1.table_rest3 values('now+1s', 1) d1.table_rest4 values('now+1s', 1) ' 127.0.0.1:7111/rest/sql -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'insert into d1.table_rest1 values('now+1s', 1) d1.table_rest2 values('now+1s', 1) d1.table_rest3 values('now+1s', 1) d1.table_rest4 values('now+1s', 1) ' 127.0.0.1:7111/rest/sql -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'insert into d1.table_rest1 values('now+1s', 1) d1.table_rest2 values('now+1s', 1) d1.table_rest3 values('now+1s', 1) d1.table_rest4 values('now+1s', 1) ' 127.0.0.1:7111/rest/sql -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'insert into d1.table_rest1 values('now+1s', 1) d1.table_rest2 values('now+1s', 1) d1.table_rest3 values('now+1s', 1) d1.table_rest4 values('now+1s', 1) ' 127.0.0.1:7111/rest/sql -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'insert into d1.table_rest1 values('now+1s', 1) d1.table_rest2 values('now+1s', 1) d1.table_rest3 values('now+1s', 1) d1.table_rest4 values('now+1s', 1) ' 127.0.0.1:7111/rest/sql - -print =============== step5 - query data - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'select * from d1.table_rest1' 127.0.0.1:7111/rest/sql -print curl 127.0.0.1:7111/rest/sql -----> $system_content - -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/http/restful_limit.sim b/tests/script/general/http/restful_limit.sim deleted file mode 100644 index 48a4fdf7d3..0000000000 --- a/tests/script/general/http/restful_limit.sim +++ /dev/null @@ -1,46 +0,0 @@ -system sh/stop_dnodes.sh -sleep 2000 -system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/exec.sh -n dnode1 -s start - -sleep 2000 -sql connect - -print ============================ dnode1 start - -print =============== step1 - prepare data - -$dbPrefix = db -$tbPrefix = tb -$mtPrefix = st - -print =============== step1 -$i = 0 -$db = $dbPrefix . $i -$mt = $mtPrefix . $i - -sql create database $db -sql use $db -sql create table $mt (ts timestamp, tbcol bigint, t1 bigint, t2 bigint, t3 bigint, t4 bigint, t5 bigint, t6 bigint, t7 bigint, t8 bigint, t9 bigint, t0 nchar(20)) TAGS(tgcol bigint) - -$i = 0 -while $i < 2 - $tb = $tbPrefix . $i - sql create table $tb using $mt tags( 0 ) - $x = 0 - while $x < 2000 - $ms = $x . m - sql insert into $tb values (now + $ms , 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, '你好' ) - $x = $x + 1 - endw - $i = $i + 1 -endw - -system_content curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from db0.st0 limit 100' 127.0.0.1:7111/rest/sql -print curl 127.0.0.1:7111/rest/sql -----> $system_content - -#system_content curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d "select * from db0.st0 where tbname in ('tb0', 'tb1') limit 1000" 127.0.0.1:7111/rest/sql -#print curl 127.0.0.1:7111/rest/sql -----> $system_content - -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/http/taos.json b/tests/script/general/http/taos.json deleted file mode 100644 index abc25f19a2..0000000000 --- a/tests/script/general/http/taos.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "hostIp": "192.168.100.128", - "connNum": 1, - "insertModel": "insertTable", - "waitTime": 0, - "tableDesc": "ts timestamp, i int", - "tablePrefix": "t", - "tablePerConn": 1, - "tableCreate": true, - "tableStart": 1, - "dbName": "db", - "dbReplica": 1, - "dbKeep": 3650, - "dbDays": 7, - "metricsName": "mt", - "tagNum": 10, - "dataNum": 1000, - "dataBegin": 1485878400000, - "dataInterval": 1000, - "dataBatch": 1000, - "dataInsert": true, - "dataRandom": false -} diff --git a/tests/script/general/http/taos_cloud.json b/tests/script/general/http/taos_cloud.json deleted file mode 100644 index c54186b65c..0000000000 --- a/tests/script/general/http/taos_cloud.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "hostIp": "192.168.100.128", - "connNum": 10, - "insertModel": "insertTable", - "waitTime": 0, - "tableDesc": "ts timestamp, i int", - "tablePrefix": "t", - "tablePerConn": 1, - "tableCreate": true, - "tableStart": 1, - "dbName": "db", - "dbKeep": 3650, - "metricsName": "mt", - "tagNum": 10, - "dataNum": 1000, - "dataBegin": 1517414400000, - "dataInterval": 1000, - "dataBatch": 1000, - "dataInsert": true, - "dataRandom": false -} diff --git a/tests/script/general/http/telegraf.json b/tests/script/general/http/telegraf.json deleted file mode 100644 index baf7c5e711..0000000000 --- a/tests/script/general/http/telegraf.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "hostIp": "192.168.0.1", - "machineNum": 100, - "loopNum": 100, - "dbName": "db", - "dataBegin": 1485878400000 -} diff --git a/tests/script/general/http/telegraf.req b/tests/script/general/http/telegraf.req deleted file mode 100644 index 1a3b732999..0000000000 --- a/tests/script/general/http/telegraf.req +++ /dev/null @@ -1,621 +0,0 @@ -{ - "metrics": [{ - "fields": { - "result_code": 2, - "result_type": "connection_failed" - }, - "name": "net_response", - "tags": { - "host": "panshi-gsl", - "port": "80", - "protocol": "tcp", - "result": "connection_failed", - "server": "localhost" - }, - "timestamp": 1536750390000 - },{ - "fields": { - "load1": 0.27, - "load15": 0.09, - "load5": 0.22, - "n_cpus": 4, - "n_users": 4 - }, - "name": "system", - "tags": { - "host": "panshi-gsl" - }, - "timestamp": 1536750390000 - },{ - "fields": { - "uptime": 122 - }, - "name": "system", - "tags": { - "host": "panshi-gsl" - }, - "timestamp": 1536750390000 - }, - { - "fields": { - "uptime_format": " 0:02" - }, - "name": "system", - "tags": { - "host": "panshi-gsl" - }, - "timestamp": 1536750390000 - }, - { - "fields": { - "bytes_recv": 19964, - "bytes_sent": 11077, - "drop_in": 0, - "drop_out": 0, - "err_in": 0, - "err_out": 0, - "packets_recv": 237, - "packets_sent": 124 - }, - "name": "net", - "tags": { - "host": "panshi-gsl", - "interface": "eth0" - }, - "timestamp": 1536750390000 - }, - { - "fields": { - "usage_guest": 0, - "usage_guest_nice": 0, - "usage_idle": 99.39879759519036, - "usage_iowait": 0, - "usage_irq": 0, - "usage_nice": 0, - "usage_softirq": 0, - "usage_steal": 0, - "usage_system": 0.300601202404812, - "usage_user": 0.30060120240480753 - }, - "name": "cpu", - "tags": { - "cpu": "cpu0", - "host": "panshi-gsl" - }, - "timestamp": 1536750390000 - }, - { - "fields": { - "usage_guest": 0, - "usage_guest_nice": 0, - "usage_idle": 99.79959919839683, - "usage_iowait": 0, - "usage_irq": 0, - "usage_nice": 0, - "usage_softirq": 0, - "usage_steal": 0, - "usage_system": 0.2004008016032065, - "usage_user": 0 - }, - "name": "cpu", - "tags": { - "cpu": "cpu1", - "host": "panshi-gsl" - }, - "timestamp": 1536750390000 - }, - { - "fields": { - "usage_guest": 0, - "usage_guest_nice": 0, - "usage_idle": 99.7999999999999, - "usage_iowait": 0, - "usage_irq": 0, - "usage_nice": 0, - "usage_softirq": 0, - "usage_steal": 0, - "usage_system": 0.10000000000000217, - "usage_user": 0.09999999999999995 - }, - "name": "cpu", - "tags": { - "cpu": "cpu2", - "host": "panshi-gsl" - }, - "timestamp": 1536750390000 - }, - { - "fields": { - "usage_guest": 0, - "usage_guest_nice": 0, - "usage_idle": 99.7999999999999, - "usage_iowait": 0, - "usage_irq": 0, - "usage_nice": 0, - "usage_softirq": 0.09999999999999988, - "usage_steal": 0, - "usage_system": 0.09999999999999995, - "usage_user": 0 - }, - "name": "cpu", - "tags": { - "cpu": "cpu3", - "host": "panshi-gsl" - }, - "timestamp": 1536750390000 - }, - { - "fields": { - "usage_guest": 0, - "usage_guest_nice": 0, - "usage_idle": 99.74956173303289, - "usage_iowait": 0, - "usage_irq": 0, - "usage_nice": 0, - "usage_softirq": 0.025043826696719312, - "usage_steal": 0, - "usage_system": 0.12521913348359823, - "usage_user": 0.10017530678687725 - }, - "name": "cpu", - "tags": { - "cpu": "cpu-total", - "host": "panshi-gsl" - }, - "timestamp": 1536750390000 - }, - { - "fields": { - "icmp_inaddrmaskreps": 0, - "icmp_inaddrmasks": 0, - "icmp_indestunreachs": 1, - "icmp_inechoreps": 0, - "icmp_inechos": 0, - "icmp_inerrors": 0, - "icmp_inmsgs": 1, - "icmp_inparmprobs": 0, - "icmp_inredirects": 0, - "icmp_insrcquenchs": 0, - "icmp_intimeexcds": 0, - "icmp_intimestampreps": 0, - "icmp_intimestamps": 0, - "icmp_outaddrmaskreps": 0, - "icmp_outaddrmasks": 0, - "icmp_outdestunreachs": 5, - "icmp_outechoreps": 0, - "icmp_outechos": 0, - "icmp_outerrors": 0, - "icmp_outmsgs": 5, - "icmp_outparmprobs": 0, - "icmp_outredirects": 0, - "icmp_outsrcquenchs": 0, - "icmp_outtimeexcds": 0, - "icmp_outtimestampreps": 0, - "icmp_outtimestamps": 0, - "icmpmsg_intype3": 1, - "icmpmsg_outtype3": 5, - "ip_defaultttl": 64, - "ip_forwarding": 2, - "ip_forwdatagrams": 0, - "ip_fragcreates": 0, - "ip_fragfails": 0, - "ip_fragoks": 0, - "ip_inaddrerrors": 0, - "ip_indelivers": 132, - "ip_indiscards": 0, - "ip_inhdrerrors": 0, - "ip_inreceives": 132, - "ip_inunknownprotos": 0, - "ip_outdiscards": 0, - "ip_outnoroutes": 40, - "ip_outrequests": 134, - "ip_reasmfails": 0, - "ip_reasmoks": 0, - "ip_reasmreqds": 0, - "ip_reasmtimeout": 0, - "tcp_activeopens": 11, - "tcp_attemptfails": 11, - "tcp_currestab": 0, - "tcp_estabresets": 0, - "tcp_inerrs": 0, - "tcp_insegs": 22, - "tcp_maxconn": -1, - "tcp_outrsts": 11, - "tcp_outsegs": 22, - "tcp_passiveopens": 0, - "tcp_retranssegs": 0, - "tcp_rtoalgorithm": 1, - "tcp_rtomax": 120000, - "tcp_rtomin": 200, - "udp_indatagrams": 101, - "udp_inerrors": 0, - "udp_noports": 5, - "udp_outdatagrams": 109, - "udp_rcvbuferrors": 0, - "udp_sndbuferrors": 0, - "udplite_indatagrams": 0, - "udplite_inerrors": 0, - "udplite_noports": 0, - "udplite_outdatagrams": 0, - "udplite_rcvbuferrors": 0, - "udplite_sndbuferrors": 0 - }, - "name": "net", - "tags": { - "host": "panshi-gsl", - "interface": "all" - }, - "timestamp": 1536750390000 - }, - { - "fields": { - "io_time": 44, - "iops_in_progress": 0, - "read_bytes": 569344, - "read_time": 44, - "reads": 108, - "weighted_io_time": 44, - "write_bytes": 0, - "write_time": 0, - "writes": 0 - }, - "name": "diskio", - "tags": { - "host": "panshi-gsl", - "name": "sda5" - }, - "timestamp": 1536750390000 - }, - { - "fields": { - "io_time": 4900, - "iops_in_progress": 0, - "read_bytes": 262294528, - "read_time": 13607, - "reads": 8122, - "weighted_io_time": 27970, - "write_bytes": 12054528, - "write_time": 14369, - "writes": 1073 - }, - "name": "diskio", - "tags": { - "host": "panshi-gsl", - "name": "sda" - }, - "timestamp": 1536750390000 - }, - { - "fields": { - "io_time": 4874, - "iops_in_progress": 0, - "read_bytes": 261522432, - "read_time": 13538, - "reads": 7963, - "weighted_io_time": 27901, - "write_bytes": 12054528, - "write_time": 14369, - "writes": 1073 - }, - "name": "diskio", - "tags": { - "host": "panshi-gsl", - "name": "sda1" - }, - "timestamp": 1536750390000 - }, - { - "fields": { - "io_time": 15, - "iops_in_progress": 0, - "read_bytes": 2048, - "read_time": 15, - "reads": 2, - "weighted_io_time": 15, - "write_bytes": 0, - "write_time": 0, - "writes": 0 - }, - "name": "diskio", - "tags": { - "host": "panshi-gsl", - "name": "sda2" - }, - "timestamp": 1536750390000 - }, - { - "fields": { - "free": 4106956800, - "inodes_free": 526746, - "inodes_total": 752192, - "inodes_used": 225446, - "total": 12112691200, - "used": 7390433280, - "used_percent": 64.27922535963918 - }, - "name": "disk", - "tags": { - "device": "rootfs", - "fstype": "rootfs", - "host": "panshi-gsl", - "mode": "rw", - "path": "/" - }, - "timestamp": 1536750390000 - }, - { - "fields": { - "free": 4106956800, - "inodes_free": 526746, - "inodes_total": 752192, - "inodes_used": 225446, - "total": 12112691200, - "used": 7390433280, - "used_percent": 64.27922535963918 - }, - "name": "disk", - "tags": { - "device": "disk/by-uuid/d4becabf-b49c-4c1c-a4ea-b2f593018766", - "fstype": "ext3", - "host": "panshi-gsl", - "mode": "rw", - "path": "/" - }, - "timestamp": 1536750390000 - }, - { - "fields": { - "boot_time": 1536750268, - "context_switches": 135890, - "entropy_avail": 514, - "interrupts": 112317, - "processes_forked": 3595 - }, - "name": "kernel", - "tags": { - "host": "panshi-gsl" - }, - "timestamp": 1536750390000 - }, - { - "fields": { - "active": 211845120, - "available": 3829399552, - "available_percent": 92.37877479393148, - "buffered": 12083200, - "cached": 254320640, - "commit_limit": 2648322048, - "committed_as": 549027840, - "dirty": 49152, - "free": 3562995712, - "high_free": 0, - "high_total": 0, - "huge_page_size": 2097152, - "huge_pages_free": 0, - "huge_pages_total": 0, - "inactive": 214351872, - "low_free": 0, - "low_total": 0, - "mapped": 81797120, - "page_tables": 13062144, - "shared": 2682880, - "slab": 33386496, - "swap_cached": 0, - "swap_free": 575660032, - "swap_total": 575660032, - "total": 4145324032, - "used": 315924480, - "used_percent": 7.621225206068523, - "vmalloc_chunk": 35184198369280, - "vmalloc_total": 35184372087808, - "vmalloc_used": 168906752, - "wired": 0, - "write_back": 0, - "write_back_tmp": 0 - }, - "name": "mem", - "tags": { - "host": "panshi-gsl" - }, - "timestamp": 1536750390000 - }, - { - "fields": { - "IpExtInBcastOctets": 1291, - "IpExtInBcastPkts": 6, - "IpExtInMcastOctets": 0, - "IpExtInMcastPkts": 0, - "IpExtInNoRoutes": 0, - "IpExtInOctets": 12160, - "IpExtInTruncatedPkts": 0, - "IpExtOutBcastOctets": 0, - "IpExtOutBcastPkts": 0, - "IpExtOutMcastOctets": 0, - "IpExtOutMcastPkts": 0, - "IpExtOutOctets": 9881, - "TcpExtArpFilter": 0, - "TcpExtDelayedACKLocked": 0, - "TcpExtDelayedACKLost": 0, - "TcpExtDelayedACKs": 0, - "TcpExtEmbryonicRsts": 0, - "TcpExtListenDrops": 0, - "TcpExtListenOverflows": 0, - "TcpExtLockDroppedIcmps": 0, - "TcpExtOfoPruned": 0, - "TcpExtOutOfWindowIcmps": 0, - "TcpExtPAWSActive": 0, - "TcpExtPAWSEstab": 0, - "TcpExtPAWSPassive": 0, - "TcpExtPruneCalled": 0, - "TcpExtRcvPruned": 0, - "TcpExtSyncookiesFailed": 0, - "TcpExtSyncookiesRecv": 0, - "TcpExtSyncookiesSent": 0, - "TcpExtTCPAbortFailed": 0, - "TcpExtTCPAbortOnClose": 0, - "TcpExtTCPAbortOnData": 0, - "TcpExtTCPAbortOnLinger": 0, - "TcpExtTCPAbortOnMemory": 0, - "TcpExtTCPAbortOnSyn": 0, - "TcpExtTCPAbortOnTimeout": 0, - "TcpExtTCPDSACKIgnoredNoUndo": 0, - "TcpExtTCPDSACKIgnoredOld": 0, - "TcpExtTCPDSACKOfoRecv": 0, - "TcpExtTCPDSACKOfoSent": 0, - "TcpExtTCPDSACKOldSent": 0, - "TcpExtTCPDSACKRecv": 0, - "TcpExtTCPDSACKUndo": 0, - "TcpExtTCPDirectCopyFromBacklog": 0, - "TcpExtTCPDirectCopyFromPrequeue": 0, - "TcpExtTCPFACKReorder": 0, - "TcpExtTCPFastRetrans": 0, - "TcpExtTCPForwardRetrans": 0, - "TcpExtTCPFullUndo": 0, - "TcpExtTCPHPAcks": 0, - "TcpExtTCPHPHits": 0, - "TcpExtTCPHPHitsToUser": 0, - "TcpExtTCPLoss": 0, - "TcpExtTCPLossFailures": 0, - "TcpExtTCPLossUndo": 0, - "TcpExtTCPLostRetransmit": 0, - "TcpExtTCPMD5NotFound": 0, - "TcpExtTCPMD5Unexpected": 0, - "TcpExtTCPMemoryPressures": 0, - "TcpExtTCPPartialUndo": 0, - "TcpExtTCPPrequeueDropped": 0, - "TcpExtTCPPrequeued": 0, - "TcpExtTCPPureAcks": 0, - "TcpExtTCPRcvCollapsed": 0, - "TcpExtTCPRenoFailures": 0, - "TcpExtTCPRenoRecovery": 0, - "TcpExtTCPRenoRecoveryFail": 0, - "TcpExtTCPRenoReorder": 0, - "TcpExtTCPSACKDiscard": 0, - "TcpExtTCPSACKReneging": 0, - "TcpExtTCPSACKReorder": 0, - "TcpExtTCPSackFailures": 0, - "TcpExtTCPSackMerged": 0, - "TcpExtTCPSackRecovery": 0, - "TcpExtTCPSackRecoveryFail": 0, - "TcpExtTCPSackShiftFallback": 0, - "TcpExtTCPSackShifted": 0, - "TcpExtTCPSchedulerFailed": 0, - "TcpExtTCPSlowStartRetrans": 0, - "TcpExtTCPSpuriousRTOs": 0, - "TcpExtTCPTSReorder": 0, - "TcpExtTCPTimeouts": 0, - "TcpExtTW": 0, - "TcpExtTWKilled": 0, - "TcpExtTWRecycled": 0 - }, - "name": "nstat", - "tags": { - "host": "panshi-gsl", - "name": "netstat" - }, - "timestamp": 1536750390000 - }, - { - "fields": { - "IcmpInAddrMaskReps": 0, - "IcmpInAddrMasks": 0, - "IcmpInDestUnreachs": 1, - "IcmpInEchoReps": 0, - "IcmpInEchos": 0, - "IcmpInErrors": 0, - "IcmpInMsgs": 1, - "IcmpInParmProbs": 0, - "IcmpInRedirects": 0, - "IcmpInSrcQuenchs": 0, - "IcmpInTimeExcds": 0, - "IcmpInTimestampReps": 0, - "IcmpInTimestamps": 0, - "IcmpMsgInType3": 1, - "IcmpMsgOutType3": 5, - "IcmpOutAddrMaskReps": 0, - "IcmpOutAddrMasks": 0, - "IcmpOutDestUnreachs": 5, - "IcmpOutEchoReps": 0, - "IcmpOutEchos": 0, - "IcmpOutErrors": 0, - "IcmpOutMsgs": 5, - "IcmpOutParmProbs": 0, - "IcmpOutRedirects": 0, - "IcmpOutSrcQuenchs": 0, - "IcmpOutTimeExcds": 0, - "IcmpOutTimestampReps": 0, - "IcmpOutTimestamps": 0, - "IpDefaultTTL": 64, - "IpForwDatagrams": 0, - "IpForwarding": 2, - "IpFragCreates": 0, - "IpFragFails": 0, - "IpFragOKs": 0, - "IpInAddrErrors": 0, - "IpInDelivers": 132, - "IpInDiscards": 0, - "IpInHdrErrors": 0, - "IpInReceives": 132, - "IpInUnknownProtos": 0, - "IpOutDiscards": 0, - "IpOutNoRoutes": 40, - "IpOutRequests": 134, - "IpReasmFails": 0, - "IpReasmOKs": 0, - "IpReasmReqds": 0, - "IpReasmTimeout": 0, - "TcpActiveOpens": 12, - "TcpAttemptFails": 12, - "TcpCurrEstab": 0, - "TcpEstabResets": 0, - "TcpInErrs": 0, - "TcpInSegs": 24, - "TcpMaxConn": -1, - "TcpOutRsts": 12, - "TcpOutSegs": 24, - "TcpPassiveOpens": 0, - "TcpRetransSegs": 0, - "TcpRtoAlgorithm": 1, - "TcpRtoMax": 120000, - "TcpRtoMin": 200, - "UdpInDatagrams": 101, - "UdpInErrors": 0, - "UdpLiteInDatagrams": 0, - "UdpLiteInErrors": 0, - "UdpLiteNoPorts": 0, - "UdpLiteOutDatagrams": 0, - "UdpLiteRcvbufErrors": 0, - "UdpLiteSndbufErrors": 0, - "UdpNoPorts": 5, - "UdpOutDatagrams": 109, - "UdpRcvbufErrors": 0, - "UdpSndbufErrors": 0 - }, - "name": "nstat", - "tags": { - "host": "panshi-gsl", - "name": "snmp" - }, - "timestamp": 1536750390000 - }, - { - "fields": { - "blocked": 0, - "dead": 0, - "idle": 0, - "paging": 0, - "running": 1, - "sleeping": 181, - "stopped": 0, - "total": 182, - "total_threads": 280, - "unknown": 0, - "zombies": 0 - }, - "name": "processes", - "tags": { - "host": "panshi-gsl" - }, - "timestamp": 1536750390000 - }] -} \ No newline at end of file diff --git a/tests/script/general/http/telegraf.sim b/tests/script/general/http/telegraf.sim deleted file mode 100644 index 9fc153b232..0000000000 --- a/tests/script/general/http/telegraf.sim +++ /dev/null @@ -1,292 +0,0 @@ -system sh/stop_dnodes.sh -sleep 2000 -system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c http -v 1 -system sh/cfg.sh -n dnode1 -c httpEnableRecordSql -v 1 -system sh/cfg.sh -n dnode1 -c telegrafUseFieldNum -v 0 -system sh/exec.sh -n dnode1 -s start - -sleep 2000 -sql connect - -print ============================ dnode1 start - -print =============== step1 - parse -system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":1535784122}' 127.0.0.1:7111/telegraf/ -print $system_content - -if $system_content != @{"status":"error","code":4448,"desc":"database name can not be null"}@ then - return -1 -endi - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'select * from d1.table_admin' -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":1535784122}' 127.0.0.1:7111/telegraf/ -print $system_content - -if $system_content != @{"status":"error","code":4448,"desc":"database name can not be null"}@ then - return -1 -endi - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'select * from d1.table_admin' -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":1535784122}' 127.0.0.1:7111/telegraf/d123456789012345678901234567890123456 -print $system_content - -if $system_content != @{"status":"error","code":4449,"desc":"database name too long"}@ then - return -1 -endi - -system_content curl -u root:taosdata -d '[]' 127.0.0.1:7111/telegraf/db/root/taosdata1 -print $system_content - -if $system_content != @{"status":"error","code":4453,"desc":"metric name not find"}@ then - return -1 -endi - -system_content curl -u root:taosdata -d '{}' 127.0.0.1:7111/telegraf/db/root/taosdata1 -print $system_content - -if $system_content != @{"status":"error","code":4453,"desc":"metric name not find"}@ then - return -1 -endi - -system_content curl -u root:taosdata -d '[{}]' 127.0.0.1:7111/telegraf/db/root/taosdata1 -print $system_content - -if $system_content != @{"status":"error","code":4453,"desc":"metric name not find"}@ then - return -1 -endi - -system_content curl -u root:taosdata -d '{"metrics": []}' 127.0.0.1:7111/telegraf/db/root/taosdata1 -print $system_content - -if $system_content != @{"status":"error","code":4451,"desc":"metrics size is 0"}@ then - return -1 -endi - -system_content curl -u root:taosdata -d '{"metrics": [{}]}' 127.0.0.1:7111/telegraf/db/root/taosdata1 -print $system_content - -if $system_content != @{"status":"error","code":4453,"desc":"metric name not find"}@ then - return -1 -endi - -system_content curl -u root:taosdata -d '{"metrics": 12}' 127.0.0.1:7111/telegraf/db/root/taosdata1 -print $system_content - -if $system_content != @{"status":"error","code":4451,"desc":"metrics size is 0"}@ then - return -1 -endi - -#system_content curl -u root:taosdata -d '{"metrics": [{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{}]}' 127.0.0.1:7111/telegraf/db/root/taosdata1 -#print $system_content - -#if $system_content != @{"status":"error","code":4452,"desc":"metrics size can not more than 50"}@ then -# return -1 -#endi - -system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":1535784122}' 127.0.0.1:7111/telegraf/db/root/taosdata1 -print $system_content - -if $system_content != @{"status":"error","code":4453,"desc":"metric name not find"}@ then - return -1 -endi - -system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":111,"tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":1535784122}' 127.0.0.1:7111/telegraf/db/root/taosdata1 -print $system_content - -if $system_content != @{"status":"error","code":4454,"desc":"metric name type should be string"}@ then - return -1 -endi - -system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"","tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":1535784122}' 127.0.0.1:7111/telegraf/db/root/taosdata1 -print $system_content - -if $system_content != @{"status":"error","code":4455,"desc":"metric name length is 0"}@ then - return -1 -endi - -system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234","tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":1535784122}' 127.0.0.1:7111/telegraf/db/root/taosdata1 -print $system_content - -if $system_content != @{"status":"error","code":4456,"desc":"metric name length too long"}@ then - return -1 -endi - -system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":"windows","instance":"1","objectname":"Processor"}}' 127.0.0.1:7111/telegraf/db/root/taosdata1 -print $system_content - -if $system_content != @{"status":"error","code":4457,"desc":"timestamp not find"}@ then - return -1 -endi - -system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":""}' 127.0.0.1:7111/telegraf/db/root/taosdata1 -print $system_content - -if $system_content != @{"status":"error","code":4458,"desc":"timestamp type should be integer"}@ then - return -1 -endi - -system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":-1}' 127.0.0.1:7111/telegraf/db/root/taosdata1 -print $system_content - -if $system_content != @{"status":"error","code":4459,"desc":"timestamp value smaller than 0"}@ then - return -1 -endi - -system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","timestamp":1535784122}' 127.0.0.1:7111/telegraf/db/root/taosdata1 -print $system_content - -if $system_content != @{"status":"error","code":4460,"desc":"tags not find"}@ then - return -1 -endi - -system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{},"timestamp":1535784122}' 127.0.0.1:7111/telegraf/db/root/taosdata1 -print $system_content - -if $system_content != @{"status":"error","code":4461,"desc":"tags size is 0"}@ then - return -1 -endi - -system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":"","timestamp":1535784122}' 127.0.0.1:7111/telegraf/db/root/taosdata1 -print $system_content - -if $system_content != @{"status":"error","code":4461,"desc":"tags size is 0"}@ then - return -1 -endi - -#system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":"windows","instance":"1","objectname":"Processor","host":"windows","instance":"1","objectname":"Processor","objectname":"Processor"},"timestamp":1535784122}' 127.0.0.1:7111/telegraf/db/root/taosdata -#print $system_content - -#if $system_content != @{"status":"error","code":4461,"desc":"tags size too long"}@ then -# return -1 -#endi - -system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{},"timestamp":1535784122}' 127.0.0.1:7111/telegraf/db/root/taosdata1 -print $system_content - -if $system_content != @{"status":"error","code":4461,"desc":"tags size is 0"}@ then - return -1 -endi - -system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"":"windows"},"timestamp":1535784122}' 127.0.0.1:7111/telegraf/db/root/taosdata1 -print $system_content - -if $system_content != @{"status":"error","code":4464,"desc":"tag name is null"}@ then - return -1 -endi - -#system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host111111111111222222222222222222222":""},"timestamp":1535784122}' 127.0.0.1:7111/telegraf/db/root/taosdata1 -#print $system_content - -#if $system_content != @{"status":"error","code":4465,"desc":"tag name length too long"}@ then -# return -1 -#endi - -system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":true},"timestamp":1535784122}' 127.0.0.1:7111/telegraf/db/root/taosdata1 -print $system_content - -if $system_content != @{"status":"error","code":4466,"desc":"tag value type should be number or string"}@ then - return -1 -endi - -system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":""},"timestamp":1535784122}' 127.0.0.1:7111/telegraf/db/root/taosdata1 -print $system_content - -if $system_content != @{"status":"error","code":4467,"desc":"tag value is null"}@ then - return -1 -endi - -system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"5022":"111"},"timestamp":1535784122}' 127.0.0.1:7111/telegraf/db/root/taosdata1 -print $system_content - -if $system_content != @{"status":"error","code":4468,"desc":"table is null"}@ then - return -1 -endi - -system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":"host111111111111222222222222222222222host111111111111222222222222222222222host111111111111222222222222222222222host111111111111222222222222222222222host111111111111222222222222222222222host111111111111222222222222222222222host111111111111222222222222222222222host111111111111222222222222222222222"},"timestamp":1535784122}' 127.0.0.1:7111/telegraf/db/root/taosdata1 -print $system_content - -if $system_content != @{"status":"error","code":4469,"desc":"table name length too long"}@ then - return -1 -endi - -system_content curl -u root:taosdata -d '{"fields":{},"name":"win_cpu","tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":1535784122}' 127.0.0.1:7111/telegraf/db/root/taosdata1 -print $system_content - -if $system_content != @{"status":"error","code":4471,"desc":"fields size is 0"}@ then - return -1 -endi - -system_content curl -u root:taosdata -d '{"fields":{"":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":1535784122}' 127.0.0.1:7111/telegraf/db/root/taosdata1 -print $system_content - -if $system_content != @{"status":"error","code":4474,"desc":"field name is null"}@ then - return -1 -endi - -system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":"","Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":1535784122}' 127.0.0.1:7111/telegraf/db/root/taosdata1 -print $system_content - -if $system_content != @{"status":"error","code":4477,"desc":"field value is null"}@ then - return -1 -endi - -system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":true,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":1535784122}' 127.0.0.1:7111/telegraf/db/root/taosdata1 -print $system_content - -if $system_content != @{"status":"error","code":4476,"desc":"field value type should be number or string"}@ then - return -1 -endi - -system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":1564641722000}' 127.0.0.1:7111/telegraf/db - -print $system_content - -if $system_content != @{"metrics":[{"metric":"win_cpu","stable":"win_cpu","table":"win_cpu_windows_1_Processor","timestamp":"1564641722000","affected_rows":1,"status":"succ"}]}@ then - return -1 -endi - -sleep 2000 - -print =============== step2 - insert single data -system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":1564641722000}' 127.0.0.1:7111/telegraf/db/ - -print $system_content - -#if $system_content != @{"metrics":[{"metric":"win_cpu","stable":"win_cpu","table":"win_cpu_windows_1_Processor","timestamp":"1564641722000","affected_rows":1,"status":"succ"}]}@ then -# return -1 -#endi - -system_content curl -u root:taosdata -d 'select * from db.win_cpu_windows_1_Processor' 127.0.0.1:7111/rest/sql/ - -print $system_content - -#if $system_content != @{"status":"succ","head":["ts","f_percent_dpc_time","f_percent_idle_time","f_percent_interrupt_time","f_percent_privileged_time","f_percent_processor_time","f_percent_user_time"],"data":[["2019-08-01 06:42:02.000",0.000000000,95.598305000,0.000000000,0.000000000,0.000000000,0.000000000]],"rows":1}@ then -# return -1 -#endi - -print =============== step3 - multi-query data -system_content curl -u root:taosdata -d '{"metrics": [{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":"window1","instance":"1","objectname":"Processor"},"timestamp":1564641723000},{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":"window2","instance":"1","objectname":"Processor"},"timestamp":1564641723000}]}' 127.0.0.1:7111/telegraf/db/ - -print $system_content - -if $system_content != @{"metrics":[{"metric":"win_cpu","stable":"win_cpu","table":"win_cpu_window1_1_Processor","timestamp":"1564641723000","affected_rows":1,"status":"succ"},{"metric":"win_cpu","stable":"win_cpu","table":"win_cpu_window2_1_Processor","timestamp":"1564641723000","affected_rows":1,"status":"succ"}]}@ then - return -1 -endi - -system_content curl -u root:taosdata -d 'select * from db.win_cpu_window1_1_Processor' 127.0.0.1:7111/rest/sql/ - -print $system_content - -#if $system_content != @{"status":"succ","head":["ts","f_percent_dpc_time","f_percent_idle_time","f_percent_interrupt_time","f_percent_privileged_time","f_percent_processor_time","f_percent_user_time"],"data":[["2019-08-01 14:42:03.000",0.000000000,95.598305000,0.000000000,0.000000000,0.000000000,0.000000000]],"rows":1}@ then -# return -1 -#endi - -system_content curl -u root:taosdata -d 'select count(*) from db.win_cpu' 127.0.0.1:7111/rest/sql/ - -print $system_content - -if $system_content != @{"status":"succ","head":["count(*)"],"column_meta":[["count(*)",5,8]],"data":[[3]],"rows":1}@ then - return -1 -endi - -system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/http/telegrafTest.go b/tests/script/general/http/telegrafTest.go deleted file mode 100644 index caa6493c82..0000000000 --- a/tests/script/general/http/telegrafTest.go +++ /dev/null @@ -1,238 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package main - -import ( - "bytes" - "encoding/json" - "fmt" - "strings" - "io/ioutil" - "net/http" - "os" - "sync" - "sync/atomic" - "time" - "flag" -) - -var ( - token string - url string - config Config - request int64 - period int64 - errorNum int64 - template string -) - -type Config struct { - HostIp string `json:"hostIp"` - MachineNum int `json:"machineNum"` - LoopNum int `json:"loopNum"` - DbName string `json:"dbName"` - DataBegin int64 `json:"dataBegin"` -} - -type TokenResult struct { - Status string `json:"status"` - Code int `json:"code"` - Desc string `json:"desc"` -} - -type JsonResult struct { - Status string `json:"status"` - Code int `json:"code"` -} - -func readConf(filename string) { - file, err := os.Open(filename) - if err != nil { - println("%s not found", filename) - panic(err) - } - defer file.Close() - - dec := json.NewDecoder(file) - err = dec.Decode(&config) - if err != nil { - println("%s parse error", filename) - panic(err) - } - - request = 0 - period = 0 - errorNum = 0 - - fmt.Println("================config parameters======================") - fmt.Println("HostIp:", config.HostIp) - fmt.Println("MachineNum:", config.MachineNum) - fmt.Println("LoopNum:", config.LoopNum) - fmt.Println("dbName:", config.DbName) - fmt.Println("dataBegin:", config.DataBegin) - - fmt.Println("================http token=============================") - token, err = getToken() - url = fmt.Sprintf("http://%s:%d/telegraf/%s", config.HostIp, 6020, config.DbName) - - fmt.Println("httpToken:", token) - fmt.Println("httpUrl:", url) - - if err != nil { - panic(err) - } -} - -func readReq(filename string) { - file, err := os.Open(filename) - if err != nil { - println("%s not found", filename) - panic(err) - } - defer file.Close() - - data, _ := ioutil.ReadAll(file) - - template = string(data[:]) - - //fmt.Println(template) -} - -func getToken() (string, error) { - resp, err := http.Get(fmt.Sprintf("http://%s:%d/rest/login/%s/%s", config.HostIp, 6020, "root", "taosdata")) - if err != nil { - return "", err - } - - defer resp.Body.Close() - - var tokenResult TokenResult - - data, err := ioutil.ReadAll(resp.Body) - - if err != nil { - return "", err - } - - err = json.Unmarshal(data, &tokenResult) - if err != nil { - return "", err - } - - if tokenResult.Status != "succ" { - fmt.Println("get http token failed") - fmt.Println(tokenResult) - return "", err - } - - return tokenResult.Desc, nil -} - -func exec(client *http.Client, sql string) { - for times := 0; times < 10; times++ { - - req, err1 := http.NewRequest("POST", url, bytes.NewReader([]byte(sql))) - if err1 != nil { - continue - } - req.Header.Add("Authorization", "Taosd "+token) - - begin := time.Now() - resp, err := client.Do(req) - - if err != nil { - continue - } - - data, err := ioutil.ReadAll(resp.Body) - if err != nil { - fmt.Println(data) - resp.Body.Close() - continue - } - - spend := (time.Since(begin).Nanoseconds()) - - var jsonResult JsonResult - err = json.Unmarshal(data, &jsonResult) - if err != nil { - fmt.Println("parse json error: ", string(data[:])) - resp.Body.Close() - continue - } - - - atomic.AddInt64(&request, 1) - atomic.AddInt64(&period, spend) - - if request%1000 == 0 && request != 0 { - requestAvg := float64(period) / float64(1000000) / float64(request) - qps := float64(1000) / float64(requestAvg) * float64(config.MachineNum) - dps := qps * float64(22) - fmt.Println("====== req:", request, ", error:", errorNum, ", qps:", int64(qps), ", wait:", int64(requestAvg), "ms", ", data per second:", int64(dps)) - } - return - } - //fmt.Println("xxxx>sql:", sql, ", retryTimes:", 10) - fmt.Println("exec sql failed") - errorNum++ -} - -func writeData(wg *sync.WaitGroup, tbIndex int) { - defer wg.Done() - client := &http.Client{} - - tbName := fmt.Sprintf("t%d", tbIndex) - - for j := 0; j < config.LoopNum; j++ { - tmVal := fmt.Sprintf("%d", int64(j)*int64(10000) + config.DataBegin) - //fmt.Println(tmVal) - - req1 := strings.Replace(template, "panshi-gsl", tbName, -1) - req2 := strings.Replace(req1, "1536750390000", tmVal, -1) - - //fmt.Println(req2) - exec(client, req2) - } -} - -func main() { - filename := flag.String("config", "telegraf.json", "config file name") - - flag.Parse() - - readReq("telegraf.req") - - readConf(*filename) - - fmt.Println("\n================telegraf test start======================") - - var wg sync.WaitGroup - - for i := 0; i < config.MachineNum; i++ { - wg.Add(1) - go writeData(&wg, i) - } - - wg.Wait() - - fmt.Println("\n================telegraf test stop ======================") - - requestAvg := float64(period) / float64(1000000) / float64(request) - qps := float64(1000) / float64(requestAvg) * float64(config.MachineNum) - dps := qps * float64(22) - fmt.Println("====== req:", request, ", error:", errorNum, ", qps:", int64(qps), ", wait:", int64(requestAvg), "ms", ", data per second:", int64(dps)) -} diff --git a/tests/script/general/http/telegraf_help.txt b/tests/script/general/http/telegraf_help.txt deleted file mode 100644 index a2e6d6e49c..0000000000 --- a/tests/script/general/http/telegraf_help.txt +++ /dev/null @@ -1,24 +0,0 @@ -1.±àÒë -½«main.go±àÒë³É¿ÉÖ´ÐÐÎļþ£¬telegrafTest -2.Ö´ÐÐ -./telegrafTest -3.²ÎÊý telegraf.jsonµÄÅäÖÃÎļþ - hostIp TDengine»úÆ÷IPµØÖ· - machineNum Ä£ÄâµÄ»úÆ÷ÊýÁ¿ - loopNum ÿ¸ö»úÆ÷·¢ËÍÇëÇóµÄ´ÎÊý - dbName ´´½¨µÄÊý¾Ý¿âÃû³Æ - dataBegin Éú³ÉÄ£ÄâÊý¾ÝµÄ¿ªÊ¼Ê±¼ä´Á -4.telegraf.req£¨Í¨³£²»±ä£© - 1£©Ìæ»»panshi-gslºÍ1536750380000ÕâÁ½¸ö×Ö·û´®£¬Éú³ÉhttpÇëÇó - 2£©Í¨¹ýhttp·¢Ë͸øTDengine -5.×¢Òâ - 1£©ÐÞ¸ÄÅäÖÃÎļþ/etc/taos/taos.cfg - httpDebugFlag ÉèÖÃΪ131£¨131-httpDebug,135-httpDebug£© - httpMaxThreads °´ÕÕ»úÆ÷Çé¿öÉèÖã¬ÉèÖ÷¶Î§Îª£¨1-cpu cores) - httpCacheSessions ÉèÖÃΪ¿ÉÄܵÄhttpÁ¬½ÓµÄ2±¶ - maxVnodeConnections ÉèÖÃΪ100000 - maxMeterConnections ÉèÖÃΪ100000 - maxShellConns ÉèÖÃΪ100000 - maxMgmtConnections ÉèÖÃΪ100000 - 2£©ÉèÖÃlinux²ÎÊý£¬ÀýÈç - ulimit -n 600000 \ No newline at end of file diff --git a/tests/script/general/http/testSuite.sim b/tests/script/general/http/testSuite.sim deleted file mode 100644 index f35362bf07..0000000000 --- a/tests/script/general/http/testSuite.sim +++ /dev/null @@ -1,10 +0,0 @@ -run general/http/autocreate.sim -run general/http/chunked.sim -run general/http/restful.sim -run general/http/restful_insert.sim -run general/http/restful_limit.sim -run general/http/restful_full.sim -run general/http/prepare.sim -run general/http/telegraf.sim -run general/http/grafana_bug.sim -run general/http/grafana.sim From 536e3aeb9db32a654cb0172687111d8d420ef317 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 27 Dec 2021 18:42:33 +0800 Subject: [PATCH 03/21] feature/qnode --- include/common/tmsg.h | 12 +- source/dnode/vnode/impl/src/vnodeQuery.c | 20 +- source/libs/qworker/inc/qworkerInt.h | 28 +- source/libs/qworker/src/qworker.c | 577 ++++++++++++---------- source/libs/qworker/test/qworkerTests.cpp | 35 +- source/libs/scheduler/inc/schedulerInt.h | 2 +- source/libs/scheduler/src/scheduler.c | 10 +- 7 files changed, 401 insertions(+), 283 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 92b453bc1f..a49ce5f6ce 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1002,7 +1002,7 @@ typedef struct { } SUpdateTagValRsp; typedef struct SSubQueryMsg { - uint64_t schedulerId; + uint64_t sId; uint64_t queryId; uint64_t taskId; uint32_t contentLen; @@ -1010,7 +1010,7 @@ typedef struct SSubQueryMsg { } SSubQueryMsg; typedef struct SResReadyMsg { - uint64_t schedulerId; + uint64_t sId; uint64_t queryId; uint64_t taskId; } SResReadyMsg; @@ -1020,13 +1020,13 @@ typedef struct SResReadyRsp { } SResReadyRsp; typedef struct SResFetchMsg { - uint64_t schedulerId; + uint64_t sId; uint64_t queryId; uint64_t taskId; } SResFetchMsg; typedef struct SSchTasksStatusMsg { - uint64_t schedulerId; + uint64_t sId; } SSchTasksStatusMsg; typedef struct STaskStatus { @@ -1041,7 +1041,7 @@ typedef struct SSchedulerStatusRsp { } SSchedulerStatusRsp; typedef struct STaskCancelMsg { - uint64_t schedulerId; + uint64_t sId; uint64_t queryId; uint64_t taskId; } STaskCancelMsg; @@ -1051,7 +1051,7 @@ typedef struct STaskCancelRsp { } STaskCancelRsp; typedef struct STaskDropMsg { - uint64_t schedulerId; + uint64_t sId; uint64_t queryId; uint64_t taskId; } STaskDropMsg; diff --git a/source/dnode/vnode/impl/src/vnodeQuery.c b/source/dnode/vnode/impl/src/vnodeQuery.c index 31481bf7c4..cc743d658e 100644 --- a/source/dnode/vnode/impl/src/vnodeQuery.c +++ b/source/dnode/vnode/impl/src/vnodeQuery.c @@ -22,13 +22,27 @@ int vnodeQueryOpen(SVnode *pVnode) { int vnodeProcessQueryReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { vInfo("query message is processed"); - qWorkerProcessQueryMsg(pVnode, pVnode->pQuery, pMsg); - return 0; + return qWorkerProcessQueryMsg(pVnode, pVnode->pQuery, pMsg); } int vnodeProcessFetchReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { vInfo("fetch message is processed"); - qWorkerProcessFetchMsg(pVnode, pVnode->pQuery, pMsg); + switch (pMsg->msgType) { + case TDMT_VND_FETCH: + return qWorkerProcessFetchMsg(pVnode, pVnode->pQuery, pMsg); + case TDMT_VND_RES_READY: + return qWorkerProcessReadyMsg(pVnode, pVnode->pQuery, pMsg); + case TDMT_VND_TASKS_STATUS: + return qWorkerProcessStatusMsg(pVnode, pVnode->pQuery, pMsg); + case TDMT_VND_CANCEL_TASK: + return qWorkerProcessCancelMsg(pVnode, pVnode->pQuery, pMsg); + case TDMT_VND_DROP_TASK: + return qWorkerProcessDropMsg(pVnode, pVnode->pQuery, pMsg); + default: + vError("unknown msg type:%d in fetch queue", pMsg->msgType); + return TSDB_CODE_VND_APP_ERROR; + break; + } return 0; } diff --git a/source/libs/qworker/inc/qworkerInt.h b/source/libs/qworker/inc/qworkerInt.h index 6f454e2f81..825e891c8b 100644 --- a/source/libs/qworker/inc/qworkerInt.h +++ b/source/libs/qworker/inc/qworkerInt.h @@ -42,25 +42,41 @@ enum { QW_WRITE, }; -typedef struct SQWorkerTaskStatus { +enum { + QW_EXIST_ACQUIRE = 1, + QW_EXIST_RET_ERR, +}; + +enum { + QW_NOT_EXIST_RET_ERR = 1, + QW_NOT_EXIST_ADD, +}; + +enum { + QW_ADD_RET_ERR = 1, + QW_ADD_ACQUIRE, +}; + + +typedef struct SQWTaskStatus { SRWLatch lock; int32_t code; int8_t status; int8_t ready; bool cancel; bool drop; -} SQWorkerTaskStatus; +} SQWTaskStatus; typedef struct SQWorkerResCache { SRWLatch lock; void *data; } SQWorkerResCache; -typedef struct SQWorkerSchStatus { +typedef struct SQWSchStatus { int32_t lastAccessTs; // timestamp in second SRWLatch tasksLock; SHashObj *tasksHash; // key:queryId+taskId, value: SQWorkerTaskStatus -} SQWorkerSchStatus; +} SQWSchStatus; // Qnode/Vnode level task management typedef struct SQWorkerMgmt { @@ -71,7 +87,7 @@ typedef struct SQWorkerMgmt { SHashObj *resHash; //key: queryId+taskId, value: SQWorkerResCache } SQWorkerMgmt; -#define QW_GOT_RES_DATA(data) (false) +#define QW_GOT_RES_DATA(data) (true) #define QW_LOW_RES_DATA(data) (false) #define QW_TASK_NOT_EXIST(code) (TSDB_CODE_QRY_SCH_NOT_EXIST == (code) || TSDB_CODE_QRY_TASK_NOT_EXIST == (code)) @@ -89,6 +105,8 @@ typedef struct SQWorkerMgmt { #define QW_LOCK(type, _lock) (QW_READ == (type) ? taosRLockLatch(_lock) : taosWLockLatch(_lock)) #define QW_UNLOCK(type, _lock) (QW_READ == (type) ? taosRUnLockLatch(_lock) : taosWUnLockLatch(_lock)) +static int32_t qwAcquireScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t sId, SQWSchStatus **sch, int32_t nOpt); + #ifdef __cplusplus } diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 149f46273c..7a861a0b8f 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -4,38 +4,42 @@ #include "qworkerInt.h" #include "planner.h" -int32_t qwCheckStatusSwitch(int8_t oriStatus, int8_t newStatus) { +int32_t qwValidateStatus(int8_t oriStatus, int8_t newStatus) { int32_t code = 0; if (oriStatus == newStatus) { - if (newStatus == JOB_TASK_STATUS_CANCELLING) { - return TSDB_CODE_SUCCESS; - } - QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } switch (oriStatus) { case JOB_TASK_STATUS_NULL: - if (newStatus != JOB_TASK_STATUS_EXECUTING && newStatus != JOB_TASK_STATUS_FAILED ) { + if (newStatus != JOB_TASK_STATUS_EXECUTING + && newStatus != JOB_TASK_STATUS_FAILED + && newStatus != JOB_TASK_STATUS_NOT_START) { QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } break; case JOB_TASK_STATUS_NOT_START: - if (newStatus != JOB_TASK_STATUS_EXECUTING && newStatus != JOB_TASK_STATUS_FAILED) { + if (newStatus != JOB_TASK_STATUS_CANCELLED) { QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } break; case JOB_TASK_STATUS_EXECUTING: - if (newStatus != JOB_TASK_STATUS_SUCCEED && newStatus != JOB_TASK_STATUS_FAILED && newStatus != JOB_TASK_STATUS_CANCELLING) { + if (newStatus != JOB_TASK_STATUS_PARTIAL_SUCCEED + && newStatus != JOB_TASK_STATUS_FAILED + && newStatus != JOB_TASK_STATUS_CANCELLING + && newStatus != JOB_TASK_STATUS_CANCELLED + && newStatus != JOB_TASK_STATUS_DROPPING) { QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } break; case JOB_TASK_STATUS_PARTIAL_SUCCEED: - if (newStatus != JOB_TASK_STATUS_EXECUTING && newStatus != JOB_TASK_STATUS_CANCELLING) { + if (newStatus != JOB_TASK_STATUS_EXECUTING + && newStatus != JOB_TASK_STATUS_SUCCEED + && newStatus != JOB_TASK_STATUS_CANCELLED) { QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } @@ -49,6 +53,10 @@ int32_t qwCheckStatusSwitch(int8_t oriStatus, int8_t newStatus) { break; case JOB_TASK_STATUS_CANCELLED: + case JOB_TASK_STATUS_DROPPING: + QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + break; + default: qError("invalid task status:%d", oriStatus); return TSDB_CODE_QRY_APP_ERROR; @@ -58,17 +66,17 @@ int32_t qwCheckStatusSwitch(int8_t oriStatus, int8_t newStatus) { _return: - qError("invalid task status:%d", oriStatus); + qError("invalid task status, from %d to %d", oriStatus, newStatus); QW_ERR_RET(code); } -int32_t qwUpdateTaskInfo(SQWorkerTaskStatus *task, int8_t type, void *data) { +int32_t qwUpdateTaskInfo(SQWTaskStatus *task, int8_t type, void *data) { int32_t code = 0; switch (type) { case QW_TASK_INFO_STATUS: { int8_t newStatus = *(int8_t *)data; - QW_ERR_RET(qwCheckStatusSwitch(task->status, newStatus)); + QW_ERR_RET(qwValidateStatus(task->status, newStatus)); task->status = newStatus; break; } @@ -80,9 +88,9 @@ int32_t qwUpdateTaskInfo(SQWorkerTaskStatus *task, int8_t type, void *data) { return TSDB_CODE_SUCCESS; } -int32_t qwAddTaskResult(SQWorkerMgmt *mgmt, uint64_t queryId, uint64_t taskId, void *data) { - char id[sizeof(queryId) + sizeof(taskId)] = {0}; - QW_SET_QTID(id, queryId, taskId); +int32_t qwAddTaskResCache(SQWorkerMgmt *mgmt, uint64_t qId, uint64_t tId, void *data) { + char id[sizeof(qId) + sizeof(tId)] = {0}; + QW_SET_QTID(id, qId, tId); SQWorkerResCache resCache = {0}; resCache.data = data; @@ -90,7 +98,7 @@ int32_t qwAddTaskResult(SQWorkerMgmt *mgmt, uint64_t queryId, uint64_t taskId, v QW_LOCK(QW_WRITE, &mgmt->resLock); if (0 != taosHashPut(mgmt->resHash, id, sizeof(id), &resCache, sizeof(SQWorkerResCache))) { QW_UNLOCK(QW_WRITE, &mgmt->resLock); - qError("taosHashPut queryId[%"PRIx64"] taskId[%"PRIx64"] to resHash failed", queryId, taskId); + qError("taosHashPut queryId[%"PRIx64"] taskId[%"PRIx64"] to resHash failed", qId, tId); return TSDB_CODE_QRY_APP_ERROR; } @@ -99,37 +107,8 @@ int32_t qwAddTaskResult(SQWorkerMgmt *mgmt, uint64_t queryId, uint64_t taskId, v return TSDB_CODE_SUCCESS; } - -int32_t qwGetTaskResult(SQWorkerMgmt *mgmt, uint64_t queryId, uint64_t taskId, void **data) { - char id[sizeof(queryId) + sizeof(taskId)] = {0}; - QW_SET_QTID(id, queryId, taskId); - - SQWorkerResCache *resCache = taosHashGet(mgmt->resHash, id, sizeof(id)); - if (NULL == resCache) { - qError("no task res for queryId[%"PRIx64"] taskId[%"PRIx64"]", queryId, taskId); - return TSDB_CODE_QRY_APP_ERROR; - } - - *data = resCache->data; - - return TSDB_CODE_SUCCESS; -} - - -static FORCE_INLINE int32_t qwAcquireScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t schedulerId, SQWorkerSchStatus **sch) { - QW_LOCK(rwType, &mgmt->schLock); - *sch = taosHashGet(mgmt->schHash, &schedulerId, sizeof(schedulerId)); - if (NULL == (*sch)) { - QW_LOCK(rwType, &mgmt->schLock); - return TSDB_CODE_QRY_SCH_NOT_EXIST; - } - - return TSDB_CODE_SUCCESS; -} - - -static FORCE_INLINE int32_t qwInsertAndAcquireScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t schedulerId, SQWorkerSchStatus **sch) { - SQWorkerSchStatus newSch = {0}; +static int32_t qwAddScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t sId, SQWSchStatus **sch) { + SQWSchStatus newSch = {0}; newSch.tasksHash = taosHashInit(mgmt->cfg.maxSchTaskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); if (NULL == newSch.tasksHash) { qError("taosHashInit %d failed", mgmt->cfg.maxSchTaskNum); @@ -138,19 +117,18 @@ static FORCE_INLINE int32_t qwInsertAndAcquireScheduler(int32_t rwType, SQWorker while (true) { QW_LOCK(QW_WRITE, &mgmt->schLock); - int32_t code = taosHashPut(mgmt->schHash, &schedulerId, sizeof(schedulerId), &newSch, sizeof(newSch)); + int32_t code = taosHashPut(mgmt->schHash, &sId, sizeof(sId), &newSch, sizeof(newSch)); if (0 != code) { if (!HASH_NODE_EXIST(code)) { QW_UNLOCK(QW_WRITE, &mgmt->schLock); - qError("taosHashPut schedulerId[%"PRIx64"] to scheduleHash failed", schedulerId); + qError("taosHashPut sId[%"PRIx64"] to scheduleHash failed", sId); taosHashCleanup(newSch.tasksHash); return TSDB_CODE_QRY_APP_ERROR; } } QW_UNLOCK(QW_WRITE, &mgmt->schLock); - if (TSDB_CODE_SUCCESS == qwAcquireScheduler(rwType, mgmt, schedulerId, sch)) { - taosHashCleanup(newSch.tasksHash); + if (TSDB_CODE_SUCCESS == qwAcquireScheduler(rwType, mgmt, sId, sch, QW_NOT_EXIST_ADD)) { return TSDB_CODE_SUCCESS; } } @@ -159,63 +137,122 @@ static FORCE_INLINE int32_t qwInsertAndAcquireScheduler(int32_t rwType, SQWorker } +static int32_t qwAcquireScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t sId, SQWSchStatus **sch, int32_t nOpt) { + QW_LOCK(rwType, &mgmt->schLock); + *sch = taosHashGet(mgmt->schHash, &sId, sizeof(sId)); + if (NULL == (*sch)) { + QW_UNLOCK(rwType, &mgmt->schLock); + + if (QW_NOT_EXIST_ADD == nOpt) { + return qwAddScheduler(rwType, mgmt, sId, sch); + } else if (QW_NOT_EXIST_RET_ERR == nOpt) { + return TSDB_CODE_QRY_SCH_NOT_EXIST; + } else { + assert(0); + } + } + + return TSDB_CODE_SUCCESS; +} + + + static FORCE_INLINE void qwReleaseScheduler(int32_t rwType, SQWorkerMgmt *mgmt) { QW_UNLOCK(rwType, &mgmt->schLock); } -static FORCE_INLINE int32_t qwAcquireTask(int32_t rwType, SQWorkerSchStatus *sch, uint64_t queryId, uint64_t taskId, SQWorkerTaskStatus **task) { - char id[sizeof(queryId) + sizeof(taskId)] = {0}; - QW_SET_QTID(id, queryId, taskId); +static int32_t qwAcquireTaskImpl(int32_t rwType, SQWSchStatus *sch, uint64_t qId, uint64_t tId, SQWTaskStatus **task) { + char id[sizeof(qId) + sizeof(tId)] = {0}; + QW_SET_QTID(id, qId, tId); QW_LOCK(rwType, &sch->tasksLock); *task = taosHashGet(sch->tasksHash, id, sizeof(id)); if (NULL == (*task)) { QW_UNLOCK(rwType, &sch->tasksLock); + return TSDB_CODE_QRY_TASK_NOT_EXIST; } return TSDB_CODE_SUCCESS; } -static FORCE_INLINE int32_t qwInsertAndAcquireTask(int32_t rwType, SQWorkerSchStatus *sch, uint64_t queryId, uint64_t taskId, int8_t status, bool *inserted, SQWorkerTaskStatus **task) { - char id[sizeof(queryId) + sizeof(taskId)] = {0}; - QW_SET_QTID(id, queryId, taskId); +static int32_t qwAcquireTask(int32_t rwType, SQWSchStatus *sch, uint64_t qId, uint64_t tId, SQWTaskStatus **task) { + return qwAcquireTaskImpl(rwType, sch, qId, tId, task); +} + + +static FORCE_INLINE void qwReleaseTask(int32_t rwType, SQWSchStatus *sch) { + QW_UNLOCK(rwType, &sch->tasksLock); +} + + +int32_t qwAddTaskToSch(int32_t rwType, SQWSchStatus *sch, uint64_t qId, uint64_t tId, int8_t status, int32_t eOpt, SQWTaskStatus **task) { + int32_t code = 0; + + char id[sizeof(qId) + sizeof(tId)] = {0}; + QW_SET_QTID(id, qId, tId); + + SQWTaskStatus ntask = {0}; + ntask.status = status; while (true) { - *inserted = false; - QW_LOCK(QW_WRITE, &sch->tasksLock); - int32_t code = taosHashPut(sch->tasksHash, id, sizeof(id), &status, sizeof(status)); + int32_t code = taosHashPut(sch->tasksHash, id, sizeof(id), &ntask, sizeof(ntask)); if (0 != code) { QW_UNLOCK(QW_WRITE, &sch->tasksLock); if (HASH_NODE_EXIST(code)) { - if (qwAcquireTask(rwType, sch, queryId, taskId, task)) { - continue; + if (QW_EXIST_ACQUIRE == eOpt && rwType && task) { + if (qwAcquireTask(rwType, sch, qId, tId, task)) { + continue; + } + } else if (QW_EXIST_RET_ERR == eOpt) { + return TSDB_CODE_QRY_TASK_ALREADY_EXIST; + } else { + assert(0); } break; } else { - qError("taosHashPut queryId[%"PRIx64"] taskId[%"PRIx64"] to scheduleHash failed", queryId, taskId); + qError("taosHashPut queryId[%"PRIx64"] taskId[%"PRIx64"] to scheduleHash failed", qId, tId); return TSDB_CODE_QRY_APP_ERROR; } } + QW_UNLOCK(QW_WRITE, &sch->tasksLock); - *inserted = true; - - if (TSDB_CODE_SUCCESS == qwAcquireTask(rwType, sch, queryId, taskId, task)) { - return TSDB_CODE_SUCCESS; + if (rwType && task) { + if (TSDB_CODE_SUCCESS == qwAcquireTask(rwType, sch, qId, tId, task)) { + return TSDB_CODE_SUCCESS; + } + } else { + break; } - } + } return TSDB_CODE_SUCCESS; } -static FORCE_INLINE void qwReleaseTask(int32_t rwType, SQWorkerSchStatus *sch) { - QW_UNLOCK(rwType, &sch->tasksLock); +static int32_t qwAddTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int32_t status, int32_t eOpt, SQWSchStatus **sch, SQWTaskStatus **task) { + SQWSchStatus *tsch = NULL; + QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &tsch, QW_NOT_EXIST_ADD)); + + int32_t code = qwAddTaskToSch(QW_READ, tsch, qId, tId, status, eOpt, task); + if (code) { + qwReleaseScheduler(QW_WRITE, mgmt); + } + + if (NULL == task) { + qwReleaseScheduler(QW_READ, mgmt); + } else if (sch) { + *sch = tsch; + } + + QW_RET(code); } + + static FORCE_INLINE int32_t qwAcquireTaskResCache(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t queryId, uint64_t taskId, SQWorkerResCache **res) { char id[sizeof(queryId) + sizeof(taskId)] = {0}; QW_SET_QTID(id, queryId, taskId); @@ -235,27 +272,24 @@ static FORCE_INLINE void qwReleaseTaskResCache(int32_t rwType, SQWorkerMgmt *mgm } -int32_t qwGetSchTasksStatus(SQWorkerMgmt *mgmt, uint64_t schedulerId, SSchedulerStatusRsp **rsp) { - SQWorkerSchStatus *schStatus = NULL; +int32_t qwGetSchTasksStatus(SQWorkerMgmt *mgmt, uint64_t sId, SSchedulerStatusRsp **rsp) { + SQWSchStatus *sch = NULL; int32_t taskNum = 0; - if (qwAcquireScheduler(QW_READ, mgmt, schedulerId, &schStatus)) { - qWarn("no scheduler for schedulerId[%"PRIx64"]", schedulerId); - } else { - schStatus->lastAccessTs = taosGetTimestampSec(); + QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch, QW_NOT_EXIST_RET_ERR)); + + sch->lastAccessTs = taosGetTimestampSec(); - QW_LOCK(QW_READ, &schStatus->tasksLock); - taskNum = taosHashGetSize(schStatus->tasksHash); - } + QW_LOCK(QW_READ, &sch->tasksLock); + + taskNum = taosHashGetSize(sch->tasksHash); int32_t size = sizeof(SSchedulerStatusRsp) + sizeof((*rsp)->status[0]) * taskNum; *rsp = calloc(1, size); if (NULL == *rsp) { qError("calloc %d failed", size); - if (schStatus) { - QW_UNLOCK(QW_READ, &schStatus->tasksLock); - qwReleaseScheduler(QW_READ, mgmt); - } + QW_UNLOCK(QW_READ, &sch->tasksLock); + qwReleaseScheduler(QW_READ, mgmt); return TSDB_CODE_QRY_OUT_OF_MEMORY; } @@ -264,23 +298,19 @@ int32_t qwGetSchTasksStatus(SQWorkerMgmt *mgmt, uint64_t schedulerId, SScheduler size_t keyLen = 0; int32_t i = 0; - if (schStatus) { - void *pIter = taosHashIterate(schStatus->tasksHash, NULL); - while (pIter) { - SQWorkerTaskStatus *taskStatus = (SQWorkerTaskStatus *)pIter; - taosHashGetKey(pIter, &key, &keyLen); + void *pIter = taosHashIterate(sch->tasksHash, NULL); + while (pIter) { + SQWTaskStatus *taskStatus = (SQWTaskStatus *)pIter; + taosHashGetKey(pIter, &key, &keyLen); - QW_GET_QTID(key, (*rsp)->status[i].queryId, (*rsp)->status[i].taskId); - (*rsp)->status[i].status = taskStatus->status; - - pIter = taosHashIterate(schStatus->tasksHash, pIter); - } - } + QW_GET_QTID(key, (*rsp)->status[i].queryId, (*rsp)->status[i].taskId); + (*rsp)->status[i].status = taskStatus->status; + + pIter = taosHashIterate(sch->tasksHash, pIter); + } - if (schStatus) { - QW_UNLOCK(QW_READ, &schStatus->tasksLock); - qwReleaseScheduler(QW_READ, mgmt); - } + QW_UNLOCK(QW_READ, &sch->tasksLock); + qwReleaseScheduler(QW_READ, mgmt); (*rsp)->num = taskNum; @@ -289,115 +319,81 @@ int32_t qwGetSchTasksStatus(SQWorkerMgmt *mgmt, uint64_t schedulerId, SScheduler -int32_t qwUpdateSchLastAccess(SQWorkerMgmt *mgmt, uint64_t schedulerId) { - SQWorkerSchStatus *schStatus = NULL; +int32_t qwUpdateSchLastAccess(SQWorkerMgmt *mgmt, uint64_t sId) { + SQWSchStatus *sch = NULL; - QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, schedulerId, &schStatus)); + QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch, QW_NOT_EXIST_RET_ERR)); - schStatus->lastAccessTs = taosGetTimestampSec(); + sch->lastAccessTs = taosGetTimestampSec(); qwReleaseScheduler(QW_READ, mgmt); return TSDB_CODE_SUCCESS; } - -int32_t qwGetTaskStatus(SQWorkerMgmt *mgmt, uint64_t schedulerId, uint64_t queryId, uint64_t taskId, int8_t *taskStatus) { - SQWorkerSchStatus *sch = NULL; - SQWorkerTaskStatus *task = NULL; +int32_t qwUpdateTaskStatus(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int8_t status) { + SQWSchStatus *sch = NULL; + SQWTaskStatus *task = NULL; int32_t code = 0; - - QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, schedulerId, &sch)); - QW_ERR_JRET(qwAcquireTask(QW_READ, sch, queryId, taskId, &task)); + QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch, QW_NOT_EXIST_RET_ERR)); - *taskStatus = task->status; - -_return: - if (task) { - qwReleaseTask(QW_READ, sch); - } - - if (sch) { - qwReleaseScheduler(QW_READ, mgmt); - } - - QW_RET(code); -} - - -int32_t qwSwitchTaskStatus(SQWorkerMgmt *mgmt, uint64_t schedulerId, uint64_t queryId, uint64_t taskId, int8_t taskStatus) { - SQWorkerSchStatus *sch = NULL; - SQWorkerTaskStatus *task = NULL; - int32_t code = 0; - bool inserted = false; - - if (qwAcquireScheduler(QW_READ, mgmt, schedulerId, &sch)) { - if (qwCheckStatusSwitch(JOB_TASK_STATUS_NULL, taskStatus)) { - qError("switch status error, not start to %d", taskStatus); - QW_ERR_RET(TSDB_CODE_QRY_APP_ERROR); - } - - QW_ERR_RET(qwInsertAndAcquireScheduler(QW_READ, mgmt, schedulerId, &sch)); - } - - if (qwAcquireTask(QW_READ, sch, queryId, taskId, &task)) { - if (qwCheckStatusSwitch(JOB_TASK_STATUS_NOT_START, taskStatus)) { - qwReleaseScheduler(QW_READ, mgmt); - qError("switch status error, not start to %d", taskStatus); - QW_ERR_RET(TSDB_CODE_QRY_APP_ERROR); - } - - QW_ERR_JRET(qwInsertAndAcquireTask(QW_READ, sch, queryId, taskId, taskStatus, &inserted, &task)); - - if (inserted) { - qwReleaseTask(QW_READ, sch); - qwReleaseScheduler(QW_READ, mgmt); - return TSDB_CODE_SUCCESS; - } - - QW_LOCK(QW_WRITE, &task->lock); - code = qwUpdateTaskInfo(task, QW_TASK_INFO_STATUS, &taskStatus); - QW_UNLOCK(QW_WRITE, &task->lock); - - qwReleaseTask(QW_READ, sch); - qwReleaseScheduler(QW_READ, mgmt); - - QW_RET(code); - } + QW_ERR_JRET(qwAcquireTask(QW_READ, sch, qId, tId, &task)); QW_LOCK(QW_WRITE, &task->lock); - code = qwUpdateTaskInfo(task, QW_TASK_INFO_STATUS, &taskStatus); + qwUpdateTaskInfo(task, QW_TASK_INFO_STATUS, &status); QW_UNLOCK(QW_WRITE, &task->lock); - + _return: qwReleaseTask(QW_READ, sch); - qwReleaseScheduler(QW_READ, mgmt); - + qwReleaseScheduler(QW_READ, mgmt); + QW_RET(code); } -int32_t qwCancelTask(SQWorkerMgmt *mgmt, uint64_t schedulerId, uint64_t queryId, uint64_t taskId) { - SQWorkerSchStatus *sch = NULL; - SQWorkerTaskStatus *task = NULL; +int32_t qwGetTaskStatus(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_t taskId, int8_t *taskStatus) { + SQWSchStatus *sch = NULL; + SQWTaskStatus *task = NULL; int32_t code = 0; - - if (TSDB_CODE_SUCCESS != qwAcquireScheduler(QW_READ, mgmt, schedulerId, &sch)) { - QW_ERR_RET(qwSwitchTaskStatus(mgmt, schedulerId, queryId, taskId, JOB_TASK_STATUS_NOT_START)); - - QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, schedulerId, &sch)); + + if (qwAcquireScheduler(QW_READ, mgmt, sId, &sch, QW_NOT_EXIST_RET_ERR)) { + *taskStatus = JOB_TASK_STATUS_NULL; + return TSDB_CODE_SUCCESS; } if (qwAcquireTask(QW_READ, sch, queryId, taskId, &task)) { - code = qwSwitchTaskStatus(mgmt, schedulerId, queryId, taskId, JOB_TASK_STATUS_NOT_START); + qwReleaseScheduler(QW_READ, mgmt); + + *taskStatus = JOB_TASK_STATUS_NULL; + return TSDB_CODE_SUCCESS; + } + + *taskStatus = task->status; + + qwReleaseTask(QW_READ, sch); + qwReleaseScheduler(QW_READ, mgmt); + + QW_RET(code); +} + + +int32_t qwCancelTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_t taskId) { + SQWSchStatus *sch = NULL; + SQWTaskStatus *task = NULL; + int32_t code = 0; + + QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch, QW_NOT_EXIST_ADD)); + + if (qwAcquireTask(QW_READ, sch, queryId, taskId, &task)) { + qwReleaseScheduler(QW_READ, mgmt); + + code = qwAddTask(mgmt, sId, queryId, taskId, JOB_TASK_STATUS_NOT_START, QW_EXIST_ACQUIRE, &sch, &task); if (code) { qwReleaseScheduler(QW_READ, mgmt); QW_ERR_RET(code); } - - QW_ERR_JRET(qwAcquireTask(QW_READ, sch, queryId, taskId, &task)); } QW_LOCK(QW_WRITE, &task->lock); @@ -423,6 +419,7 @@ int32_t qwCancelTask(SQWorkerMgmt *mgmt, uint64_t schedulerId, uint64_t queryId, } QW_UNLOCK(QW_WRITE, &task->lock); + qwReleaseTask(QW_READ, sch); qwReleaseScheduler(QW_READ, mgmt); @@ -449,9 +446,9 @@ _return: -int32_t qwDropTask(SQWorkerMgmt *mgmt, uint64_t schedulerId, uint64_t queryId, uint64_t taskId) { - SQWorkerSchStatus *sch = NULL; - SQWorkerTaskStatus *task = NULL; +int32_t qwDropTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_t taskId) { + SQWSchStatus *sch = NULL; + SQWTaskStatus *task = NULL; int32_t code = 0; char id[sizeof(queryId) + sizeof(taskId)] = {0}; QW_SET_QTID(id, queryId, taskId); @@ -462,15 +459,15 @@ int32_t qwDropTask(SQWorkerMgmt *mgmt, uint64_t schedulerId, uint64_t queryId, u } QW_UNLOCK(QW_WRITE, &mgmt->resLock); - if (TSDB_CODE_SUCCESS != qwAcquireScheduler(QW_WRITE, mgmt, schedulerId, &sch)) { - qWarn("scheduler %"PRIx64" doesn't exist", schedulerId); + if (TSDB_CODE_SUCCESS != qwAcquireScheduler(QW_WRITE, mgmt, sId, &sch, QW_NOT_EXIST_RET_ERR)) { + qWarn("scheduler %"PRIx64" doesn't exist", sId); return TSDB_CODE_SUCCESS; } if (qwAcquireTask(QW_WRITE, sch, queryId, taskId, &task)) { qwReleaseScheduler(QW_WRITE, mgmt); - qWarn("scheduler %"PRIx64" queryId %"PRIx64" taskId:%"PRIx64" doesn't exist", schedulerId, queryId, taskId); + qWarn("scheduler %"PRIx64" queryId %"PRIx64" taskId:%"PRIx64" doesn't exist", sId, queryId, taskId); return TSDB_CODE_SUCCESS; } @@ -483,21 +480,21 @@ int32_t qwDropTask(SQWorkerMgmt *mgmt, uint64_t schedulerId, uint64_t queryId, u } -int32_t qwCancelDropTask(SQWorkerMgmt *mgmt, uint64_t schedulerId, uint64_t queryId, uint64_t taskId) { - SQWorkerSchStatus *sch = NULL; - SQWorkerTaskStatus *task = NULL; +int32_t qwCancelDropTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_t taskId) { + SQWSchStatus *sch = NULL; + SQWTaskStatus *task = NULL; int32_t code = 0; - if (TSDB_CODE_SUCCESS != qwAcquireScheduler(QW_READ, mgmt, schedulerId, &sch)) { - qWarn("scheduler %"PRIx64" doesn't exist", schedulerId); - return TSDB_CODE_SUCCESS; - } + QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch, QW_NOT_EXIST_ADD)); if (qwAcquireTask(QW_READ, sch, queryId, taskId, &task)) { qwReleaseScheduler(QW_READ, mgmt); - qWarn("scheduler %"PRIx64" queryId %"PRIx64" taskId:%"PRIx64" doesn't exist", schedulerId, queryId, taskId); - return TSDB_CODE_SUCCESS; + code = qwAddTask(mgmt, sId, queryId, taskId, JOB_TASK_STATUS_NOT_START, QW_EXIST_ACQUIRE, &sch, &task); + if (code) { + qwReleaseScheduler(QW_READ, mgmt); + QW_ERR_RET(code); + } } QW_LOCK(QW_WRITE, &task->lock); @@ -508,7 +505,7 @@ int32_t qwCancelDropTask(SQWorkerMgmt *mgmt, uint64_t schedulerId, uint64_t quer int8_t newStatus = 0; if (task->status == JOB_TASK_STATUS_EXECUTING) { - newStatus = JOB_TASK_STATUS_CANCELLING; + newStatus = JOB_TASK_STATUS_DROPPING; QW_ERR_JRET(qwUpdateTaskInfo(task, QW_TASK_INFO_STATUS, &newStatus)); } else if (task->status == JOB_TASK_STATUS_CANCELLING || task->status == JOB_TASK_STATUS_DROPPING || task->status == JOB_TASK_STATUS_NOT_START) { QW_UNLOCK(QW_WRITE, &task->lock); @@ -521,7 +518,7 @@ int32_t qwCancelDropTask(SQWorkerMgmt *mgmt, uint64_t schedulerId, uint64_t quer qwReleaseTask(QW_READ, sch); qwReleaseScheduler(QW_READ, mgmt); - QW_ERR_RET(qwDropTask(mgmt, schedulerId, queryId, taskId)); + QW_ERR_RET(qwDropTask(mgmt, sId, queryId, taskId)); return TSDB_CODE_SUCCESS; } @@ -673,12 +670,12 @@ int32_t qwBuildAndSendDropRsp(SRpcMsg *pMsg, int32_t code) { -int32_t qwCheckAndSendReadyRsp(SQWorkerMgmt *mgmt, uint64_t schedulerId, uint64_t queryId, uint64_t taskId, SRpcMsg *pMsg, int32_t rspCode) { - SQWorkerSchStatus *sch = NULL; - SQWorkerTaskStatus *task = NULL; +int32_t qwCheckAndSendReadyRsp(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_t taskId, SRpcMsg *pMsg, int32_t rspCode) { + SQWSchStatus *sch = NULL; + SQWTaskStatus *task = NULL; int32_t code = 0; - QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, schedulerId, &sch)); + QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch, QW_NOT_EXIST_RET_ERR)); QW_ERR_JRET(qwAcquireTask(QW_READ, sch, queryId, taskId, &task)); @@ -717,12 +714,12 @@ _return: QW_RET(code); } -int32_t qwSetAndSendReadyRsp(SQWorkerMgmt *mgmt, uint64_t schedulerId, uint64_t queryId, uint64_t taskId, SRpcMsg *pMsg) { - SQWorkerSchStatus *sch = NULL; - SQWorkerTaskStatus *task = NULL; +int32_t qwSetAndSendReadyRsp(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_t taskId, SRpcMsg *pMsg) { + SQWSchStatus *sch = NULL; + SQWTaskStatus *task = NULL; int32_t code = 0; - QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, schedulerId, &sch)); + QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch, QW_NOT_EXIST_RET_ERR)); QW_ERR_JRET(qwAcquireTask(QW_READ, sch, queryId, taskId, &task)); @@ -756,15 +753,15 @@ _return: QW_RET(code); } -int32_t qwCheckTaskCancelDrop( SQWorkerMgmt *mgmt, uint64_t schedulerId, uint64_t queryId, uint64_t taskId, bool *needStop) { - SQWorkerSchStatus *sch = NULL; - SQWorkerTaskStatus *task = NULL; +int32_t qwCheckTaskCancelDrop( SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_t taskId, bool *needStop) { + SQWSchStatus *sch = NULL; + SQWTaskStatus *task = NULL; int32_t code = 0; int8_t status = JOB_TASK_STATUS_CANCELLED; *needStop = false; - if (qwAcquireScheduler(QW_READ, mgmt, schedulerId, &sch)) { + if (qwAcquireScheduler(QW_READ, mgmt, sId, &sch, QW_NOT_EXIST_RET_ERR)) { return TSDB_CODE_SUCCESS; } @@ -776,11 +773,13 @@ int32_t qwCheckTaskCancelDrop( SQWorkerMgmt *mgmt, uint64_t schedulerId, uint64_ QW_LOCK(QW_READ, &task->lock); if ((!task->cancel) && (!task->drop)) { + qError("no cancel or drop, but task:%"PRIx64" exists", taskId); + QW_UNLOCK(QW_READ, &task->lock); qwReleaseTask(QW_READ, sch); qwReleaseScheduler(QW_READ, mgmt); - return TSDB_CODE_SUCCESS; + QW_RET(TSDB_CODE_QRY_APP_ERROR); } QW_UNLOCK(QW_READ, &task->lock); @@ -791,30 +790,40 @@ int32_t qwCheckTaskCancelDrop( SQWorkerMgmt *mgmt, uint64_t schedulerId, uint64_ QW_LOCK(QW_WRITE, &task->lock); qwUpdateTaskInfo(task, QW_TASK_INFO_STATUS, &status); QW_UNLOCK(QW_WRITE, &task->lock); - } else if (task->drop) { + } + + if (task->drop) { qwReleaseTask(QW_READ, sch); qwReleaseScheduler(QW_READ, mgmt); - qwDropTask(mgmt, schedulerId, queryId, taskId); + return qwDropTask(mgmt, sId, queryId, taskId); } + qwReleaseTask(QW_READ, sch); + qwReleaseScheduler(QW_READ, mgmt); + return TSDB_CODE_SUCCESS; } -int32_t qwHandleFetch(SQWorkerResCache *res, SQWorkerMgmt *mgmt, uint64_t schedulerId, uint64_t queryId, uint64_t taskId, SRpcMsg *pMsg) { - SQWorkerSchStatus *sch = NULL; - SQWorkerTaskStatus *task = NULL; +int32_t qwHandleFetch(SQWorkerResCache *res, SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_t taskId, SRpcMsg *pMsg) { + SQWSchStatus *sch = NULL; + SQWTaskStatus *task = NULL; int32_t code = 0; int32_t needRsp = true; void *data = NULL; - QW_ERR_JRET(qwAcquireScheduler(QW_READ, mgmt, schedulerId, &sch)); + QW_ERR_JRET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch, QW_NOT_EXIST_RET_ERR)); QW_ERR_JRET(qwAcquireTask(QW_READ, sch, queryId, taskId, &task)); QW_LOCK(QW_READ, &task->lock); - if (task->status != JOB_TASK_STATUS_EXECUTING && task->status != JOB_TASK_STATUS_PARTIAL_SUCCEED && task->status != JOB_TASK_STATUS_SUCCEED) { + if (task->cancel || task->drop) { + qError("task is already cancelled or dropped"); + QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + } + + if (task->status != JOB_TASK_STATUS_EXECUTING && task->status != JOB_TASK_STATUS_PARTIAL_SUCCEED) { qError("invalid status %d for fetch", task->status); QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } @@ -854,37 +863,46 @@ _return: QW_RET(code); } -int32_t qwQueryPostProcess(SQWorkerMgmt *mgmt, uint64_t schedulerId, uint64_t queryId, uint64_t taskId, int8_t status, int32_t errCode) { - SQWorkerSchStatus *sch = NULL; - SQWorkerTaskStatus *task = NULL; +int32_t qwQueryPostProcess(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int8_t status, int32_t errCode) { + SQWSchStatus *sch = NULL; + SQWTaskStatus *task = NULL; int32_t code = 0; int8_t newStatus = JOB_TASK_STATUS_CANCELLED; - code = qwAcquireScheduler(QW_READ, mgmt, schedulerId, &sch); + code = qwAcquireScheduler(QW_READ, mgmt, sId, &sch, QW_NOT_EXIST_ADD); if (code) { - qError("schedulerId:%"PRIx64" not in cache", schedulerId); + qError("sId:%"PRIx64" not in cache", sId); QW_ERR_RET(code); } - code = qwAcquireTask(QW_READ, sch, queryId, taskId, &task); + code = qwAcquireTask(QW_READ, sch, qId, tId, &task); if (code) { qwReleaseScheduler(QW_READ, mgmt); - qError("schedulerId:%"PRIx64" queryId:%"PRIx64" taskId:%"PRIx64" not in cache", schedulerId, queryId, taskId); - QW_ERR_RET(code); + + if (JOB_TASK_STATUS_PARTIAL_SUCCEED == status || JOB_TASK_STATUS_SUCCEED == status) { + qError("sId:%"PRIx64" queryId:%"PRIx64" taskId:%"PRIx64" not in cache", sId, qId, tId); + QW_ERR_RET(code); + } + + QW_ERR_RET(qwAddTask(mgmt, sId, qId, tId, status, QW_EXIST_ACQUIRE, &sch, &task)); } if (task->cancel) { QW_LOCK(QW_WRITE, &task->lock); qwUpdateTaskInfo(task, QW_TASK_INFO_STATUS, &newStatus); QW_UNLOCK(QW_WRITE, &task->lock); - } else if (task->drop) { + } + + if (task->drop) { qwReleaseTask(QW_READ, sch); qwReleaseScheduler(QW_READ, mgmt); - qwDropTask(mgmt, schedulerId, queryId, taskId); + qwDropTask(mgmt, sId, qId, tId); return TSDB_CODE_SUCCESS; - } else { + } + + if (!(task->cancel || task->drop)) { QW_LOCK(QW_WRITE, &task->lock); qwUpdateTaskInfo(task, QW_TASK_INFO_STATUS, &status); task->code = errCode; @@ -938,24 +956,24 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } + int32_t code = 0; SSubQueryMsg *msg = pMsg->pCont; if (NULL == msg || pMsg->contLen <= sizeof(*msg)) { qError("invalid query msg"); - QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + QW_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); } - msg->schedulerId = htobe64(msg->schedulerId); + msg->sId = htobe64(msg->sId); msg->queryId = htobe64(msg->queryId); msg->taskId = htobe64(msg->taskId); msg->contentLen = ntohl(msg->contentLen); bool queryDone = false; - bool queryRsp = false; + bool queryRsped = false; bool needStop = false; SSubplan *plan = NULL; - int32_t code = 0; - QW_ERR_JRET(qwCheckTaskCancelDrop(qWorkerMgmt, msg->schedulerId, msg->queryId, msg->taskId, &needStop)); + QW_ERR_JRET(qwCheckTaskCancelDrop(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, &needStop)); if (needStop) { qWarn("task need stop"); QW_ERR_JRET(TSDB_CODE_QRY_TASK_CANCELLED); @@ -963,7 +981,7 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { code = qStringToSubplan(msg->msg, &plan); if (TSDB_CODE_SUCCESS != code) { - qError("schId:%"PRIx64",qId:%"PRIx64",taskId:%"PRIx64" string to subplan failed, code:%d", msg->schedulerId, msg->queryId, msg->taskId, code); + qError("schId:%"PRIx64",qId:%"PRIx64",taskId:%"PRIx64" string to subplan failed, code:%d", msg->sId, msg->queryId, msg->taskId, code); QW_ERR_JRET(code); } @@ -974,12 +992,12 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { if (code) { QW_ERR_JRET(code); } else { - QW_ERR_JRET(qwSwitchTaskStatus(qWorkerMgmt, msg->schedulerId, msg->queryId, msg->taskId, JOB_TASK_STATUS_EXECUTING)); + QW_ERR_JRET(qwAddTask(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, JOB_TASK_STATUS_EXECUTING, QW_EXIST_RET_ERR, NULL, NULL)); } QW_ERR_JRET(qwBuildAndSendQueryRsp(pMsg, TSDB_CODE_SUCCESS)); - queryRsp = true; + queryRsped = true; //TODO call executer to execute subquery code = 0; @@ -990,29 +1008,29 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { if (code) { QW_ERR_JRET(code); } else { - QW_ERR_JRET(qwAddTaskResult(qWorkerMgmt, msg->queryId, msg->taskId, data)); + QW_ERR_JRET(qwAddTaskResCache(qWorkerMgmt, msg->queryId, msg->taskId, data)); - QW_ERR_JRET(qwSwitchTaskStatus(qWorkerMgmt, msg->schedulerId, msg->queryId, msg->taskId, JOB_TASK_STATUS_PARTIAL_SUCCEED)); - } + QW_ERR_JRET(qwUpdateTaskStatus(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, JOB_TASK_STATUS_PARTIAL_SUCCEED)); + } _return: - if (queryRsp) { - code = qwCheckAndSendReadyRsp(qWorkerMgmt, msg->schedulerId, msg->queryId, msg->taskId, pMsg, code); + if (queryRsped) { + code = qwCheckAndSendReadyRsp(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, pMsg, code); } else { code = qwBuildAndSendQueryRsp(pMsg, code); } int8_t status = 0; - if (TSDB_CODE_SUCCESS != code || queryDone) { - if (code) { - status = JOB_TASK_STATUS_FAILED; //TODO set CANCELLED from code - } else { - status = JOB_TASK_STATUS_SUCCEED; - } - - qwQueryPostProcess(qWorkerMgmt, msg->schedulerId, msg->queryId, msg->taskId, status, code); + if (TSDB_CODE_SUCCESS != code) { + status = JOB_TASK_STATUS_FAILED; + } else if (queryDone) { + status = JOB_TASK_STATUS_SUCCEED; + } else { + status = JOB_TASK_STATUS_PARTIAL_SUCCEED; } + + qwQueryPostProcess(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, status, code); QW_RET(code); } @@ -1023,12 +1041,16 @@ int32_t qWorkerProcessReadyMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg){ } SResReadyMsg *msg = pMsg->pCont; - if (NULL == msg || pMsg->contLen <= sizeof(*msg)) { + if (NULL == msg || pMsg->contLen < sizeof(*msg)) { qError("invalid task status msg"); QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - QW_ERR_RET(qwSetAndSendReadyRsp(qWorkerMgmt, msg->schedulerId, msg->queryId, msg->taskId, pMsg)); + msg->sId = htobe64(msg->sId); + msg->queryId = htobe64(msg->queryId); + msg->taskId = htobe64(msg->taskId); + + QW_ERR_RET(qwSetAndSendReadyRsp(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, pMsg)); return TSDB_CODE_SUCCESS; } @@ -1040,14 +1062,16 @@ int32_t qWorkerProcessStatusMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { int32_t code = 0; SSchTasksStatusMsg *msg = pMsg->pCont; - if (NULL == msg || pMsg->contLen <= sizeof(*msg)) { + if (NULL == msg || pMsg->contLen < sizeof(*msg)) { qError("invalid task status msg"); QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } + msg->sId = htobe64(msg->sId); + SSchedulerStatusRsp *sStatus = NULL; - QW_ERR_JRET(qwGetSchTasksStatus(qWorkerMgmt, msg->schedulerId, &sStatus)); + QW_ERR_JRET(qwGetSchTasksStatus(qWorkerMgmt, msg->sId, &sStatus)); _return: @@ -1062,11 +1086,15 @@ int32_t qWorkerProcessFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { } SResFetchMsg *msg = pMsg->pCont; - if (NULL == msg || pMsg->contLen <= sizeof(*msg)) { + if (NULL == msg || pMsg->contLen < sizeof(*msg)) { QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - QW_ERR_RET(qwUpdateSchLastAccess(qWorkerMgmt, msg->schedulerId)); + msg->sId = htobe64(msg->sId); + msg->queryId = htobe64(msg->queryId); + msg->taskId = htobe64(msg->taskId); + + QW_ERR_RET(qwUpdateSchLastAccess(qWorkerMgmt, msg->sId)); void *data = NULL; SQWorkerResCache *res = NULL; @@ -1074,7 +1102,7 @@ int32_t qWorkerProcessFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { QW_ERR_RET(qwAcquireTaskResCache(QW_READ, qWorkerMgmt, msg->queryId, msg->taskId, &res)); - QW_ERR_JRET(qwHandleFetch(res, qWorkerMgmt, msg->schedulerId, msg->queryId, msg->taskId, pMsg)); + QW_ERR_JRET(qwHandleFetch(res, qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, pMsg)); _return: @@ -1090,12 +1118,16 @@ int32_t qWorkerProcessCancelMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { int32_t code = 0; STaskCancelMsg *msg = pMsg->pCont; - if (NULL == msg || pMsg->contLen <= sizeof(*msg)) { + if (NULL == msg || pMsg->contLen < sizeof(*msg)) { qError("invalid task cancel msg"); QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - QW_ERR_JRET(qwCancelTask(qWorkerMgmt, msg->schedulerId, msg->queryId, msg->taskId)); + msg->sId = htobe64(msg->sId); + msg->queryId = htobe64(msg->queryId); + msg->taskId = htobe64(msg->taskId); + + QW_ERR_JRET(qwCancelTask(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId)); _return: @@ -1111,12 +1143,16 @@ int32_t qWorkerProcessDropMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { int32_t code = 0; STaskDropMsg *msg = pMsg->pCont; - if (NULL == msg || pMsg->contLen <= sizeof(*msg)) { + if (NULL == msg || pMsg->contLen < sizeof(*msg)) { qError("invalid task drop msg"); QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - QW_ERR_JRET(qwCancelDropTask(qWorkerMgmt, msg->schedulerId, msg->queryId, msg->taskId)); + msg->sId = htobe64(msg->sId); + msg->queryId = htobe64(msg->queryId); + msg->taskId = htobe64(msg->taskId); + + QW_ERR_JRET(qwCancelDropTask(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId)); _return: @@ -1125,6 +1161,31 @@ _return: return TSDB_CODE_SUCCESS; } +int32_t qWorkerContinueQuery(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { + int32_t code = 0; + int8_t status = 0; + bool queryDone = false; + uint64_t sId, qId, tId; + + //TODO call executer to continue execute subquery + code = 0; + void *data = NULL; + queryDone = false; + //TODO call executer to continue execute subquery + + if (TSDB_CODE_SUCCESS != code) { + status = JOB_TASK_STATUS_FAILED; + } else if (queryDone) { + status = JOB_TASK_STATUS_SUCCEED; + } else { + status = JOB_TASK_STATUS_PARTIAL_SUCCEED; + } + + code = qwQueryPostProcess(qWorkerMgmt, sId, qId, tId, status, code); + + QW_RET(code); +} + void qWorkerDestroy(void **qWorkerMgmt) { if (NULL == qWorkerMgmt || NULL == *qWorkerMgmt) { diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index 4b54b77544..d63d40b4f4 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -40,6 +40,11 @@ int32_t qwtStringToPlan(const char* str, SSubplan** subplan) { return 0; } +void qwtRpcSendResponse(const SRpcMsg *pRsp) { + return; +} + + void stubSetStringToPlan() { static Stub stub; @@ -54,6 +59,20 @@ void stubSetStringToPlan() { } } +void stubSetRpcSendResponse() { + static Stub stub; + stub.set(rpcSendResponse, qwtRpcSendResponse); + { + AddrAny any("libplanner.so"); + std::map result; + any.get_global_func_addr_dynsym("^rpcSendResponse$", result); + for (const auto& f : result) { + stub.set(f.second, qwtRpcSendResponse); + } + } +} + + } @@ -68,30 +87,35 @@ TEST(testCase, normalCase) { SRpcMsg dropRpc = {0}; SSubQueryMsg *queryMsg = (SSubQueryMsg *)calloc(1, sizeof(SSubQueryMsg) + 100); queryMsg->queryId = htobe64(1); - queryMsg->schedulerId = htobe64(1); + queryMsg->sId = htobe64(1); queryMsg->taskId = htobe64(1); queryMsg->contentLen = htonl(100); queryRpc.pCont = queryMsg; + queryRpc.contLen = sizeof(SSubQueryMsg) + 100; SResReadyMsg readyMsg = {0}; - readyMsg.schedulerId = htobe64(1); + readyMsg.sId = htobe64(1); readyMsg.queryId = htobe64(1); readyMsg.taskId = htobe64(1); readyRpc.pCont = &readyMsg; + readyRpc.contLen = sizeof(SResReadyMsg); SResFetchMsg fetchMsg = {0}; - fetchMsg.schedulerId = htobe64(1); + fetchMsg.sId = htobe64(1); fetchMsg.queryId = htobe64(1); fetchMsg.taskId = htobe64(1); fetchRpc.pCont = &fetchMsg; + fetchRpc.contLen = sizeof(SResFetchMsg); STaskDropMsg dropMsg = {0}; - dropMsg.schedulerId = htobe64(1); + dropMsg.sId = htobe64(1); dropMsg.queryId = htobe64(1); dropMsg.taskId = htobe64(1); dropRpc.pCont = &dropMsg; + dropRpc.contLen = sizeof(STaskDropMsg); stubSetStringToPlan(); + stubSetRpcSendResponse(); code = qWorkerInit(NULL, &mgmt); ASSERT_EQ(code, 0); @@ -107,7 +131,8 @@ TEST(testCase, normalCase) { code = qWorkerProcessDropMsg(mockPointer, mgmt, &dropRpc); ASSERT_EQ(code, 0); - + + qWorkerDestroy(&mgmt); } diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index 2381a1dd49..d5833c3adf 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -38,7 +38,7 @@ enum { typedef struct SSchedulerMgmt { uint64_t taskId; - uint64_t schedulerId; + uint64_t sId; SSchedulerCfg cfg; SHashObj *jobs; // key: queryId, value: SQueryJob* } SSchedulerMgmt; diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 503383f4b1..3cea97381f 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -343,7 +343,7 @@ int32_t schAsyncSendMsg(SSchJob *job, SSchTask *task, int32_t msgType) { SSubQueryMsg *pMsg = msg; - pMsg->schedulerId = htobe64(schMgmt.schedulerId); + pMsg->sId = htobe64(schMgmt.sId); pMsg->queryId = htobe64(job->queryId); pMsg->taskId = htobe64(task->taskId); pMsg->contentLen = htonl(task->msgLen); @@ -359,7 +359,7 @@ int32_t schAsyncSendMsg(SSchJob *job, SSchTask *task, int32_t msgType) { } SResReadyMsg *pMsg = msg; - pMsg->schedulerId = htobe64(schMgmt.schedulerId); + pMsg->sId = htobe64(schMgmt.sId); pMsg->queryId = htobe64(job->queryId); pMsg->taskId = htobe64(task->taskId); break; @@ -376,7 +376,7 @@ int32_t schAsyncSendMsg(SSchJob *job, SSchTask *task, int32_t msgType) { } SResFetchMsg *pMsg = msg; - pMsg->schedulerId = htobe64(schMgmt.schedulerId); + pMsg->sId = htobe64(schMgmt.sId); pMsg->queryId = htobe64(job->queryId); pMsg->taskId = htobe64(task->taskId); break; @@ -390,7 +390,7 @@ int32_t schAsyncSendMsg(SSchJob *job, SSchTask *task, int32_t msgType) { } STaskDropMsg *pMsg = msg; - pMsg->schedulerId = htobe64(schMgmt.schedulerId); + pMsg->sId = htobe64(schMgmt.sId); pMsg->queryId = htobe64(job->queryId); pMsg->taskId = htobe64(task->taskId); break; @@ -717,7 +717,7 @@ int32_t schedulerInit(SSchedulerCfg *cfg) { SCH_ERR_LRET(TSDB_CODE_QRY_OUT_OF_MEMORY, "init %d schduler jobs failed", schMgmt.cfg.maxJobNum); } - schMgmt.schedulerId = 1; //TODO GENERATE A UUID + schMgmt.sId = 1; //TODO GENERATE A UUID return TSDB_CODE_SUCCESS; } From b63266b05eef3996fe7051c80b57d567a0a5e7e2 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 27 Dec 2021 02:43:27 -0800 Subject: [PATCH 04/21] add qnode snode bnode --- include/dnode/bnode/bnode.h | 86 ++++++++++++++++++ include/dnode/mgmt/dnode.h | 1 + include/dnode/qnode/qnode.h | 91 +++++++++++++------ include/dnode/snode/snode.h | 87 ++++++++++++++++++ source/dnode/CMakeLists.txt | 2 + source/dnode/bnode/CMakeLists.txt | 14 +++ source/dnode/bnode/inc/bndInt.h | 45 +++++++++ source/dnode/bnode/src/bnode.c | 27 ++++++ source/dnode/mgmt/daemon/src/daemon.c | 1 + source/dnode/mgmt/impl/CMakeLists.txt | 3 + source/dnode/mgmt/impl/inc/dndBnode.h | 35 +++++++ source/dnode/mgmt/impl/inc/dndInt.h | 47 +++++++++- source/dnode/mgmt/impl/inc/dndQnode.h | 36 ++++++++ source/dnode/mgmt/impl/inc/dndSnode.h | 35 +++++++ source/dnode/mgmt/impl/src/dnode.c | 43 ++++++++- .../dnode/mgmt/impl/test/sut/src/server.cpp | 1 + source/dnode/qnode/CMakeLists.txt | 7 ++ .../dnode/qnode/inc/{qnodeInt.h => qndInt.h} | 17 ++++ source/dnode/qnode/src/qnode.c | 23 ++++- source/dnode/snode/CMakeLists.txt | 14 +++ source/dnode/snode/inc/sndInt.h | 44 +++++++++ source/dnode/snode/src/snode.c | 30 ++++++ 22 files changed, 654 insertions(+), 35 deletions(-) create mode 100644 include/dnode/bnode/bnode.h create mode 100644 include/dnode/snode/snode.h create mode 100644 source/dnode/bnode/CMakeLists.txt create mode 100644 source/dnode/bnode/inc/bndInt.h create mode 100644 source/dnode/bnode/src/bnode.c create mode 100644 source/dnode/mgmt/impl/inc/dndBnode.h create mode 100644 source/dnode/mgmt/impl/inc/dndQnode.h create mode 100644 source/dnode/mgmt/impl/inc/dndSnode.h rename source/dnode/qnode/inc/{qnodeInt.h => qndInt.h} (70%) create mode 100644 source/dnode/snode/CMakeLists.txt create mode 100644 source/dnode/snode/inc/sndInt.h create mode 100644 source/dnode/snode/src/snode.c diff --git a/include/dnode/bnode/bnode.h b/include/dnode/bnode/bnode.h new file mode 100644 index 0000000000..74574f5462 --- /dev/null +++ b/include/dnode/bnode/bnode.h @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_BNODE_H_ +#define _TD_BNODE_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +/* ------------------------ TYPES EXPOSED ------------------------ */ +typedef struct SDnode SDnode; +typedef struct SBnode SBnode; +typedef void (*SendMsgToDnodeFp)(SDnode *pDnode, struct SEpSet *epSet, struct SRpcMsg *rpcMsg); +typedef void (*SendMsgToMnodeFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg); +typedef void (*SendRedirectMsgFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg); + +typedef struct { + int64_t numOfErrors; +} SBnodeLoad; + +typedef struct { + int32_t sver; +} SBnodeCfg; + +typedef struct { + int32_t dnodeId; + int64_t clusterId; + SBnodeCfg cfg; + SDnode *pDnode; + SendMsgToDnodeFp sendMsgToDnodeFp; + SendMsgToMnodeFp sendMsgToMnodeFp; + SendRedirectMsgFp sendRedirectMsgFp; +} SBnodeOpt; + +/* ------------------------ SBnode ------------------------ */ +/** + * @brief Start one Bnode in Dnode. + * + * @param pOption Option of the bnode. + * @return SBnode* The bnode object. + */ +SBnode *bndOpen(const SBnodeOpt *pOption); + +/** + * @brief Stop Bnode in Dnode. + * + * @param pBnode The bnode object to close. + */ +void bndClose(SBnode *pBnode); + +/** + * @brief Get the statistical information of Bnode + * + * @param pBnode The bnode object. + * @param pLoad Statistics of the bnode. + * @return int32_t 0 for success, -1 for failure. + */ +int32_t bndGetLoad(SBnode *pBnode, SBnodeLoad *pLoad); + +/** + * @brief Process a query message. + * + * @param pBnode The bnode object. + * @param pMsgs The array of SRpcMsg + * @return int32_t 0 for success, -1 for failure + */ +int32_t bndProcessWMsgs(SBnode *pBnode, SArray *pMsgs); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_BNODE_H_*/ \ No newline at end of file diff --git a/include/dnode/mgmt/dnode.h b/include/dnode/mgmt/dnode.h index b7f3cea7fc..e12ce09422 100644 --- a/include/dnode/mgmt/dnode.h +++ b/include/dnode/mgmt/dnode.h @@ -29,6 +29,7 @@ typedef struct { int32_t sver; int16_t numOfCores; int16_t numOfSupportVnodes; + int16_t numOfCommitThreads; int8_t enableTelem; int32_t statusInterval; float numOfThreadsPerCore; diff --git a/include/dnode/qnode/qnode.h b/include/dnode/qnode/qnode.h index aa4c3af392..29a3d1af20 100644 --- a/include/dnode/qnode/qnode.h +++ b/include/dnode/qnode/qnode.h @@ -19,52 +19,83 @@ #ifdef __cplusplus extern "C" { #endif -#include "trpc.h" + +/* ------------------------ TYPES EXPOSED ------------------------ */ +typedef struct SDnode SDnode; +typedef struct SQnode SQnode; +typedef void (*SendMsgToDnodeFp)(SDnode *pDnode, struct SEpSet *epSet, struct SRpcMsg *rpcMsg); +typedef void (*SendMsgToMnodeFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg); +typedef void (*SendRedirectMsgFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg); typedef struct { - uint64_t numOfStartTask; - uint64_t numOfStopTask; - uint64_t numOfRecvedFetch; - uint64_t numOfSentHb; - uint64_t numOfSentFetch; - uint64_t numOfTaskInQueue; - uint64_t numOfFetchInQueue; - uint64_t numOfErrors; -} SQnodeStat; + int64_t numOfStartTask; + int64_t numOfStopTask; + int64_t numOfRecvedFetch; + int64_t numOfSentHb; + int64_t numOfSentFetch; + int64_t numOfTaskInQueue; + int64_t numOfFetchInQueue; + int64_t numOfErrors; +} SQnodeLoad; +typedef struct { + int32_t sver; +} SQnodeCfg; +typedef struct { + int32_t dnodeId; + int64_t clusterId; + SQnodeCfg cfg; + SDnode *pDnode; + SendMsgToDnodeFp sendMsgToDnodeFp; + SendMsgToMnodeFp sendMsgToMnodeFp; + SendRedirectMsgFp sendRedirectMsgFp; +} SQnodeOpt; + +/* ------------------------ SQnode ------------------------ */ /** - * Start one Qnode in Dnode. - * @return Error Code. - */ -int32_t qnodeStart(); - -/** - * Stop Qnode in Dnode. + * @brief Start one Qnode in Dnode. * - * @param qnodeId Qnode ID to stop, -1 for all Qnodes. + * @param pOption Option of the qnode. + * @return SQnode* The qnode object. */ -void qnodeStop(int64_t qnodeId); +SQnode *qndOpen(const SQnodeOpt *pOption); - /** - * Get the statistical information of Qnode + * @brief Stop Qnode in Dnode. * - * @param qnodeId Qnode ID to get statistics, -1 for all - * @param stat Statistical information. - * @return Error Code. + * @param pQnode The qnode object to close. */ -int32_t qnodeGetStatistics(int64_t qnodeId, SQnodeStat *stat); +void qndClose(SQnode *pQnode); /** - * Interface for processing Qnode messages. - * - * @param pMsg Message to be processed. - * @return Error code + * @brief Get the statistical information of Qnode + * + * @param pQnode The qnode object. + * @param pLoad Statistics of the qnode. + * @return int32_t 0 for success, -1 for failure. */ -void qnodeProcessReq(SRpcMsg *pMsg); +int32_t qndGetLoad(SQnode *pQnode, SQnodeLoad *pLoad); +/** + * @brief Process a query message. + * + * @param pQnode The qnode object. + * @param pMsg The request message + * @param pRsp The response message + * @return int32_t 0 for success, -1 for failure + */ +int32_t qndProcessQueryReq(SQnode *pQnode, SRpcMsg *pMsg, SRpcMsg **pRsp); +/** + * @brief Process a fetch message. + * + * @param pQnode The qnode object. + * @param pMsg The request message + * @param pRsp The response message + * @return int32_t 0 for success, -1 for failure + */ +int32_t qndProcessFetchReq(SQnode *pQnode, SRpcMsg *pMsg, SRpcMsg **pRsp); #ifdef __cplusplus } diff --git a/include/dnode/snode/snode.h b/include/dnode/snode/snode.h new file mode 100644 index 0000000000..1d30bd1e43 --- /dev/null +++ b/include/dnode/snode/snode.h @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_SNODE_H_ +#define _TD_SNODE_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +/* ------------------------ TYPES EXPOSED ------------------------ */ +typedef struct SDnode SDnode; +typedef struct SSnode SSnode; +typedef void (*SendMsgToDnodeFp)(SDnode *pDnode, struct SEpSet *epSet, struct SRpcMsg *rpcMsg); +typedef void (*SendMsgToMnodeFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg); +typedef void (*SendRedirectMsgFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg); + +typedef struct { + int64_t numOfErrors; +} SSnodeLoad; + +typedef struct { + int32_t sver; +} SSnodeCfg; + +typedef struct { + int32_t dnodeId; + int64_t clusterId; + SSnodeCfg cfg; + SDnode *pDnode; + SendMsgToDnodeFp sendMsgToDnodeFp; + SendMsgToMnodeFp sendMsgToMnodeFp; + SendRedirectMsgFp sendRedirectMsgFp; +} SSnodeOpt; + +/* ------------------------ SSnode ------------------------ */ +/** + * @brief Start one Snode in Dnode. + * + * @param pOption Option of the snode. + * @return SSnode* The snode object. + */ +SSnode *sndOpen(const SSnodeOpt *pOption); + +/** + * @brief Stop Snode in Dnode. + * + * @param pSnode The snode object to close. + */ +void sndClose(SSnode *pSnode); + +/** + * @brief Get the statistical information of Snode + * + * @param pSnode The snode object. + * @param pLoad Statistics of the snode. + * @return int32_t 0 for success, -1 for failure. + */ +int32_t sndGetLoad(SSnode *pSnode, SSnodeLoad *pLoad); + +/** + * @brief Process a query message. + * + * @param pSnode The snode object. + * @param pMsg The request message + * @param pRsp The response message + * @return int32_t 0 for success, -1 for failure + */ +int32_t sndProcessWriteMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg **pRsp); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_SNODE_H_*/ \ No newline at end of file diff --git a/source/dnode/CMakeLists.txt b/source/dnode/CMakeLists.txt index af132dea80..87e4c5fc46 100644 --- a/source/dnode/CMakeLists.txt +++ b/source/dnode/CMakeLists.txt @@ -1,4 +1,6 @@ add_subdirectory(mnode) add_subdirectory(vnode) add_subdirectory(qnode) +add_subdirectory(snode) +add_subdirectory(bnode) add_subdirectory(mgmt) \ No newline at end of file diff --git a/source/dnode/bnode/CMakeLists.txt b/source/dnode/bnode/CMakeLists.txt new file mode 100644 index 0000000000..a284437450 --- /dev/null +++ b/source/dnode/bnode/CMakeLists.txt @@ -0,0 +1,14 @@ +aux_source_directory(src BNODE_SRC) +add_library(bnode ${BNODE_SRC}) +target_include_directories( + bnode + PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/bnode" + private "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) +target_link_libraries( + bnode + PRIVATE transport + PRIVATE os + PRIVATE common + PRIVATE util +) \ No newline at end of file diff --git a/source/dnode/bnode/inc/bndInt.h b/source/dnode/bnode/inc/bndInt.h new file mode 100644 index 0000000000..d44c520a26 --- /dev/null +++ b/source/dnode/bnode/inc/bndInt.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_BNODE_INT_H_ +#define _TD_BNODE_INT_H_ + +#include "os.h" + +#include "tarray.h" +#include "tlog.h" +#include "tmsg.h" +#include "trpc.h" + +#include "bnode.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SBnode { + int32_t dnodeId; + int64_t clusterId; + SBnodeCfg cfg; + SendMsgToDnodeFp sendMsgToDnodeFp; + SendMsgToMnodeFp sendMsgToMnodeFp; + SendRedirectMsgFp sendRedirectMsgFp; +} SBnode; + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_BNODE_INT_H_*/ \ No newline at end of file diff --git a/source/dnode/bnode/src/bnode.c b/source/dnode/bnode/src/bnode.c new file mode 100644 index 0000000000..40b22dd58d --- /dev/null +++ b/source/dnode/bnode/src/bnode.c @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "bndInt.h" + +SBnode *bndOpen(const SBnodeOpt *pOption) { + SBnode *pBnode = calloc(1, sizeof(SBnode)); + return pBnode; +} + +void bndClose(SBnode *pBnode) { free(pBnode); } + +int32_t bndGetLoad(SBnode *pBnode, SBnodeLoad *pLoad) { return 0; } + +int32_t bndProcessWMsgs(SBnode *pBnode, SArray *pMsgs) { return 0; } diff --git a/source/dnode/mgmt/daemon/src/daemon.c b/source/dnode/mgmt/daemon/src/daemon.c index 75c2ff00e0..6c4fae406e 100644 --- a/source/dnode/mgmt/daemon/src/daemon.c +++ b/source/dnode/mgmt/daemon/src/daemon.c @@ -140,6 +140,7 @@ void dmnInitOption(SDnodeOpt *pOption) { pOption->sver = 30000000; //3.0.0.0 pOption->numOfCores = tsNumOfCores; pOption->numOfSupportVnodes = 1; + pOption->numOfCommitThreads = 1; pOption->statusInterval = tsStatusInterval; pOption->numOfThreadsPerCore = tsNumOfThreadsPerCore; pOption->ratioOfQueryCores = tsRatioOfQueryCores; diff --git a/source/dnode/mgmt/impl/CMakeLists.txt b/source/dnode/mgmt/impl/CMakeLists.txt index 51131ede6a..866eced51a 100644 --- a/source/dnode/mgmt/impl/CMakeLists.txt +++ b/source/dnode/mgmt/impl/CMakeLists.txt @@ -5,6 +5,9 @@ target_link_libraries( PUBLIC cjson PUBLIC mnode PUBLIC vnode + PUBLIC qnode + PUBLIC snode + PUBLIC bnode PUBLIC wal PUBLIC sync PUBLIC taos diff --git a/source/dnode/mgmt/impl/inc/dndBnode.h b/source/dnode/mgmt/impl/inc/dndBnode.h new file mode 100644 index 0000000000..a350eae2d4 --- /dev/null +++ b/source/dnode/mgmt/impl/inc/dndBnode.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DND_BNODE_H_ +#define _TD_DND_BNODE_H_ + +#ifdef __cplusplus +extern "C" { +#endif +#include "dndInt.h" + +int32_t dndInitBnode(SDnode *pDnode); +void dndCleanupBnode(SDnode *pDnode); + +ioid dndProcessBnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); +int32_t dndProcessCreateBnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg); +int32_t dndProcessDropBnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DND_BNODE_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/impl/inc/dndInt.h b/source/dnode/mgmt/impl/inc/dndInt.h index d6e9a6b4a1..b7a414672e 100644 --- a/source/dnode/mgmt/impl/inc/dndInt.h +++ b/source/dnode/mgmt/impl/inc/dndInt.h @@ -20,8 +20,11 @@ extern "C" { #endif -#include "cJSON.h" #include "os.h" + +#include "cJSON.h" +#include "tcache.h" +#include "tcrc32c.h" #include "tep.h" #include "thash.h" #include "tlockfree.h" @@ -34,7 +37,11 @@ extern "C" { #include "tworker.h" #include "dnode.h" + +#include "bnode.h" #include "mnode.h" +#include "qnode.h" +#include "snode.h" #include "vnode.h" extern int32_t dDebugFlag; @@ -93,6 +100,41 @@ typedef struct { SWorkerPool syncPool; } SMnodeMgmt; +typedef struct { + int32_t refCount; + int8_t deployed; + int8_t dropped; + char *file; + SQnode *pQnode; + SRWLatch latch; + taos_queue pQueryQ; + taos_queue pFetchQ; + SWorkerPool queryPool; + SWorkerPool fetchPool; +} SQnodeMgmt; + +typedef struct { + int32_t refCount; + int8_t deployed; + int8_t dropped; + char *file; + SSnode *pSnode; + SRWLatch latch; + taos_queue pWriteQ; + SWorkerPool writePool; +} SSnodeMgmt; + +typedef struct { + int32_t refCount; + int8_t deployed; + int8_t dropped; + char *file; + SBnode *pBnode; + SRWLatch latch; + taos_queue pWriteQ; + SMWorkerPool writePool; +} SBnodeMgmt; + typedef struct { SHashObj *hash; int32_t openVnodes; @@ -117,6 +159,9 @@ typedef struct SDnode { FileFd lockFd; SDnodeMgmt dmgmt; SMnodeMgmt mmgmt; + SQnodeMgmt qmgmt; + SSnodeMgmt smgmt; + SBnodeMgmt bmgmt; SVnodesMgmt vmgmt; STransMgmt tmgmt; SStartupMsg startup; diff --git a/source/dnode/mgmt/impl/inc/dndQnode.h b/source/dnode/mgmt/impl/inc/dndQnode.h new file mode 100644 index 0000000000..951209a193 --- /dev/null +++ b/source/dnode/mgmt/impl/inc/dndQnode.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DND_QNODE_H_ +#define _TD_DND_QNODE_H_ + +#ifdef __cplusplus +extern "C" { +#endif +#include "dndInt.h" + +int32_t dndInitQnode(SDnode *pDnode); +void dndCleanupQnode(SDnode *pDnode); + +ioid dndProcessQnodeQueryMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); +void dndProcessQnodeFetchMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); +int32_t dndProcessCreateQnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg); +int32_t dndProcessDropQnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DND_QNODE_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/impl/inc/dndSnode.h b/source/dnode/mgmt/impl/inc/dndSnode.h new file mode 100644 index 0000000000..8cb883794d --- /dev/null +++ b/source/dnode/mgmt/impl/inc/dndSnode.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DND_SNODE_H_ +#define _TD_DND_SNODE_H_ + +#ifdef __cplusplus +extern "C" { +#endif +#include "dndInt.h" + +int32_t dndInitSnode(SDnode *pDnode); +void dndCleanupSnode(SDnode *pDnode); + +void dndProcessSnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); +int32_t dndProcessCreateSnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg); +int32_t dndProcessDropSnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DND_SNODE_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/impl/src/dnode.c b/source/dnode/mgmt/impl/src/dnode.c index a4996ecb3b..c8ab35dd96 100644 --- a/source/dnode/mgmt/impl/src/dnode.c +++ b/source/dnode/mgmt/impl/src/dnode.c @@ -19,8 +19,6 @@ #include "dndTransport.h" #include "dndVnodes.h" #include "sync.h" -#include "tcache.h" -#include "tcrc32c.h" #include "wal.h" EStat dndGetStat(SDnode *pDnode) { return pDnode->stat; } @@ -93,6 +91,15 @@ static int32_t dndInitEnv(SDnode *pDnode, SDnodeOpt *pOption) { snprintf(path, sizeof(path), "%s%sdnode", pOption->dataDir, TD_DIRSEP); pDnode->dir.dnode = tstrdup(path); + snprintf(path, sizeof(path), "%s%sqnode", pOption->dataDir, TD_DIRSEP); + pDnode->dir.dnode = tstrdup(path); + + snprintf(path, sizeof(path), "%s%ssnode", pOption->dataDir, TD_DIRSEP); + pDnode->dir.dnode = tstrdup(path); + + snprintf(path, sizeof(path), "%s%sbnode", pOption->dataDir, TD_DIRSEP); + pDnode->dir.dnode = tstrdup(path); + if (pDnode->dir.mnode == NULL || pDnode->dir.vnodes == NULL || pDnode->dir.dnode == NULL) { dError("failed to malloc dir object"); terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -117,6 +124,24 @@ static int32_t dndInitEnv(SDnode *pDnode, SDnodeOpt *pOption) { return -1; } + if (taosMkDir(pDnode->dir.qnode) != 0) { + dError("failed to create dir:%s since %s", pDnode->dir.qnode, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + if (taosMkDir(pDnode->dir.snode) != 0) { + dError("failed to create dir:%s since %s", pDnode->dir.snode, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + if (taosMkDir(pDnode->dir.bnode) != 0) { + dError("failed to create dir:%s since %s", pDnode->dir.bnode, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + memcpy(&pDnode->opt, pOption, sizeof(SDnodeOpt)); return 0; } @@ -134,6 +159,18 @@ static void dndCleanupEnv(SDnode *pDnode) { tfree(pDnode->dir.dnode); } + if (pDnode->dir.qnode != NULL) { + tfree(pDnode->dir.qnode); + } + + if (pDnode->dir.bnode != NULL) { + tfree(pDnode->dir.snode); + } + + if (pDnode->dir.snode != NULL) { + tfree(pDnode->dir.bnode); + } + if (pDnode->lockFd >= 0) { taosUnLockFile(pDnode->lockFd); taosCloseFile(pDnode->lockFd); @@ -176,7 +213,7 @@ SDnode *dndInit(SDnodeOpt *pOption) { return NULL; } - if (vnodeInit(1) != 0) { + if (vnodeInit(pDnode->opt.numOfCommitThreads) != 0) { dError("failed to init vnode env"); dndCleanup(pDnode); return NULL; diff --git a/source/dnode/mgmt/impl/test/sut/src/server.cpp b/source/dnode/mgmt/impl/test/sut/src/server.cpp index a001748a12..f29b2fad1d 100644 --- a/source/dnode/mgmt/impl/test/sut/src/server.cpp +++ b/source/dnode/mgmt/impl/test/sut/src/server.cpp @@ -27,6 +27,7 @@ SDnodeOpt TestServer::BuildOption(const char* path, const char* fqdn, uint16_t p option.sver = 1; option.numOfCores = 1; option.numOfSupportVnodes = 1; + option.numOfCommitThreads = 1; option.statusInterval = 1; option.numOfThreadsPerCore = 1; option.ratioOfQueryCores = 1; diff --git a/source/dnode/qnode/CMakeLists.txt b/source/dnode/qnode/CMakeLists.txt index d9f932a631..f6f78f7357 100644 --- a/source/dnode/qnode/CMakeLists.txt +++ b/source/dnode/qnode/CMakeLists.txt @@ -4,4 +4,11 @@ target_include_directories( qnode PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/qnode" private "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) +target_link_libraries( + qnode + PRIVATE transport + PRIVATE os + PRIVATE common + PRIVATE util ) \ No newline at end of file diff --git a/source/dnode/qnode/inc/qnodeInt.h b/source/dnode/qnode/inc/qndInt.h similarity index 70% rename from source/dnode/qnode/inc/qnodeInt.h rename to source/dnode/qnode/inc/qndInt.h index 43df47845b..e9f1229a9d 100644 --- a/source/dnode/qnode/inc/qnodeInt.h +++ b/source/dnode/qnode/inc/qndInt.h @@ -16,10 +16,27 @@ #ifndef _TD_QNODE_INT_H_ #define _TD_QNODE_INT_H_ +#include "os.h" + +#include "tlog.h" +#include "tmsg.h" +#include "trpc.h" + +#include "qnode.h" + #ifdef __cplusplus extern "C" { #endif +typedef struct SQnode { + int32_t dnodeId; + int64_t clusterId; + SQnodeCfg cfg; + SendMsgToDnodeFp sendMsgToDnodeFp; + SendMsgToMnodeFp sendMsgToMnodeFp; + SendRedirectMsgFp sendRedirectMsgFp; +} SQnode; + #ifdef __cplusplus } #endif diff --git a/source/dnode/qnode/src/qnode.c b/source/dnode/qnode/src/qnode.c index 6dea4a4e57..ff1a460228 100644 --- a/source/dnode/qnode/src/qnode.c +++ b/source/dnode/qnode/src/qnode.c @@ -11,4 +11,25 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . - */ \ No newline at end of file + */ + +#include "qndInt.h" + +SQnode *qndOpen(const SQnodeOpt *pOption) { + SQnode *pQnode = calloc(1, sizeof(SQnode)); + return pQnode; +} + +void qndClose(SQnode *pQnode) { free(pQnode); } + +int32_t qndGetLoad(SQnode *pQnode, SQnodeLoad *pLoad) { return 0; } + +int32_t qndProcessQueryReq(SQnode *pQnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { + *pRsp = NULL; + return 0; +} + +int32_t qndProcessFetchReq(SQnode *pQnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { + *pRsp = NULL; + return 0; +} diff --git a/source/dnode/snode/CMakeLists.txt b/source/dnode/snode/CMakeLists.txt new file mode 100644 index 0000000000..a94dd9edd8 --- /dev/null +++ b/source/dnode/snode/CMakeLists.txt @@ -0,0 +1,14 @@ +aux_source_directory(src SNODE_SRC) +add_library(snode ${SNODE_SRC}) +target_include_directories( + snode + PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/snode" + private "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) +target_link_libraries( + snode + PRIVATE transport + PRIVATE os + PRIVATE common + PRIVATE util +) \ No newline at end of file diff --git a/source/dnode/snode/inc/sndInt.h b/source/dnode/snode/inc/sndInt.h new file mode 100644 index 0000000000..8827c92eef --- /dev/null +++ b/source/dnode/snode/inc/sndInt.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_SNODE_INT_H_ +#define _TD_SNODE_INT_H_ + +#include "os.h" + +#include "tlog.h" +#include "tmsg.h" +#include "trpc.h" + +#include "snode.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SSnode { + int32_t dnodeId; + int64_t clusterId; + SSnodeCfg cfg; + SendMsgToDnodeFp sendMsgToDnodeFp; + SendMsgToMnodeFp sendMsgToMnodeFp; + SendRedirectMsgFp sendRedirectMsgFp; +} SSnode; + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_SNODE_INT_H_*/ \ No newline at end of file diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c new file mode 100644 index 0000000000..3423ce41e2 --- /dev/null +++ b/source/dnode/snode/src/snode.c @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "sndInt.h" + +SSnode *sndOpen(const SSnodeOpt *pOption) { + SSnode *pSnode = calloc(1, sizeof(SSnode)); + return pSnode; +} + +void sndClose(SSnode *pSnode) { free(pSnode); } + +int32_t sndGetLoad(SSnode *pSnode, SSnodeLoad *pLoad) { return 0; } + +int32_t sndProcessWriteMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { + *pRsp = NULL; + return 0; +} From 7e3e6022db74290fa243cce37766745d63ac75b5 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 27 Dec 2021 19:06:26 +0800 Subject: [PATCH 05/21] complete index write/search --- source/libs/index/inc/indexInt.h | 17 ++++- source/libs/index/inc/index_cache.h | 6 ++ source/libs/index/inc/index_fst.h | 2 + source/libs/index/inc/index_tfile.h | 10 +++ source/libs/index/src/index.c | 87 +++++++++++++++++++++- source/libs/index/src/index_cache.c | 52 +++++++++++++ source/libs/index/src/index_tfile.c | 111 ++++++++++++++++++++++++++++ 7 files changed, 282 insertions(+), 3 deletions(-) diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index a8f231da0a..048c9e804e 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -51,6 +51,8 @@ struct SIndex { int64_t suid; // current super table id, -1 is normal table int32_t cVersion; // current version allocated to cache + char* path; + SIndexStat stat; pthread_mutex_t mtx; }; @@ -87,12 +89,23 @@ typedef struct SIndexTermQuery { EIndexQueryType qType; } SIndexTermQuery; -typedef struct Iterate { - void* iter; +typedef struct Iterate Iterate; + +typedef struct IterateValue { int8_t type; char* colVal; SArray* val; +} IterateValue; + +typedef struct Iterate { + void* iter; + IterateValue val; + bool (*next)(Iterate* iter); + IterateValue* (*getValue)(Iterate* iter); } Iterate; + +void iterateValueDestroy(IterateValue* iv, bool destroy); + extern void* indexQhandle; int indexFlushCacheTFile(SIndex* sIdx, void*); diff --git a/source/libs/index/inc/index_cache.h b/source/libs/index/inc/index_cache.h index 07b5b8d564..27f5a6fb20 100644 --- a/source/libs/index/inc/index_cache.h +++ b/source/libs/index/inc/index_cache.h @@ -41,6 +41,7 @@ typedef struct IndexCache { } IndexCache; +#define CACHE_VERSION(cache) atomic_load_32(&cache->version) typedef struct CacheTerm { // key int32_t nColVal; @@ -57,6 +58,9 @@ IndexCache* indexCacheCreate(SIndex* idx, const char* colName, int8_t type); void indexCacheDestroy(void* cache); +Iterate* indexCacheIteratorCreate(IndexCache* cache); +void indexCacheIteratorDestroy(Iterate* iiter); + int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid); // int indexCacheGet(void *cache, uint64_t *rst); @@ -66,6 +70,8 @@ void indexCacheRef(IndexCache* cache); void indexCacheUnRef(IndexCache* cache); void indexCacheDebug(IndexCache* cache); + +void indexCacheDestroySkiplist(SSkipList* slt); #ifdef __cplusplus } #endif diff --git a/source/libs/index/inc/index_fst.h b/source/libs/index/inc/index_fst.h index 3c572787fc..73c79b2619 100644 --- a/source/libs/index/inc/index_fst.h +++ b/source/libs/index/inc/index_fst.h @@ -319,6 +319,8 @@ bool streamWithStateSeekMin(StreamWithState* sws, FstBoundWithData* min); StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallback callback); FstStreamBuilder* fstStreamBuilderCreate(Fst* fst, AutomationCtx* aut); + +void fstStreamBuilderDestroy(FstStreamBuilder* b); // set up bound range // refator, simple code by marco diff --git a/source/libs/index/inc/index_tfile.h b/source/libs/index/inc/index_tfile.h index 550492ba50..f97a3126c8 100644 --- a/source/libs/index/inc/index_tfile.h +++ b/source/libs/index/inc/index_tfile.h @@ -113,6 +113,8 @@ int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SArr void tfileReaderRef(TFileReader* reader); void tfileReaderUnRef(TFileReader* reader); +TFileWriter* tfileWriterOpen(char* path, uint64_t suid, int32_t version, const char* colName, uint8_t type); +void tfileWriteClose(TFileWriter* tw); TFileWriter* tfileWriterCreate(WriterCtx* ctx, TFileHeader* header); void tfileWriterDestroy(TFileWriter* tw); int tfileWriterPut(TFileWriter* tw, void* data); @@ -123,6 +125,14 @@ IndexTFile* indexTFileCreate(const char* path); int indexTFilePut(void* tfile, SIndexTerm* term, uint64_t uid); int indexTFileSearch(void* tfile, SIndexTermQuery* query, SArray* result); +Iterate* tfileIteratorCreate(TFileReader* reader); +void tfileIteratorDestroy(Iterate* iterator); + +TFileValue* tfileValueCreate(char* val); + +int tfileValuePush(TFileValue* tf, uint64_t val); +void tfileValueDestroy(TFileValue* tf); + #ifdef __cplusplus } diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 3f871af01d..23df5f9f9a 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -75,9 +75,12 @@ int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { sIdx->tindex = indexTFileCreate(path); sIdx->colObj = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); sIdx->cVersion = 1; + sIdx->path = calloc(1, strlen(path) + 1); + memcpy(sIdx->path, path, strlen(path)); pthread_mutex_init(&sIdx->mtx, NULL); *index = sIdx; + return 0; #endif @@ -361,14 +364,96 @@ static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oType } return 0; } + int indexFlushCacheTFile(SIndex* sIdx, void* cache) { if (sIdx == NULL) { return -1; } indexWarn("suid %" PRIu64 " merge cache into tindex", sIdx->suid); - IndexCache* pCache = (IndexCache*)cache; + IndexCache* pCache = (IndexCache*)cache; TFileReader* pReader = tfileGetReaderByCol(sIdx->tindex, pCache->colName); + // handle flush + Iterate* cacheIter = indexCacheIteratorCreate(pCache); + Iterate* tfileIter = tfileIteratorCreate(pReader); + + SArray* result = taosArrayInit(1024, sizeof(void*)); + + bool cn = cacheIter->next(cacheIter); + bool tn = tfileIter->next(tfileIter); + while (cn == true && tn == true) { + IterateValue* cv = cacheIter->getValue(cacheIter); + IterateValue* tv = tfileIter->getValue(tfileIter); + + // dump value + int comp = strcmp(cv->colVal, tv->colVal); + if (comp == 0) { + TFileValue* tfv = tfileValueCreate(cv->colVal); + taosArrayAddAll(tfv->tableId, cv->val); + taosArrayAddAll(tfv->tableId, tv->val); + taosArrayPush(result, &tfv); + + cn = cacheIter->next(cacheIter); + tn = tfileIter->next(tfileIter); + continue; + } else if (comp < 0) { + TFileValue* tfv = tfileValueCreate(cv->colVal); + taosArrayAddAll(tfv->tableId, cv->val); + taosArrayPush(result, &tfv); + + // copy to final Result; + cn = cacheIter->next(cacheIter); + } else { + TFileValue* tfv = tfileValueCreate(tv->colVal); + taosArrayPush(result, &tfv); + taosArrayAddAll(tfv->tableId, tv->val); + // copy to final result + tn = tfileIter->next(tfileIter); + } + } + while (cn == true) { + IterateValue* cv = cacheIter->getValue(cacheIter); + TFileValue* tfv = tfileValueCreate(cv->colVal); + taosArrayAddAll(tfv->tableId, cv->val); + taosArrayPush(result, &tfv); + cn = cacheIter->next(cacheIter); + } + while (tn == true) { + IterateValue* tv = tfileIter->getValue(tfileIter); + TFileValue* tfv = tfileValueCreate(tv->colVal); + taosArrayAddAll(tfv->tableId, tv->val); + taosArrayPush(result, &tfv); + tn = tfileIter->next(tfileIter); + } + + int32_t version = CACHE_VERSION(pCache); + uint8_t colType = pCache->type; + + TFileWriter* tw = tfileWriterOpen(sIdx->path, sIdx->suid, version, pCache->colName, colType); + if (tw == NULL) { + indexError("faile to open file to write"); + } else { + int ret = tfileWriterPut(tw, result); + if (ret != 0) { indexError("faile to write into tindex "); } + } + // not free later, just put int table cache + SSkipList* timm = (SSkipList*)pCache->imm; + pCache->imm = NULL; // or throw int bg thread + indexCacheDestroySkiplist(timm); + + tfileWriteClose(tw); + indexCacheIteratorDestroy(cacheIter); + tfileIteratorDestroy(tfileIter); + tfileReaderUnRef(pReader); indexCacheUnRef(pCache); return 0; } +void iterateValueDestroy(IterateValue* value, bool destroy) { + if (destroy) { + taosArrayDestroy(value->val); + } else { + taosArrayClear(value->val); + } + free(value->colVal); + value->colVal = NULL; +} diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c index 8181c17505..54aee8858b 100644 --- a/source/libs/index/src/index_cache.c +++ b/source/libs/index/src/index_cache.c @@ -94,6 +94,16 @@ void indexCacheDebug(IndexCache* cache) { tSkipListDestroyIter(iter); } +void indexCacheDestroySkiplist(SSkipList* slt) { + SSkipListIterator* iter = tSkipListCreateIter(slt); + while (tSkipListIterNext(iter)) { + SSkipListNode* node = tSkipListIterGet(iter); + CacheTerm* ct = (CacheTerm*)SL_GET_NODE_DATA(node); + if (ct != NULL) {} + } + tSkipListDestroyIter(iter); +} + void indexCacheDestroy(void* cache) { IndexCache* pCache = cache; if (pCache == NULL) { return; } @@ -108,6 +118,48 @@ static void doMergeWork(SSchedMsg* msg) { SIndex* sidx = (SIndex*)pCache->index; indexFlushCacheTFile(sidx, pCache); } +static bool indexCacheIteratorNext(Iterate* itera) { + SSkipListIterator* iter = itera->iter; + if (iter == NULL) { return false; } + + IterateValue* iv = &itera->val; + iterateValueDestroy(iv, false); + + bool next = tSkipListIterNext(iter); + if (next) { + SSkipListNode* node = tSkipListIterGet(iter); + CacheTerm* ct = (CacheTerm*)SL_GET_NODE_DATA(node); + + iv->type = ct->operaType; + iv->colVal = ct->colVal; + + taosArrayPush(iv->val, &ct->uid); + } + + return next; +} + +static IterateValue* indexCacheIteratorGetValue(Iterate* iter) { + return &iter->val; +} +Iterate* indexCacheIteratorCreate(IndexCache* cache) { + Iterate* iiter = calloc(1, sizeof(Iterate)); + if (iiter == NULL) { return NULL; } + + iiter->val.val = taosArrayInit(1, sizeof(uint64_t)); + iiter->iter = cache->imm != NULL ? tSkipListCreateIter(cache->imm) : NULL; + iiter->next = indexCacheIteratorNext; + iiter->getValue = indexCacheIteratorGetValue; + + return iiter; +} +void indexCacheIteratorDestroy(Iterate* iter) { + if (iter == NULL) { return; } + + tSkipListDestroyIter(iter->iter); + iterateValueDestroy(&iter->val, true); + free(iter); +} int indexCacheSchedToMerge(IndexCache* pCache) { SSchedMsg schedMsg = {0}; diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index 0dfb14cc8d..fc31ff3c29 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -23,6 +23,13 @@ #include "taosdef.h" #include "tcompare.h" +typedef struct TFileFstIter { + FstStreamBuilder* fb; + StreamWithState* st; + AutomationCtx* ctx; + TFileReader* rdr; +} TFileFstIter; + #define TF_TABLE_TATOAL_SIZE(sz) (sizeof(sz) + sz * sizeof(uint64_t)) static int tfileStrCompare(const void* a, const void* b); @@ -184,6 +191,23 @@ int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SArray* resul return ret; } +TFileWriter* tfileWriterOpen(char* path, uint64_t suid, int32_t version, const char* colName, uint8_t colType) { + char filename[128] = {0}; + int32_t coldId = 1; + tfileGenFileName(filename, suid, coldId, version); + + char fullname[256] = {0}; + snprintf(fullname, sizeof(fullname), "%s/%s", path, filename); + WriterCtx* wcx = writerCtxCreate(TFile, fullname, true, 1024 * 1024); + + TFileHeader tfh = {0}; + tfh.suid = suid; + tfh.version = version; + memcpy(tfh.colName, colName, strlen(colName)); + tfh.colType = colType; + + return tfileWriterCreate(wcx, &tfh); +} TFileWriter* tfileWriterCreate(WriterCtx* ctx, TFileHeader* header) { // char pathBuf[128] = {0}; // sprintf(pathBuf, "%s/% " PRIu64 "-%d-%d.tindex", path, suid, colId, version); @@ -279,6 +303,11 @@ int tfileWriterPut(TFileWriter* tw, void* data) { tw->fb = NULL; return 0; } +void tfileWriteClose(TFileWriter* tw) { + if (tw == NULL) { return; } + writerCtxDestroy(tw->ctx); + free(tw); +} void tfileWriterDestroy(TFileWriter* tw) { if (tw == NULL) { return; } @@ -314,6 +343,71 @@ int indexTFilePut(void* tfile, SIndexTerm* term, uint64_t uid) { return 0; } +static bool tfileIteratorNext(Iterate* iiter) { + IterateValue* iv = &iiter->val; + iterateValueDestroy(iv, false); + // SArray* tblIds = iv->val; + + char* colVal = NULL; + uint64_t offset = 0; + + TFileFstIter* tIter = iiter->iter; + StreamWithStateResult* rt = streamWithStateNextWith(tIter->st, NULL); + if (rt == NULL) { return false; } + + int32_t sz = 0; + char* ch = (char*)fstSliceData(&rt->data, &sz); + colVal = calloc(1, sz + 1); + memcpy(colVal, ch, sz); + + offset = (uint64_t)(rt->out.out); + + swsResultDestroy(rt); + // set up iterate value + if (tfileReaderLoadTableIds(tIter->rdr, offset, iv->val) != 0) { return false; } + + iv->colVal = colVal; + + // std::string key(ch, sz); +} + +static IterateValue* tifileIterateGetValue(Iterate* iter) { + return &iter->val; +} + +static TFileFstIter* tfileFstIteratorCreate(TFileReader* reader) { + TFileFstIter* tIter = calloc(1, sizeof(Iterate)); + if (tIter == NULL) { return NULL; } + tIter->ctx = automCtxCreate(NULL, AUTOMATION_ALWAYS); + tIter->fb = fstSearch(reader->fst, tIter->ctx); + tIter->st = streamBuilderIntoStream(tIter->fb); + tIter->rdr = reader; + return tIter; +} + +Iterate* tfileIteratorCreate(TFileReader* reader) { + Iterate* iter = calloc(1, sizeof(Iterate)); + + iter->iter = tfileFstIteratorCreate(reader); + if (iter->iter == NULL) { return NULL; } + + iter->next = tfileIteratorNext; + iter->getValue = tifileIterateGetValue; + return iter; +} +void tfileIteratorDestroy(Iterate* iter) { + if (iter == NULL) { return; } + IterateValue* iv = &iter->val; + iterateValueDestroy(iv, true); + + TFileFstIter* tIter = iter->iter; + streamWithStateDestroy(tIter->st); + fstStreamBuilderDestroy(tIter->fb); + automCtxDestroy(tIter->ctx); + + free(iter); +} + TFileReader* tfileGetReaderByCol(IndexTFile* tf, char* colName) { if (tf == NULL) { return NULL; } TFileCacheKey key = {.suid = 0, .colType = TSDB_DATA_TYPE_BINARY, .colName = colName, .nColName = strlen(colName)}; @@ -334,6 +428,23 @@ static int tfileValueCompare(const void* a, const void* b, const void* param) { return fn(av->colVal, bv->colVal); } + +TFileValue* tfileValueCreate(char* val) { + TFileValue* tf = calloc(1, sizeof(TFileValue)); + if (tf == NULL) { return NULL; } + + tf->tableId = taosArrayInit(32, sizeof(uint64_t)); + return tf; +} +int tfileValuePush(TFileValue* tf, uint64_t val) { + if (tf == NULL) { return -1; } + taosArrayPush(tf->tableId, &val); + return 0; +} +void tfileValueDestroy(TFileValue* tf) { + taosArrayDestroy(tf->tableId); + free(tf); +} static void tfileSerialTableIdsToBuf(char* buf, SArray* ids) { int sz = taosArrayGetSize(ids); SERIALIZE_VAR_TO_BUF(buf, sz, int32_t); From 6d050231d8ec597b47baca95b9e7701146ae9bd4 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 27 Dec 2021 04:33:23 -0800 Subject: [PATCH 06/21] add qnode mgmt --- include/common/tmsg.h | 12 + include/util/taoserror.h | 150 ++------ source/dnode/mgmt/impl/inc/dndInt.h | 1 - source/dnode/mgmt/impl/inc/dndQnode.h | 2 +- source/dnode/mgmt/impl/src/dndMnode.c | 1 + source/dnode/mgmt/impl/src/dndQnode.c | 498 ++++++++++++++++++++++++++ source/dnode/mgmt/impl/src/dnode.c | 45 +-- source/util/src/terror.c | 144 +------- tests/script/general/table/basic1.sim | 13 +- tests/script/unique/dnode/basic1.sim | 34 +- 10 files changed, 582 insertions(+), 318 deletions(-) create mode 100644 source/dnode/mgmt/impl/src/dndQnode.c diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 6727dd3289..b3e8cf7d26 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -852,6 +852,18 @@ typedef struct { int32_t dnodeId; } SDropMnodeInMsg; +typedef struct { + int32_t dnodeId; +} SCreateQnodeInMsg, SDropQnodeInMsg; + +typedef struct { + int32_t dnodeId; +} SCreateSnodeInMsg, SDropSnodeInMsg; + +typedef struct { + int32_t dnodeId; +} SCreateBnodeInMsg, SDropBnodeInMsg; + typedef struct { int32_t dnodeId; int32_t vgId; diff --git a/include/util/taoserror.h b/include/util/taoserror.h index b6e8964b98..408a656e83 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -256,9 +256,27 @@ int32_t* taosGetErrno(); #define TSDB_CODE_DND_MNODE_ID_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x0423) #define TSDB_CODE_DND_MNODE_READ_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0424) #define TSDB_CODE_DND_MNODE_WRITE_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0425) -#define TSDB_CODE_DND_VNODE_TOO_MANY_VNODES TAOS_DEF_ERROR_CODE(0, 0x0430) -#define TSDB_CODE_DND_VNODE_READ_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0431) -#define TSDB_CODE_DND_VNODE_WRITE_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0432) +#define TSDB_CODE_DND_QNODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0430) +#define TSDB_CODE_DND_QNODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0431) +#define TSDB_CODE_DND_QNODE_ID_INVALID TAOS_DEF_ERROR_CODE(0, 0x0432) +#define TSDB_CODE_DND_QNODE_ID_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x0433) +#define TSDB_CODE_DND_QNODE_READ_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0434) +#define TSDB_CODE_DND_QNODE_WRITE_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0435) +#define TSDB_CODE_DND_SNODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0440) +#define TSDB_CODE_DND_SNODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0441) +#define TSDB_CODE_DND_SNODE_ID_INVALID TAOS_DEF_ERROR_CODE(0, 0x0442) +#define TSDB_CODE_DND_SNODE_ID_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x0443) +#define TSDB_CODE_DND_SNODE_READ_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0444) +#define TSDB_CODE_DND_SNODE_WRITE_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0445) +#define TSDB_CODE_DND_BNODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0450) +#define TSDB_CODE_DND_BNODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0451) +#define TSDB_CODE_DND_BNODE_ID_INVALID TAOS_DEF_ERROR_CODE(0, 0x0452) +#define TSDB_CODE_DND_BNODE_ID_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x0453) +#define TSDB_CODE_DND_BNODE_READ_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0454) +#define TSDB_CODE_DND_BNODE_WRITE_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0455) +#define TSDB_CODE_DND_VNODE_TOO_MANY_VNODES TAOS_DEF_ERROR_CODE(0, 0x0460) +#define TSDB_CODE_DND_VNODE_READ_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0461) +#define TSDB_CODE_DND_VNODE_WRITE_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0462) // vnode #define TSDB_CODE_VND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0500) //"Action in progress") @@ -377,132 +395,6 @@ int32_t* taosGetErrno(); #define TSDB_CODE_WAL_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x1001) //"WAL file is corrupted") #define TSDB_CODE_WAL_SIZE_LIMIT TAOS_DEF_ERROR_CODE(0, 0x1002) //"WAL size exceeds limit") -// http -#define TSDB_CODE_HTTP_SERVER_OFFLINE TAOS_DEF_ERROR_CODE(0, 0x1100) //"http server is not onlin") -#define TSDB_CODE_HTTP_UNSUPPORT_URL TAOS_DEF_ERROR_CODE(0, 0x1101) //"url is not support") -#define TSDB_CODE_HTTP_INVALID_URL TAOS_DEF_ERROR_CODE(0, 0x1102) //invalid url format") -#define TSDB_CODE_HTTP_NO_ENOUGH_MEMORY TAOS_DEF_ERROR_CODE(0, 0x1103) //"no enough memory") -#define TSDB_CODE_HTTP_REQUSET_TOO_BIG TAOS_DEF_ERROR_CODE(0, 0x1104) //"request size is too big") -#define TSDB_CODE_HTTP_NO_AUTH_INFO TAOS_DEF_ERROR_CODE(0, 0x1105) //"no auth info input") -#define TSDB_CODE_HTTP_NO_MSG_INPUT TAOS_DEF_ERROR_CODE(0, 0x1106) //"request is empty") -#define TSDB_CODE_HTTP_NO_SQL_INPUT TAOS_DEF_ERROR_CODE(0, 0x1107) //"no sql input") -#define TSDB_CODE_HTTP_NO_EXEC_USEDB TAOS_DEF_ERROR_CODE(0, 0x1108) //"no need to execute use db cmd") -#define TSDB_CODE_HTTP_SESSION_FULL TAOS_DEF_ERROR_CODE(0, 0x1109) //"session list was full") -#define TSDB_CODE_HTTP_GEN_TAOSD_TOKEN_ERR TAOS_DEF_ERROR_CODE(0, 0x110A) //"generate taosd token error") -#define TSDB_CODE_HTTP_INVALID_MULTI_REQUEST TAOS_DEF_ERROR_CODE(0, 0x110B) //"size of multi request is 0") -#define TSDB_CODE_HTTP_CREATE_GZIP_FAILED TAOS_DEF_ERROR_CODE(0, 0x110C) //"failed to create gzip") -#define TSDB_CODE_HTTP_FINISH_GZIP_FAILED TAOS_DEF_ERROR_CODE(0, 0x110D) //"failed to finish gzip") -#define TSDB_CODE_HTTP_LOGIN_FAILED TAOS_DEF_ERROR_CODE(0, 0x110E) //"failed to login") - -#define TSDB_CODE_HTTP_INVALID_VERSION TAOS_DEF_ERROR_CODE(0, 0x1120) //"invalid http version") -#define TSDB_CODE_HTTP_INVALID_CONTENT_LENGTH TAOS_DEF_ERROR_CODE(0, 0x1121) //"invalid content length") -#define TSDB_CODE_HTTP_INVALID_AUTH_TYPE TAOS_DEF_ERROR_CODE(0, 0x1122) //"invalid type of Authorization") -#define TSDB_CODE_HTTP_INVALID_AUTH_FORMAT TAOS_DEF_ERROR_CODE(0, 0x1123) //"invalid format of Authorization") -#define TSDB_CODE_HTTP_INVALID_BASIC_AUTH TAOS_DEF_ERROR_CODE(0, 0x1124) //"invalid basic Authorization") -#define TSDB_CODE_HTTP_INVALID_TAOSD_AUTH TAOS_DEF_ERROR_CODE(0, 0x1125) //"invalid taosd Authorization") -#define TSDB_CODE_HTTP_PARSE_METHOD_FAILED TAOS_DEF_ERROR_CODE(0, 0x1126) //"failed to parse method") -#define TSDB_CODE_HTTP_PARSE_TARGET_FAILED TAOS_DEF_ERROR_CODE(0, 0x1127) //"failed to parse target") -#define TSDB_CODE_HTTP_PARSE_VERSION_FAILED TAOS_DEF_ERROR_CODE(0, 0x1128) //"failed to parse http version") -#define TSDB_CODE_HTTP_PARSE_SP_FAILED TAOS_DEF_ERROR_CODE(0, 0x1129) //"failed to parse sp") -#define TSDB_CODE_HTTP_PARSE_STATUS_FAILED TAOS_DEF_ERROR_CODE(0, 0x112A) //"failed to parse status") -#define TSDB_CODE_HTTP_PARSE_PHRASE_FAILED TAOS_DEF_ERROR_CODE(0, 0x112B) //"failed to parse phrase") -#define TSDB_CODE_HTTP_PARSE_CRLF_FAILED TAOS_DEF_ERROR_CODE(0, 0x112C) //"failed to parse crlf") -#define TSDB_CODE_HTTP_PARSE_HEADER_FAILED TAOS_DEF_ERROR_CODE(0, 0x112D) //"failed to parse header") -#define TSDB_CODE_HTTP_PARSE_HEADER_KEY_FAILED TAOS_DEF_ERROR_CODE(0, 0x112E) //"failed to parse header key") -#define TSDB_CODE_HTTP_PARSE_HEADER_VAL_FAILED TAOS_DEF_ERROR_CODE(0, 0x112F) //"failed to parse header val") -#define TSDB_CODE_HTTP_PARSE_CHUNK_SIZE_FAILED TAOS_DEF_ERROR_CODE(0, 0x1130) //"failed to parse chunk size") -#define TSDB_CODE_HTTP_PARSE_CHUNK_FAILED TAOS_DEF_ERROR_CODE(0, 0x1131) //"failed to parse chunk") -#define TSDB_CODE_HTTP_PARSE_END_FAILED TAOS_DEF_ERROR_CODE(0, 0x1132) //"failed to parse end section") -#define TSDB_CODE_HTTP_PARSE_INVALID_STATE TAOS_DEF_ERROR_CODE(0, 0x1134) //"invalid parse state") -#define TSDB_CODE_HTTP_PARSE_ERROR_STATE TAOS_DEF_ERROR_CODE(0, 0x1135) //"failed to parse error section") - -#define TSDB_CODE_HTTP_GC_QUERY_NULL TAOS_DEF_ERROR_CODE(0, 0x1150) //"query size is 0") -#define TSDB_CODE_HTTP_GC_QUERY_SIZE TAOS_DEF_ERROR_CODE(0, 0x1151) //"query size can not more than 100") -#define TSDB_CODE_HTTP_GC_REQ_PARSE_ERROR TAOS_DEF_ERROR_CODE(0, 0x1152) //"parse grafana json error") - -#define TSDB_CODE_HTTP_TG_DB_NOT_INPUT TAOS_DEF_ERROR_CODE(0, 0x1160) //"database name can not be null") -#define TSDB_CODE_HTTP_TG_DB_TOO_LONG TAOS_DEF_ERROR_CODE(0, 0x1161) //"database name too long") -#define TSDB_CODE_HTTP_TG_INVALID_JSON TAOS_DEF_ERROR_CODE(0, 0x1162) //"invalid telegraf json fromat") -#define TSDB_CODE_HTTP_TG_METRICS_NULL TAOS_DEF_ERROR_CODE(0, 0x1163) //"metrics size is 0") -#define TSDB_CODE_HTTP_TG_METRICS_SIZE TAOS_DEF_ERROR_CODE(0, 0x1164) //"metrics size can not more than 1K") -#define TSDB_CODE_HTTP_TG_METRIC_NULL TAOS_DEF_ERROR_CODE(0, 0x1165) //"metric name not find") -#define TSDB_CODE_HTTP_TG_METRIC_TYPE TAOS_DEF_ERROR_CODE(0, 0x1166) //"metric name type should be string") -#define TSDB_CODE_HTTP_TG_METRIC_NAME_NULL TAOS_DEF_ERROR_CODE(0, 0x1167) //"metric name length is 0") -#define TSDB_CODE_HTTP_TG_METRIC_NAME_LONG TAOS_DEF_ERROR_CODE(0, 0x1168) //"metric name length too long") -#define TSDB_CODE_HTTP_TG_TIMESTAMP_NULL TAOS_DEF_ERROR_CODE(0, 0x1169) //"timestamp not find") -#define TSDB_CODE_HTTP_TG_TIMESTAMP_TYPE TAOS_DEF_ERROR_CODE(0, 0x116A) //"timestamp type should be integer") -#define TSDB_CODE_HTTP_TG_TIMESTAMP_VAL_NULL TAOS_DEF_ERROR_CODE(0, 0x116B) //"timestamp value smaller than 0") -#define TSDB_CODE_HTTP_TG_TAGS_NULL TAOS_DEF_ERROR_CODE(0, 0x116C) //"tags not find") -#define TSDB_CODE_HTTP_TG_TAGS_SIZE_0 TAOS_DEF_ERROR_CODE(0, 0x116D) //"tags size is 0") -#define TSDB_CODE_HTTP_TG_TAGS_SIZE_LONG TAOS_DEF_ERROR_CODE(0, 0x116E) //"tags size too long") -#define TSDB_CODE_HTTP_TG_TAG_NULL TAOS_DEF_ERROR_CODE(0, 0x116F) //"tag is null") -#define TSDB_CODE_HTTP_TG_TAG_NAME_NULL TAOS_DEF_ERROR_CODE(0, 0x1170) //"tag name is null") -#define TSDB_CODE_HTTP_TG_TAG_NAME_SIZE TAOS_DEF_ERROR_CODE(0, 0x1171) //"tag name length too long") -#define TSDB_CODE_HTTP_TG_TAG_VALUE_TYPE TAOS_DEF_ERROR_CODE(0, 0x1172) //"tag value type should be number or string") -#define TSDB_CODE_HTTP_TG_TAG_VALUE_NULL TAOS_DEF_ERROR_CODE(0, 0x1173) //"tag value is null") -#define TSDB_CODE_HTTP_TG_TABLE_NULL TAOS_DEF_ERROR_CODE(0, 0x1174) //"table is null") -#define TSDB_CODE_HTTP_TG_TABLE_SIZE TAOS_DEF_ERROR_CODE(0, 0x1175) //"table name length too long") -#define TSDB_CODE_HTTP_TG_FIELDS_NULL TAOS_DEF_ERROR_CODE(0, 0x1176) //"fields not find") -#define TSDB_CODE_HTTP_TG_FIELDS_SIZE_0 TAOS_DEF_ERROR_CODE(0, 0x1177) //"fields size is 0") -#define TSDB_CODE_HTTP_TG_FIELDS_SIZE_LONG TAOS_DEF_ERROR_CODE(0, 0x1178) //"fields size too long") -#define TSDB_CODE_HTTP_TG_FIELD_NULL TAOS_DEF_ERROR_CODE(0, 0x1179) //"field is null") -#define TSDB_CODE_HTTP_TG_FIELD_NAME_NULL TAOS_DEF_ERROR_CODE(0, 0x117A) //"field name is null") -#define TSDB_CODE_HTTP_TG_FIELD_NAME_SIZE TAOS_DEF_ERROR_CODE(0, 0x117B) //"field name length too long") -#define TSDB_CODE_HTTP_TG_FIELD_VALUE_TYPE TAOS_DEF_ERROR_CODE(0, 0x117C) //"field value type should be number or string") -#define TSDB_CODE_HTTP_TG_FIELD_VALUE_NULL TAOS_DEF_ERROR_CODE(0, 0x117D) //"field value is null") -#define TSDB_CODE_HTTP_TG_HOST_NOT_STRING TAOS_DEF_ERROR_CODE(0, 0x117E) //"host type should be string") -#define TSDB_CODE_HTTP_TG_STABLE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x117F) //"stable not exist") - -#define TSDB_CODE_HTTP_OP_DB_NOT_INPUT TAOS_DEF_ERROR_CODE(0, 0x1190) //"database name can not be null") -#define TSDB_CODE_HTTP_OP_DB_TOO_LONG TAOS_DEF_ERROR_CODE(0, 0x1191) //"database name too long") -#define TSDB_CODE_HTTP_OP_INVALID_JSON TAOS_DEF_ERROR_CODE(0, 0x1192) //"invalid opentsdb json fromat") -#define TSDB_CODE_HTTP_OP_METRICS_NULL TAOS_DEF_ERROR_CODE(0, 0x1193) //"metrics size is 0") -#define TSDB_CODE_HTTP_OP_METRICS_SIZE TAOS_DEF_ERROR_CODE(0, 0x1194) //"metrics size can not more than 10K") -#define TSDB_CODE_HTTP_OP_METRIC_NULL TAOS_DEF_ERROR_CODE(0, 0x1195) //"metric name not find") -#define TSDB_CODE_HTTP_OP_METRIC_TYPE TAOS_DEF_ERROR_CODE(0, 0x1196) //"metric name type should be string") -#define TSDB_CODE_HTTP_OP_METRIC_NAME_NULL TAOS_DEF_ERROR_CODE(0, 0x1197) //"metric name length is 0") -#define TSDB_CODE_HTTP_OP_METRIC_NAME_LONG TAOS_DEF_ERROR_CODE(0, 0x1198) //"metric name length can not more than 22") -#define TSDB_CODE_HTTP_OP_TIMESTAMP_NULL TAOS_DEF_ERROR_CODE(0, 0x1199) //"timestamp not find") -#define TSDB_CODE_HTTP_OP_TIMESTAMP_TYPE TAOS_DEF_ERROR_CODE(0, 0x119A) //"timestamp type should be integer") -#define TSDB_CODE_HTTP_OP_TIMESTAMP_VAL_NULL TAOS_DEF_ERROR_CODE(0, 0x119B) //"timestamp value smaller than 0") -#define TSDB_CODE_HTTP_OP_TAGS_NULL TAOS_DEF_ERROR_CODE(0, 0x119C) //"tags not find") -#define TSDB_CODE_HTTP_OP_TAGS_SIZE_0 TAOS_DEF_ERROR_CODE(0, 0x119D) //"tags size is 0") -#define TSDB_CODE_HTTP_OP_TAGS_SIZE_LONG TAOS_DEF_ERROR_CODE(0, 0x119E) //"tags size too long") -#define TSDB_CODE_HTTP_OP_TAG_NULL TAOS_DEF_ERROR_CODE(0, 0x119F) //"tag is null") -#define TSDB_CODE_HTTP_OP_TAG_NAME_NULL TAOS_DEF_ERROR_CODE(0, 0x11A0) //"tag name is null") -#define TSDB_CODE_HTTP_OP_TAG_NAME_SIZE TAOS_DEF_ERROR_CODE(0, 0x11A1) //"tag name length too long") -#define TSDB_CODE_HTTP_OP_TAG_VALUE_TYPE TAOS_DEF_ERROR_CODE(0, 0x11A2) //"tag value type should be boolean number or string") -#define TSDB_CODE_HTTP_OP_TAG_VALUE_NULL TAOS_DEF_ERROR_CODE(0, 0x11A3) //"tag value is null") -#define TSDB_CODE_HTTP_OP_TAG_VALUE_TOO_LONG TAOS_DEF_ERROR_CODE(0, 0x11A4) //"tag value can not more than 64") -#define TSDB_CODE_HTTP_OP_VALUE_NULL TAOS_DEF_ERROR_CODE(0, 0x11A5) //"value not find") -#define TSDB_CODE_HTTP_OP_VALUE_TYPE TAOS_DEF_ERROR_CODE(0, 0x11A6) //"value type should be boolean number or string") - -#define TSDB_CODE_HTTP_REQUEST_JSON_ERROR TAOS_DEF_ERROR_CODE(0, 0x1F00) //"http request json error") - -// odbc -#define TSDB_CODE_ODBC_OOM TAOS_DEF_ERROR_CODE(0, 0x2100) //"out of memory") -#define TSDB_CODE_ODBC_CONV_CHAR_NOT_NUM TAOS_DEF_ERROR_CODE(0, 0x2101) //"convertion not a valid literal input") -#define TSDB_CODE_ODBC_CONV_UNDEF TAOS_DEF_ERROR_CODE(0, 0x2102) //"convertion undefined") -#define TSDB_CODE_ODBC_CONV_TRUNC_FRAC TAOS_DEF_ERROR_CODE(0, 0x2103) //"convertion fractional truncated") -#define TSDB_CODE_ODBC_CONV_TRUNC TAOS_DEF_ERROR_CODE(0, 0x2104) //"convertion truncated") -#define TSDB_CODE_ODBC_CONV_NOT_SUPPORT TAOS_DEF_ERROR_CODE(0, 0x2105) //"convertion not supported") -#define TSDB_CODE_ODBC_CONV_OOR TAOS_DEF_ERROR_CODE(0, 0x2106) //"convertion numeric value out of range") -#define TSDB_CODE_ODBC_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x2107) //"out of range") -#define TSDB_CODE_ODBC_NOT_SUPPORT TAOS_DEF_ERROR_CODE(0, 0x2108) //"not supported yet") -#define TSDB_CODE_ODBC_INVALID_HANDLE TAOS_DEF_ERROR_CODE(0, 0x2109) //"invalid handle") -#define TSDB_CODE_ODBC_NO_RESULT TAOS_DEF_ERROR_CODE(0, 0x210a) //"no result set") -#define TSDB_CODE_ODBC_NO_FIELDS TAOS_DEF_ERROR_CODE(0, 0x210b) //"no fields returned") -#define TSDB_CODE_ODBC_INVALID_CURSOR TAOS_DEF_ERROR_CODE(0, 0x210c) //"invalid cursor") -#define TSDB_CODE_ODBC_STATEMENT_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x210d) //"statement not ready") -#define TSDB_CODE_ODBC_CONNECTION_BUSY TAOS_DEF_ERROR_CODE(0, 0x210e) //"connection still busy") -#define TSDB_CODE_ODBC_BAD_CONNSTR TAOS_DEF_ERROR_CODE(0, 0x210f) //"bad connection string") -#define TSDB_CODE_ODBC_BAD_ARG TAOS_DEF_ERROR_CODE(0, 0x2110) //"bad argument") -#define TSDB_CODE_ODBC_CONV_NOT_VALID_TS TAOS_DEF_ERROR_CODE(0, 0x2111) //"not a valid timestamp") -#define TSDB_CODE_ODBC_CONV_SRC_TOO_LARGE TAOS_DEF_ERROR_CODE(0, 0x2112) //"src too large") -#define TSDB_CODE_ODBC_CONV_SRC_BAD_SEQ TAOS_DEF_ERROR_CODE(0, 0x2113) //"src bad sequence") -#define TSDB_CODE_ODBC_CONV_SRC_INCOMPLETE TAOS_DEF_ERROR_CODE(0, 0x2114) //"src incomplete") -#define TSDB_CODE_ODBC_CONV_SRC_GENERAL TAOS_DEF_ERROR_CODE(0, 0x2115) //"src general") - // tfs #define TSDB_CODE_FS_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x2200) //"tfs out of memory") #define TSDB_CODE_FS_INVLD_CFG TAOS_DEF_ERROR_CODE(0, 0x2201) //"tfs invalid mount config") diff --git a/source/dnode/mgmt/impl/inc/dndInt.h b/source/dnode/mgmt/impl/inc/dndInt.h index b7a414672e..9679a03c3e 100644 --- a/source/dnode/mgmt/impl/inc/dndInt.h +++ b/source/dnode/mgmt/impl/inc/dndInt.h @@ -59,7 +59,6 @@ typedef void (*DndMsgFp)(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEps); typedef struct { char *dnode; char *mnode; - char *qnode; char *snode; char *bnode; char *vnodes; diff --git a/source/dnode/mgmt/impl/inc/dndQnode.h b/source/dnode/mgmt/impl/inc/dndQnode.h index 951209a193..677c234679 100644 --- a/source/dnode/mgmt/impl/inc/dndQnode.h +++ b/source/dnode/mgmt/impl/inc/dndQnode.h @@ -24,7 +24,7 @@ extern "C" { int32_t dndInitQnode(SDnode *pDnode); void dndCleanupQnode(SDnode *pDnode); -ioid dndProcessQnodeQueryMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); +void dndProcessQnodeQueryMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); void dndProcessQnodeFetchMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); int32_t dndProcessCreateQnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg); int32_t dndProcessDropQnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg); diff --git a/source/dnode/mgmt/impl/src/dndMnode.c b/source/dnode/mgmt/impl/src/dndMnode.c index 50da49d325..59f809489e 100644 --- a/source/dnode/mgmt/impl/src/dndMnode.c +++ b/source/dnode/mgmt/impl/src/dndMnode.c @@ -796,6 +796,7 @@ void dndCleanupMnode(SDnode *pDnode) { if (pMgmt->pMnode) dndStopMnodeWorker(pDnode); tfree(pMgmt->file); mndClose(pMgmt->pMnode); + pMgmt->pMnode = NULL; dInfo("dnode-mnode is cleaned up"); } diff --git a/source/dnode/mgmt/impl/src/dndQnode.c b/source/dnode/mgmt/impl/src/dndQnode.c new file mode 100644 index 0000000000..08044ea263 --- /dev/null +++ b/source/dnode/mgmt/impl/src/dndQnode.c @@ -0,0 +1,498 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dndQnode.h" +#include "dndDnode.h" +#include "dndTransport.h" + +static int32_t dndInitQnodeQueryWorker(SDnode *pDnode); +static int32_t dndInitQnodeFetchWorker(SDnode *pDnode); +static void dndCleanupQnodeQueryWorker(SDnode *pDnode); +static void dndCleanupQnodeFetchWorker(SDnode *pDnode); +static int32_t dndAllocQnodeQueryQueue(SDnode *pDnode); +static int32_t dndAllocQnodeFetchQueue(SDnode *pDnode); +static void dndFreeQnodeQueryQueue(SDnode *pDnode); +static void dndFreeQnodeFetchQueue(SDnode *pDnode); + +static void dndProcessQnodeQueue(SDnode *pDnode, SRpcMsg *pMsg); +static int32_t dndWriteQnodeMsgToQueue(SQnode *pQnode, taos_queue pQueue, SRpcMsg *pRpcMsg); + +static int32_t dndStartQnodeWorker(SDnode *pDnode); +static void dndStopQnodeWorker(SDnode *pDnode); + +static SQnode *dndAcquireQnode(SDnode *pDnode); +static void dndReleaseQnode(SDnode *pDnode, SQnode *pQnode); + +static int32_t dndReadQnodeFile(SDnode *pDnode); +static int32_t dndWriteQnodeFile(SDnode *pDnode); + +static int32_t dndOpenQnode(SDnode *pDnode); +static int32_t dndDropQnode(SDnode *pDnode); + +static SQnode *dndAcquireQnode(SDnode *pDnode) { + SQnodeMgmt *pMgmt = &pDnode->qmgmt; + SQnode *pQnode = NULL; + int32_t refCount = 0; + + taosRLockLatch(&pMgmt->latch); + if (pMgmt->deployed && !pMgmt->dropped) { + refCount = atomic_add_fetch_32(&pMgmt->refCount, 1); + pQnode = pMgmt->pQnode; + } else { + terrno = TSDB_CODE_DND_QNODE_NOT_DEPLOYED; + } + taosRUnLockLatch(&pMgmt->latch); + + if (pQnode != NULL) { + dTrace("acquire qnode, refCount:%d", refCount); + } + return pQnode; +} + +static void dndReleaseQnode(SDnode *pDnode, SQnode *pQnode) { + SQnodeMgmt *pMgmt = &pDnode->qmgmt; + int32_t refCount = 0; + + taosRLockLatch(&pMgmt->latch); + if (pQnode != NULL) { + refCount = atomic_sub_fetch_32(&pMgmt->refCount, 1); + } + taosRUnLockLatch(&pMgmt->latch); + + if (pQnode != NULL) { + dTrace("release qnode, refCount:%d", refCount); + } +} + +static int32_t dndReadQnodeFile(SDnode *pDnode) { + SQnodeMgmt *pMgmt = &pDnode->qmgmt; + int32_t code = TSDB_CODE_DND_QNODE_READ_FILE_ERROR; + int32_t len = 0; + int32_t maxLen = 4096; + char *content = calloc(1, maxLen + 1); + cJSON *root = NULL; + + FILE *fp = fopen(pMgmt->file, "r"); + if (fp == NULL) { + dDebug("file %s not exist", pMgmt->file); + code = 0; + goto PRASE_MNODE_OVER; + } + + len = (int32_t)fread(content, 1, maxLen, fp); + if (len <= 0) { + dError("failed to read %s since content is null", pMgmt->file); + goto PRASE_MNODE_OVER; + } + + content[len] = 0; + root = cJSON_Parse(content); + if (root == NULL) { + dError("failed to read %s since invalid json format", pMgmt->file); + goto PRASE_MNODE_OVER; + } + + cJSON *deployed = cJSON_GetObjectItem(root, "deployed"); + if (!deployed || deployed->type != cJSON_Number) { + dError("failed to read %s since deployed not found", pMgmt->file); + goto PRASE_MNODE_OVER; + } + pMgmt->deployed = deployed->valueint; + + cJSON *dropped = cJSON_GetObjectItem(root, "dropped"); + if (!dropped || dropped->type != cJSON_Number) { + dError("failed to read %s since dropped not found", pMgmt->file); + goto PRASE_MNODE_OVER; + } + pMgmt->dropped = dropped->valueint; + + code = 0; + dDebug("succcessed to read file %s, deployed:%d dropped:%d", pMgmt->file, pMgmt->deployed, pMgmt->dropped); + +PRASE_MNODE_OVER: + if (content != NULL) free(content); + if (root != NULL) cJSON_Delete(root); + if (fp != NULL) fclose(fp); + + terrno = code; + return code; +} + +static int32_t dndWriteQnodeFile(SDnode *pDnode) { + SQnodeMgmt *pMgmt = &pDnode->qmgmt; + + char file[PATH_MAX + 20] = {0}; + snprintf(file, sizeof(file), "%s.bak", pMgmt->file); + + FILE *fp = fopen(file, "w"); + if (fp == NULL) { + terrno = TSDB_CODE_DND_QNODE_WRITE_FILE_ERROR; + dError("failed to write %s since %s", file, terrstr()); + return -1; + } + + int32_t len = 0; + int32_t maxLen = 4096; + char *content = calloc(1, maxLen + 1); + + len += snprintf(content + len, maxLen - len, "{\n"); + len += snprintf(content + len, maxLen - len, " \"deployed\": %d,\n", pMgmt->deployed); + len += snprintf(content + len, maxLen - len, " \"dropped\": %d\n", pMgmt->dropped); + len += snprintf(content + len, maxLen - len, "}\n"); + + fwrite(content, 1, len, fp); + taosFfetchFile(fileno(fp)); + fclose(fp); + free(content); + + if (taosRenameFile(file, pMgmt->file) != 0) { + terrno = TSDB_CODE_DND_QNODE_WRITE_FILE_ERROR; + dError("failed to rename %s since %s", pMgmt->file, terrstr()); + return -1; + } + + dInfo("successed to write %s, deployed:%d dropped:%d", pMgmt->file, pMgmt->deployed, pMgmt->dropped); + return 0; +} + +static int32_t dndStartQnodeWorker(SDnode *pDnode) { + if (dndInitQnodeQueryWorker(pDnode) != 0) { + dError("failed to start qnode query worker since %s", terrstr()); + return -1; + } + + if (dndInitQnodeFetchWorker(pDnode) != 0) { + dError("failed to start qnode fetch worker since %s", terrstr()); + return -1; + } + + if (dndAllocQnodeQueryQueue(pDnode) != 0) { + dError("failed to alloc qnode query queue since %s", terrstr()); + return -1; + } + + if (dndAllocQnodeFetchQueue(pDnode) != 0) { + dError("failed to alloc qnode fetch queue since %s", terrstr()); + return -1; + } + + return 0; +} + +static void dndStopQnodeWorker(SDnode *pDnode) { + SQnodeMgmt *pMgmt = &pDnode->qmgmt; + + taosWLockLatch(&pMgmt->latch); + pMgmt->deployed = 0; + taosWUnLockLatch(&pMgmt->latch); + + while (pMgmt->refCount > 1) taosMsleep(10); + while (!taosQueueEmpty(pMgmt->pQueryQ)) taosMsleep(10); + while (!taosQueueEmpty(pMgmt->pFetchQ)) taosMsleep(10); + + dndCleanupQnodeQueryWorker(pDnode); + dndCleanupQnodeFetchWorker(pDnode); + + dndFreeQnodeQueryQueue(pDnode); + dndFreeQnodeFetchQueue(pDnode); +} + +static void dndBuildQnodeOption(SDnode *pDnode, SQnodeOpt *pOption) { + pOption->pDnode = pDnode; + pOption->sendMsgToDnodeFp = dndSendMsgToDnode; + pOption->sendMsgToMnodeFp = dndSendMsgToMnode; + pOption->sendRedirectMsgFp = dndSendRedirectMsg; + pOption->dnodeId = dndGetDnodeId(pDnode); + pOption->clusterId = dndGetClusterId(pDnode); + pOption->cfg.sver = pDnode->opt.sver; +} + +static int32_t dndOpenQnode(SDnode *pDnode) { + SQnodeMgmt *pMgmt = &pDnode->qmgmt; + SQnodeOpt option = {0}; + dndBuildQnodeOption(pDnode, &option); + + SQnode *pQnode = qndOpen(&option); + if (pQnode == NULL) { + dError("failed to open qnode since %s", terrstr()); + return -1; + } + pMgmt->deployed = 1; + + int32_t code = dndWriteQnodeFile(pDnode); + if (code != 0) { + dError("failed to write qnode file since %s", terrstr()); + code = terrno; + pMgmt->deployed = 0; + qndClose(pQnode); + // qndDestroy(pDnode->dir.qnode); + terrno = code; + return -1; + } + + code = dndStartQnodeWorker(pDnode); + if (code != 0) { + dError("failed to start qnode worker since %s", terrstr()); + code = terrno; + pMgmt->deployed = 0; + dndStopQnodeWorker(pDnode); + qndClose(pQnode); + // qndDestroy(pDnode->dir.qnode); + terrno = code; + return -1; + } + + taosWLockLatch(&pMgmt->latch); + pMgmt->pQnode = pQnode; + pMgmt->deployed = 1; + taosWUnLockLatch(&pMgmt->latch); + + dInfo("qnode open successfully"); + return 0; +} + +static int32_t dndDropQnode(SDnode *pDnode) { + SQnodeMgmt *pMgmt = &pDnode->qmgmt; + + SQnode *pQnode = dndAcquireQnode(pDnode); + if (pQnode == NULL) { + dError("failed to drop qnode since %s", terrstr()); + return -1; + } + + taosRLockLatch(&pMgmt->latch); + pMgmt->dropped = 1; + taosRUnLockLatch(&pMgmt->latch); + + if (dndWriteQnodeFile(pDnode) != 0) { + taosRLockLatch(&pMgmt->latch); + pMgmt->dropped = 0; + taosRUnLockLatch(&pMgmt->latch); + + dndReleaseQnode(pDnode, pQnode); + dError("failed to drop qnode since %s", terrstr()); + return -1; + } + + dndReleaseQnode(pDnode, pQnode); + dndStopQnodeWorker(pDnode); + dndWriteQnodeFile(pDnode); + qndClose(pQnode); + pMgmt->pQnode = NULL; + // qndDestroy(pDnode->dir.qnode); + + return 0; +} + +int32_t dndProcessCreateQnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { + SCreateQnodeInMsg *pMsg = pRpcMsg->pCont; + pMsg->dnodeId = htonl(pMsg->dnodeId); + + if (pMsg->dnodeId != dndGetDnodeId(pDnode)) { + terrno = TSDB_CODE_DND_QNODE_ID_INVALID; + return -1; + } else { + return dndOpenQnode(pDnode); + } +} + +int32_t dndProcessDropQnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { + SDropQnodeInMsg *pMsg = pRpcMsg->pCont; + pMsg->dnodeId = htonl(pMsg->dnodeId); + + if (pMsg->dnodeId != dndGetDnodeId(pDnode)) { + terrno = TSDB_CODE_DND_QNODE_ID_INVALID; + return -1; + } else { + return dndDropQnode(pDnode); + } +} + +static void dndProcessQnodeQueue(SDnode *pDnode, SRpcMsg *pMsg) { + SQnodeMgmt *pMgmt = &pDnode->qmgmt; + SRpcMsg *pRsp = NULL; + int32_t code = 0; + + SQnode *pQnode = dndAcquireQnode(pDnode); + if (pQnode == NULL) { + code = -1; + } else { + code = qndProcessQueryReq(pQnode, pMsg, &pRsp); + } + + if (pRsp != NULL) { + pRsp->ahandle = pMsg->ahandle; + rpcSendResponse(pRsp); + free(pRsp); + } else { + if (code != 0) code = terrno; + SRpcMsg rpcRsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code}; + rpcSendResponse(&rpcRsp); + } + + rpcFreeCont(pMsg->pCont); + taosFreeQitem(pMsg); +} + +static int32_t dndWriteQnodeMsgToQueue(SQnode *pQnode, taos_queue pQueue, SRpcMsg *pRpcMsg) { + int32_t code = 0; + + if (pQnode == NULL || pQueue == NULL) { + code = TSDB_CODE_DND_QNODE_NOT_DEPLOYED; + } else { + SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg)); + if (pMsg == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + } else { + *pMsg = *pRpcMsg; + if (taosWriteQitem(pQueue, pMsg) != 0) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + } + + if (code != 0) { + if (pRpcMsg->msgType & 1u) { + SRpcMsg rsp = {.handle = pRpcMsg->handle, .code = code}; + rpcSendResponse(&rsp); + } + rpcFreeCont(pRpcMsg->pCont); + } +} + +void dndProcessQnodeQueryMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { + SQnodeMgmt *pMgmt = &pDnode->qmgmt; + SQnode *pQnode = dndAcquireQnode(pDnode); + dndWriteQnodeMsgToQueue(pQnode, pMgmt->pQueryQ, pMsg); + dndReleaseQnode(pDnode, pQnode); +} + +void dndProcessQnodeFetchMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { + SQnodeMgmt *pMgmt = &pDnode->qmgmt; + SQnode *pQnode = dndAcquireQnode(pDnode); + dndWriteQnodeMsgToQueue(pQnode, pMgmt->pFetchQ, pMsg); + dndReleaseQnode(pDnode, pQnode); +} + +static int32_t dndAllocQnodeQueryQueue(SDnode *pDnode) { + SQnodeMgmt *pMgmt = &pDnode->qmgmt; + pMgmt->pQueryQ = tWorkerAllocQueue(&pMgmt->queryPool, pDnode, (FProcessItem)dndProcessQnodeQueue); + if (pMgmt->pQueryQ == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + +static void dndFreeQnodeQueryQueue(SDnode *pDnode) { + SQnodeMgmt *pMgmt = &pDnode->qmgmt; + tWorkerFreeQueue(&pMgmt->queryPool, pMgmt->pQueryQ); + pMgmt->pQueryQ = NULL; +} + +static int32_t dndInitQnodeQueryWorker(SDnode *pDnode) { + SQnodeMgmt *pMgmt = &pDnode->qmgmt; + SWorkerPool *pPool = &pMgmt->queryPool; + pPool->name = "qnode-query"; + pPool->min = 0; + pPool->max = 1; + if (tWorkerInit(pPool) != 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + dDebug("qnode query worker is initialized"); + return 0; +} + +static void dndCleanupQnodeQueryWorker(SDnode *pDnode) { + SQnodeMgmt *pMgmt = &pDnode->qmgmt; + tWorkerCleanup(&pMgmt->queryPool); + dDebug("qnode query worker is closed"); +} + +static int32_t dndAllocQnodeFetchQueue(SDnode *pDnode) { + SQnodeMgmt *pMgmt = &pDnode->qmgmt; + pMgmt->pFetchQ = tWorkerAllocQueue(&pMgmt->queryPool, pDnode, (FProcessItem)dndProcessQnodeQueue); + if (pMgmt->pFetchQ == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + +static void dndFreeQnodeFetchQueue(SDnode *pDnode) { + SQnodeMgmt *pMgmt = &pDnode->qmgmt; + tWorkerFreeQueue(&pMgmt->fetchPool, pMgmt->pFetchQ); + pMgmt->pFetchQ = NULL; +} + +static int32_t dndInitQnodeFetchWorker(SDnode *pDnode) { + SQnodeMgmt *pMgmt = &pDnode->qmgmt; + SWorkerPool *pPool = &pMgmt->fetchPool; + pPool->name = "qnode-fetch"; + pPool->min = 0; + pPool->max = 1; + if (tWorkerInit(pPool) != 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + dDebug("qnode fetch worker is initialized"); + return 0; +} + +static void dndCleanupQnodeFetchWorker(SDnode *pDnode) { + SQnodeMgmt *pMgmt = &pDnode->qmgmt; + tWorkerCleanup(&pMgmt->fetchPool); + dDebug("qnode fetch worker is closed"); +} + +int32_t dndInitQnode(SDnode *pDnode) { + dInfo("dnode-qnode start to init"); + SQnodeMgmt *pMgmt = &pDnode->qmgmt; + taosInitRWLatch(&pMgmt->latch); + + char path[PATH_MAX]; + snprintf(path, PATH_MAX, "%s/qnode.json", pDnode->dir.dnode); + pMgmt->file = strdup(path); + if (pMgmt->file == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + if (dndReadQnodeFile(pDnode) != 0) { + return -1; + } + + if (pMgmt->dropped) return 0; + if (!pMgmt->deployed) return 0; + + return dndOpenQnode(pDnode); +} + +void dndCleanupQnode(SDnode *pDnode) { + SQnodeMgmt *pMgmt = &pDnode->qmgmt; + + dInfo("dnode-qnode start to clean up"); + if (pMgmt->pQnode) dndStopQnodeWorker(pDnode); + tfree(pMgmt->file); + qndClose(pMgmt->pQnode); + pMgmt->pQnode = NULL; + dInfo("dnode-qnode is cleaned up"); +} diff --git a/source/dnode/mgmt/impl/src/dnode.c b/source/dnode/mgmt/impl/src/dnode.c index c8ab35dd96..88a96dadc4 100644 --- a/source/dnode/mgmt/impl/src/dnode.c +++ b/source/dnode/mgmt/impl/src/dnode.c @@ -84,21 +84,14 @@ static int32_t dndInitEnv(SDnode *pDnode, SDnodeOpt *pOption) { char path[PATH_MAX + 100]; snprintf(path, sizeof(path), "%s%smnode", pOption->dataDir, TD_DIRSEP); pDnode->dir.mnode = tstrdup(path); - snprintf(path, sizeof(path), "%s%svnode", pOption->dataDir, TD_DIRSEP); pDnode->dir.vnodes = tstrdup(path); - snprintf(path, sizeof(path), "%s%sdnode", pOption->dataDir, TD_DIRSEP); pDnode->dir.dnode = tstrdup(path); - - snprintf(path, sizeof(path), "%s%sqnode", pOption->dataDir, TD_DIRSEP); - pDnode->dir.dnode = tstrdup(path); - snprintf(path, sizeof(path), "%s%ssnode", pOption->dataDir, TD_DIRSEP); - pDnode->dir.dnode = tstrdup(path); - + pDnode->dir.snode = tstrdup(path); snprintf(path, sizeof(path), "%s%sbnode", pOption->dataDir, TD_DIRSEP); - pDnode->dir.dnode = tstrdup(path); + pDnode->dir.bnode = tstrdup(path); if (pDnode->dir.mnode == NULL || pDnode->dir.vnodes == NULL || pDnode->dir.dnode == NULL) { dError("failed to malloc dir object"); @@ -124,12 +117,6 @@ static int32_t dndInitEnv(SDnode *pDnode, SDnodeOpt *pOption) { return -1; } - if (taosMkDir(pDnode->dir.qnode) != 0) { - dError("failed to create dir:%s since %s", pDnode->dir.qnode, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - if (taosMkDir(pDnode->dir.snode) != 0) { dError("failed to create dir:%s since %s", pDnode->dir.snode, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); @@ -147,29 +134,11 @@ static int32_t dndInitEnv(SDnode *pDnode, SDnodeOpt *pOption) { } static void dndCleanupEnv(SDnode *pDnode) { - if (pDnode->dir.mnode != NULL) { - tfree(pDnode->dir.mnode); - } - - if (pDnode->dir.vnodes != NULL) { - tfree(pDnode->dir.vnodes); - } - - if (pDnode->dir.dnode != NULL) { - tfree(pDnode->dir.dnode); - } - - if (pDnode->dir.qnode != NULL) { - tfree(pDnode->dir.qnode); - } - - if (pDnode->dir.bnode != NULL) { - tfree(pDnode->dir.snode); - } - - if (pDnode->dir.snode != NULL) { - tfree(pDnode->dir.bnode); - } + tfree(pDnode->dir.mnode); + tfree(pDnode->dir.vnodes); + tfree(pDnode->dir.dnode); + tfree(pDnode->dir.snode); + tfree(pDnode->dir.bnode); if (pDnode->lockFd >= 0) { taosUnLockFile(pDnode->lockFd); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 65057501b9..451b7e4fa4 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -256,6 +256,24 @@ TAOS_DEFINE_ERROR(TSDB_CODE_DND_MNODE_ID_INVALID, "Mnode Id invalid") TAOS_DEFINE_ERROR(TSDB_CODE_DND_MNODE_ID_NOT_FOUND, "Mnode Id not found") TAOS_DEFINE_ERROR(TSDB_CODE_DND_MNODE_READ_FILE_ERROR, "Read mnode.json error") TAOS_DEFINE_ERROR(TSDB_CODE_DND_MNODE_WRITE_FILE_ERROR, "Write mnode.json error") +TAOS_DEFINE_ERROR(TSDB_CODE_DND_QNODE_ALREADY_DEPLOYED, "Qnode already deployed") +TAOS_DEFINE_ERROR(TSDB_CODE_DND_QNODE_NOT_DEPLOYED, "Qnode not deployed") +TAOS_DEFINE_ERROR(TSDB_CODE_DND_QNODE_ID_INVALID, "Qnode Id invalid") +TAOS_DEFINE_ERROR(TSDB_CODE_DND_QNODE_ID_NOT_FOUND, "Qnode Id not found") +TAOS_DEFINE_ERROR(TSDB_CODE_DND_QNODE_READ_FILE_ERROR, "Read qnode.json error") +TAOS_DEFINE_ERROR(TSDB_CODE_DND_QNODE_WRITE_FILE_ERROR, "Write qnode.json error") +TAOS_DEFINE_ERROR(TSDB_CODE_DND_SNODE_ALREADY_DEPLOYED, "Snode already deployed") +TAOS_DEFINE_ERROR(TSDB_CODE_DND_SNODE_NOT_DEPLOYED, "Snode not deployed") +TAOS_DEFINE_ERROR(TSDB_CODE_DND_SNODE_ID_INVALID, "Snode Id invalid") +TAOS_DEFINE_ERROR(TSDB_CODE_DND_SNODE_ID_NOT_FOUND, "Snode Id not found") +TAOS_DEFINE_ERROR(TSDB_CODE_DND_SNODE_READ_FILE_ERROR, "Read snode.json error") +TAOS_DEFINE_ERROR(TSDB_CODE_DND_SNODE_WRITE_FILE_ERROR, "Write snode.json error") +TAOS_DEFINE_ERROR(TSDB_CODE_DND_BNODE_ALREADY_DEPLOYED, "Bnode already deployed") +TAOS_DEFINE_ERROR(TSDB_CODE_DND_BNODE_NOT_DEPLOYED, "Bnode not deployed") +TAOS_DEFINE_ERROR(TSDB_CODE_DND_BNODE_ID_INVALID, "Bnode Id invalid") +TAOS_DEFINE_ERROR(TSDB_CODE_DND_BNODE_ID_NOT_FOUND, "Bnode Id not found") +TAOS_DEFINE_ERROR(TSDB_CODE_DND_BNODE_READ_FILE_ERROR, "Read bnode.json error") +TAOS_DEFINE_ERROR(TSDB_CODE_DND_BNODE_WRITE_FILE_ERROR, "Write bnode.json error") TAOS_DEFINE_ERROR(TSDB_CODE_DND_VNODE_TOO_MANY_VNODES, "Too many vnode directories") TAOS_DEFINE_ERROR(TSDB_CODE_DND_VNODE_READ_FILE_ERROR, "Read vnodes.json error") TAOS_DEFINE_ERROR(TSDB_CODE_DND_VNODE_WRITE_FILE_ERROR, "Write vnodes.json error") @@ -360,132 +378,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_WAL_APP_ERROR, "Unexpected generic er TAOS_DEFINE_ERROR(TSDB_CODE_WAL_FILE_CORRUPTED, "WAL file is corrupted") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_SIZE_LIMIT, "WAL size exceeds limit") -// http -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_SERVER_OFFLINE, "http server is not onlin") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_UNSUPPORT_URL, "url is not support") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_URL, "invalid url format") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_NO_ENOUGH_MEMORY, "no enough memory") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_REQUSET_TOO_BIG, "request size is too big") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_NO_AUTH_INFO, "no auth info input") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_NO_MSG_INPUT, "request is empty") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_NO_SQL_INPUT, "no sql input") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_NO_EXEC_USEDB, "no need to execute use db cmd") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_SESSION_FULL, "session list was full") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_GEN_TAOSD_TOKEN_ERR, "generate taosd token error") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_MULTI_REQUEST, "size of multi request is 0") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_CREATE_GZIP_FAILED, "failed to create gzip") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_FINISH_GZIP_FAILED, "failed to finish gzip") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_LOGIN_FAILED, "failed to login") - -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_VERSION, "invalid http version") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_CONTENT_LENGTH, "invalid content length") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_AUTH_TYPE, "invalid type of Authorization") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_AUTH_FORMAT, "invalid format of Authorization") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_BASIC_AUTH, "invalid basic Authorization") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_TAOSD_AUTH, "invalid taosd Authorization") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_METHOD_FAILED, "failed to parse method") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_TARGET_FAILED, "failed to parse target") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_VERSION_FAILED, "failed to parse http version") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_SP_FAILED, "failed to parse sp") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_STATUS_FAILED, "failed to parse status") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_PHRASE_FAILED, "failed to parse phrase") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_CRLF_FAILED, "failed to parse crlf") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_HEADER_FAILED, "failed to parse header") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_HEADER_KEY_FAILED, "failed to parse header key") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_HEADER_VAL_FAILED, "failed to parse header val") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_CHUNK_SIZE_FAILED, "failed to parse chunk size") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_CHUNK_FAILED, "failed to parse chunk") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_END_FAILED, "failed to parse end section") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_INVALID_STATE, "invalid parse state") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_ERROR_STATE, "failed to parse error section") - -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_GC_QUERY_NULL, "query size is 0") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_GC_QUERY_SIZE, "query size can not more than 100") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_GC_REQ_PARSE_ERROR, "parse grafana json error") - -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_DB_NOT_INPUT, "database name can not be null") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_DB_TOO_LONG, "database name too long") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_INVALID_JSON, "invalid telegraf json fromat") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_METRICS_NULL, "metrics size is 0") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_METRICS_SIZE, "metrics size can not more than 1K") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_METRIC_NULL, "metric name not find") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_METRIC_TYPE, "metric name type should be string") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_METRIC_NAME_NULL, "metric name length is 0") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_METRIC_NAME_LONG, "metric name length too long") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TIMESTAMP_NULL, "timestamp not find") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TIMESTAMP_TYPE, "timestamp type should be integer") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TIMESTAMP_VAL_NULL, "timestamp value smaller than 0") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAGS_NULL, "tags not find") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAGS_SIZE_0, "tags size is 0") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAGS_SIZE_LONG, "tags size too long") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAG_NULL, "tag is null") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAG_NAME_NULL, "tag name is null") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAG_NAME_SIZE, "tag name length too long") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAG_VALUE_TYPE, "tag value type should be number or string") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAG_VALUE_NULL, "tag value is null") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TABLE_NULL, "table is null") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TABLE_SIZE, "table name length too long") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELDS_NULL, "fields not find") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELDS_SIZE_0, "fields size is 0") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELDS_SIZE_LONG, "fields size too long") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELD_NULL, "field is null") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELD_NAME_NULL, "field name is null") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELD_NAME_SIZE, "field name length too long") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELD_VALUE_TYPE, "field value type should be number or string") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELD_VALUE_NULL, "field value is null") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_HOST_NOT_STRING, "host type should be string") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_STABLE_NOT_EXIST, "stable not exist") - -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_DB_NOT_INPUT, "database name can not be null") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_DB_TOO_LONG, "database name too long") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_INVALID_JSON, "invalid opentsdb json fromat") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_METRICS_NULL, "metrics size is 0") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_METRICS_SIZE, "metrics size can not more than 10K") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_METRIC_NULL, "metric name not find") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_METRIC_TYPE, "metric name type should be string") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_METRIC_NAME_NULL, "metric name length is 0") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_METRIC_NAME_LONG, "metric name length can not more than 22") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TIMESTAMP_NULL, "timestamp not find") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TIMESTAMP_TYPE, "timestamp type should be integer") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TIMESTAMP_VAL_NULL, "timestamp value smaller than 0") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAGS_NULL, "tags not find") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAGS_SIZE_0, "tags size is 0") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAGS_SIZE_LONG, "tags size too long") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_NULL, "tag is null") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_NAME_NULL, "tag name is null") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_NAME_SIZE, "tag name length too long") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_VALUE_TYPE, "tag value type should be boolean, number or string") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_VALUE_NULL, "tag value is null") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_VALUE_TOO_LONG, "tag value can not more than 64") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_VALUE_NULL, "value not find") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_VALUE_TYPE, "value type should be boolean, number or string") - -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_REQUEST_JSON_ERROR, "http request json error") - -// odbc -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_OOM, "out of memory") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_CHAR_NOT_NUM, "convertion not a valid literal input") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_UNDEF, "convertion undefined") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_TRUNC_FRAC, "convertion fractional truncated") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_TRUNC, "convertion truncated") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_NOT_SUPPORT, "convertion not supported") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_OOR, "convertion numeric value out of range") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_OUT_OF_RANGE, "out of range") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_NOT_SUPPORT, "not supported yet") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_INVALID_HANDLE, "invalid handle") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_NO_RESULT, "no result set") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_NO_FIELDS, "no fields returned") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_INVALID_CURSOR, "invalid cursor") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_STATEMENT_NOT_READY, "statement not ready") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONNECTION_BUSY, "connection still busy") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_BAD_CONNSTR, "bad connection string") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_BAD_ARG, "bad argument") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_NOT_VALID_TS, "not a valid timestamp") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_SRC_TOO_LARGE, "src too large") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_SRC_BAD_SEQ, "src bad sequence") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_SRC_INCOMPLETE, "src incomplete") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_SRC_GENERAL, "src general") - // tfs TAOS_DEFINE_ERROR(TSDB_CODE_FS_OUT_OF_MEMORY, "tfs out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_FS_INVLD_CFG, "tfs invalid mount config") diff --git a/tests/script/general/table/basic1.sim b/tests/script/general/table/basic1.sim index 04e44f7308..5892a81f2e 100644 --- a/tests/script/general/table/basic1.sim +++ b/tests/script/general/table/basic1.sim @@ -30,7 +30,18 @@ if $rows != 2 then endi print $data00 $data01 $data02 -print $data00 $data11 $data12 +print $data10 $data11 $data12 + +sql drop table st2 + +sql show stables +if $rows != 1 then + return -1 +endi + +print --> +print $data00 $data01 $data02 +print $data10 $data11 $data12 return diff --git a/tests/script/unique/dnode/basic1.sim b/tests/script/unique/dnode/basic1.sim index 5382e88e53..0d1b75f3c0 100644 --- a/tests/script/unique/dnode/basic1.sim +++ b/tests/script/unique/dnode/basic1.sim @@ -15,7 +15,7 @@ if $data00 != 1 then return -1 endi -if $data01 != 192.168.0.1 then +if $data02 != 0 then return -1 endi @@ -28,7 +28,7 @@ if $data00 != 1 then return -1 endi -if $data01 != 192.168.0.1 then +if $data02 != master then return -1 endi @@ -43,18 +43,10 @@ if $data00 != 1 then return -1 endi -if $data01 != 192.168.0.1 then - return -1 -endi - if $data10 != 2 then return -1 endi -if $data11 != 192.168.0.2 then - return -1 -endi - sql show mnodes; if $rows != 1 then return -1 @@ -64,27 +56,25 @@ if $data00 != 1 then return -1 endi -if $data01 != 192.168.0.1 then +if $data01 != master then return -1 endi print =============== create table sql create database d1; sql create database d2; -sql create database d3; -sql create database d4; -sql create database d5; -sql create table d1.t1 (ts timestamp, i int); -sql create table d2.t1 (ts timestamp, i int); -sql create table d3.t1 (ts timestamp, i int); -sql create table d4.t1 (ts timestamp, i int); - -sql show dnodes; -if $data05 != 2 then +sql show databases +if $rows != 2 then return -1 endi -if $data15 != 2 then +sql use d1 +sql show vgroups; + +if $rows != 2 then return -1 endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT \ No newline at end of file From a53d1d95fb6e1cd8e543b509143dc8d4e7536bc3 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 27 Dec 2021 20:45:30 +0800 Subject: [PATCH 07/21] add dag serialize and deserialize --- include/libs/planner/planner.h | 8 +++- source/libs/planner/src/physicalPlanJson.c | 50 ++++++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index b3262953b5..f9c25c3ccc 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -21,6 +21,7 @@ extern "C" { #endif #include "tmsg.h" +#include "tarray.h" #define QUERY_TYPE_MERGE 1 #define QUERY_TYPE_PARTIAL 2 @@ -158,6 +159,8 @@ int32_t qStringToSubplan(const char* str, SSubplan** subplan); void qDestroySubplan(SSubplan* pSubplan); +char* qDagSerialize(const SQueryDag* pDag); + /** * Destroy the physical plan. * @param pQueryPhyNode @@ -165,8 +168,11 @@ void qDestroySubplan(SSubplan* pSubplan); */ void qDestroyQueryDag(SQueryDag* pDag); +char* qDagToString(const SQueryDag* pDag); +SQueryDag* qStringToDag(const char* pStr); + #ifdef __cplusplus } #endif -#endif /*_TD_PLANNER_H_*/ \ No newline at end of file +#endif /*_TD_PLANNER_H_*/ diff --git a/source/libs/planner/src/physicalPlanJson.c b/source/libs/planner/src/physicalPlanJson.c index 5a1b2a6da2..945c354284 100644 --- a/source/libs/planner/src/physicalPlanJson.c +++ b/source/libs/planner/src/physicalPlanJson.c @@ -857,3 +857,53 @@ int32_t stringToSubplan(const char* str, SSubplan** subplan) { *subplan = subplanFromJson(json); return (NULL == *subplan ? TSDB_CODE_FAILED : TSDB_CODE_SUCCESS); } + +cJSON* qDagToJson(const SQueryDag* pDag) { + cJSON* pRoot = cJSON_CreateObject(); + cJSON_AddNumberToObject(pRoot, "numOfSubplans", pDag->numOfSubplans); + cJSON_AddNumberToObject(pRoot, "queryId", pDag->queryId); + cJSON *pLevels = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "pSubplans", pLevels); + size_t level = taosArrayGetSize(pDag->pSubplans); + for(size_t i = 0; i < level; i++) { + const SArray* pSubplans = (const SArray*)taosArrayGetP(pDag->pSubplans, i); + size_t num = taosArrayGetSize(pSubplans); + cJSON* plansOneLevel = cJSON_CreateArray(); + cJSON_AddItemToArray(pLevels, plansOneLevel); + for(size_t j = 0; j < num; j++) { + cJSON* pSubplan = subplanToJson((const SSubplan*)taosArrayGetP(pSubplans, j)); + cJSON_AddItemToArray(plansOneLevel, pSubplan); + } + } + return pRoot; +} + +char* qDagToString(const SQueryDag* pDag) { + cJSON* pRoot = qDagToJson(pDag); + return cJSON_Print(pRoot); +} + +SQueryDag* qJsonToDag(const cJSON* pRoot) { + SQueryDag* pDag = malloc(sizeof(SQueryDag)); + if(pDag == NULL) { + return NULL; + } + pDag->numOfSubplans = cJSON_GetNumberValue(cJSON_GetObjectItem(pRoot, "numOfSubplans")); + pDag->pSubplans = taosArrayInit(0, sizeof(SArray)); + cJSON* pLevels = cJSON_GetObjectItem(pRoot, "numOfSubplans"); + int level = cJSON_GetArraySize(pLevels); + for(int i = 0; i < level; i++) { + cJSON* pItem = cJSON_GetArrayItem(pLevels, i); + int sz = cJSON_GetArraySize(pItem); + for(int j = 0; j < sz; j++) { + cJSON* pSubplanJson = cJSON_GetArrayItem(pItem, j); + SSubplan* pSubplan = subplanFromJson(pSubplanJson); + } + } + return NULL; +} + +SQueryDag* qStringToDag(const char* pStr) { + cJSON* pRoot = cJSON_Parse(pStr); + return qJsonToDag(pRoot); +} From 938bd01b17ee68f8df6dfd3166fd0529df489965 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 27 Dec 2021 19:57:24 -0800 Subject: [PATCH 08/21] add qnode --- include/dnode/qnode/qnode.h | 14 +- include/util/taoserror.h | 1 + source/dnode/mgmt/impl/inc/dndInt.h | 29 ++- source/dnode/mgmt/impl/inc/dndWorker.h | 33 ++++ source/dnode/mgmt/impl/src/dndQnode.c | 257 ++++++------------------- source/dnode/mgmt/impl/src/dndWorker.c | 85 ++++++++ source/util/src/terror.c | 1 + 7 files changed, 198 insertions(+), 222 deletions(-) create mode 100644 source/dnode/mgmt/impl/inc/dndWorker.h create mode 100644 source/dnode/mgmt/impl/src/dndWorker.c diff --git a/include/dnode/qnode/qnode.h b/include/dnode/qnode/qnode.h index 29a3d1af20..8084175a90 100644 --- a/include/dnode/qnode/qnode.h +++ b/include/dnode/qnode/qnode.h @@ -78,24 +78,14 @@ void qndClose(SQnode *pQnode); int32_t qndGetLoad(SQnode *pQnode, SQnodeLoad *pLoad); /** - * @brief Process a query message. + * @brief Process a query or fetch message. * * @param pQnode The qnode object. * @param pMsg The request message * @param pRsp The response message * @return int32_t 0 for success, -1 for failure */ -int32_t qndProcessQueryReq(SQnode *pQnode, SRpcMsg *pMsg, SRpcMsg **pRsp); - -/** - * @brief Process a fetch message. - * - * @param pQnode The qnode object. - * @param pMsg The request message - * @param pRsp The response message - * @return int32_t 0 for success, -1 for failure - */ -int32_t qndProcessFetchReq(SQnode *pQnode, SRpcMsg *pMsg, SRpcMsg **pRsp); +int32_t qndProcessMsg(SQnode *pQnode, SRpcMsg *pMsg, SRpcMsg **pRsp); #ifdef __cplusplus } diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 408a656e83..ba3e122db3 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -69,6 +69,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_CHECKSUM_ERROR TAOS_DEF_ERROR_CODE(0, 0x0107) #define TSDB_CODE_INVALID_MSG TAOS_DEF_ERROR_CODE(0, 0x0108) #define TSDB_CODE_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0109) +#define TSDB_CODE_INVALID_PARA TAOS_DEF_ERROR_CODE(0, 0x010A) #define TSDB_CODE_REF_NO_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0110) #define TSDB_CODE_REF_FULL TAOS_DEF_ERROR_CODE(0, 0x0111) #define TSDB_CODE_REF_ID_REMOVED TAOS_DEF_ERROR_CODE(0, 0x0112) diff --git a/source/dnode/mgmt/impl/inc/dndInt.h b/source/dnode/mgmt/impl/inc/dndInt.h index 9679a03c3e..0d37828ecd 100644 --- a/source/dnode/mgmt/impl/inc/dndInt.h +++ b/source/dnode/mgmt/impl/inc/dndInt.h @@ -54,8 +54,20 @@ extern int32_t dDebugFlag; #define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("DND ", dDebugFlag, __VA_ARGS__); }} typedef enum { DND_STAT_INIT, DND_STAT_RUNNING, DND_STAT_STOPPED } EStat; +typedef enum { DND_WORKER_SINGLE, DND_WORKER_MULTI } EDndWorkerType; typedef void (*DndMsgFp)(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEps); +typedef struct { + EDndWorkerType type; + const char *name; + int32_t minNum; + int32_t maxNum; + FProcessItem fp; + SDnode *pDnode; + taos_queue queue; + SWorkerPool pool; +} SDnodeWorker; + typedef struct { char *dnode; char *mnode; @@ -100,16 +112,13 @@ typedef struct { } SMnodeMgmt; typedef struct { - int32_t refCount; - int8_t deployed; - int8_t dropped; - char *file; - SQnode *pQnode; - SRWLatch latch; - taos_queue pQueryQ; - taos_queue pFetchQ; - SWorkerPool queryPool; - SWorkerPool fetchPool; + int32_t refCount; + int8_t deployed; + int8_t dropped; + SQnode *pQnode; + SRWLatch latch; + SDnodeWorker queryWorker; + SDnodeWorker fetchWorker; } SQnodeMgmt; typedef struct { diff --git a/source/dnode/mgmt/impl/inc/dndWorker.h b/source/dnode/mgmt/impl/inc/dndWorker.h new file mode 100644 index 0000000000..237c0518e8 --- /dev/null +++ b/source/dnode/mgmt/impl/inc/dndWorker.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DND_WORKER_H_ +#define _TD_DND_WORKER_H_ + +#ifdef __cplusplus +extern "C" { +#endif +#include "dndInt.h" + +int32_t dndInitWorker(SDnode *pDnode, SDnodeWorker *pWorker, EDndWorkerType type, const char *name, int32_t minNum, + int32_t maxNum, FProcessItem fp); +void dndCleanupWorker(SDnodeWorker *pWorker); +int32_t dndWriteMsgToWorker(SDnodeWorker *pWorker, void *pCont, int32_t contLen); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DND_WORKER_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/impl/src/dndQnode.c b/source/dnode/mgmt/impl/src/dndQnode.c index 08044ea263..8c76bf95a6 100644 --- a/source/dnode/mgmt/impl/src/dndQnode.c +++ b/source/dnode/mgmt/impl/src/dndQnode.c @@ -17,30 +17,9 @@ #include "dndQnode.h" #include "dndDnode.h" #include "dndTransport.h" +#include "dndWorker.h" -static int32_t dndInitQnodeQueryWorker(SDnode *pDnode); -static int32_t dndInitQnodeFetchWorker(SDnode *pDnode); -static void dndCleanupQnodeQueryWorker(SDnode *pDnode); -static void dndCleanupQnodeFetchWorker(SDnode *pDnode); -static int32_t dndAllocQnodeQueryQueue(SDnode *pDnode); -static int32_t dndAllocQnodeFetchQueue(SDnode *pDnode); -static void dndFreeQnodeQueryQueue(SDnode *pDnode); -static void dndFreeQnodeFetchQueue(SDnode *pDnode); - -static void dndProcessQnodeQueue(SDnode *pDnode, SRpcMsg *pMsg); -static int32_t dndWriteQnodeMsgToQueue(SQnode *pQnode, taos_queue pQueue, SRpcMsg *pRpcMsg); - -static int32_t dndStartQnodeWorker(SDnode *pDnode); -static void dndStopQnodeWorker(SDnode *pDnode); - -static SQnode *dndAcquireQnode(SDnode *pDnode); -static void dndReleaseQnode(SDnode *pDnode, SQnode *pQnode); - -static int32_t dndReadQnodeFile(SDnode *pDnode); -static int32_t dndWriteQnodeFile(SDnode *pDnode); - -static int32_t dndOpenQnode(SDnode *pDnode); -static int32_t dndDropQnode(SDnode *pDnode); +static void dndProcessQnodeQueue(SDnode *pDnode, SRpcMsg *pMsg); static SQnode *dndAcquireQnode(SDnode *pDnode) { SQnodeMgmt *pMgmt = &pDnode->qmgmt; @@ -85,44 +64,47 @@ static int32_t dndReadQnodeFile(SDnode *pDnode) { char *content = calloc(1, maxLen + 1); cJSON *root = NULL; - FILE *fp = fopen(pMgmt->file, "r"); + char file[PATH_MAX + 20]; + snprintf(file, PATH_MAX + 20, "%s/qnode.json", pDnode->dir.dnode); + + FILE *fp = fopen(file, "r"); if (fp == NULL) { - dDebug("file %s not exist", pMgmt->file); + dDebug("file %s not exist", file); code = 0; - goto PRASE_MNODE_OVER; + goto PRASE_QNODE_OVER; } len = (int32_t)fread(content, 1, maxLen, fp); if (len <= 0) { - dError("failed to read %s since content is null", pMgmt->file); - goto PRASE_MNODE_OVER; + dError("failed to read %s since content is null", file); + goto PRASE_QNODE_OVER; } content[len] = 0; root = cJSON_Parse(content); if (root == NULL) { - dError("failed to read %s since invalid json format", pMgmt->file); - goto PRASE_MNODE_OVER; + dError("failed to read %s since invalid json format", file); + goto PRASE_QNODE_OVER; } cJSON *deployed = cJSON_GetObjectItem(root, "deployed"); if (!deployed || deployed->type != cJSON_Number) { - dError("failed to read %s since deployed not found", pMgmt->file); - goto PRASE_MNODE_OVER; + dError("failed to read %s since deployed not found", file); + goto PRASE_QNODE_OVER; } pMgmt->deployed = deployed->valueint; cJSON *dropped = cJSON_GetObjectItem(root, "dropped"); if (!dropped || dropped->type != cJSON_Number) { - dError("failed to read %s since dropped not found", pMgmt->file); - goto PRASE_MNODE_OVER; + dError("failed to read %s since dropped not found", file); + goto PRASE_QNODE_OVER; } pMgmt->dropped = dropped->valueint; code = 0; - dDebug("succcessed to read file %s, deployed:%d dropped:%d", pMgmt->file, pMgmt->deployed, pMgmt->dropped); + dDebug("succcessed to read file %s, deployed:%d dropped:%d", file, pMgmt->deployed, pMgmt->dropped); -PRASE_MNODE_OVER: +PRASE_QNODE_OVER: if (content != NULL) free(content); if (root != NULL) cJSON_Delete(root); if (fp != NULL) fclose(fp); @@ -134,8 +116,8 @@ PRASE_MNODE_OVER: static int32_t dndWriteQnodeFile(SDnode *pDnode) { SQnodeMgmt *pMgmt = &pDnode->qmgmt; - char file[PATH_MAX + 20] = {0}; - snprintf(file, sizeof(file), "%s.bak", pMgmt->file); + char file[PATH_MAX + 20]; + snprintf(file, PATH_MAX + 20, "%s/qnode.json", pDnode->dir.dnode); FILE *fp = fopen(file, "w"); if (fp == NULL) { @@ -154,41 +136,34 @@ static int32_t dndWriteQnodeFile(SDnode *pDnode) { len += snprintf(content + len, maxLen - len, "}\n"); fwrite(content, 1, len, fp); - taosFfetchFile(fileno(fp)); + taosFsyncFile(fileno(fp)); fclose(fp); free(content); - if (taosRenameFile(file, pMgmt->file) != 0) { + if (taosRenameFile(file, file) != 0) { terrno = TSDB_CODE_DND_QNODE_WRITE_FILE_ERROR; - dError("failed to rename %s since %s", pMgmt->file, terrstr()); + dError("failed to rename %s since %s", file, terrstr()); return -1; } - dInfo("successed to write %s, deployed:%d dropped:%d", pMgmt->file, pMgmt->deployed, pMgmt->dropped); + dInfo("successed to write %s, deployed:%d dropped:%d", file, pMgmt->deployed, pMgmt->dropped); return 0; } static int32_t dndStartQnodeWorker(SDnode *pDnode) { - if (dndInitQnodeQueryWorker(pDnode) != 0) { + SQnodeMgmt *pMgmt = &pDnode->qmgmt; + if (dndInitWorker(pDnode, &pMgmt->queryWorker, DND_WORKER_SINGLE, "qnode-query", 0, 1, + (FProcessItem)dndProcessQnodeQueue) != 0) { dError("failed to start qnode query worker since %s", terrstr()); return -1; } - if (dndInitQnodeFetchWorker(pDnode) != 0) { + if (dndInitWorker(pDnode, &pMgmt->fetchWorker, DND_WORKER_SINGLE, "qnode-fetch", 0, 1, + (FProcessItem)dndProcessQnodeQueue) != 0) { dError("failed to start qnode fetch worker since %s", terrstr()); return -1; } - if (dndAllocQnodeQueryQueue(pDnode) != 0) { - dError("failed to alloc qnode query queue since %s", terrstr()); - return -1; - } - - if (dndAllocQnodeFetchQueue(pDnode) != 0) { - dError("failed to alloc qnode fetch queue since %s", terrstr()); - return -1; - } - return 0; } @@ -199,15 +174,12 @@ static void dndStopQnodeWorker(SDnode *pDnode) { pMgmt->deployed = 0; taosWUnLockLatch(&pMgmt->latch); - while (pMgmt->refCount > 1) taosMsleep(10); - while (!taosQueueEmpty(pMgmt->pQueryQ)) taosMsleep(10); - while (!taosQueueEmpty(pMgmt->pFetchQ)) taosMsleep(10); + while (pMgmt->refCount > 1) { + taosMsleep(10); + } - dndCleanupQnodeQueryWorker(pDnode); - dndCleanupQnodeFetchWorker(pDnode); - - dndFreeQnodeQueryQueue(pDnode); - dndFreeQnodeFetchQueue(pDnode); + dndCleanupWorker(&pMgmt->queryWorker); + dndCleanupWorker(&pMgmt->fetchWorker); } static void dndBuildQnodeOption(SDnode *pDnode, SQnodeOpt *pOption) { @@ -230,28 +202,17 @@ static int32_t dndOpenQnode(SDnode *pDnode) { dError("failed to open qnode since %s", terrstr()); return -1; } - pMgmt->deployed = 1; - int32_t code = dndWriteQnodeFile(pDnode); - if (code != 0) { - dError("failed to write qnode file since %s", terrstr()); - code = terrno; - pMgmt->deployed = 0; + if (dndStartQnodeWorker(pDnode) != 0) { + dError("failed to start qnode worker since %s", terrstr()); qndClose(pQnode); - // qndDestroy(pDnode->dir.qnode); - terrno = code; return -1; } - code = dndStartQnodeWorker(pDnode); - if (code != 0) { - dError("failed to start qnode worker since %s", terrstr()); - code = terrno; - pMgmt->deployed = 0; + if (dndWriteQnodeFile(pDnode) != 0) { + dError("failed to write qnode file since %s", terrstr()); dndStopQnodeWorker(pDnode); qndClose(pQnode); - // qndDestroy(pDnode->dir.qnode); - terrno = code; return -1; } @@ -289,7 +250,6 @@ static int32_t dndDropQnode(SDnode *pDnode) { dndReleaseQnode(pDnode, pQnode); dndStopQnodeWorker(pDnode); - dndWriteQnodeFile(pDnode); qndClose(pQnode); pMgmt->pQnode = NULL; // qndDestroy(pDnode->dir.qnode); @@ -324,13 +284,11 @@ int32_t dndProcessDropQnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { static void dndProcessQnodeQueue(SDnode *pDnode, SRpcMsg *pMsg) { SQnodeMgmt *pMgmt = &pDnode->qmgmt; SRpcMsg *pRsp = NULL; - int32_t code = 0; + int32_t code = TSDB_CODE_DND_QNODE_NOT_DEPLOYED; SQnode *pQnode = dndAcquireQnode(pDnode); - if (pQnode == NULL) { - code = -1; - } else { - code = qndProcessQueryReq(pQnode, pMsg, &pRsp); + if (pQnode != NULL) { + code = qndProcessMsg(pQnode, pMsg, &pRsp); } if (pRsp != NULL) { @@ -347,135 +305,36 @@ static void dndProcessQnodeQueue(SDnode *pDnode, SRpcMsg *pMsg) { taosFreeQitem(pMsg); } -static int32_t dndWriteQnodeMsgToQueue(SQnode *pQnode, taos_queue pQueue, SRpcMsg *pRpcMsg) { - int32_t code = 0; +static void dndWriteQnodeMsgToWorker(SDnode *pDnode, SDnodeWorker *pWorker, SRpcMsg *pMsg) { + int32_t code = TSDB_CODE_DND_QNODE_NOT_DEPLOYED; - if (pQnode == NULL || pQueue == NULL) { - code = TSDB_CODE_DND_QNODE_NOT_DEPLOYED; - } else { - SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg)); - if (pMsg == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - } else { - *pMsg = *pRpcMsg; - if (taosWriteQitem(pQueue, pMsg) != 0) { - code = TSDB_CODE_OUT_OF_MEMORY; - } - } + SQnode *pQnode = dndAcquireQnode(pDnode); + if (pQnode != NULL) { + code = dndWriteMsgToWorker(pWorker, pMsg, sizeof(SRpcMsg)); } + dndReleaseQnode(pDnode, pQnode); if (code != 0) { - if (pRpcMsg->msgType & 1u) { - SRpcMsg rsp = {.handle = pRpcMsg->handle, .code = code}; + if (pMsg->msgType & 1u) { + SRpcMsg rsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code}; rpcSendResponse(&rsp); } - rpcFreeCont(pRpcMsg->pCont); + rpcFreeCont(pMsg->pCont); } } void dndProcessQnodeQueryMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - SQnodeMgmt *pMgmt = &pDnode->qmgmt; - SQnode *pQnode = dndAcquireQnode(pDnode); - dndWriteQnodeMsgToQueue(pQnode, pMgmt->pQueryQ, pMsg); - dndReleaseQnode(pDnode, pQnode); + dndWriteQnodeMsgToWorker(pDnode, &pDnode->qmgmt.queryWorker, pMsg); } void dndProcessQnodeFetchMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - SQnodeMgmt *pMgmt = &pDnode->qmgmt; - SQnode *pQnode = dndAcquireQnode(pDnode); - dndWriteQnodeMsgToQueue(pQnode, pMgmt->pFetchQ, pMsg); - dndReleaseQnode(pDnode, pQnode); -} - -static int32_t dndAllocQnodeQueryQueue(SDnode *pDnode) { - SQnodeMgmt *pMgmt = &pDnode->qmgmt; - pMgmt->pQueryQ = tWorkerAllocQueue(&pMgmt->queryPool, pDnode, (FProcessItem)dndProcessQnodeQueue); - if (pMgmt->pQueryQ == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - return 0; -} - -static void dndFreeQnodeQueryQueue(SDnode *pDnode) { - SQnodeMgmt *pMgmt = &pDnode->qmgmt; - tWorkerFreeQueue(&pMgmt->queryPool, pMgmt->pQueryQ); - pMgmt->pQueryQ = NULL; -} - -static int32_t dndInitQnodeQueryWorker(SDnode *pDnode) { - SQnodeMgmt *pMgmt = &pDnode->qmgmt; - SWorkerPool *pPool = &pMgmt->queryPool; - pPool->name = "qnode-query"; - pPool->min = 0; - pPool->max = 1; - if (tWorkerInit(pPool) != 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - dDebug("qnode query worker is initialized"); - return 0; -} - -static void dndCleanupQnodeQueryWorker(SDnode *pDnode) { - SQnodeMgmt *pMgmt = &pDnode->qmgmt; - tWorkerCleanup(&pMgmt->queryPool); - dDebug("qnode query worker is closed"); -} - -static int32_t dndAllocQnodeFetchQueue(SDnode *pDnode) { - SQnodeMgmt *pMgmt = &pDnode->qmgmt; - pMgmt->pFetchQ = tWorkerAllocQueue(&pMgmt->queryPool, pDnode, (FProcessItem)dndProcessQnodeQueue); - if (pMgmt->pFetchQ == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - return 0; -} - -static void dndFreeQnodeFetchQueue(SDnode *pDnode) { - SQnodeMgmt *pMgmt = &pDnode->qmgmt; - tWorkerFreeQueue(&pMgmt->fetchPool, pMgmt->pFetchQ); - pMgmt->pFetchQ = NULL; -} - -static int32_t dndInitQnodeFetchWorker(SDnode *pDnode) { - SQnodeMgmt *pMgmt = &pDnode->qmgmt; - SWorkerPool *pPool = &pMgmt->fetchPool; - pPool->name = "qnode-fetch"; - pPool->min = 0; - pPool->max = 1; - if (tWorkerInit(pPool) != 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - dDebug("qnode fetch worker is initialized"); - return 0; -} - -static void dndCleanupQnodeFetchWorker(SDnode *pDnode) { - SQnodeMgmt *pMgmt = &pDnode->qmgmt; - tWorkerCleanup(&pMgmt->fetchPool); - dDebug("qnode fetch worker is closed"); + dndWriteQnodeMsgToWorker(pDnode, &pDnode->qmgmt.queryWorker, pMsg); } int32_t dndInitQnode(SDnode *pDnode) { - dInfo("dnode-qnode start to init"); SQnodeMgmt *pMgmt = &pDnode->qmgmt; taosInitRWLatch(&pMgmt->latch); - char path[PATH_MAX]; - snprintf(path, PATH_MAX, "%s/qnode.json", pDnode->dir.dnode); - pMgmt->file = strdup(path); - if (pMgmt->file == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - if (dndReadQnodeFile(pDnode) != 0) { return -1; } @@ -488,11 +347,9 @@ int32_t dndInitQnode(SDnode *pDnode) { void dndCleanupQnode(SDnode *pDnode) { SQnodeMgmt *pMgmt = &pDnode->qmgmt; - - dInfo("dnode-qnode start to clean up"); - if (pMgmt->pQnode) dndStopQnodeWorker(pDnode); - tfree(pMgmt->file); - qndClose(pMgmt->pQnode); - pMgmt->pQnode = NULL; - dInfo("dnode-qnode is cleaned up"); + if (pMgmt->pQnode) { + dndStopQnodeWorker(pDnode); + qndClose(pMgmt->pQnode); + pMgmt->pQnode = NULL; + } } diff --git a/source/dnode/mgmt/impl/src/dndWorker.c b/source/dnode/mgmt/impl/src/dndWorker.c new file mode 100644 index 0000000000..da0e3a9319 --- /dev/null +++ b/source/dnode/mgmt/impl/src/dndWorker.c @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dndWorker.h" + +int32_t dndInitWorker(SDnode *pDnode, SDnodeWorker *pWorker, EDndWorkerType type, const char *name, int32_t minNum, + int32_t maxNum, FProcessItem fp) { + if (pDnode == NULL || pWorker == NULL || name == NULL || minNum < 0 || maxNum <= 0 || fp == NULL) { + terrno = TSDB_CODE_INVALID_PARA; + return -1; + } + + pWorker->type = type; + pWorker->name = name; + pWorker->minNum = minNum; + pWorker->maxNum = maxNum; + pWorker->fp = fp; + pWorker->pDnode = pDnode; + + if (pWorker->type == DND_WORKER_SINGLE) { + SWorkerPool *pPool = &pWorker->pool; + pPool->min = minNum; + pPool->max = maxNum; + if (tWorkerInit(pPool) != 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + pWorker->queue = tWorkerAllocQueue(&pPool, pDnode, fp); + if (pWorker->queue == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + } else { + terrno = TSDB_CODE_INVALID_PARA; + } + + return 0; +} + +void dndCleanupWorker(SDnodeWorker *pWorker) { + if (pWorker->type == DND_WORKER_SINGLE) { + while (!taosQueueEmpty(pWorker->queue)) { + taosMsleep(10); + } + tWorkerCleanup(&pWorker->pool); + tWorkerFreeQueue(&pWorker->pool, pWorker->queue); + } +} + +int32_t dndWriteMsgToWorker(SDnodeWorker *pWorker, void *pCont, int32_t contLen) { + if (pWorker == NULL || pWorker->queue == NULL) { + terrno = TSDB_CODE_INVALID_PARA; + return -1; + } + + void *pMsg = taosAllocateQitem(contLen); + if (pMsg == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + memcpy(pMsg, pCont, contLen); + + if (taosWriteQitem(pWorker, pMsg) != 0) { + taosFreeItem(pMsg); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + return 0; +} \ No newline at end of file diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 451b7e4fa4..3a923c6653 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -79,6 +79,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_FILE_CORRUPTED, "Data file corrupted") TAOS_DEFINE_ERROR(TSDB_CODE_CHECKSUM_ERROR, "Checksum error") TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_MSG, "Invalid config message") TAOS_DEFINE_ERROR(TSDB_CODE_MSG_NOT_PROCESSED, "Message not processed") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_PARA, "Invalid parameters") TAOS_DEFINE_ERROR(TSDB_CODE_REF_NO_MEMORY, "Ref out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_REF_FULL, "too many Ref Objs") TAOS_DEFINE_ERROR(TSDB_CODE_REF_ID_REMOVED, "Ref ID is removed") From a536e6080047f5032851c9725a6382e9170eb4bd Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 27 Dec 2021 21:43:11 -0800 Subject: [PATCH 09/21] add show dnodes case --- include/common/tmsg.h | 2 +- source/dnode/mgmt/impl/test/dnode/dnode.cpp | 12 ++++--- source/dnode/mgmt/impl/test/mnode/mnode.cpp | 3 +- source/dnode/mnode/impl/src/mndDnode.c | 29 ++++++++--------- source/libs/parser/src/astToMsg.c | 4 +-- tests/script/unique/dnode/basic1.sim | 22 +++++++++++-- tests/tsim/src/simSystem.c | 36 +-------------------- 7 files changed, 48 insertions(+), 60 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index f51075a89a..f57b753d93 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -826,7 +826,7 @@ typedef struct SShowRsp { } SShowRsp; typedef struct { - char ep[TSDB_FQDN_LEN]; // end point, hostname:port + char fqdn[TSDB_FQDN_LEN]; // end point, hostname:port int32_t port; } SCreateDnodeMsg; diff --git a/source/dnode/mgmt/impl/test/dnode/dnode.cpp b/source/dnode/mgmt/impl/test/dnode/dnode.cpp index 4009a83cd0..dc352c5a3f 100644 --- a/source/dnode/mgmt/impl/test/dnode/dnode.cpp +++ b/source/dnode/mgmt/impl/test/dnode/dnode.cpp @@ -91,7 +91,8 @@ TEST_F(DndTestDnode, 03_Create_Drop_Restart_Dnode) { int32_t contLen = sizeof(SCreateDnodeMsg); SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); - strcpy(pReq->ep, "localhost:9042"); + strcpy(pReq->fqdn, "localhost"); + pReq->port = htonl(9042); SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_DNODE, pReq, contLen); ASSERT_NE(pMsg, nullptr); @@ -148,7 +149,8 @@ TEST_F(DndTestDnode, 03_Create_Drop_Restart_Dnode) { int32_t contLen = sizeof(SCreateDnodeMsg); SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); - strcpy(pReq->ep, "localhost:9043"); + strcpy(pReq->fqdn, "localhost"); + pReq->port = htonl(9043); SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_DNODE, pReq, contLen); ASSERT_NE(pMsg, nullptr); @@ -159,7 +161,8 @@ TEST_F(DndTestDnode, 03_Create_Drop_Restart_Dnode) { int32_t contLen = sizeof(SCreateDnodeMsg); SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); - strcpy(pReq->ep, "localhost:9044"); + strcpy(pReq->fqdn, "localhost"); + pReq->port = htonl(904); SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_DNODE, pReq, contLen); ASSERT_NE(pMsg, nullptr); @@ -170,7 +173,8 @@ TEST_F(DndTestDnode, 03_Create_Drop_Restart_Dnode) { int32_t contLen = sizeof(SCreateDnodeMsg); SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); - strcpy(pReq->ep, "localhost:9045"); + strcpy(pReq->fqdn, "localhost"); + pReq->port = htonl(9045); SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_DNODE, pReq, contLen); ASSERT_NE(pMsg, nullptr); diff --git a/source/dnode/mgmt/impl/test/mnode/mnode.cpp b/source/dnode/mgmt/impl/test/mnode/mnode.cpp index 0d89542288..a6cec93539 100644 --- a/source/dnode/mgmt/impl/test/mnode/mnode.cpp +++ b/source/dnode/mgmt/impl/test/mnode/mnode.cpp @@ -102,7 +102,8 @@ TEST_F(DndTestMnode, 04_Create_Mnode) { int32_t contLen = sizeof(SCreateDnodeMsg); SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); - strcpy(pReq->ep, "localhost:9062"); + strcpy(pReq->fqdn, "localhost"); + pReq->port = htonl(9062); SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_DNODE, pReq, contLen); ASSERT_NE(pMsg, nullptr); diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 153d75ffd0..56559cbea1 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -386,20 +386,16 @@ static int32_t mndCreateDnode(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDnodeMsg * dnodeObj.id = sdbGetMaxId(pMnode->pSdb, SDB_DNODE); dnodeObj.createdTime = taosGetTimestampMs(); dnodeObj.updateTime = dnodeObj.createdTime; - taosGetFqdnPortFromEp(pCreate->ep, dnodeObj.fqdn, &dnodeObj.port); - - if (dnodeObj.fqdn[0] == 0 || dnodeObj.port <= 0) { - terrno = TSDB_CODE_MND_INVALID_DNODE_EP; - mError("dnode:%s, failed to create since %s", pCreate->ep, terrstr()); - return terrno; - } + dnodeObj.port = pCreate->port; + memcpy(dnodeObj.fqdn, pCreate->fqdn, TSDB_FQDN_LEN); + snprintf(dnodeObj.ep, "%s:%u", dnodeObj.fqdn, dnodeObj.port); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pMsg->rpcMsg); if (pTrans == NULL) { - mError("dnode:%s, failed to create since %s", pCreate->ep, terrstr()); + mError("dnode:%s, failed to create since %s", dnodeObj.ep, terrstr()); return -1; } - mDebug("trans:%d, used to create dnode:%s", pTrans->id, pCreate->ep); + mDebug("trans:%d, used to create dnode:%s", pTrans->id, dnodeObj.ep); SSdbRaw *pRedoRaw = mndDnodeActionEncode(&dnodeObj); if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { @@ -423,17 +419,20 @@ static int32_t mndProcessCreateDnodeMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; SCreateDnodeMsg *pCreate = pMsg->rpcMsg.pCont; - mDebug("dnode:%s, start to create", pCreate->ep); + mDebug("dnode:%s:%d, start to create", pCreate->fqdn, pCreate->port); - if (pCreate->ep[0] == 0) { + pCreate->port = htonl(pCreate->port); + if (pCreate->fqdn[0] == 0 || pCreate->port <= 0 || pCreate->port > UINT16_MAX) { terrno = TSDB_CODE_MND_INVALID_DNODE_EP; - mError("dnode:%s, failed to create since %s", pCreate->ep, terrstr()); + mError("dnode:%s:%d, failed to create since %s", pCreate->fqdn, pCreate->port, terrstr()); return -1; } - SDnodeObj *pDnode = mndAcquireDnodeByEp(pMnode, pCreate->ep); + char ep[TSDB_EP_LEN]; + snprintf(ep, TSDB_EP_LEN, "%s:%d", pCreate->fqdn, pCreate->port); + SDnodeObj *pDnode = mndAcquireDnodeByEp(pMnode, ep); if (pDnode != NULL) { - mError("dnode:%d, already exist", pDnode->id); + mError("dnode:%d, already exist, %s:%u", pDnode->id, pCreate->fqdn, pCreate->port); mndReleaseDnode(pMnode, pDnode); terrno = TSDB_CODE_MND_DNODE_ALREADY_EXIST; return -1; @@ -442,7 +441,7 @@ static int32_t mndProcessCreateDnodeMsg(SMnodeMsg *pMsg) { int32_t code = mndCreateDnode(pMnode, pMsg, pCreate); if (code != 0) { - mError("dnode:%s, failed to create since %s", pCreate->ep, terrstr()); + mError("dnode:%s:%d, failed to create since %s", pCreate->fqdn, pCreate->port, terrstr()); return -1; } diff --git a/source/libs/parser/src/astToMsg.c b/source/libs/parser/src/astToMsg.c index 48bdf8d3e4..2f80af225a 100644 --- a/source/libs/parser/src/astToMsg.c +++ b/source/libs/parser/src/astToMsg.c @@ -414,8 +414,8 @@ SCreateDnodeMsg *buildCreateDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMs return NULL; } - strncpy(pCreate->ep, id->z, id->n); - pCreate->port = val; + strncpy(pCreate->fqdn, id->z, id->n); + pCreate->port = htonl(val); *len = sizeof(SCreateDnodeMsg); return pCreate; diff --git a/tests/script/unique/dnode/basic1.sim b/tests/script/unique/dnode/basic1.sim index 0d1b75f3c0..730864ef26 100644 --- a/tests/script/unique/dnode/basic1.sim +++ b/tests/script/unique/dnode/basic1.sim @@ -33,7 +33,9 @@ if $data02 != master then endi print =============== create dnodes -sql create dnode $hostname2 +sql create dnode $hostname port 7200 +sleep 2000 + sql show dnodes; if $rows != 2 then return -1 @@ -47,6 +49,22 @@ if $data10 != 2 then return -1 endi +if $data02 != 0 then + return -1 +endi + +if $data12 != 0 then + return -1 +endi + +if $data04 != ready then + return -1 +endi + +if $data14 != ready then + return -1 +endi + sql show mnodes; if $rows != 1 then return -1 @@ -56,7 +74,7 @@ if $data00 != 1 then return -1 endi -if $data01 != master then +if $data02 != master then return -1 endi diff --git a/tests/tsim/src/simSystem.c b/tests/tsim/src/simSystem.c index cb61e6b814..016b6500ed 100644 --- a/tests/tsim/src/simSystem.c +++ b/tests/tsim/src/simSystem.c @@ -42,41 +42,7 @@ char *simParseArbitratorName(char *varName) { char *simParseHostName(char *varName) { static char hostName[140]; - - int32_t index = atoi(varName + 8); - int32_t port = 7100; - switch (index) { - case 1: - port = 7100; - break; - case 2: - port = 7200; - break; - case 3: - port = 7300; - break; - case 4: - port = 7400; - break; - case 5: - port = 7500; - break; - case 6: - port = 7600; - break; - case 7: - port = 7700; - break; - case 8: - port = 7800; - break; - case 9: - port = 7900; - break; - } - - sprintf(hostName, "'%s:%d'", simHostName, port); - // simInfo("hostName:%s", hostName); + sprintf(hostName, "%s", simHostName); return hostName; } From d5f5fc00bf2ca9210a1353dd0cabb3905e2adf7a Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 27 Dec 2021 20:46:30 +0800 Subject: [PATCH 10/21] add dag serialize and deserialize --- include/libs/planner/planner.h | 4 +-- source/libs/planner/src/physicalPlanJson.c | 37 ++++++++++++++++++++-- source/libs/planner/test/phyPlanTests.cpp | 15 ++++++--- 3 files changed, 46 insertions(+), 10 deletions(-) diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index f9c25c3ccc..3ca923b0aa 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -132,7 +132,7 @@ typedef struct SSubplan { typedef struct SQueryDag { uint64_t queryId; int32_t numOfSubplans; - SArray *pSubplans; // Element is SArray*, and nested element is SSubplan. The execution level of subplan, starting from 0. + SArray *pSubplans; // SArray*>. The execution level of subplan, starting from 0. } SQueryDag; struct SQueryNode; @@ -159,8 +159,6 @@ int32_t qStringToSubplan(const char* str, SSubplan** subplan); void qDestroySubplan(SSubplan* pSubplan); -char* qDagSerialize(const SQueryDag* pDag); - /** * Destroy the physical plan. * @param pQueryPhyNode diff --git a/source/libs/planner/src/physicalPlanJson.c b/source/libs/planner/src/physicalPlanJson.c index 945c354284..9b2c9c1796 100644 --- a/source/libs/planner/src/physicalPlanJson.c +++ b/source/libs/planner/src/physicalPlanJson.c @@ -860,18 +860,33 @@ int32_t stringToSubplan(const char* str, SSubplan** subplan) { cJSON* qDagToJson(const SQueryDag* pDag) { cJSON* pRoot = cJSON_CreateObject(); + if(pRoot == NULL) { + return NULL; + } cJSON_AddNumberToObject(pRoot, "numOfSubplans", pDag->numOfSubplans); cJSON_AddNumberToObject(pRoot, "queryId", pDag->queryId); cJSON *pLevels = cJSON_CreateArray(); + if(pLevels == NULL) { + cJSON_Delete(pRoot); + return NULL; + } cJSON_AddItemToObject(pRoot, "pSubplans", pLevels); size_t level = taosArrayGetSize(pDag->pSubplans); for(size_t i = 0; i < level; i++) { const SArray* pSubplans = (const SArray*)taosArrayGetP(pDag->pSubplans, i); size_t num = taosArrayGetSize(pSubplans); cJSON* plansOneLevel = cJSON_CreateArray(); + if(plansOneLevel == NULL) { + cJSON_Delete(pRoot); + return NULL; + } cJSON_AddItemToArray(pLevels, plansOneLevel); for(size_t j = 0; j < num; j++) { cJSON* pSubplan = subplanToJson((const SSubplan*)taosArrayGetP(pSubplans, j)); + if(pSubplan == NULL) { + cJSON_Delete(pRoot); + return NULL; + } cJSON_AddItemToArray(plansOneLevel, pSubplan); } } @@ -889,18 +904,34 @@ SQueryDag* qJsonToDag(const cJSON* pRoot) { return NULL; } pDag->numOfSubplans = cJSON_GetNumberValue(cJSON_GetObjectItem(pRoot, "numOfSubplans")); + pDag->queryId = cJSON_GetNumberValue(cJSON_GetObjectItem(pRoot, "queryId")); pDag->pSubplans = taosArrayInit(0, sizeof(SArray)); - cJSON* pLevels = cJSON_GetObjectItem(pRoot, "numOfSubplans"); + if (pDag->pSubplans == NULL) { + free(pDag); + return NULL; + } + cJSON* pLevels = cJSON_GetObjectItem(pRoot, "pSubplans"); int level = cJSON_GetArraySize(pLevels); for(int i = 0; i < level; i++) { + SArray* plansOneLevel = taosArrayInit(0, sizeof(void*)); + if(plansOneLevel == NULL) { + for(int j = 0; j < i; j++) { + taosArrayDestroy(taosArrayGetP(pDag->pSubplans, j)); + } + taosArrayDestroy(pDag->pSubplans); + free(pDag); + return NULL; + } cJSON* pItem = cJSON_GetArrayItem(pLevels, i); int sz = cJSON_GetArraySize(pItem); for(int j = 0; j < sz; j++) { cJSON* pSubplanJson = cJSON_GetArrayItem(pItem, j); SSubplan* pSubplan = subplanFromJson(pSubplanJson); - } + taosArrayPush(plansOneLevel, &pSubplan); + } + taosArrayPush(pDag->pSubplans, plansOneLevel); } - return NULL; + return pDag; } SQueryDag* qStringToDag(const char* pStr) { diff --git a/source/libs/planner/test/phyPlanTests.cpp b/source/libs/planner/test/phyPlanTests.cpp index d0f6fd5adf..02d06bb49d 100644 --- a/source/libs/planner/test/phyPlanTests.cpp +++ b/source/libs/planner/test/phyPlanTests.cpp @@ -83,7 +83,7 @@ protected: } } - SQueryDag* reslut() { + SQueryDag* result() { return dag_.get(); } @@ -149,16 +149,23 @@ TEST_F(PhyPlanTest, tableScanTest) { pushScan("test", "t1", QNODE_TABLESCAN); ASSERT_EQ(run(), TSDB_CODE_SUCCESS); explain(); - SQueryDag* dag = reslut(); + SQueryDag* dag = result(); // todo check } +TEST_F(PhyPlanTest, serializeTest) { + pushScan("test", "t1", QNODE_TABLESCAN); + ASSERT_EQ(run(), TSDB_CODE_SUCCESS); + SQueryDag* dag = result(); + cout << qDagToString(dag) << endl; +} + // select * from supertable TEST_F(PhyPlanTest, superTableScanTest) { pushScan("test", "st1", QNODE_TABLESCAN); ASSERT_EQ(run(), TSDB_CODE_SUCCESS); explain(); - SQueryDag* dag = reslut(); + SQueryDag* dag = result(); // todo check } @@ -166,6 +173,6 @@ TEST_F(PhyPlanTest, superTableScanTest) { TEST_F(PhyPlanTest, insertTest) { ASSERT_EQ(run("test", "insert into t1 values (now, 1, \"beijing\")"), TSDB_CODE_SUCCESS); explain(); - SQueryDag* dag = reslut(); + SQueryDag* dag = result(); // todo check } From 80718aee3460e1b3eb25fec7328fce1d98cdceec Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 28 Dec 2021 14:36:44 +0800 Subject: [PATCH 11/21] refactor index write --- source/libs/index/inc/index_cache.h | 3 +- source/libs/index/src/index.c | 4 +-- source/libs/index/src/index_cache.c | 51 ++++++++++++++++++++--------- 3 files changed, 38 insertions(+), 20 deletions(-) diff --git a/source/libs/index/inc/index_cache.h b/source/libs/index/inc/index_cache.h index 27f5a6fb20..0e7405869a 100644 --- a/source/libs/index/inc/index_cache.h +++ b/source/libs/index/inc/index_cache.h @@ -39,6 +39,7 @@ typedef struct IndexCache { int32_t nTerm; int8_t type; + pthread_mutex_t mtx; } IndexCache; #define CACHE_VERSION(cache) atomic_load_32(&cache->version) @@ -71,7 +72,7 @@ void indexCacheUnRef(IndexCache* cache); void indexCacheDebug(IndexCache* cache); -void indexCacheDestroySkiplist(SSkipList* slt); +void indexCacheDestroyImm(IndexCache* cache); #ifdef __cplusplus } #endif diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 23df5f9f9a..06e7e8ba44 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -436,9 +436,7 @@ int indexFlushCacheTFile(SIndex* sIdx, void* cache) { if (ret != 0) { indexError("faile to write into tindex "); } } // not free later, just put int table cache - SSkipList* timm = (SSkipList*)pCache->imm; - pCache->imm = NULL; // or throw int bg thread - indexCacheDestroySkiplist(timm); + indexCacheDestroyImm(pCache); tfileWriteClose(tw); indexCacheIteratorDestroy(cacheIter); diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c index 54aee8858b..3f99d04bc9 100644 --- a/source/libs/index/src/index_cache.c +++ b/source/libs/index/src/index_cache.c @@ -20,7 +20,7 @@ #define MAX_INDEX_KEY_LEN 256 // test only, change later -#define CACH_LIMIT 1000000 +#define MEM_TERM_LIMIT 1000000 // ref index_cache.h:22 //#define CACHE_KEY_LEN(p) \ // (sizeof(int32_t) + sizeof(uint16_t) + sizeof(p->colType) + sizeof(p->nColVal) + p->nColVal + sizeof(uint64_t) + sizeof(p->operType)) @@ -78,6 +78,7 @@ IndexCache* indexCacheCreate(SIndex* idx, const char* colName, int8_t type) { cache->index = idx; cache->version = 0; + pthread_mutex_init(&cache->mtx, NULL); indexCacheRef(cache); return cache; } @@ -103,13 +104,21 @@ void indexCacheDestroySkiplist(SSkipList* slt) { } tSkipListDestroyIter(iter); } +void indexCacheDestroyImm(IndexCache* cache) { + pthread_mutex_lock(&cache->mtx); + SSkipList* timm = (SSkipList*)cache->imm; + cache->imm = NULL; // or throw int bg thread + pthread_mutex_unlock(&cache->mtx); + indexCacheDestroySkiplist(timm); +} void indexCacheDestroy(void* cache) { IndexCache* pCache = cache; if (pCache == NULL) { return; } tSkipListDestroy(pCache->mem); tSkipListDestroy(pCache->imm); free(pCache->colName); + free(pCache); } @@ -170,6 +179,27 @@ int indexCacheSchedToMerge(IndexCache* pCache) { taosScheduleTask(indexQhandle, &schedMsg); } +static void indexCacheMakeRoomForWrite(IndexCache* cache) { + while (true) { + if (cache->nTerm < MEM_TERM_LIMIT) { + cache->nTerm += 1; + break; + } else if (cache->imm != NULL) { + // TODO: wake up by condition variable + pthread_mutex_unlock(&cache->mtx); + taosMsleep(50); + pthread_mutex_lock(&cache->mtx); + } else { + cache->imm = cache->mem; + cache->mem = indexInternalCacheCreate(cache->type); + cache->nTerm = 1; + // sched to merge + // unref cache in bgwork + indexCacheSchedToMerge(cache); + } + } +} + int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) { if (cache == NULL) { return -1; } @@ -188,23 +218,12 @@ int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) { ct->uid = uid; ct->operaType = term->operType; + // ugly code, refactor later + pthread_mutex_lock(&pCache->mtx); + indexCacheMakeRoomForWrite(pCache); tSkipListPut(pCache->mem, (char*)ct); - pCache->nTerm += 1; + pthread_mutex_unlock(&pCache->mtx); - if (pCache->nTerm >= CACH_LIMIT) { - pCache->nTerm = 0; - - while (pCache->imm != NULL) { - // do nothong - } - - pCache->imm = pCache->mem; - pCache->mem = indexInternalCacheCreate(pCache->type); - - // sched to merge - // unref cache int bgwork - indexCacheSchedToMerge(pCache); - } indexCacheUnRef(pCache); return 0; // encode end From 6f7eecb215949771a157ffe95e89fa81690036f6 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 27 Dec 2021 22:57:20 -0800 Subject: [PATCH 12/21] add dnodes sim --- tests/script/general/user/testSuite.sim | 6 - tests/script/jenkins/basic.txt | 3 + tests/script/jenkins/simple.txt | 231 -------------- tests/script/jenkins/unique.txt | 127 -------- tests/script/jenkins/wbasic.txt | 393 ------------------------ tests/script/sh/deploy.sh | 21 +- tests/script/tmp/182.sim | 41 --- tests/script/tmp/dnodes.sim | 78 +++++ tests/script/tmp/http.sim | 35 --- tests/script/tmp/mnodes.sim | 115 ------- tests/script/tmp/prepare.sim | 38 --- 11 files changed, 82 insertions(+), 1006 deletions(-) delete mode 100644 tests/script/general/user/testSuite.sim delete mode 100644 tests/script/jenkins/simple.txt delete mode 100644 tests/script/jenkins/unique.txt delete mode 100644 tests/script/jenkins/wbasic.txt delete mode 100644 tests/script/tmp/182.sim create mode 100644 tests/script/tmp/dnodes.sim delete mode 100644 tests/script/tmp/http.sim delete mode 100644 tests/script/tmp/mnodes.sim diff --git a/tests/script/general/user/testSuite.sim b/tests/script/general/user/testSuite.sim deleted file mode 100644 index b4bf52cfd1..0000000000 --- a/tests/script/general/user/testSuite.sim +++ /dev/null @@ -1,6 +0,0 @@ -run general/user/basic1.sim -run general/user/monitor.sim -run general/user/pass_alter.sim -run general/user/pass_len.sim -run general/user/user_create.sim -run general/user/user_len.sim \ No newline at end of file diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 0ef43aaced..f960abb4e0 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -10,4 +10,7 @@ # ---- table ./test.sh -f general/table/basic1.sim +# ---- dnode +./test.sh -f unique/dnode/basic1.sim + #======================b1-end=============== diff --git a/tests/script/jenkins/simple.txt b/tests/script/jenkins/simple.txt deleted file mode 100644 index 88ff4d6601..0000000000 --- a/tests/script/jenkins/simple.txt +++ /dev/null @@ -1,231 +0,0 @@ -cd ../../../debug; cmake .. -cd ../../../debug; make - -#./test.sh -f general/alter/cached_schema_after_alter.sim -./test.sh -f general/alter/count.sim -./test.sh -f general/alter/import.sim -#./test.sh -f general/alter/insert1.sim -./test.sh -f general/alter/insert2.sim -./test.sh -f general/alter/metrics.sim -./test.sh -f general/alter/table.sim - -./test.sh -f general/cache/new_metrics.sim -./test.sh -f general/cache/restart_metrics.sim -./test.sh -f general/cache/restart_table.sim - -./test.sh -f general/connection/connection.sim - -./test.sh -f general/column/commit.sim -./test.sh -f general/column/metrics.sim -./test.sh -f general/column/table.sim - -./test.sh -f general/compress/commitlog.sim -./test.sh -f general/compress/compress.sim -./test.sh -f general/compress/compress2.sim -./test.sh -f general/compress/uncompress.sim - -./test.sh -f general/compute/avg.sim -./test.sh -f general/compute/bottom.sim -./test.sh -f general/compute/count.sim -./test.sh -f general/compute/diff.sim -./test.sh -f general/compute/diff2.sim -./test.sh -f general/compute/first.sim -./test.sh -f general/compute/interval.sim -./test.sh -f general/compute/last.sim -./test.sh -f general/compute/leastsquare.sim -./test.sh -f general/compute/max.sim -./test.sh -f general/compute/min.sim -./test.sh -f general/compute/null.sim -./test.sh -f general/compute/percentile.sim -./test.sh -f general/compute/stddev.sim -./test.sh -f general/compute/sum.sim -./test.sh -f general/compute/top.sim - -./test.sh -f general/db/alter_option.sim -./test.sh -f general/db/alter_tables_d2.sim -./test.sh -f general/db/alter_tables_v1.sim -./test.sh -f general/db/alter_tables_v4.sim -./test.sh -f general/db/alter_vgroups.sim -./test.sh -f general/db/basic.sim -./test.sh -f general/db/basic1.sim -./test.sh -f general/db/basic2.sim -./test.sh -f general/db/basic3.sim -./test.sh -f general/db/basic4.sim -./test.sh -f general/db/basic5.sim -./test.sh -f general/db/delete_reuse1.sim -./test.sh -f general/db/delete_reuse2.sim -./test.sh -f general/db/delete_reusevnode.sim -./test.sh -f general/db/delete_reusevnode2.sim -./test.sh -f general/db/delete_writing1.sim -./test.sh -f general/db/delete_writing2.sim -./test.sh -f general/db/delete.sim -./test.sh -f general/db/len.sim -./test.sh -f general/db/repeat.sim -./test.sh -f general/db/tables.sim -./test.sh -f general/db/vnodes.sim - -./test.sh -f general/field/2.sim -./test.sh -f general/field/3.sim -./test.sh -f general/field/4.sim -./test.sh -f general/field/5.sim -./test.sh -f general/field/6.sim -./test.sh -f general/field/bigint.sim -./test.sh -f general/field/binary.sim -./test.sh -f general/field/bool.sim -./test.sh -f general/field/single.sim -./test.sh -f general/field/smallint.sim -./test.sh -f general/field/tinyint.sim - -./test.sh -f general/http/restful.sim -./test.sh -f general/http/restful_insert.sim -./test.sh -f general/http/restful_limit.sim -./test.sh -f general/http/restful_full.sim -./test.sh -f general/http/prepare.sim -./test.sh -f general/http/telegraf.sim -./test.sh -f general/http/grafana_bug.sim -./test.sh -f general/http/grafana.sim - -./test.sh -f general/import/basic.sim -./test.sh -f general/import/commit.sim -./test.sh -f general/import/large.sim -./test.sh -f general/import/replica1.sim - -./test.sh -f general/insert/basic.sim -./test.sh -f general/insert/insert_drop.sim -./test.sh -f general/insert/query_block1_memory.sim -./test.sh -f general/insert/query_block2_memory.sim -./test.sh -f general/insert/query_block1_file.sim -./test.sh -f general/insert/query_block2_file.sim -./test.sh -f general/insert/query_file_memory.sim -./test.sh -f general/insert/query_multi_file.sim -./test.sh -f general/insert/tcp.sim - -./test.sh -f general/parser/alter.sim -./test.sh -f general/parser/alter1.sim -./test.sh -f general/parser/alter_stable.sim -./test.sh -f general/parser/auto_create_tb.sim -./test.sh -f general/parser/auto_create_tb_drop_tb.sim -./test.sh -f general/parser/col_arithmetic_operation.sim -./test.sh -f general/parser/columnValue.sim -./test.sh -f general/parser/commit.sim -./test.sh -f general/parser/create_db.sim -./test.sh -f general/parser/create_mt.sim -./test.sh -f general/parser/create_tb.sim -./test.sh -f general/parser/dbtbnameValidate.sim -./test.sh -f general/parser/import_commit1.sim -./test.sh -f general/parser/import_commit2.sim -./test.sh -f general/parser/import_commit3.sim -./test.sh -f general/parser/insert_tb.sim -./test.sh -f general/parser/first_last.sim -#./test.sh -f general/parser/import_file.sim -./test.sh -f general/parser/lastrow.sim -./test.sh -f general/parser/nchar.sim -./test.sh -f general/parser/null_char.sim -./test.sh -f general/parser/single_row_in_tb.sim -./test.sh -f general/parser/select_from_cache_disk.sim -./test.sh -f general/parser/mixed_blocks.sim -./test.sh -f general/parser/selectResNum.sim -./test.sh -f general/parser/limit.sim -./test.sh -f general/parser/limit1.sim -./test.sh -f general/parser/limit1_tblocks100.sim -./test.sh -f general/parser/select_across_vnodes.sim -./test.sh -f general/parser/slimit1.sim -./test.sh -f general/parser/tbnameIn.sim -./test.sh -f general/parser/projection_limit_offset.sim -./test.sh -f general/parser/limit2.sim -./test.sh -f general/parser/fill.sim -./test.sh -f general/parser/fill_stb.sim -./test.sh -f general/parser/where.sim -./test.sh -f general/parser/slimit.sim -./test.sh -f general/parser/select_with_tags.sim -./test.sh -f general/parser/interp.sim -./test.sh -f general/parser/tags_dynamically_specifiy.sim -./test.sh -f general/parser/groupby.sim -./test.sh -f general/parser/set_tag_vals.sim -#./test.sh -f general/parser/slimit_alter_tags.sim -./test.sh -f general/parser/join.sim -./test.sh -f general/parser/join_multivnode.sim -./test.sh -f general/parser/binary_escapeCharacter.sim -./test.sh -f general/parser/bug.sim -./test.sh -f general/parser/repeatAlter.sim - -./test.sh -f general/stable/disk.sim -./test.sh -f general/stable/dnode3.sim -./test.sh -f general/stable/metrics.sim -./test.sh -f general/stable/refcount.sim -./test.sh -f general/stable/show.sim -./test.sh -f general/stable/values.sim -./test.sh -f general/stable/vnode3.sim - -./test.sh -f general/table/autocreate.sim -./test.sh -f general/table/basic1.sim -./test.sh -f general/table/basic2.sim -./test.sh -f general/table/basic3.sim -./test.sh -f general/table/bigint.sim -./test.sh -f general/table/binary.sim -./test.sh -f general/table/bool.sim -./test.sh -f general/table/column_name.sim -./test.sh -f general/table/column_num.sim -./test.sh -f general/table/column_value.sim -./test.sh -f general/table/column2.sim -./test.sh -f general/table/date.sim -./test.sh -f general/table/db.table.sim -./test.sh -f general/table/delete_reuse1.sim -./test.sh -f general/table/delete_reuse2.sim -./test.sh -f general/table/delete_writing.sim -./test.sh -f general/table/describe.sim -./test.sh -f general/table/double.sim -./test.sh -f general/table/fill.sim -./test.sh -f general/table/float.sim -./test.sh -f general/table/int.sim -./test.sh -f general/table/limit.sim -./test.sh -f general/table/smallint.sim -./test.sh -f general/table/table_len.sim -./test.sh -f general/table/table.sim -./test.sh -f general/table/tinyint.sim -./test.sh -f general/table/vgroup.sim - -./test.sh -f general/tag/3.sim -./test.sh -f general/tag/4.sim -./test.sh -f general/tag/5.sim -./test.sh -f general/tag/6.sim -./test.sh -f general/tag/add.sim -./test.sh -f general/tag/bigint.sim -./test.sh -f general/tag/binary_binary.sim -./test.sh -f general/tag/binary.sim -./test.sh -f general/tag/bool_binary.sim -./test.sh -f general/tag/bool_int.sim -./test.sh -f general/tag/bool.sim -./test.sh -f general/tag/change.sim -./test.sh -f general/tag/column.sim -#./test.sh -f general/tag/commit.sim -./test.sh -f general/tag/create.sim -./test.sh -f general/tag/delete.sim -./test.sh -f general/tag/double.sim -./test.sh -f general/tag/filter.sim -./test.sh -f general/tag/float.sim -./test.sh -f general/tag/int_binary.sim -./test.sh -f general/tag/int_float.sim -./test.sh -f general/tag/int.sim -./test.sh -f general/tag/set.sim -./test.sh -f general/tag/smallint.sim -./test.sh -f general/tag/tinyint.sim - -./test.sh -f general/user/authority.sim -./test.sh -f general/user/monitor.sim -./test.sh -f general/user/pass_alter.sim -./test.sh -f general/user/pass_len.sim -./test.sh -f general/user/user_create.sim -./test.sh -f general/user/user_len.sim - -./test.sh -f general/vector/metrics_field.sim -./test.sh -f general/vector/metrics_mix.sim -./test.sh -f general/vector/metrics_query.sim -./test.sh -f general/vector/metrics_tag.sim -./test.sh -f general/vector/metrics_time.sim -./test.sh -f general/vector/multi.sim -./test.sh -f general/vector/single.sim -./test.sh -f general/vector/table_field.sim -./test.sh -f general/vector/table_mix.sim -./test.sh -f general/vector/table_query.sim -./test.sh -f general/vector/table_time.sim diff --git a/tests/script/jenkins/unique.txt b/tests/script/jenkins/unique.txt deleted file mode 100644 index 372bdc9d9d..0000000000 --- a/tests/script/jenkins/unique.txt +++ /dev/null @@ -1,127 +0,0 @@ -cd ../../../debug; cmake .. -cd ../../../debug; make - -./test.sh -f unique/big/balance.sim -./test.sh -f unique/big/maxvnodes.sim -./test.sh -f unique/big/tcp.sim - -./test.sh -f unique/cluster/alter.sim -./test.sh -f unique/cluster/balance1.sim -./test.sh -f unique/cluster/balance2.sim -./test.sh -f unique/cluster/balance3.sim -./test.sh -f unique/cluster/cache.sim -./test.sh -f unique/cluster/vgroup100.sim - -./test.sh -f unique/column/replica3.sim - -./test.sh -f unique/db/commit.sim -./test.sh -f unique/db/delete.sim -./test.sh -f unique/db/delete_part.sim -./test.sh -f unique/db/replica_add12.sim -./test.sh -f unique/db/replica_add13.sim -./test.sh -f unique/db/replica_add23.sim -./test.sh -f unique/db/replica_reduce21.sim -./test.sh -f unique/db/replica_reduce32.sim -./test.sh -f unique/db/replica_reduce31.sim -./test.sh -f unique/db/replica_part.sim - -./test.sh -f unique/dnode/alternativeRole.sim -./test.sh -f unique/dnode/monitor.sim -./test.sh -f unique/dnode/monitor_bug.sim -./test.sh -f unique/dnode/simple.sim -./test.sh -f unique/dnode/balance1.sim -./test.sh -f unique/dnode/balance2.sim -./test.sh -f unique/dnode/balance3.sim -./test.sh -f unique/dnode/balancex.sim -./test.sh -f unique/dnode/data1.sim -./test.sh -f unique/dnode/m2.sim -./test.sh -f unique/dnode/m3.sim -./test.sh -f unique/dnode/lossdata.sim -./test.sh -f unique/dnode/offline1.sim -./test.sh -f unique/dnode/offline2.sim -./test.sh -f unique/dnode/offline3.sim -./test.sh -f unique/dnode/reason.sim -./test.sh -f unique/dnode/remove1.sim -./test.sh -f unique/dnode/remove2.sim -./test.sh -f unique/dnode/vnode_clean.sim - -./test.sh -f unique/http/admin.sim -./test.sh -f unique/http/opentsdb.sim - -./test.sh -f unique/import/replica2.sim -./test.sh -f unique/import/replica3.sim - -./test.sh -f unique/stable/balance_replica1.sim -./test.sh -f unique/stable/dnode2_stop.sim -./test.sh -f unique/stable/dnode2.sim -./test.sh -f unique/stable/dnode3.sim -./test.sh -f unique/stable/replica2_dnode4.sim -./test.sh -f unique/stable/replica2_vnode3.sim -./test.sh -f unique/stable/replica3_dnode6.sim -./test.sh -f unique/stable/replica3_vnode3.sim - -./test.sh -f unique/mnode/mgmt20.sim -./test.sh -f unique/mnode/mgmt21.sim -./test.sh -f unique/mnode/mgmt22.sim -./test.sh -f unique/mnode/mgmt23.sim -./test.sh -f unique/mnode/mgmt24.sim -./test.sh -f unique/mnode/mgmt25.sim -./test.sh -f unique/mnode/mgmt26.sim -./test.sh -f unique/mnode/mgmt30.sim -./test.sh -f unique/mnode/mgmt33.sim -./test.sh -f unique/mnode/mgmt34.sim -./test.sh -f unique/mnode/mgmtr2.sim - -./test.sh -f unique/vnode/many.sim -./test.sh -f unique/vnode/replica2_basic2.sim -./test.sh -f unique/vnode/replica2_repeat.sim -./test.sh -f unique/vnode/replica3_basic.sim -./test.sh -f unique/vnode/replica3_repeat.sim -./test.sh -f unique/vnode/replica3_vgroup.sim - -./test.sh -f unique/arbitrator/check_cluster_cfg_para.sim -#./test.sh -f unique/arbitrator/dn2_mn1_cache_file_sync.sim -./test.sh -f unique/arbitrator/dn3_mn1_full_createTableFail.sim -./test.sh -f unique/arbitrator/dn3_mn1_multiCreateDropTable.sim -#./test.sh -f unique/arbitrator/dn3_mn1_nw_disable_timeout_autoDropDnode.sim -#./test.sh -f unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim -./test.sh -f unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim -./test.sh -f unique/arbitrator/dn3_mn1_replica_change.sim -#./test.sh -f unique/arbitrator/dn3_mn1_stopDnode_timeout.sim -# lower the priority while file corruption -#./test.sh -f unique/arbitrator/dn3_mn1_vnode_change.sim -#./test.sh -f unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim -#./test.sh -f unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim -#./test.sh -f unique/arbitrator/dn3_mn1_vnode_createErrData_online.sim -./test.sh -f unique/arbitrator/dn3_mn1_vnode_noCorruptFile_offline.sim -./test.sh -f unique/arbitrator/dn3_mn1_vnode_delDir.sim -./test.sh -f unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim -./test.sh -f unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim -./test.sh -f unique/arbitrator/dn3_mn1_vnode_nomaster.sim -./test.sh -f unique/arbitrator/dn3_mn2_killDnode.sim -./test.sh -f unique/arbitrator/insert_duplicationTs.sim -./test.sh -f unique/arbitrator/offline_replica2_alterTable_online.sim -./test.sh -f unique/arbitrator/offline_replica2_alterTag_online.sim -./test.sh -f unique/arbitrator/offline_replica2_createTable_online.sim -./test.sh -f unique/arbitrator/offline_replica2_dropDb_online.sim -./test.sh -f unique/arbitrator/offline_replica2_dropTable_online.sim -./test.sh -f unique/arbitrator/offline_replica3_alterTable_online.sim -./test.sh -f unique/arbitrator/offline_replica3_alterTag_online.sim -./test.sh -f unique/arbitrator/offline_replica3_createTable_online.sim -./test.sh -f unique/arbitrator/offline_replica3_dropDb_online.sim -./test.sh -f unique/arbitrator/offline_replica3_dropTable_online.sim -./test.sh -f unique/arbitrator/replica_changeWithArbitrator.sim -./test.sh -f unique/arbitrator/sync_replica2_alterTable_add.sim -./test.sh -f unique/arbitrator/sync_replica2_alterTable_drop.sim - -./test.sh -f unique/arbitrator/sync_replica2_dropDb.sim -./test.sh -f unique/arbitrator/sync_replica2_dropTable.sim -./test.sh -f unique/arbitrator/sync_replica3_alterTable_add.sim -./test.sh -f unique/arbitrator/sync_replica3_alterTable_drop.sim -./test.sh -f unique/arbitrator/sync_replica3_dropDb.sim -./test.sh -f unique/arbitrator/sync_replica3_dropTable.sim - -./test.sh -f unique/migrate/mn2_vn2_repl2_rmMnodeDir.sim -./test.sh -f unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim -./test.sh -f unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir_stopAll_starAll.sim -./test.sh -f unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim diff --git a/tests/script/jenkins/wbasic.txt b/tests/script/jenkins/wbasic.txt deleted file mode 100644 index 984b15fc5a..0000000000 --- a/tests/script/jenkins/wbasic.txt +++ /dev/null @@ -1,393 +0,0 @@ -wtest.bat -f issue/TD-2677.sim -wtest.bat -f issue/TD-2680.sim -wtest.bat -f issue/TD-2713.sim - -wtest.bat -f general/alter/cached_schema_after_alter.sim -wtest.bat -f general/alter/count.sim -wtest.bat -f general/alter/dnode.sim -wtest.bat -f general/alter/import.sim -wtest.bat -f general/alter/insert1.sim -wtest.bat -f general/alter/insert2.sim -wtest.bat -f general/alter/metrics.sim -wtest.bat -f general/alter/table.sim - -wtest.bat -f general/cache/new_metrics.sim -wtest.bat -f general/cache/restart_metrics.sim -wtest.bat -f general/cache/restart_table.sim - -wtest.bat -f general/connection/connection.sim - -wtest.bat -f general/column/commit.sim -wtest.bat -f general/column/metrics.sim -wtest.bat -f general/column/table.sim - -wtest.bat -f general/compress/commitlog.sim -wtest.bat -f general/compress/compress.sim -wtest.bat -f general/compress/compress2.sim -wtest.bat -f general/compress/uncompress.sim - -wtest.bat -f general/compute/avg.sim -wtest.bat -f general/compute/bottom.sim -wtest.bat -f general/compute/count.sim -wtest.bat -f general/compute/diff.sim -wtest.bat -f general/compute/diff2.sim -wtest.bat -f general/compute/first.sim -wtest.bat -f general/compute/interval.sim -wtest.bat -f general/compute/last.sim -wtest.bat -f general/compute/leastsquare.sim -wtest.bat -f general/compute/max.sim -wtest.bat -f general/compute/min.sim -wtest.bat -f general/compute/null.sim -wtest.bat -f general/compute/percentile.sim -wtest.bat -f general/compute/stddev.sim -wtest.bat -f general/compute/sum.sim -wtest.bat -f general/compute/top.sim - -wtest.bat -f general/db/alter_option.sim -wtest.bat -f general/db/alter_tables_d2.sim -wtest.bat -f general/db/alter_tables_v1.sim -wtest.bat -f general/db/alter_tables_v4.sim -wtest.bat -f general/db/alter_vgroups.sim -wtest.bat -f general/db/basic.sim -wtest.bat -f general/db/basic1.sim -wtest.bat -f general/db/basic2.sim -wtest.bat -f general/db/basic3.sim -wtest.bat -f general/db/basic4.sim -wtest.bat -f general/db/basic5.sim -wtest.bat -f general/db/delete_reuse1.sim -wtest.bat -f general/db/delete_reuse2.sim -wtest.bat -f general/db/delete_reusevnode.sim -wtest.bat -f general/db/delete_reusevnode2.sim -wtest.bat -f general/db/delete_writing1.sim -wtest.bat -f general/db/delete_writing2.sim -wtest.bat -f general/db/delete.sim -wtest.bat -f general/db/len.sim -wtest.bat -f general/db/repeat.sim -wtest.bat -f general/db/tables.sim -wtest.bat -f general/db/vnodes.sim -wtest.bat -f general/db/nosuchfile.sim - -wtest.bat -f general/field/2.sim -wtest.bat -f general/field/3.sim -wtest.bat -f general/field/4.sim -wtest.bat -f general/field/5.sim -wtest.bat -f general/field/6.sim -wtest.bat -f general/field/bigint.sim -wtest.bat -f general/field/binary.sim -wtest.bat -f general/field/bool.sim -wtest.bat -f general/field/single.sim -wtest.bat -f general/field/smallint.sim -wtest.bat -f general/field/tinyint.sim - -wtest.bat -f general/http/autocreate.sim -wtest.bat -f general/http/chunked.sim -wtest.bat -f general/http/gzip.sim -wtest.bat -f general/http/restful.sim -wtest.bat -f general/http/restful_insert.sim -wtest.bat -f general/http/restful_limit.sim -wtest.bat -f general/http/restful_full.sim -wtest.bat -f general/http/prepare.sim -wtest.bat -f general/http/telegraf.sim -wtest.bat -f general/http/grafana_bug.sim -wtest.bat -f general/http/grafana.sim - -wtest.bat -f general/import/basic.sim -wtest.bat -f general/import/commit.sim -wtest.bat -f general/import/large.sim -wtest.bat -f general/import/replica1.sim - -wtest.bat -f general/insert/basic.sim -wtest.bat -f general/insert/insert_drop.sim -wtest.bat -f general/insert/query_block1_memory.sim -wtest.bat -f general/insert/query_block2_memory.sim -wtest.bat -f general/insert/query_block1_file.sim -wtest.bat -f general/insert/query_block2_file.sim -wtest.bat -f general/insert/query_file_memory.sim -wtest.bat -f general/insert/query_multi_file.sim -wtest.bat -f general/insert/tcp.sim - -wtest.bat -f general/parser/alter.sim -wtest.bat -f general/parser/alter1.sim -wtest.bat -f general/parser/alter_stable.sim -wtest.bat -f general/parser/auto_create_tb.sim -wtest.bat -f general/parser/auto_create_tb_drop_tb.sim -wtest.bat -f general/parser/col_arithmetic_operation.sim -wtest.bat -f general/parser/columnValue.sim -wtest.bat -f general/parser/commit.sim -wtest.bat -f general/parser/create_db.sim -wtest.bat -f general/parser/create_mt.sim -wtest.bat -f general/parser/create_tb.sim -wtest.bat -f general/parser/dbtbnameValidate.sim -wtest.bat -f general/parser/import_commit1.sim -wtest.bat -f general/parser/import_commit2.sim -wtest.bat -f general/parser/import_commit3.sim -wtest.bat -f general/parser/insert_tb.sim -wtest.bat -f general/parser/first_last.sim -wtest.bat -f general/parser/lastrow.sim -wtest.bat -f general/parser/nchar.sim -wtest.bat -f general/parser/null_char.sim -wtest.bat -f general/parser/single_row_in_tb.sim -wtest.bat -f general/parser/select_from_cache_disk.sim -wtest.bat -f general/parser/mixed_blocks.sim -wtest.bat -f general/parser/selectResNum.sim -wtest.bat -f general/parser/limit.sim -wtest.bat -f general/parser/limit1.sim -wtest.bat -f general/parser/limit1_tblocks100.sim -wtest.bat -f general/parser/select_across_vnodes.sim -wtest.bat -f general/parser/slimit1.sim -wtest.bat -f general/parser/tbnameIn.sim -wtest.bat -f general/parser/projection_limit_offset.sim -wtest.bat -f general/parser/limit2.sim -wtest.bat -f general/parser/fill.sim -wtest.bat -f general/parser/fill_stb.sim -wtest.bat -f general/parser/where.sim -wtest.bat -f general/parser/slimit.sim -wtest.bat -f general/parser/select_with_tags.sim -wtest.bat -f general/parser/interp.sim -wtest.bat -f general/parser/tags_dynamically_specifiy.sim -wtest.bat -f general/parser/groupby.sim -wtest.bat -f general/parser/set_tag_vals.sim -wtest.bat -f general/parser/tags_filter.sim -wtest.bat -f general/parser/slimit_alter_tags.sim -wtest.bat -f general/parser/join.sim -wtest.bat -f general/parser/join_multivnode.sim -wtest.bat -f general/parser/binary_escapeCharacter.sim -wtest.bat -f general/parser/repeatAlter.sim -wtest.bat -f general/parser/union.sim -wtest.bat -f general/parser/topbot.sim -wtest.bat -f general/parser/function.sim - -wtest.bat -f general/stable/disk.sim -wtest.bat -f general/stable/dnode3.sim -wtest.bat -f general/stable/metrics.sim -wtest.bat -f general/stable/refcount.sim -wtest.bat -f general/stable/show.sim -wtest.bat -f general/stable/values.sim -wtest.bat -f general/stable/vnode3.sim - -wtest.bat -f general/table/autocreate.sim -wtest.bat -f general/table/basic1.sim -wtest.bat -f general/table/basic2.sim -wtest.bat -f general/table/basic3.sim -wtest.bat -f general/table/bigint.sim -wtest.bat -f general/table/binary.sim -wtest.bat -f general/table/bool.sim -wtest.bat -f general/table/column_name.sim -wtest.bat -f general/table/column_num.sim -wtest.bat -f general/table/column_value.sim -wtest.bat -f general/table/column2.sim -wtest.bat -f general/table/date.sim -wtest.bat -f general/table/db.table.sim -wtest.bat -f general/table/delete_reuse1.sim -wtest.bat -f general/table/delete_reuse2.sim -wtest.bat -f general/table/delete_writing.sim -wtest.bat -f general/table/describe.sim -wtest.bat -f general/table/double.sim -wtest.bat -f general/table/fill.sim -wtest.bat -f general/table/float.sim -wtest.bat -f general/table/int.sim -wtest.bat -f general/table/limit.sim -wtest.bat -f general/table/smallint.sim -wtest.bat -f general/table/table_len.sim -wtest.bat -f general/table/table.sim -wtest.bat -f general/table/tinyint.sim -wtest.bat -f general/table/vgroup.sim - -wtest.bat -f general/tag/3.sim -wtest.bat -f general/tag/4.sim -wtest.bat -f general/tag/5.sim -wtest.bat -f general/tag/6.sim -wtest.bat -f general/tag/add.sim -wtest.bat -f general/tag/bigint.sim -wtest.bat -f general/tag/binary_binary.sim -wtest.bat -f general/tag/binary.sim -wtest.bat -f general/tag/bool_binary.sim -wtest.bat -f general/tag/bool_int.sim -wtest.bat -f general/tag/bool.sim -wtest.bat -f general/tag/change.sim -wtest.bat -f general/tag/column.sim -wtest.bat -f general/tag/commit.sim -wtest.bat -f general/tag/create.sim -wtest.bat -f general/tag/delete.sim -wtest.bat -f general/tag/double.sim -wtest.bat -f general/tag/filter.sim -wtest.bat -f general/tag/float.sim -wtest.bat -f general/tag/int_binary.sim -wtest.bat -f general/tag/int_float.sim -wtest.bat -f general/tag/int.sim -wtest.bat -f general/tag/set.sim -wtest.bat -f general/tag/smallint.sim -wtest.bat -f general/tag/tinyint.sim - -wtest.bat -f general/user/authority.sim -wtest.bat -f general/user/monitor.sim -wtest.bat -f general/user/pass_alter.sim -wtest.bat -f general/user/pass_len.sim -wtest.bat -f general/user/user_create.sim -wtest.bat -f general/user/user_len.sim - -wtest.bat -f general/vector/metrics_field.sim -wtest.bat -f general/vector/metrics_mix.sim -wtest.bat -f general/vector/metrics_query.sim -wtest.bat -f general/vector/metrics_tag.sim -wtest.bat -f general/vector/metrics_time.sim -wtest.bat -f general/vector/multi.sim -wtest.bat -f general/vector/single.sim -wtest.bat -f general/vector/table_field.sim -wtest.bat -f general/vector/table_mix.sim -wtest.bat -f general/vector/table_query.sim -wtest.bat -f general/vector/table_time.sim - -wtest.bat -f general/wal/sync.sim -wtest.bat -f general/wal/kill.sim -wtest.bat -f general/wal/maxtables.sim - -wtest.bat -f unique/account/account_create.sim -wtest.bat -f unique/account/account_delete.sim -wtest.bat -f unique/account/account_len.sim -wtest.bat -f unique/account/authority.sim -wtest.bat -f unique/account/basic.sim -wtest.bat -f unique/account/paras.sim -wtest.bat -f unique/account/pass_alter.sim -wtest.bat -f unique/account/pass_len.sim -wtest.bat -f unique/account/usage.sim -wtest.bat -f unique/account/user_create.sim -wtest.bat -f unique/account/user_len.sim - -wtest.bat -f unique/big/balance.sim -wtest.bat -f unique/big/maxvnodes.sim -wtest.bat -f unique/big/tcp.sim - -wtest.bat -f unique/cluster/alter.sim -wtest.bat -f unique/cluster/balance1.sim -wtest.bat -f unique/cluster/balance2.sim -wtest.bat -f unique/cluster/balance3.sim -wtest.bat -f unique/cluster/cache.sim -wtest.bat -f unique/cluster/vgroup100.sim - -wtest.bat -f unique/column/replica3.sim - -wtest.bat -f unique/db/commit.sim -wtest.bat -f unique/db/delete.sim -wtest.bat -f unique/db/delete_part.sim -wtest.bat -f unique/db/replica_add12.sim -wtest.bat -f unique/db/replica_add13.sim -wtest.bat -f unique/db/replica_add23.sim -wtest.bat -f unique/db/replica_reduce21.sim -wtest.bat -f unique/db/replica_reduce32.sim -wtest.bat -f unique/db/replica_reduce31.sim -wtest.bat -f unique/db/replica_part.sim - -wtest.bat -f unique/dnode/alternativeRole.sim -wtest.bat -f unique/dnode/monitor.sim -wtest.bat -f unique/dnode/monitor_bug.sim -wtest.bat -f unique/dnode/simple.sim -wtest.bat -f unique/dnode/balance1.sim -wtest.bat -f unique/dnode/balance2.sim -wtest.bat -f unique/dnode/balance3.sim -wtest.bat -f unique/dnode/balancex.sim -wtest.bat -f unique/dnode/data1.sim -wtest.bat -f unique/dnode/m2.sim -wtest.bat -f unique/dnode/m3.sim -wtest.bat -f unique/dnode/lossdata.sim -wtest.bat -f unique/dnode/offline1.sim -wtest.bat -f unique/dnode/offline2.sim -wtest.bat -f unique/dnode/offline3.sim -wtest.bat -f unique/dnode/reason.sim -wtest.bat -f unique/dnode/remove1.sim -wtest.bat -f unique/dnode/remove2.sim -wtest.bat -f unique/dnode/vnode_clean.sim - -wtest.bat -f unique/http/admin.sim -wtest.bat -f unique/http/opentsdb.sim - -wtest.bat -f unique/import/replica2.sim -wtest.bat -f unique/import/replica3.sim - -wtest.bat -f unique/stable/balance_replica1.sim -wtest.bat -f unique/stable/dnode2_stop.sim -wtest.bat -f unique/stable/dnode2.sim -wtest.bat -f unique/stable/dnode3.sim -wtest.bat -f unique/stable/replica2_dnode4.sim -wtest.bat -f unique/stable/replica2_vnode3.sim -wtest.bat -f unique/stable/replica3_dnode6.sim -wtest.bat -f unique/stable/replica3_vnode3.sim - -wtest.bat -f unique/mnode/mgmt20.sim -wtest.bat -f unique/mnode/mgmt21.sim -wtest.bat -f unique/mnode/mgmt22.sim -wtest.bat -f unique/mnode/mgmt23.sim -wtest.bat -f unique/mnode/mgmt24.sim -wtest.bat -f unique/mnode/mgmt25.sim -wtest.bat -f unique/mnode/mgmt26.sim -wtest.bat -f unique/mnode/mgmt30.sim -wtest.bat -f unique/mnode/mgmt33.sim -wtest.bat -f unique/mnode/mgmt34.sim -wtest.bat -f unique/mnode/mgmtr2.sim - -wtest.bat -f unique/vnode/many.sim -wtest.bat -f unique/vnode/replica2_basic2.sim -wtest.bat -f unique/vnode/replica2_repeat.sim -wtest.bat -f unique/vnode/replica3_basic.sim -wtest.bat -f unique/vnode/replica3_repeat.sim -wtest.bat -f unique/vnode/replica3_vgroup.sim - -wtest.bat -f general/stream/metrics_del.sim -wtest.bat -f general/stream/metrics_replica1_vnoden.sim -wtest.bat -f general/stream/restart_stream.sim -wtest.bat -f general/stream/stream_3.sim -wtest.bat -f general/stream/stream_restart.sim -wtest.bat -f general/stream/table_del.sim -wtest.bat -f general/stream/table_replica1_vnoden.sim - -wtest.bat -f unique/arbitrator/check_cluster_cfg_para.sim -#wtest.bat -f unique/arbitrator/dn2_mn1_cache_file_sync.sim -wtest.bat -f unique/arbitrator/dn3_mn1_full_createTableFail.sim -wtest.bat -f unique/arbitrator/dn3_mn1_multiCreateDropTable.sim -#wtest.bat -f unique/arbitrator/dn3_mn1_nw_disable_timeout_autoDropDnode.sim -#wtest.bat -f unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim -wtest.bat -f unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim -wtest.bat -f unique/arbitrator/dn3_mn1_replica_change.sim -#wtest.bat -f unique/arbitrator/dn3_mn1_stopDnode_timeout.sim -# lower the priority while file corruption -#wtest.bat -f unique/arbitrator/dn3_mn1_vnode_change.sim -#wtest.bat -f unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim -#wtest.bat -f unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim -#wtest.bat -f unique/arbitrator/dn3_mn1_vnode_createErrData_online.sim -wtest.bat -f unique/arbitrator/dn3_mn1_vnode_noCorruptFile_offline.sim -wtest.bat -f unique/arbitrator/dn3_mn1_vnode_delDir.sim -wtest.bat -f unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim -wtest.bat -f unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim -wtest.bat -f unique/arbitrator/dn3_mn1_vnode_nomaster.sim -wtest.bat -f unique/arbitrator/dn3_mn2_killDnode.sim -wtest.bat -f unique/arbitrator/insert_duplicationTs.sim -wtest.bat -f unique/arbitrator/offline_replica2_alterTable_online.sim -wtest.bat -f unique/arbitrator/offline_replica2_alterTag_online.sim -wtest.bat -f unique/arbitrator/offline_replica2_createTable_online.sim -wtest.bat -f unique/arbitrator/offline_replica2_dropDb_online.sim -wtest.bat -f unique/arbitrator/offline_replica2_dropTable_online.sim -wtest.bat -f unique/arbitrator/offline_replica3_alterTable_online.sim -wtest.bat -f unique/arbitrator/offline_replica3_alterTag_online.sim -wtest.bat -f unique/arbitrator/offline_replica3_createTable_online.sim -wtest.bat -f unique/arbitrator/offline_replica3_dropDb_online.sim -wtest.bat -f unique/arbitrator/offline_replica3_dropTable_online.sim -wtest.bat -f unique/arbitrator/replica_changeWithArbitrator.sim -wtest.bat -f unique/arbitrator/sync_replica2_alterTable_add.sim -wtest.bat -f unique/arbitrator/sync_replica2_alterTable_drop.sim - -wtest.bat -f unique/arbitrator/sync_replica2_dropDb.sim -wtest.bat -f unique/arbitrator/sync_replica2_dropTable.sim -wtest.bat -f unique/arbitrator/sync_replica3_alterTable_add.sim -wtest.bat -f unique/arbitrator/sync_replica3_alterTable_drop.sim -wtest.bat -f unique/arbitrator/sync_replica3_dropDb.sim -wtest.bat -f unique/arbitrator/sync_replica3_dropTable.sim - -wtest.bat -f unique/migrate/mn2_vn2_repl2_rmMnodeDir.sim -wtest.bat -f unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim -wtest.bat -f unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir_stopAll_starAll.sim -wtest.bat -f unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim - -wtest.bat -f general/connection/test_old_data.sim -wtest.bat -f unique/dnode/datatrans_3node.sim -wtest.bat -f unique/dnode/datatrans_3node_2.sim - diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh index 03ce1c2888..0ccc85c347 100755 --- a/tests/script/sh/deploy.sh +++ b/tests/script/sh/deploy.sh @@ -118,49 +118,30 @@ fi echo " " >> $TAOS_CFG echo "firstEp ${HOSTNAME}:7100" >> $TAOS_CFG echo "secondEp ${HOSTNAME}:7200" >> $TAOS_CFG +echo "fqdn ${HOSTNAME}" >> $TAOS_CFG echo "serverPort ${NODE}" >> $TAOS_CFG echo "dataDir $DATA_DIR" >> $TAOS_CFG echo "logDir $LOG_DIR" >> $TAOS_CFG echo "debugFlag 0" >> $TAOS_CFG echo "mDebugFlag 143" >> $TAOS_CFG -echo "sdbDebugFlag 143" >> $TAOS_CFG echo "dDebugFlag 143" >> $TAOS_CFG echo "vDebugFlag 143" >> $TAOS_CFG echo "tsdbDebugFlag 143" >> $TAOS_CFG echo "cDebugFlag 143" >> $TAOS_CFG echo "jnidebugFlag 143" >> $TAOS_CFG -echo "odbcdebugFlag 143" >> $TAOS_CFG -echo "httpDebugFlag 143" >> $TAOS_CFG -echo "monDebugFlag 143" >> $TAOS_CFG -echo "mqttDebugFlag 143" >> $TAOS_CFG echo "qdebugFlag 143" >> $TAOS_CFG echo "rpcDebugFlag 143" >> $TAOS_CFG echo "tmrDebugFlag 131" >> $TAOS_CFG echo "udebugFlag 143" >> $TAOS_CFG echo "sdebugFlag 143" >> $TAOS_CFG echo "wdebugFlag 143" >> $TAOS_CFG -echo "cqdebugFlag 143" >> $TAOS_CFG -echo "monitor 0" >> $TAOS_CFG -echo "monitorInterval 1" >> $TAOS_CFG -echo "http 0" >> $TAOS_CFG echo "slaveQuery 0" >> $TAOS_CFG echo "numOfThreadsPerCore 2.0" >> $TAOS_CFG echo "defaultPass taosdata" >> $TAOS_CFG echo "numOfLogLines 20000000" >> $TAOS_CFG -echo "mnodeEqualVnodeNum 0" >> $TAOS_CFG -echo "balanceInterval 1" >> $TAOS_CFG -echo "clog 2" >> $TAOS_CFG -#echo "cache 1" >> $TAOS_CFG -echo "days 10" >> $TAOS_CFG echo "statusInterval 1" >> $TAOS_CFG -echo "maxVgroupsPerDb 4" >> $TAOS_CFG -echo "minTablesPerVnode 4" >> $TAOS_CFG -echo "maxTablesPerVnode 1000" >> $TAOS_CFG -echo "tableIncStepPerVnode 10000" >> $TAOS_CFG echo "asyncLog 0" >> $TAOS_CFG -echo "numOfMnodes 1" >> $TAOS_CFG echo "locale en_US.UTF-8" >> $TAOS_CFG echo "fsync 0" >> $TAOS_CFG echo "telemetryReporting 0" >> $TAOS_CFG echo " " >> $TAOS_CFG - diff --git a/tests/script/tmp/182.sim b/tests/script/tmp/182.sim deleted file mode 100644 index a178282cf8..0000000000 --- a/tests/script/tmp/182.sim +++ /dev/null @@ -1,41 +0,0 @@ -system sh/stop_dnodes.sh - -system sh/deploy.sh -n dnode1 -i 1 -system sh/deploy.sh -n dnode2 -i 2 -system sh/deploy.sh -n dnode3 -i 3 -system sh/deploy.sh -n dnode4 -i 4 - -system sh/cfg.sh -n dnode1 -c walLevel -v 2 -system sh/cfg.sh -n dnode2 -c walLevel -v 2 -system sh/cfg.sh -n dnode3 -c walLevel -v 2 -system sh/cfg.sh -n dnode4 -c walLevel -v 2 - -system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1 -system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1 -system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1 -system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1 - -system sh/cfg.sh -n dnode1 -c activeCode -v eglxDLzRpslJWl7OxrPZ2K3sQ5631AP9SVpezsaz2dhJWl7OxrPZ2ElaXs7Gs9nYSVpezsaz2djGIj5StnQ3ZvLHcsE8cwcN - -system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 -system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 -system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 -system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 - -system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 100000 -system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 100000 -system sh/cfg.sh -n dnode3 -c maxTablesPerVnode -v 100000 -system sh/cfg.sh -n dnode4 -c maxTablesPerVnode -v 100000 - -system sh/cfg.sh -n dnode1 -c http -v 1 -system sh/cfg.sh -n dnode2 -c http -v 1 -system sh/cfg.sh -n dnode3 -c http -v 1 -system sh/cfg.sh -n dnode4 -c http -v 1 - -system sh/cfg.sh -n dnode1 -c httpMaxThreads -v 4 - -system sh/cfg.sh -n dnode1 -c firstEp -v 127.0.0.1:6030 -system sh/cfg.sh -n dnode1 -c secondEp -v 127.0.0.1:6030 -system sh/cfg.sh -n dnode1 -c serverPort -v 6030 -system sh/cfg.sh -n dnode1 -c fqdn -v 127.0.0.1 -system sh/exec.sh -n dnode1 -s start diff --git a/tests/script/tmp/dnodes.sim b/tests/script/tmp/dnodes.sim new file mode 100644 index 0000000000..f5146620c5 --- /dev/null +++ b/tests/script/tmp/dnodes.sim @@ -0,0 +1,78 @@ +system sh/stop_dnodes.sh + + +############## config parameter ##################### +$node1 = 192.168.101.174 +$node2 = 192.168.0.202 +$node2 = 192.168.0.203 +$node3 = 192.168.0.204 + +$first = 1 +$num = 5 +$self = $node1 + +############### deploy firstEp ##################### + +$firstEp = $node1 . :7100 +$firstPort = 7100 +if $first == 1 then + system sh/deploy.sh -n dnode1 -i 1 + system sh/cfg.sh -n dnode1 -c firstEp -v $firstEp + system sh/cfg.sh -n dnode1 -c secondEp -v $firstEp + system sh/cfg.sh -n dnode1 -c fqdn -v $node1 + system sh/cfg.sh -n dnode1 -c serverPort -v $firstPort + + system sh/exec.sh -n dnode1 -s start + sql connect + + $i = 0 + while $i < $num + $port = $i * 100 + $port = $port + 8000 + $i = $i + 1 + sql create dnode $node1 port $port + endw + + $i = 0 + while $i < $num + $port = $i * 100 + $port = $port + 8000 + $i = $i + 1 + sql create dnode $node2 port $port + endw + + $i = 0 + while $i < $num + $port = $i * 100 + $port = $port + 8000 + $i = $i + 1 + sql create dnode $node3 port $port + endw + + $i = 0 + while $i < $num + $port = $i * 100 + $port = $port + 8000 + $i = $i + 1 + sql create dnode $node4 port $port + endw +endi + +############### deploy nodes ##################### + +$i = 0 +while $i < $num + $index = $i + 80 + $port = $i * 100 + $port = $port + 8000 + $dnodename = dnode . $index + $i = $i + 1 + + system sh/deploy.sh -n $dnodename -i 1 + system sh/cfg.sh -n $dnodename -c firstEp -v $firstEp + system sh/cfg.sh -n $dnodename -c secondEp -v $firstEp + system sh/cfg.sh -n $dnodename -c fqdn -v $self + system sh/cfg.sh -n $dnodename -c serverPort -v $port + + #system sh/exec.sh -n $dnodename -s start +endw diff --git a/tests/script/tmp/http.sim b/tests/script/tmp/http.sim deleted file mode 100644 index a5212905ae..0000000000 --- a/tests/script/tmp/http.sim +++ /dev/null @@ -1,35 +0,0 @@ -system sh/stop_dnodes.sh - -system sh/deploy.sh -n dnode1 -i 1 -system sh/deploy.sh -n dnode2 -i 2 -system sh/deploy.sh -n dnode3 -i 3 -system sh/deploy.sh -n dnode4 -i 4 - -system sh/cfg.sh -n dnode1 -c walLevel -v 2 -system sh/cfg.sh -n dnode2 -c walLevel -v 2 -system sh/cfg.sh -n dnode3 -c walLevel -v 2 -system sh/cfg.sh -n dnode4 -c walLevel -v 2 - -system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1 -system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1 -system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1 -system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1 - -system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 -system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 -system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 -system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 - -system sh/cfg.sh -n dnode1 -c http -v 1 -system sh/cfg.sh -n dnode2 -c http -v 1 -system sh/cfg.sh -n dnode3 -c http -v 1 - -system sh/cfg.sh -n dnode1 -c monitor -v 1 -system sh/cfg.sh -n dnode2 -c monitor -v 1 -system sh/cfg.sh -n dnode3 -c monitor -v 1 - -system sh/cfg.sh -n dnode1 -c monitorInterval -v 1 -system sh/cfg.sh -n dnode2 -c monitorInterval -v 1 -system sh/cfg.sh -n dnode3 -c monitorInterval -v 1 - -system sh/exec.sh -n dnode1 -s start \ No newline at end of file diff --git a/tests/script/tmp/mnodes.sim b/tests/script/tmp/mnodes.sim deleted file mode 100644 index 8bca76c38b..0000000000 --- a/tests/script/tmp/mnodes.sim +++ /dev/null @@ -1,115 +0,0 @@ -system sh/stop_dnodes.sh - -system sh/deploy.sh -n dnode1 -i 1 -system sh/deploy.sh -n dnode2 -i 2 -system sh/deploy.sh -n dnode3 -i 3 - -system sh/cfg.sh -n dnode1 -c numOfMnodes -v 3 -system sh/cfg.sh -n dnode2 -c numOfMnodes -v 3 -system sh/cfg.sh -n dnode3 -c numOfMnodes -v 3 - -system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 -system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 -system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 - -system sh/cfg.sh -n dnode1 -c http -v 0 -system sh/cfg.sh -n dnode2 -c http -v 0 -system sh/cfg.sh -n dnode3 -c http -v 0 - -system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 20000 -system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 20000 -system sh/cfg.sh -n dnode3 -c maxTablesPerVnode -v 20000 - -system sh/cfg.sh -n dnode1 -c minTablesPerVnode -v 1000 -system sh/cfg.sh -n dnode2 -c minTablesPerVnode -v 1000 -system sh/cfg.sh -n dnode3 -c minTablesPerVnode -v 1000 - -system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 20 -system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 20 -system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 20 - -system sh/cfg.sh -n dnode1 -c replica -v 3 -system sh/cfg.sh -n dnode2 -c replica -v 3 -system sh/cfg.sh -n dnode3 -c replica -v 3 - -print ============== deploy - -system sh/exec.sh -n dnode1 -s start -sql connect - -sql create dnode $hostname2 -sql create dnode $hostname3 -system sh/exec.sh -n dnode2 -s start -system sh/exec.sh -n dnode3 -s start - -print =============== step1 -$x = 0 -step1: - $x = $x + 1 - sleep 1000 - if $x == 10 then - return -1 - endi - -sql show dnodes -print dnode1 $data4_1 -print dnode2 $data4_2 -print dnode3 $data4_3 - -if $data4_1 != ready then - goto step1 -endi -if $data4_2 != ready then - goto step1 -endi -if $data4_3 != ready then - goto step1 -endi - -sql show mnodes -$mnode1Role = $data2_1 -print mnode1Role $mnode1Role -$mnode2Role = $data2_2 -print mnode2Role $mnode2Role -$mnode3Role = $data2_3 -print mnode3Role $mnode3Role - -if $mnode1Role != master then - goto step1 -endi -if $mnode2Role != slave then - goto step1 -endi -if $mnode3Role != slave then - goto step1 -endi - -$x = 1 -show2: - -print =============== step $x -sql show mnodes -print $data0_1 $data2_1 - -sql show dnodes -print dnode1 $data4_1 -print dnode2 $data4_2 -print dnode3 $data4_3 - -if $data4_1 != ready then - goto step1 -endi -if $data4_2 != ready then - goto step1 -endi -if $data4_3 != ready then - goto step1 -endi - -$x = $x + 1 -sleep 3000 -if $x == 100000 then - return -1 -endi - -goto show2 diff --git a/tests/script/tmp/prepare.sim b/tests/script/tmp/prepare.sim index 343c422e9f..28675be808 100644 --- a/tests/script/tmp/prepare.sim +++ b/tests/script/tmp/prepare.sim @@ -9,41 +9,3 @@ system sh/cfg.sh -n dnode1 -c walLevel -v 2 system sh/cfg.sh -n dnode2 -c walLevel -v 2 system sh/cfg.sh -n dnode3 -c walLevel -v 2 system sh/cfg.sh -n dnode4 -c walLevel -v 2 - -system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1 -system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1 -system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1 -system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1 - -system sh/cfg.sh -n dnode1 -c activeCode -v eglxDLzRpslJWl7OxrPZ2K3sQ5631AP9SVpezsaz2dhJWl7OxrPZ2ElaXs7Gs9nYSVpezsaz2djGIj5StnQ3ZvLHcsE8cwcN - -system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 -system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 -system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 -system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 - -system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 100000 -system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 100000 -system sh/cfg.sh -n dnode3 -c maxTablesPerVnode -v 100000 -system sh/cfg.sh -n dnode4 -c maxTablesPerVnode -v 100000 - -system sh/cfg.sh -n dnode1 -c http -v 1 -system sh/cfg.sh -n dnode2 -c http -v 1 -system sh/cfg.sh -n dnode3 -c http -v 1 -system sh/cfg.sh -n dnode4 -c http -v 1 - -return -# for crash_gen -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 10 -system sh/cfg.sh -n dnode1 -c rpcMaxTime -v 101 -system sh/cfg.sh -n dnode1 -c cache -v 2 -system sh/cfg.sh -n dnode1 -c keep -v 36500 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 - - -# for windows - -system sh/cfg.sh -n dnode1 -c firstEp -v 152.136.17.116:6030 -system sh/cfg.sh -n dnode1 -c secondEp -v 152.136.17.116:6030 -system sh/cfg.sh -n dnode1 -c serverPort -v 6030 -system sh/cfg.sh -n dnode1 -c fqdn -v 152.136.17.116 From 64d21e484d23d11db8cb38376dba12d4327cd484 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 28 Dec 2021 15:50:05 +0800 Subject: [PATCH 13/21] feature/qnode --- source/dnode/vnode/impl/inc/vnodeDef.h | 2 +- source/dnode/vnode/impl/inc/vnodeQuery.h | 3 + source/libs/qworker/inc/qworkerInt.h | 25 +- source/libs/qworker/src/qworker.c | 10 +- source/libs/qworker/test/qworkerTests.cpp | 335 ++++++++++++++++- source/libs/scheduler/inc/schedulerInt.h | 6 + source/libs/scheduler/src/scheduler.c | 354 +++++++++++------- source/libs/scheduler/test/schedulerTests.cpp | 14 +- 8 files changed, 597 insertions(+), 152 deletions(-) diff --git a/source/dnode/vnode/impl/inc/vnodeDef.h b/source/dnode/vnode/impl/inc/vnodeDef.h index c5a57b02a6..78214ce14d 100644 --- a/source/dnode/vnode/impl/inc/vnodeDef.h +++ b/source/dnode/vnode/impl/inc/vnodeDef.h @@ -73,7 +73,7 @@ struct SVnode { SVnodeSync* pSync; SVnodeFS* pFs; tsem_t canCommit; - void* pQuery; + SQHandle* pQuery; }; int vnodeScheduleTask(SVnodeTask* task); diff --git a/source/dnode/vnode/impl/inc/vnodeQuery.h b/source/dnode/vnode/impl/inc/vnodeQuery.h index 59bab42f62..d43f5b1cf1 100644 --- a/source/dnode/vnode/impl/inc/vnodeQuery.h +++ b/source/dnode/vnode/impl/inc/vnodeQuery.h @@ -22,6 +22,9 @@ extern "C" { #include "vnodeInt.h" #include "qworker.h" +typedef struct SQWorkerMgmt SQHandle; + + int vnodeQueryOpen(SVnode *pVnode); #ifdef __cplusplus diff --git a/source/libs/qworker/inc/qworkerInt.h b/source/libs/qworker/inc/qworkerInt.h index 825e891c8b..07ca91729d 100644 --- a/source/libs/qworker/inc/qworkerInt.h +++ b/source/libs/qworker/inc/qworkerInt.h @@ -102,8 +102,29 @@ typedef struct SQWorkerMgmt { #define QW_ERR_LRET(c,...) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { qError(__VA_ARGS__); terrno = _code; return _code; } } while (0) #define QW_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) -#define QW_LOCK(type, _lock) (QW_READ == (type) ? taosRLockLatch(_lock) : taosWLockLatch(_lock)) -#define QW_UNLOCK(type, _lock) (QW_READ == (type) ? taosRUnLockLatch(_lock) : taosWUnLockLatch(_lock)) +#define QW_LOCK(type, _lock) do { \ + if (QW_READ == (type)) { \ + if ((*(_lock)) < 0) assert(0); \ + taosRLockLatch(_lock); \ + qDebug("RLOCK%p, %s:%d", (_lock), __FILE__, __LINE__); \ + } else { \ + if ((*(_lock)) < 0) assert(0); \ + taosWLockLatch(_lock); \ + qDebug("WLOCK%p, %s:%d", (_lock), __FILE__, __LINE__); \ + } \ +} while (0) + +#define QW_UNLOCK(type, _lock) do { \ + if (QW_READ == (type)) { \ + if ((*(_lock)) <= 0) assert(0); \ + taosRUnLockLatch(_lock); \ + qDebug("RULOCK%p, %s:%d", (_lock), __FILE__, __LINE__); \ + } else { \ + if ((*(_lock)) <= 0) assert(0); \ + taosWUnLockLatch(_lock); \ + qDebug("WULOCK%p, %s:%d", (_lock), __FILE__, __LINE__); \ + } \ +} while (0) static int32_t qwAcquireScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t sId, SQWSchStatus **sch, int32_t nOpt); diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 7a861a0b8f..4296e82a56 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -601,6 +601,7 @@ int32_t qwBuildAndSendStatusRsp(SRpcMsg *pMsg, SSchedulerStatusRsp *sStatus) { } SRpcMsg rpcRsp = { + .msgType = pMsg->msgType + 1, .handle = pMsg->handle, .ahandle = pMsg->ahandle, .pCont = pRsp, @@ -703,10 +704,8 @@ _return: if (task) { QW_UNLOCK(QW_WRITE, &task->lock); - } - - if (sch) { qwReleaseTask(QW_READ, sch); + } qwReleaseScheduler(QW_READ, mgmt); @@ -742,9 +741,6 @@ _return: if (task) { QW_UNLOCK(QW_WRITE, &task->lock); - } - - if (sch) { qwReleaseTask(QW_READ, sch); } @@ -849,10 +845,10 @@ int32_t qwHandleFetch(SQWorkerResCache *res, SQWorkerMgmt *mgmt, uint64_t sId, u _return: if (task) { QW_UNLOCK(QW_READ, &task->lock); + qwReleaseTask(QW_READ, sch); } if (sch) { - qwReleaseTask(QW_READ, sch); qwReleaseScheduler(QW_READ, mgmt); } diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index d63d40b4f4..7bc1c4ff40 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -36,11 +36,21 @@ namespace { +bool testStop = false; + int32_t qwtStringToPlan(const char* str, SSubplan** subplan) { return 0; } void qwtRpcSendResponse(const SRpcMsg *pRsp) { + if (TDMT_VND_TASKS_STATUS_RSP == pRsp->msgType) { + SSchedulerStatusRsp *rsp = (SSchedulerStatusRsp *)pRsp->pCont; + printf("task num:%d\n", rsp->num); + for (int32_t i = 0; i < rsp->num; ++i) { + STaskStatus *task = &rsp->status[i]; + printf("qId:%"PRIx64",tId:%"PRIx64",status:%d\n", task->queryId, task->taskId, task->status); + } + } return; } @@ -72,12 +82,135 @@ void stubSetRpcSendResponse() { } } +void *queryThread(void *param) { + SRpcMsg queryRpc = {0}; + int32_t code = 0; + uint32_t n = 0; + void *mockPointer = (void *)0x1; + void *mgmt = param; + SSubQueryMsg *queryMsg = (SSubQueryMsg *)calloc(1, sizeof(SSubQueryMsg) + 100); + queryMsg->queryId = htobe64(1); + queryMsg->sId = htobe64(1); + queryMsg->taskId = htobe64(1); + queryMsg->contentLen = htonl(100); + queryRpc.pCont = queryMsg; + queryRpc.contLen = sizeof(SSubQueryMsg) + 100; + + while (!testStop) { + qWorkerProcessQueryMsg(mockPointer, mgmt, &queryRpc); + usleep(rand()%5); + if (++n % 50000 == 0) { + printf("query:%d\n", n); + } + } + + return NULL; +} + +void *readyThread(void *param) { + SRpcMsg readyRpc = {0}; + int32_t code = 0; + uint32_t n = 0; + void *mockPointer = (void *)0x1; + void *mgmt = param; + SResReadyMsg readyMsg = {0}; + readyMsg.sId = htobe64(1); + readyMsg.queryId = htobe64(1); + readyMsg.taskId = htobe64(1); + readyRpc.pCont = &readyMsg; + readyRpc.contLen = sizeof(SResReadyMsg); + + while (!testStop) { + code = qWorkerProcessReadyMsg(mockPointer, mgmt, &readyRpc); + usleep(rand()%5); + if (++n % 50000 == 0) { + printf("ready:%d\n", n); + } + } + + return NULL; +} + +void *fetchThread(void *param) { + SRpcMsg fetchRpc = {0}; + int32_t code = 0; + uint32_t n = 0; + void *mockPointer = (void *)0x1; + void *mgmt = param; + SResFetchMsg fetchMsg = {0}; + fetchMsg.sId = htobe64(1); + fetchMsg.queryId = htobe64(1); + fetchMsg.taskId = htobe64(1); + fetchRpc.pCont = &fetchMsg; + fetchRpc.contLen = sizeof(SResFetchMsg); + + while (!testStop) { + code = qWorkerProcessFetchMsg(mockPointer, mgmt, &fetchRpc); + usleep(rand()%5); + if (++n % 50000 == 0) { + printf("fetch:%d\n", n); + } + } + + return NULL; +} + +void *dropThread(void *param) { + SRpcMsg dropRpc = {0}; + int32_t code = 0; + uint32_t n = 0; + void *mockPointer = (void *)0x1; + void *mgmt = param; + STaskDropMsg dropMsg = {0}; + dropMsg.sId = htobe64(1); + dropMsg.queryId = htobe64(1); + dropMsg.taskId = htobe64(1); + dropRpc.pCont = &dropMsg; + dropRpc.contLen = sizeof(STaskDropMsg); + + while (!testStop) { + code = qWorkerProcessDropMsg(mockPointer, mgmt, &dropRpc); + usleep(rand()%5); + if (++n % 50000 == 0) { + printf("drop:%d\n", n); + } + } + + return NULL; +} + +void *statusThread(void *param) { + SRpcMsg statusRpc = {0}; + int32_t code = 0; + uint32_t n = 0; + void *mockPointer = (void *)0x1; + void *mgmt = param; + SSchTasksStatusMsg statusMsg = {0}; + statusMsg.sId = htobe64(1); + statusRpc.pCont = &statusMsg; + statusRpc.contLen = sizeof(SSchTasksStatusMsg); + statusRpc.msgType = TDMT_VND_TASKS_STATUS; + + while (!testStop) { + statusMsg.sId = htobe64(1); + code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc); + usleep(rand()%5); + if (++n % 50000 == 0) { + printf("status:%d\n", n); + } + } + + return NULL; +} + + + } -TEST(testCase, normalCase) { +TEST(seqTest, normalCase) { void *mgmt = NULL; int32_t code = 0; void *mockPointer = (void *)0x1; @@ -85,6 +218,8 @@ TEST(testCase, normalCase) { SRpcMsg readyRpc = {0}; SRpcMsg fetchRpc = {0}; SRpcMsg dropRpc = {0}; + SRpcMsg statusRpc = {0}; + SSubQueryMsg *queryMsg = (SSubQueryMsg *)calloc(1, sizeof(SSubQueryMsg) + 100); queryMsg->queryId = htobe64(1); queryMsg->sId = htobe64(1); @@ -114,24 +249,222 @@ TEST(testCase, normalCase) { dropRpc.pCont = &dropMsg; dropRpc.contLen = sizeof(STaskDropMsg); + SSchTasksStatusMsg statusMsg = {0}; + statusMsg.sId = htobe64(1); + statusRpc.pCont = &statusMsg; + statusRpc.contLen = sizeof(SSchTasksStatusMsg); + statusRpc.msgType = TDMT_VND_TASKS_STATUS; + stubSetStringToPlan(); stubSetRpcSendResponse(); code = qWorkerInit(NULL, &mgmt); ASSERT_EQ(code, 0); + statusMsg.sId = htobe64(1); + code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc); + ASSERT_EQ(code, 0); + code = qWorkerProcessQueryMsg(mockPointer, mgmt, &queryRpc); ASSERT_EQ(code, 0); + statusMsg.sId = htobe64(1); + code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc); + ASSERT_EQ(code, 0); + code = qWorkerProcessReadyMsg(mockPointer, mgmt, &readyRpc); ASSERT_EQ(code, 0); + statusMsg.sId = htobe64(1); + code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc); + ASSERT_EQ(code, 0); + code = qWorkerProcessFetchMsg(mockPointer, mgmt, &fetchRpc); ASSERT_EQ(code, 0); + statusMsg.sId = htobe64(1); + code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc); + ASSERT_EQ(code, 0); + code = qWorkerProcessDropMsg(mockPointer, mgmt, &dropRpc); ASSERT_EQ(code, 0); + statusMsg.sId = htobe64(1); + code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc); + ASSERT_EQ(code, 0); + + qWorkerDestroy(&mgmt); +} + +TEST(seqTest, cancelFirst) { + void *mgmt = NULL; + int32_t code = 0; + void *mockPointer = (void *)0x1; + SRpcMsg queryRpc = {0}; + SRpcMsg dropRpc = {0}; + SRpcMsg statusRpc = {0}; + + SSubQueryMsg *queryMsg = (SSubQueryMsg *)calloc(1, sizeof(SSubQueryMsg) + 100); + queryMsg->queryId = htobe64(1); + queryMsg->sId = htobe64(1); + queryMsg->taskId = htobe64(1); + queryMsg->contentLen = htonl(100); + queryRpc.pCont = queryMsg; + queryRpc.contLen = sizeof(SSubQueryMsg) + 100; + + STaskDropMsg dropMsg = {0}; + dropMsg.sId = htobe64(1); + dropMsg.queryId = htobe64(1); + dropMsg.taskId = htobe64(1); + dropRpc.pCont = &dropMsg; + dropRpc.contLen = sizeof(STaskDropMsg); + + SSchTasksStatusMsg statusMsg = {0}; + statusMsg.sId = htobe64(1); + statusRpc.pCont = &statusMsg; + statusRpc.contLen = sizeof(SSchTasksStatusMsg); + statusRpc.msgType = TDMT_VND_TASKS_STATUS; + + stubSetStringToPlan(); + stubSetRpcSendResponse(); + + code = qWorkerInit(NULL, &mgmt); + ASSERT_EQ(code, 0); + + statusMsg.sId = htobe64(1); + code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc); + ASSERT_EQ(code, 0); + + code = qWorkerProcessDropMsg(mockPointer, mgmt, &dropRpc); + ASSERT_EQ(code, 0); + + statusMsg.sId = htobe64(1); + code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc); + ASSERT_EQ(code, 0); + + code = qWorkerProcessQueryMsg(mockPointer, mgmt, &queryRpc); + ASSERT_EQ(code, 0); + + statusMsg.sId = htobe64(1); + code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc); + ASSERT_EQ(code, 0); + + qWorkerDestroy(&mgmt); +} + +TEST(seqTest, randCase) { + void *mgmt = NULL; + int32_t code = 0; + void *mockPointer = (void *)0x1; + SRpcMsg queryRpc = {0}; + SRpcMsg readyRpc = {0}; + SRpcMsg fetchRpc = {0}; + SRpcMsg dropRpc = {0}; + SRpcMsg statusRpc = {0}; + + SSubQueryMsg *queryMsg = (SSubQueryMsg *)calloc(1, sizeof(SSubQueryMsg) + 100); + queryMsg->queryId = htobe64(1); + queryMsg->sId = htobe64(1); + queryMsg->taskId = htobe64(1); + queryMsg->contentLen = htonl(100); + queryRpc.pCont = queryMsg; + queryRpc.contLen = sizeof(SSubQueryMsg) + 100; + + SResReadyMsg readyMsg = {0}; + readyMsg.sId = htobe64(1); + readyMsg.queryId = htobe64(1); + readyMsg.taskId = htobe64(1); + readyRpc.pCont = &readyMsg; + readyRpc.contLen = sizeof(SResReadyMsg); + + SResFetchMsg fetchMsg = {0}; + fetchMsg.sId = htobe64(1); + fetchMsg.queryId = htobe64(1); + fetchMsg.taskId = htobe64(1); + fetchRpc.pCont = &fetchMsg; + fetchRpc.contLen = sizeof(SResFetchMsg); + + STaskDropMsg dropMsg = {0}; + dropMsg.sId = htobe64(1); + dropMsg.queryId = htobe64(1); + dropMsg.taskId = htobe64(1); + dropRpc.pCont = &dropMsg; + dropRpc.contLen = sizeof(STaskDropMsg); + + SSchTasksStatusMsg statusMsg = {0}; + statusMsg.sId = htobe64(1); + statusRpc.pCont = &statusMsg; + statusRpc.contLen = sizeof(SSchTasksStatusMsg); + statusRpc.msgType = TDMT_VND_TASKS_STATUS; + + stubSetStringToPlan(); + stubSetRpcSendResponse(); + + srand(time(NULL)); + + code = qWorkerInit(NULL, &mgmt); + ASSERT_EQ(code, 0); + + int32_t t = 0; + int32_t maxr = 10001; + while (true) { + int32_t r = rand() % maxr; + + if (r >= 0 && r < maxr/5) { + printf("Query,%d\n", t++); + code = qWorkerProcessQueryMsg(mockPointer, mgmt, &queryRpc); + } else if (r >= maxr/5 && r < maxr * 2/5) { + printf("Ready,%d\n", t++); + code = qWorkerProcessReadyMsg(mockPointer, mgmt, &readyRpc); + } else if (r >= maxr * 2/5 && r < maxr* 3/5) { + printf("Fetch,%d\n", t++); + code = qWorkerProcessFetchMsg(mockPointer, mgmt, &fetchRpc); + } else if (r >= maxr * 3/5 && r < maxr * 4/5) { + printf("Drop,%d\n", t++); + code = qWorkerProcessDropMsg(mockPointer, mgmt, &dropRpc); + } else if (r >= maxr * 4/5 && r < maxr-1) { + printf("Status,%d\n", t++); + statusMsg.sId = htobe64(1); + code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc); + ASSERT_EQ(code, 0); + } else { + printf("QUIT RAND NOW"); + break; + } + } + + qWorkerDestroy(&mgmt); +} + +TEST(seqTest, multithreadRand) { + void *mgmt = NULL; + int32_t code = 0; + void *mockPointer = (void *)0x1; + + stubSetStringToPlan(); + stubSetRpcSendResponse(); + + srand(time(NULL)); + + code = qWorkerInit(NULL, &mgmt); + ASSERT_EQ(code, 0); + + pthread_attr_t thattr; + pthread_attr_init(&thattr); + + pthread_t t1,t2,t3,t4,t5; + pthread_create(&(t1), &thattr, queryThread, mgmt); + pthread_create(&(t2), &thattr, readyThread, NULL); + pthread_create(&(t3), &thattr, fetchThread, NULL); + pthread_create(&(t4), &thattr, dropThread, NULL); + pthread_create(&(t5), &thattr, statusThread, NULL); + + int32_t t = 0; + int32_t maxr = 10001; + sleep(300); + testStop = true; + sleep(1); + qWorkerDestroy(&mgmt); } diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index d5833c3adf..c327e4cfea 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -43,6 +43,11 @@ typedef struct SSchedulerMgmt { SHashObj *jobs; // key: queryId, value: SQueryJob* } SSchedulerMgmt; +typedef struct SSchCallbackParam { + uint64_t queryId; + uint64_t taskId; +} SSchCallbackParam; + typedef struct SSchLevel { int32_t level; int8_t status; @@ -120,6 +125,7 @@ typedef struct SSchJob { extern int32_t schLaunchTask(SSchJob *job, SSchTask *task); +extern int32_t schBuildAndSendMsg(SSchJob *job, SSchTask *task, int32_t msgType); #ifdef __cplusplus } diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index e298cd65f6..4cde24e38c 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -21,36 +21,6 @@ SSchedulerMgmt schMgmt = {0}; -int32_t schBuildAndSendRequest(void *pRpc, const SEpSet* pMgmtEps, __taos_async_fn_t fp) { -/* - SRequestObj *pRequest = createRequest(pTscObj, fp, param, TSDB_SQL_CONNECT); - if (pRequest == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SRequestMsgBody body = {0}; - buildConnectMsg(pRequest, &body); - - int64_t transporterId = 0; - asyncSendMsgToServer(pTscObj->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &body, &transporterId); - - tsem_wait(&pRequest->body.rspSem); - destroyConnectMsg(&body); - - if (pRequest->code != TSDB_CODE_SUCCESS) { - const char *errorMsg = (pRequest->code == TSDB_CODE_RPC_FQDN_ERROR) ? taos_errstr(pRequest) : tstrerror(terrno); - printf("failed to connect to server, reason: %s\n\n", errorMsg); - - destroyRequest(pRequest); - taos_close(pTscObj); - pTscObj = NULL; - } else { - tscDebug("0x%"PRIx64" connection is opening, connId:%d, dnodeConn:%p", pTscObj->id, pTscObj->connId, pTscObj->pTransporter); - destroyRequest(pRequest); - } -*/ -} - int32_t schBuildTaskRalation(SSchJob *job, SHashObj *planToTask) { for (int32_t i = 0; i < job->levelNum; ++i) { SSchLevel *level = taosArrayGet(job->levels, i); @@ -312,100 +282,6 @@ int32_t schMoveTaskToFailList(SSchJob *job, SSchTask *task, bool *moved) { return TSDB_CODE_SUCCESS; } - -int32_t schAsyncSendMsg(SSchJob *job, SSchTask *task, int32_t msgType) { - int32_t msgSize = 0; - void *msg = NULL; - - switch (msgType) { - case TDMT_VND_SUBMIT: { - if (NULL == task->msg || task->msgLen <= 0) { - qError("submit msg is NULL"); - SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); - } - - msgSize = task->msgLen; - msg = task->msg; - break; - } - case TDMT_VND_QUERY: { - if (NULL == task->msg) { - qError("query msg is NULL"); - SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); - } - - msgSize = sizeof(SSubQueryMsg) + task->msgLen; - msg = calloc(1, msgSize); - if (NULL == msg) { - qError("calloc %d failed", msgSize); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - SSubQueryMsg *pMsg = msg; - - pMsg->sId = htobe64(schMgmt.sId); - pMsg->queryId = htobe64(job->queryId); - pMsg->taskId = htobe64(task->taskId); - pMsg->contentLen = htonl(task->msgLen); - memcpy(pMsg->msg, task->msg, task->msgLen); - break; - } - case TDMT_VND_RES_READY: { - msgSize = sizeof(SResReadyMsg); - msg = calloc(1, msgSize); - if (NULL == msg) { - qError("calloc %d failed", msgSize); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - SResReadyMsg *pMsg = msg; - pMsg->sId = htobe64(schMgmt.sId); - pMsg->queryId = htobe64(job->queryId); - pMsg->taskId = htobe64(task->taskId); - break; - } - case TDMT_VND_FETCH: { - if (NULL == task) { - SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR); - } - msgSize = sizeof(SResFetchMsg); - msg = calloc(1, msgSize); - if (NULL == msg) { - qError("calloc %d failed", msgSize); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - SResFetchMsg *pMsg = msg; - pMsg->sId = htobe64(schMgmt.sId); - pMsg->queryId = htobe64(job->queryId); - pMsg->taskId = htobe64(task->taskId); - break; - } - case TDMT_VND_DROP_TASK:{ - msgSize = sizeof(STaskDropMsg); - msg = calloc(1, msgSize); - if (NULL == msg) { - qError("calloc %d failed", msgSize); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - STaskDropMsg *pMsg = msg; - pMsg->sId = htobe64(schMgmt.sId); - pMsg->queryId = htobe64(job->queryId); - pMsg->taskId = htobe64(task->taskId); - break; - } - default: - qError("unknown msg type:%d", msgType); - break; - } - - //TODO SEND MSG - //taosAsyncExec(rpcSendRequest(void * shandle, const SEpSet * pEpSet, SRpcMsg * pMsg, int64_t * pRid), p, &code); - - return TSDB_CODE_SUCCESS; -} - int32_t schTaskCheckAndSetRetry(SSchJob *job, SSchTask *task, int32_t errCode, bool *needRetry) { // TODO set retry or not based on task type/errCode/retry times/job status/available eps... // TODO if needRetry, set task retry info @@ -424,7 +300,7 @@ int32_t schFetchFromRemote(SSchJob *job) { return TSDB_CODE_SUCCESS; } - SCH_ERR_JRET(schAsyncSendMsg(job, job->fetchTask, TDMT_VND_FETCH)); + SCH_ERR_JRET(schBuildAndSendMsg(job, job->fetchTask, TDMT_VND_FETCH)); return TSDB_CODE_SUCCESS; @@ -577,11 +453,11 @@ int32_t schProcessOnTaskFailure(SSchJob *job, SSchTask *task, int32_t errCode) { return TSDB_CODE_SUCCESS; } -int32_t schHandleRspMsg(SSchJob *job, SSchTask *task, int32_t msgType, char *msg, int32_t msgSize, int32_t rspCode) { +int32_t schProcessRspMsg(SSchJob *job, SSchTask *task, int32_t msgType, char *msg, int32_t msgSize, int32_t rspCode) { int32_t code = 0; switch (msgType) { - case TDMT_VND_SUBMIT: { + case TDMT_VND_SUBMIT_RSP: { SShellSubmitRspMsg *rsp = (SShellSubmitRspMsg *)msg; if (rsp->code != TSDB_CODE_SUCCESS) { SCH_ERR_JRET(schProcessOnTaskFailure(job, task, rsp->code)); @@ -595,20 +471,20 @@ int32_t schHandleRspMsg(SSchJob *job, SSchTask *task, int32_t msgType, char *msg } break; } - case TDMT_VND_QUERY: { + case TDMT_VND_QUERY_RSP: { SQueryTableRsp *rsp = (SQueryTableRsp *)msg; if (rsp->code != TSDB_CODE_SUCCESS) { SCH_ERR_JRET(schProcessOnTaskFailure(job, task, rsp->code)); } else { - code = schAsyncSendMsg(job, task, TDMT_VND_RES_READY); + code = schBuildAndSendMsg(job, task, TDMT_VND_RES_READY); if (code) { goto _task_error; } } break; } - case TDMT_VND_RES_READY: { + case TDMT_VND_RES_READY_RSP: { SResReadyRsp *rsp = (SResReadyRsp *)msg; if (rsp->code != TSDB_CODE_SUCCESS) { @@ -621,7 +497,7 @@ int32_t schHandleRspMsg(SSchJob *job, SSchTask *task, int32_t msgType, char *msg } break; } - case TDMT_VND_FETCH: { + case TDMT_VND_FETCH_RSP: { SCH_ERR_JRET(rspCode); SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)msg; @@ -631,6 +507,9 @@ int32_t schHandleRspMsg(SSchJob *job, SSchTask *task, int32_t msgType, char *msg SCH_ERR_JRET(schProcessOnDataFetched(job)); break; } + case TDMT_VND_DROP_TASK: { + + } default: qError("unknown msg type:%d received", msgType); return TSDB_CODE_QRY_INVALID_INPUT; @@ -648,6 +527,211 @@ _return: } +int32_t schHandleCallback(void* param, const SDataBuf* pMsg, int32_t msgType, int32_t rspCode) { + int32_t code = 0; + SSchCallbackParam *pParam = (SSchCallbackParam *)param; + + SSchJob **job = taosHashGet(schMgmt.jobs, &pParam->queryId, sizeof(pParam->queryId)); + if (NULL == job || NULL == (*job)) { + qError("taosHashGet queryId:%"PRIx64" not exist", pParam->queryId); + SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + + SSchTask **task = taosHashGet((*job)->execTasks, &pParam->taskId, sizeof(pParam->taskId)); + if (NULL == task || NULL == (*task)) { + qError("taosHashGet taskId:%"PRIx64" not exist", pParam->taskId); + SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + + schProcessRspMsg(*job, *task, msgType, pMsg->pData, pMsg->len, rspCode); + +_return: + tfree(param); + + SCH_RET(code); +} + +int32_t schHandleSubmitCallback(void* param, const SDataBuf* pMsg, int32_t code) { + return schHandleCallback(param, pMsg, TDMT_VND_SUBMIT_RSP, code); +} +int32_t schHandleQueryCallback(void* param, const SDataBuf* pMsg, int32_t code) { + return schHandleCallback(param, pMsg, TDMT_VND_QUERY_RSP, code); +} +int32_t schHandleFetchCallback(void* param, const SDataBuf* pMsg, int32_t code) { + return schHandleCallback(param, pMsg, TDMT_VND_FETCH_RSP, code); +} +int32_t schHandleReadyCallback(void* param, const SDataBuf* pMsg, int32_t code) { + return schHandleCallback(param, pMsg, TDMT_VND_RES_READY_RSP, code); +} +int32_t schHandleDropCallback(void* param, const SDataBuf* pMsg, int32_t code) { + SSchCallbackParam *pParam = (SSchCallbackParam *)param; + qDebug("drop task rsp received, queryId:%"PRIx64 ",taksId:%"PRIx64 ",code:%d", pParam->queryId, pParam->taskId, code); +} + +int32_t schGetCallbackFp(int32_t msgType, __async_send_cb_fn_t *fp) { + switch (msgType) { + case TDMT_VND_SUBMIT: + *fp = schHandleSubmitCallback; + break; + case TDMT_VND_QUERY: + *fp = schHandleQueryCallback; + break; + case TDMT_VND_RES_READY: + *fp = schHandleReadyCallback; + break; + case TDMT_VND_FETCH: + *fp = schHandleFetchCallback; + break; + case TDMT_VND_DROP_TASK: + *fp = schHandleDropCallback; + break; + default: + qError("unknown msg type:%d", msgType); + SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t schAsyncSendMsg(void *transport, SEpSet* epSet, uint64_t qId, uint64_t tId, int32_t msgType, void *msg, uint32_t msgSize) { + int32_t code = 0; + SMsgSendInfo* pMsgSendInfo = calloc(1, sizeof(SMsgSendInfo)); + if (NULL == pMsgSendInfo) { + qError("calloc %d failed", (int32_t)sizeof(SMsgSendInfo)); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + SSchCallbackParam *param = calloc(1, sizeof(SSchCallbackParam)); + if (NULL == param) { + qError("calloc %d failed", (int32_t)sizeof(SSchCallbackParam)); + SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + __async_send_cb_fn_t fp = NULL; + SCH_ERR_JRET(schGetCallbackFp(msgType, &fp)); + + param->queryId = qId; + param->taskId = tId; + + pMsgSendInfo->param = param; + pMsgSendInfo->msgInfo.pData = msg; + pMsgSendInfo->msgInfo.len = msgSize; + pMsgSendInfo->msgType = msgType; + + pMsgSendInfo->fp = fp; + + int64_t transporterId = 0; + SCH_ERR_JRET(asyncSendMsgToServer(transport, epSet, &transporterId, pMsgSendInfo)); + + return TSDB_CODE_SUCCESS; + +_return: + tfree(param); + tfree(pMsgSendInfo); + + SCH_RET(code); +} + + +int32_t schBuildAndSendMsg(SSchJob *job, SSchTask *task, int32_t msgType) { + uint32_t msgSize = 0; + void *msg = NULL; + int32_t code = 0; + + switch (msgType) { + case TDMT_VND_SUBMIT: { + if (NULL == task->msg || task->msgLen <= 0) { + qError("submit msg is NULL"); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + + msgSize = task->msgLen; + msg = task->msg; + break; + } + case TDMT_VND_QUERY: { + if (NULL == task->msg) { + qError("query msg is NULL"); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + + msgSize = sizeof(SSubQueryMsg) + task->msgLen; + msg = calloc(1, msgSize); + if (NULL == msg) { + qError("calloc %d failed", msgSize); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + SSubQueryMsg *pMsg = msg; + + pMsg->sId = htobe64(schMgmt.sId); + pMsg->queryId = htobe64(job->queryId); + pMsg->taskId = htobe64(task->taskId); + pMsg->contentLen = htonl(task->msgLen); + memcpy(pMsg->msg, task->msg, task->msgLen); + break; + } + case TDMT_VND_RES_READY: { + msgSize = sizeof(SResReadyMsg); + msg = calloc(1, msgSize); + if (NULL == msg) { + qError("calloc %d failed", msgSize); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + SResReadyMsg *pMsg = msg; + pMsg->sId = htobe64(schMgmt.sId); + pMsg->queryId = htobe64(job->queryId); + pMsg->taskId = htobe64(task->taskId); + break; + } + case TDMT_VND_FETCH: { + if (NULL == task) { + SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + msgSize = sizeof(SResFetchMsg); + msg = calloc(1, msgSize); + if (NULL == msg) { + qError("calloc %d failed", msgSize); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + SResFetchMsg *pMsg = msg; + pMsg->sId = htobe64(schMgmt.sId); + pMsg->queryId = htobe64(job->queryId); + pMsg->taskId = htobe64(task->taskId); + break; + } + case TDMT_VND_DROP_TASK:{ + msgSize = sizeof(STaskDropMsg); + msg = calloc(1, msgSize); + if (NULL == msg) { + qError("calloc %d failed", msgSize); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + STaskDropMsg *pMsg = msg; + pMsg->sId = htobe64(schMgmt.sId); + pMsg->queryId = htobe64(job->queryId); + pMsg->taskId = htobe64(task->taskId); + break; + } + default: + qError("unknown msg type:%d", msgType); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + break; + } + + SCH_ERR_JRET(schAsyncSendMsg(job->transport, &task->plan->execEpSet, job->queryId, task->taskId, msgType, msg, msgSize)); + + return TSDB_CODE_SUCCESS; + +_return: + + tfree(msg); + SCH_RET(code); +} int32_t schLaunchTask(SSchJob *job, SSchTask *task) { @@ -664,7 +748,7 @@ int32_t schLaunchTask(SSchJob *job, SSchTask *task) { int32_t msgType = (plan->type == QUERY_TYPE_MODIFY) ? TDMT_VND_SUBMIT : TDMT_VND_QUERY; - SCH_ERR_RET(schAsyncSendMsg(job, task, msgType)); + SCH_ERR_RET(schBuildAndSendMsg(job, task, msgType)); SCH_ERR_RET(schPushTaskToExecList(job, task)); @@ -673,6 +757,8 @@ int32_t schLaunchTask(SSchJob *job, SSchTask *task) { return TSDB_CODE_SUCCESS; } + + int32_t schLaunchJob(SSchJob *job) { SSchLevel *level = taosArrayGet(job->levels, job->levelIdx); for (int32_t i = 0; i < level->taskNum; ++i) { @@ -690,7 +776,7 @@ void schDropJobAllTasks(SSchJob *job) { while (pIter) { SSchTask *task = *(SSchTask **)pIter; - schAsyncSendMsg(job, task, TDMT_VND_DROP_TASK); + schBuildAndSendMsg(job, task, TDMT_VND_DROP_TASK); pIter = taosHashIterate(job->succTasks, pIter); } @@ -699,7 +785,7 @@ void schDropJobAllTasks(SSchJob *job) { while (pIter) { SSchTask *task = *(SSchTask **)pIter; - schAsyncSendMsg(job, task, TDMT_VND_DROP_TASK); + schBuildAndSendMsg(job, task, TDMT_VND_DROP_TASK); pIter = taosHashIterate(job->succTasks, pIter); } diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 4732429d0b..6163bc0c1a 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -36,7 +36,7 @@ namespace { -extern "C" int32_t schHandleRspMsg(SSchJob *job, SSchTask *task, int32_t msgType, char *msg, int32_t msgSize, int32_t rspCode); +extern "C" int32_t schProcessRspMsg(SSchJob *job, SSchTask *task, int32_t msgType, char *msg, int32_t msgSize, int32_t rspCode); void schtBuildQueryDag(SQueryDag *dag) { uint64_t qId = 0x0000000000000001; @@ -182,7 +182,7 @@ void *schtSendRsp(void *param) { SShellSubmitRspMsg rsp = {0}; rsp.affectedRows = 10; - schHandleRspMsg(job, task, TDMT_VND_SUBMIT, (char *)&rsp, sizeof(rsp), 0); + schProcessRspMsg(job, task, TDMT_VND_SUBMIT, (char *)&rsp, sizeof(rsp), 0); pIter = taosHashIterate(job->execTasks, pIter); } @@ -227,7 +227,7 @@ TEST(queryTest, normalCase) { SSchTask *task = *(SSchTask **)pIter; SQueryTableRsp rsp = {0}; - code = schHandleRspMsg(job, task, TDMT_VND_QUERY, (char *)&rsp, sizeof(rsp), 0); + code = schProcessRspMsg(job, task, TDMT_VND_QUERY, (char *)&rsp, sizeof(rsp), 0); ASSERT_EQ(code, 0); pIter = taosHashIterate(job->execTasks, pIter); @@ -238,7 +238,7 @@ TEST(queryTest, normalCase) { SSchTask *task = *(SSchTask **)pIter; SResReadyRsp rsp = {0}; - code = schHandleRspMsg(job, task, TDMT_VND_RES_READY, (char *)&rsp, sizeof(rsp), 0); + code = schProcessRspMsg(job, task, TDMT_VND_RES_READY, (char *)&rsp, sizeof(rsp), 0); ASSERT_EQ(code, 0); pIter = taosHashIterate(job->execTasks, pIter); @@ -249,7 +249,7 @@ TEST(queryTest, normalCase) { SSchTask *task = *(SSchTask **)pIter; SQueryTableRsp rsp = {0}; - code = schHandleRspMsg(job, task, TDMT_VND_QUERY, (char *)&rsp, sizeof(rsp), 0); + code = schProcessRspMsg(job, task, TDMT_VND_QUERY, (char *)&rsp, sizeof(rsp), 0); ASSERT_EQ(code, 0); pIter = taosHashIterate(job->execTasks, pIter); @@ -260,7 +260,7 @@ TEST(queryTest, normalCase) { SSchTask *task = *(SSchTask **)pIter; SResReadyRsp rsp = {0}; - code = schHandleRspMsg(job, task, TDMT_VND_RES_READY, (char *)&rsp, sizeof(rsp), 0); + code = schProcessRspMsg(job, task, TDMT_VND_RES_READY, (char *)&rsp, sizeof(rsp), 0); ASSERT_EQ(code, 0); pIter = taosHashIterate(job->execTasks, pIter); @@ -269,7 +269,7 @@ TEST(queryTest, normalCase) { SRetrieveTableRsp rsp = {0}; rsp.completed = 1; rsp.numOfRows = 10; - code = schHandleRspMsg(job, NULL, TDMT_VND_FETCH, (char *)&rsp, sizeof(rsp), 0); + code = schProcessRspMsg(job, NULL, TDMT_VND_FETCH, (char *)&rsp, sizeof(rsp), 0); ASSERT_EQ(code, 0); From aa4a5ad03a3839d604d8e806a5245b828926247b Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 28 Dec 2021 16:10:02 +0800 Subject: [PATCH 14/21] catalog get table meta from mnode --- source/libs/catalog/src/catalog.c | 42 +++++++++++++++++++++++++++++-- source/libs/qcom/src/querymsg.c | 2 ++ 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index b882e1a42c..107bbd6901 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -146,8 +146,44 @@ void ctgGenEpSet(SEpSet *epSet, SVgroupInfo *vgroupInfo) { } } +int32_t ctgGetTableMetaFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char *pDBName, const char* pTableName, STableMetaOutput* output) { + if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pDBName || NULL == pTableName || NULL == output) { + CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); + } -int32_t ctgGetTableMetaFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char *pDBName, const char* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* output) { + char tbFullName[TSDB_TABLE_FNAME_LEN]; + + snprintf(tbFullName, sizeof(tbFullName), "%s.%s", pDBName, pTableName); + + SBuildTableMetaInput bInput = {.vgId = 0, .tableFullName = tbFullName}; + char *msg = NULL; + SEpSet *pVnodeEpSet = NULL; + int32_t msgLen = 0; + + CTG_ERR_RET(queryBuildMsg[TDMT_MND_STB_META](&bInput, &msg, 0, &msgLen)); + + SRpcMsg rpcMsg = { + .msgType = TDMT_MND_STB_META, + .pCont = msg, + .contLen = msgLen, + }; + + SRpcMsg rpcRsp = {0}; + + rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); + + if (TSDB_CODE_SUCCESS != rpcRsp.code) { + ctgError("error rsp for table meta, code:%x", rpcRsp.code); + CTG_ERR_RET(rpcRsp.code); + } + + CTG_ERR_RET(queryProcessMsgRsp[TDMT_MND_STB_META](output, rpcRsp.pCont, rpcRsp.contLen)); + + return TSDB_CODE_SUCCESS; +} + + +int32_t ctgGetTableMetaFromVnode(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char *pDBName, const char* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* output) { if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pDBName || NULL == pTableName || NULL == vgroupInfo || NULL == output) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } @@ -515,7 +551,9 @@ int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSe STableMetaOutput output = {0}; - CTG_ERR_RET(ctgGetTableMetaFromMnode(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, &vgroupInfo, &output)); + //CTG_ERR_RET(ctgGetTableMetaFromVnode(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, &vgroupInfo, &output)); + + CTG_ERR_RET(ctgGetTableMetaFromMnode(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, &output)); CTG_ERR_RET(ctgUpdateTableMetaCache(pCatalog, &output)); diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 34db262d5d..8f753283c1 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -266,9 +266,11 @@ int32_t queryProcessTableMetaRsp(void* output, char *msg, int32_t msgSize) { void initQueryModuleMsgHandle() { queryBuildMsg[TDMT_VND_TABLE_META] = queryBuildTableMetaReqMsg; + queryBuildMsg[TDMT_MND_STB_META] = queryBuildTableMetaReqMsg; queryBuildMsg[TDMT_MND_USE_DB] = queryBuildUseDbMsg; queryProcessMsgRsp[TDMT_VND_TABLE_META] = queryProcessTableMetaRsp; + queryProcessMsgRsp[TDMT_MND_STB_META] = queryProcessTableMetaRsp; queryProcessMsgRsp[TDMT_MND_USE_DB] = queryProcessUseDBRsp; } From d60941d4e3a74e94f1f0f0fce44e5b06761aa879 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 28 Dec 2021 00:20:48 -0800 Subject: [PATCH 15/21] add bnode --- include/dnode/bnode/bnode.h | 10 +- include/dnode/snode/snode.h | 10 +- source/dnode/bnode/src/bnode.c | 4 +- source/dnode/mgmt/impl/inc/dndBnode.h | 2 +- source/dnode/mgmt/impl/inc/dndInt.h | 25 +- source/dnode/mgmt/impl/inc/dndWorker.h | 4 +- source/dnode/mgmt/impl/src/dndBnode.c | 369 +++++++++++++++++++++++++ source/dnode/mgmt/impl/src/dndQnode.c | 1 - source/dnode/mgmt/impl/src/dndSnode.c | 344 +++++++++++++++++++++++ source/dnode/mgmt/impl/src/dndWorker.c | 36 ++- source/dnode/snode/src/snode.c | 4 +- 11 files changed, 778 insertions(+), 31 deletions(-) create mode 100644 source/dnode/mgmt/impl/src/dndBnode.c create mode 100644 source/dnode/mgmt/impl/src/dndSnode.c diff --git a/include/dnode/bnode/bnode.h b/include/dnode/bnode/bnode.h index 74574f5462..23cc3ca617 100644 --- a/include/dnode/bnode/bnode.h +++ b/include/dnode/bnode/bnode.h @@ -49,10 +49,11 @@ typedef struct { /** * @brief Start one Bnode in Dnode. * + * @param path Path of the bnode. * @param pOption Option of the bnode. * @return SBnode* The bnode object. */ -SBnode *bndOpen(const SBnodeOpt *pOption); +SBnode *bndOpen(const char *path, const SBnodeOpt *pOption); /** * @brief Stop Bnode in Dnode. @@ -79,6 +80,13 @@ int32_t bndGetLoad(SBnode *pBnode, SBnodeLoad *pLoad); */ int32_t bndProcessWMsgs(SBnode *pBnode, SArray *pMsgs); +/** + * @brief Drop a bnode. + * + * @param path Path of the bnode. + */ +void bndDestroy(const char *path); + #ifdef __cplusplus } #endif diff --git a/include/dnode/snode/snode.h b/include/dnode/snode/snode.h index 1d30bd1e43..97069437f2 100644 --- a/include/dnode/snode/snode.h +++ b/include/dnode/snode/snode.h @@ -49,10 +49,11 @@ typedef struct { /** * @brief Start one Snode in Dnode. * + * @param path Path of the snode. * @param pOption Option of the snode. * @return SSnode* The snode object. */ -SSnode *sndOpen(const SSnodeOpt *pOption); +SSnode *sndOpen(const char *path, const SSnodeOpt *pOption); /** * @brief Stop Snode in Dnode. @@ -80,6 +81,13 @@ int32_t sndGetLoad(SSnode *pSnode, SSnodeLoad *pLoad); */ int32_t sndProcessWriteMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg **pRsp); +/** + * @brief Drop a snode. + * + * @param path Path of the snode. + */ +void sndDestroy(const char *path); + #ifdef __cplusplus } #endif diff --git a/source/dnode/bnode/src/bnode.c b/source/dnode/bnode/src/bnode.c index 40b22dd58d..9570bc72a0 100644 --- a/source/dnode/bnode/src/bnode.c +++ b/source/dnode/bnode/src/bnode.c @@ -15,7 +15,7 @@ #include "bndInt.h" -SBnode *bndOpen(const SBnodeOpt *pOption) { +SBnode *bndOpen(const char *path, const SBnodeOpt *pOption) { SBnode *pBnode = calloc(1, sizeof(SBnode)); return pBnode; } @@ -25,3 +25,5 @@ void bndClose(SBnode *pBnode) { free(pBnode); } int32_t bndGetLoad(SBnode *pBnode, SBnodeLoad *pLoad) { return 0; } int32_t bndProcessWMsgs(SBnode *pBnode, SArray *pMsgs) { return 0; } + +void bndDestroy(const char *path) {} diff --git a/source/dnode/mgmt/impl/inc/dndBnode.h b/source/dnode/mgmt/impl/inc/dndBnode.h index a350eae2d4..853b54ff69 100644 --- a/source/dnode/mgmt/impl/inc/dndBnode.h +++ b/source/dnode/mgmt/impl/inc/dndBnode.h @@ -24,7 +24,7 @@ extern "C" { int32_t dndInitBnode(SDnode *pDnode); void dndCleanupBnode(SDnode *pDnode); -ioid dndProcessBnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); +void dndProcessBnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); int32_t dndProcessCreateBnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg); int32_t dndProcessDropBnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg); diff --git a/source/dnode/mgmt/impl/inc/dndInt.h b/source/dnode/mgmt/impl/inc/dndInt.h index 0d37828ecd..ff96b7cfdf 100644 --- a/source/dnode/mgmt/impl/inc/dndInt.h +++ b/source/dnode/mgmt/impl/inc/dndInt.h @@ -54,18 +54,19 @@ extern int32_t dDebugFlag; #define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("DND ", dDebugFlag, __VA_ARGS__); }} typedef enum { DND_STAT_INIT, DND_STAT_RUNNING, DND_STAT_STOPPED } EStat; -typedef enum { DND_WORKER_SINGLE, DND_WORKER_MULTI } EDndWorkerType; +typedef enum { DND_WORKER_SINGLE, DND_WORKER_MULTI } EWorkerType; typedef void (*DndMsgFp)(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEps); typedef struct { - EDndWorkerType type; + EWorkerType type; const char *name; int32_t minNum; int32_t maxNum; - FProcessItem fp; + void *queueFp; SDnode *pDnode; taos_queue queue; SWorkerPool pool; + SMWorkerPool mpool; } SDnodeWorker; typedef struct { @@ -122,25 +123,21 @@ typedef struct { } SQnodeMgmt; typedef struct { - int32_t refCount; - int8_t deployed; - int8_t dropped; - char *file; - SSnode *pSnode; - SRWLatch latch; - taos_queue pWriteQ; - SWorkerPool writePool; + int32_t refCount; + int8_t deployed; + int8_t dropped; + SSnode *pSnode; + SRWLatch latch; + SDnodeWorker writeWorker; } SSnodeMgmt; typedef struct { int32_t refCount; int8_t deployed; int8_t dropped; - char *file; SBnode *pBnode; SRWLatch latch; - taos_queue pWriteQ; - SMWorkerPool writePool; + SDnodeWorker writeWorker; } SBnodeMgmt; typedef struct { diff --git a/source/dnode/mgmt/impl/inc/dndWorker.h b/source/dnode/mgmt/impl/inc/dndWorker.h index 237c0518e8..49ef88e67d 100644 --- a/source/dnode/mgmt/impl/inc/dndWorker.h +++ b/source/dnode/mgmt/impl/inc/dndWorker.h @@ -21,8 +21,8 @@ extern "C" { #endif #include "dndInt.h" -int32_t dndInitWorker(SDnode *pDnode, SDnodeWorker *pWorker, EDndWorkerType type, const char *name, int32_t minNum, - int32_t maxNum, FProcessItem fp); +int32_t dndInitWorker(SDnode *pDnode, SDnodeWorker *pWorker, EWorkerType type, const char *name, int32_t minNum, + int32_t maxNum, void *queueFp); void dndCleanupWorker(SDnodeWorker *pWorker); int32_t dndWriteMsgToWorker(SDnodeWorker *pWorker, void *pCont, int32_t contLen); diff --git a/source/dnode/mgmt/impl/src/dndBnode.c b/source/dnode/mgmt/impl/src/dndBnode.c new file mode 100644 index 0000000000..b978c1102f --- /dev/null +++ b/source/dnode/mgmt/impl/src/dndBnode.c @@ -0,0 +1,369 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dndBnode.h" +#include "dndDnode.h" +#include "dndTransport.h" +#include "dndWorker.h" + +static void dndProcessBnodeQueue(SDnode *pDnode, taos_qall qall, int32_t numOfMsgs); + +static SBnode *dndAcquireBnode(SDnode *pDnode) { + SBnodeMgmt *pMgmt = &pDnode->bmgmt; + SBnode *pBnode = NULL; + int32_t refCount = 0; + + taosRLockLatch(&pMgmt->latch); + if (pMgmt->deployed && !pMgmt->dropped) { + refCount = atomic_add_fetch_32(&pMgmt->refCount, 1); + pBnode = pMgmt->pBnode; + } else { + terrno = TSDB_CODE_DND_BNODE_NOT_DEPLOYED; + } + taosRUnLockLatch(&pMgmt->latch); + + if (pBnode != NULL) { + dTrace("acquire bnode, refCount:%d", refCount); + } + return pBnode; +} + +static void dndReleaseBnode(SDnode *pDnode, SBnode *pBnode) { + SBnodeMgmt *pMgmt = &pDnode->bmgmt; + int32_t refCount = 0; + + taosRLockLatch(&pMgmt->latch); + if (pBnode != NULL) { + refCount = atomic_sub_fetch_32(&pMgmt->refCount, 1); + } + taosRUnLockLatch(&pMgmt->latch); + + if (pBnode != NULL) { + dTrace("release bnode, refCount:%d", refCount); + } +} + +static int32_t dndReadBnodeFile(SDnode *pDnode) { + SBnodeMgmt *pMgmt = &pDnode->bmgmt; + int32_t code = TSDB_CODE_DND_BNODE_READ_FILE_ERROR; + int32_t len = 0; + int32_t maxLen = 4096; + char *content = calloc(1, maxLen + 1); + cJSON *root = NULL; + + char file[PATH_MAX + 20]; + snprintf(file, PATH_MAX + 20, "%s/bnode.json", pDnode->dir.dnode); + + FILE *fp = fopen(file, "r"); + if (fp == NULL) { + dDebug("file %s not exist", file); + code = 0; + goto PRASE_BNODE_OVER; + } + + len = (int32_t)fread(content, 1, maxLen, fp); + if (len <= 0) { + dError("failed to read %s since content is null", file); + goto PRASE_BNODE_OVER; + } + + content[len] = 0; + root = cJSON_Parse(content); + if (root == NULL) { + dError("failed to read %s since invalid json format", file); + goto PRASE_BNODE_OVER; + } + + cJSON *deployed = cJSON_GetObjectItem(root, "deployed"); + if (!deployed || deployed->type != cJSON_Number) { + dError("failed to read %s since deployed not found", file); + goto PRASE_BNODE_OVER; + } + pMgmt->deployed = deployed->valueint; + + cJSON *dropped = cJSON_GetObjectItem(root, "dropped"); + if (!dropped || dropped->type != cJSON_Number) { + dError("failed to read %s since dropped not found", file); + goto PRASE_BNODE_OVER; + } + pMgmt->dropped = dropped->valueint; + + code = 0; + dDebug("succcessed to read file %s, deployed:%d dropped:%d", file, pMgmt->deployed, pMgmt->dropped); + +PRASE_BNODE_OVER: + if (content != NULL) free(content); + if (root != NULL) cJSON_Delete(root); + if (fp != NULL) fclose(fp); + + terrno = code; + return code; +} + +static int32_t dndWriteBnodeFile(SDnode *pDnode) { + SBnodeMgmt *pMgmt = &pDnode->bmgmt; + + char file[PATH_MAX + 20]; + snprintf(file, PATH_MAX + 20, "%s/bnode.json", pDnode->dir.dnode); + + FILE *fp = fopen(file, "w"); + if (fp == NULL) { + terrno = TSDB_CODE_DND_BNODE_WRITE_FILE_ERROR; + dError("failed to write %s since %s", file, terrstr()); + return -1; + } + + int32_t len = 0; + int32_t maxLen = 4096; + char *content = calloc(1, maxLen + 1); + + len += snprintf(content + len, maxLen - len, "{\n"); + len += snprintf(content + len, maxLen - len, " \"deployed\": %d,\n", pMgmt->deployed); + len += snprintf(content + len, maxLen - len, " \"dropped\": %d\n", pMgmt->dropped); + len += snprintf(content + len, maxLen - len, "}\n"); + + fwrite(content, 1, len, fp); + taosFsyncFile(fileno(fp)); + fclose(fp); + free(content); + + if (taosRenameFile(file, file) != 0) { + terrno = TSDB_CODE_DND_BNODE_WRITE_FILE_ERROR; + dError("failed to rename %s since %s", file, terrstr()); + return -1; + } + + dInfo("successed to write %s, deployed:%d dropped:%d", file, pMgmt->deployed, pMgmt->dropped); + return 0; +} + +static int32_t dndStartBnodeWorker(SDnode *pDnode) { + SBnodeMgmt *pMgmt = &pDnode->bmgmt; + if (dndInitWorker(pDnode, &pMgmt->writeWorker, DND_WORKER_SINGLE, "bnode-write", 0, 1, + (FProcessItem)dndProcessBnodeQueue) != 0) { + dError("failed to start bnode write worker since %s", terrstr()); + return -1; + } + + return 0; +} + +static void dndStopBnodeWorker(SDnode *pDnode) { + SBnodeMgmt *pMgmt = &pDnode->bmgmt; + + taosWLockLatch(&pMgmt->latch); + pMgmt->deployed = 0; + taosWUnLockLatch(&pMgmt->latch); + + while (pMgmt->refCount > 1) { + taosMsleep(10); + } + + dndCleanupWorker(&pMgmt->writeWorker); +} + +static void dndBuildBnodeOption(SDnode *pDnode, SBnodeOpt *pOption) { + pOption->pDnode = pDnode; + pOption->sendMsgToDnodeFp = dndSendMsgToDnode; + pOption->sendMsgToMnodeFp = dndSendMsgToMnode; + pOption->sendRedirectMsgFp = dndSendRedirectMsg; + pOption->dnodeId = dndGetDnodeId(pDnode); + pOption->clusterId = dndGetClusterId(pDnode); + pOption->cfg.sver = pDnode->opt.sver; +} + +static int32_t dndOpenBnode(SDnode *pDnode) { + SBnodeMgmt *pMgmt = &pDnode->bmgmt; + SBnodeOpt option = {0}; + dndBuildBnodeOption(pDnode, &option); + + SBnode *pBnode = bndOpen(pDnode->dir.bnode, &option); + if (pBnode == NULL) { + dError("failed to open bnode since %s", terrstr()); + return -1; + } + + if (dndStartBnodeWorker(pDnode) != 0) { + dError("failed to start bnode worker since %s", terrstr()); + bndClose(pBnode); + return -1; + } + + if (dndWriteBnodeFile(pDnode) != 0) { + dError("failed to write bnode file since %s", terrstr()); + dndStopBnodeWorker(pDnode); + bndClose(pBnode); + return -1; + } + + taosWLockLatch(&pMgmt->latch); + pMgmt->pBnode = pBnode; + pMgmt->deployed = 1; + taosWUnLockLatch(&pMgmt->latch); + + dInfo("bnode open successfully"); + return 0; +} + +static int32_t dndDropBnode(SDnode *pDnode) { + SBnodeMgmt *pMgmt = &pDnode->bmgmt; + + SBnode *pBnode = dndAcquireBnode(pDnode); + if (pBnode == NULL) { + dError("failed to drop bnode since %s", terrstr()); + return -1; + } + + taosRLockLatch(&pMgmt->latch); + pMgmt->dropped = 1; + taosRUnLockLatch(&pMgmt->latch); + + if (dndWriteBnodeFile(pDnode) != 0) { + taosRLockLatch(&pMgmt->latch); + pMgmt->dropped = 0; + taosRUnLockLatch(&pMgmt->latch); + + dndReleaseBnode(pDnode, pBnode); + dError("failed to drop bnode since %s", terrstr()); + return -1; + } + + dndReleaseBnode(pDnode, pBnode); + dndStopBnodeWorker(pDnode); + bndClose(pBnode); + pMgmt->pBnode = NULL; + bndDestroy(pDnode->dir.bnode); + + return 0; +} + +int32_t dndProcessCreateBnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { + SCreateBnodeInMsg *pMsg = pRpcMsg->pCont; + pMsg->dnodeId = htonl(pMsg->dnodeId); + + if (pMsg->dnodeId != dndGetDnodeId(pDnode)) { + terrno = TSDB_CODE_DND_BNODE_ID_INVALID; + return -1; + } else { + return dndOpenBnode(pDnode); + } +} + +int32_t dndProcessDropBnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { + SDropBnodeInMsg *pMsg = pRpcMsg->pCont; + pMsg->dnodeId = htonl(pMsg->dnodeId); + + if (pMsg->dnodeId != dndGetDnodeId(pDnode)) { + terrno = TSDB_CODE_DND_BNODE_ID_INVALID; + return -1; + } else { + return dndDropBnode(pDnode); + } +} + +static void dndSendBnodeErrorRsp(SRpcMsg *pMsg, int32_t code) { + SRpcMsg rpcRsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code}; + rpcSendResponse(&rpcRsp); + rpcFreeCont(pMsg->pCont); + taosFreeQitem(pMsg); +} + +static void dndSendBnodeErrorRsps(taos_qall qall, int32_t numOfMsgs, int32_t code) { + for (int32_t i = 0; i < numOfMsgs; ++i) { + SRpcMsg *pMsg = NULL; + taosGetQitem(qall, (void **)&pMsg); + dndSendBnodeErrorRsp(pMsg, code); + } +} + +static void dndProcessBnodeQueue(SDnode *pDnode, taos_qall qall, int32_t numOfMsgs) { + SBnode *pBnode = dndAcquireBnode(pDnode); + if (pBnode == NULL) { + dndSendBnodeErrorRsps(qall, numOfMsgs, TSDB_CODE_OUT_OF_MEMORY); + return; + } + + SArray *pArray = taosArrayInit(numOfMsgs, sizeof(SRpcMsg *)); + if (pArray == NULL) { + dndReleaseBnode(pDnode, pBnode); + dndSendBnodeErrorRsps(qall, numOfMsgs, TSDB_CODE_OUT_OF_MEMORY); + return; + } + + for (int32_t i = 0; i < numOfMsgs; ++i) { + SRpcMsg *pMsg = NULL; + taosGetQitem(qall, (void **)&pMsg); + void *ptr = taosArrayPush(pArray, &pMsg); + if (ptr == NULL) { + dndSendBnodeErrorRsp(pMsg, TSDB_CODE_OUT_OF_MEMORY); + } + } + + bndProcessWMsgs(pBnode, pArray); + + for (size_t i = 0; i < numOfMsgs; i++) { + SRpcMsg *pMsg = *(SRpcMsg **)taosArrayGet(pArray, i); + rpcFreeCont(pMsg->pCont); + taosFreeQitem(pMsg); + } + taosArrayDestroy(pArray); + dndReleaseBnode(pDnode, pBnode); +} + +static void dndWriteBnodeMsgToWorker(SDnode *pDnode, SDnodeWorker *pWorker, SRpcMsg *pMsg) { + int32_t code = TSDB_CODE_DND_BNODE_NOT_DEPLOYED; + + SBnode *pBnode = dndAcquireBnode(pDnode); + if (pBnode != NULL) { + code = dndWriteMsgToWorker(pWorker, pMsg, sizeof(SRpcMsg)); + } + dndReleaseBnode(pDnode, pBnode); + + if (code != 0) { + if (pMsg->msgType & 1u) { + SRpcMsg rsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code}; + rpcSendResponse(&rsp); + } + rpcFreeCont(pMsg->pCont); + } +} + +void dndProcessBnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { + dndWriteBnodeMsgToWorker(pDnode, &pDnode->bmgmt.writeWorker, pMsg); +} + +int32_t dndInitBnode(SDnode *pDnode) { + SBnodeMgmt *pMgmt = &pDnode->bmgmt; + taosInitRWLatch(&pMgmt->latch); + + if (dndReadBnodeFile(pDnode) != 0) { + return -1; + } + + if (pMgmt->dropped) return 0; + if (!pMgmt->deployed) return 0; + + return dndOpenBnode(pDnode); +} + +void dndCleanupBnode(SDnode *pDnode) { + SBnodeMgmt *pMgmt = &pDnode->bmgmt; + if (pMgmt->pBnode) { + dndStopBnodeWorker(pDnode); + bndClose(pMgmt->pBnode); + pMgmt->pBnode = NULL; + } +} diff --git a/source/dnode/mgmt/impl/src/dndQnode.c b/source/dnode/mgmt/impl/src/dndQnode.c index 8c76bf95a6..0b92de81c1 100644 --- a/source/dnode/mgmt/impl/src/dndQnode.c +++ b/source/dnode/mgmt/impl/src/dndQnode.c @@ -252,7 +252,6 @@ static int32_t dndDropQnode(SDnode *pDnode) { dndStopQnodeWorker(pDnode); qndClose(pQnode); pMgmt->pQnode = NULL; - // qndDestroy(pDnode->dir.qnode); return 0; } diff --git a/source/dnode/mgmt/impl/src/dndSnode.c b/source/dnode/mgmt/impl/src/dndSnode.c new file mode 100644 index 0000000000..c1eb347350 --- /dev/null +++ b/source/dnode/mgmt/impl/src/dndSnode.c @@ -0,0 +1,344 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dndSnode.h" +#include "dndDnode.h" +#include "dndTransport.h" +#include "dndWorker.h" + +static void dndProcessSnodeQueue(SDnode *pDnode, SRpcMsg *pMsg); + +static SSnode *dndAcquireSnode(SDnode *pDnode) { + SSnodeMgmt *pMgmt = &pDnode->smgmt; + SSnode *pSnode = NULL; + int32_t refCount = 0; + + taosRLockLatch(&pMgmt->latch); + if (pMgmt->deployed && !pMgmt->dropped) { + refCount = atomic_add_fetch_32(&pMgmt->refCount, 1); + pSnode = pMgmt->pSnode; + } else { + terrno = TSDB_CODE_DND_SNODE_NOT_DEPLOYED; + } + taosRUnLockLatch(&pMgmt->latch); + + if (pSnode != NULL) { + dTrace("acquire snode, refCount:%d", refCount); + } + return pSnode; +} + +static void dndReleaseSnode(SDnode *pDnode, SSnode *pSnode) { + SSnodeMgmt *pMgmt = &pDnode->smgmt; + int32_t refCount = 0; + + taosRLockLatch(&pMgmt->latch); + if (pSnode != NULL) { + refCount = atomic_sub_fetch_32(&pMgmt->refCount, 1); + } + taosRUnLockLatch(&pMgmt->latch); + + if (pSnode != NULL) { + dTrace("release snode, refCount:%d", refCount); + } +} + +static int32_t dndReadSnodeFile(SDnode *pDnode) { + SSnodeMgmt *pMgmt = &pDnode->smgmt; + int32_t code = TSDB_CODE_DND_SNODE_READ_FILE_ERROR; + int32_t len = 0; + int32_t maxLen = 4096; + char *content = calloc(1, maxLen + 1); + cJSON *root = NULL; + + char file[PATH_MAX + 20]; + snprintf(file, PATH_MAX + 20, "%s/snode.json", pDnode->dir.dnode); + + FILE *fp = fopen(file, "r"); + if (fp == NULL) { + dDebug("file %s not exist", file); + code = 0; + goto PRASE_SNODE_OVER; + } + + len = (int32_t)fread(content, 1, maxLen, fp); + if (len <= 0) { + dError("failed to read %s since content is null", file); + goto PRASE_SNODE_OVER; + } + + content[len] = 0; + root = cJSON_Parse(content); + if (root == NULL) { + dError("failed to read %s since invalid json format", file); + goto PRASE_SNODE_OVER; + } + + cJSON *deployed = cJSON_GetObjectItem(root, "deployed"); + if (!deployed || deployed->type != cJSON_Number) { + dError("failed to read %s since deployed not found", file); + goto PRASE_SNODE_OVER; + } + pMgmt->deployed = deployed->valueint; + + cJSON *dropped = cJSON_GetObjectItem(root, "dropped"); + if (!dropped || dropped->type != cJSON_Number) { + dError("failed to read %s since dropped not found", file); + goto PRASE_SNODE_OVER; + } + pMgmt->dropped = dropped->valueint; + + code = 0; + dDebug("succcessed to read file %s, deployed:%d dropped:%d", file, pMgmt->deployed, pMgmt->dropped); + +PRASE_SNODE_OVER: + if (content != NULL) free(content); + if (root != NULL) cJSON_Delete(root); + if (fp != NULL) fclose(fp); + + terrno = code; + return code; +} + +static int32_t dndWriteSnodeFile(SDnode *pDnode) { + SSnodeMgmt *pMgmt = &pDnode->smgmt; + + char file[PATH_MAX + 20]; + snprintf(file, PATH_MAX + 20, "%s/snode.json", pDnode->dir.dnode); + + FILE *fp = fopen(file, "w"); + if (fp == NULL) { + terrno = TSDB_CODE_DND_SNODE_WRITE_FILE_ERROR; + dError("failed to write %s since %s", file, terrstr()); + return -1; + } + + int32_t len = 0; + int32_t maxLen = 4096; + char *content = calloc(1, maxLen + 1); + + len += snprintf(content + len, maxLen - len, "{\n"); + len += snprintf(content + len, maxLen - len, " \"deployed\": %d,\n", pMgmt->deployed); + len += snprintf(content + len, maxLen - len, " \"dropped\": %d\n", pMgmt->dropped); + len += snprintf(content + len, maxLen - len, "}\n"); + + fwrite(content, 1, len, fp); + taosFsyncFile(fileno(fp)); + fclose(fp); + free(content); + + if (taosRenameFile(file, file) != 0) { + terrno = TSDB_CODE_DND_SNODE_WRITE_FILE_ERROR; + dError("failed to rename %s since %s", file, terrstr()); + return -1; + } + + dInfo("successed to write %s, deployed:%d dropped:%d", file, pMgmt->deployed, pMgmt->dropped); + return 0; +} + +static int32_t dndStartSnodeWorker(SDnode *pDnode) { + SSnodeMgmt *pMgmt = &pDnode->smgmt; + if (dndInitWorker(pDnode, &pMgmt->writeWorker, DND_WORKER_SINGLE, "snode-write", 0, 1, + (FProcessItem)dndProcessSnodeQueue) != 0) { + dError("failed to start snode write worker since %s", terrstr()); + return -1; + } + + return 0; +} + +static void dndStopSnodeWorker(SDnode *pDnode) { + SSnodeMgmt *pMgmt = &pDnode->smgmt; + + taosWLockLatch(&pMgmt->latch); + pMgmt->deployed = 0; + taosWUnLockLatch(&pMgmt->latch); + + while (pMgmt->refCount > 1) { + taosMsleep(10); + } + + dndCleanupWorker(&pMgmt->writeWorker); +} + +static void dndBuildSnodeOption(SDnode *pDnode, SSnodeOpt *pOption) { + pOption->pDnode = pDnode; + pOption->sendMsgToDnodeFp = dndSendMsgToDnode; + pOption->sendMsgToMnodeFp = dndSendMsgToMnode; + pOption->sendRedirectMsgFp = dndSendRedirectMsg; + pOption->dnodeId = dndGetDnodeId(pDnode); + pOption->clusterId = dndGetClusterId(pDnode); + pOption->cfg.sver = pDnode->opt.sver; +} + +static int32_t dndOpenSnode(SDnode *pDnode) { + SSnodeMgmt *pMgmt = &pDnode->smgmt; + SSnodeOpt option = {0}; + dndBuildSnodeOption(pDnode, &option); + + SSnode *pSnode = sndOpen(pDnode->dir.snode, &option); + if (pSnode == NULL) { + dError("failed to open snode since %s", terrstr()); + return -1; + } + + if (dndStartSnodeWorker(pDnode) != 0) { + dError("failed to start snode worker since %s", terrstr()); + sndClose(pSnode); + return -1; + } + + if (dndWriteSnodeFile(pDnode) != 0) { + dError("failed to write snode file since %s", terrstr()); + dndStopSnodeWorker(pDnode); + sndClose(pSnode); + return -1; + } + + taosWLockLatch(&pMgmt->latch); + pMgmt->pSnode = pSnode; + pMgmt->deployed = 1; + taosWUnLockLatch(&pMgmt->latch); + + dInfo("snode open successfully"); + return 0; +} + +static int32_t dndDropSnode(SDnode *pDnode) { + SSnodeMgmt *pMgmt = &pDnode->smgmt; + + SSnode *pSnode = dndAcquireSnode(pDnode); + if (pSnode == NULL) { + dError("failed to drop snode since %s", terrstr()); + return -1; + } + + taosRLockLatch(&pMgmt->latch); + pMgmt->dropped = 1; + taosRUnLockLatch(&pMgmt->latch); + + if (dndWriteSnodeFile(pDnode) != 0) { + taosRLockLatch(&pMgmt->latch); + pMgmt->dropped = 0; + taosRUnLockLatch(&pMgmt->latch); + + dndReleaseSnode(pDnode, pSnode); + dError("failed to drop snode since %s", terrstr()); + return -1; + } + + dndReleaseSnode(pDnode, pSnode); + dndStopSnodeWorker(pDnode); + sndClose(pSnode); + pMgmt->pSnode = NULL; + sndDestroy(pDnode->dir.snode); + + return 0; +} + +int32_t dndProcessCreateSnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { + SCreateSnodeInMsg *pMsg = pRpcMsg->pCont; + pMsg->dnodeId = htonl(pMsg->dnodeId); + + if (pMsg->dnodeId != dndGetDnodeId(pDnode)) { + terrno = TSDB_CODE_DND_SNODE_ID_INVALID; + return -1; + } else { + return dndOpenSnode(pDnode); + } +} + +int32_t dndProcessDropSnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { + SDropSnodeInMsg *pMsg = pRpcMsg->pCont; + pMsg->dnodeId = htonl(pMsg->dnodeId); + + if (pMsg->dnodeId != dndGetDnodeId(pDnode)) { + terrno = TSDB_CODE_DND_SNODE_ID_INVALID; + return -1; + } else { + return dndDropSnode(pDnode); + } +} + +static void dndProcessSnodeQueue(SDnode *pDnode, SRpcMsg *pMsg) { + SSnodeMgmt *pMgmt = &pDnode->smgmt; + SRpcMsg *pRsp = NULL; + int32_t code = TSDB_CODE_DND_SNODE_NOT_DEPLOYED; + + SSnode *pSnode = dndAcquireSnode(pDnode); + if (pSnode != NULL) { + code = sndProcessMsg(pSnode, pMsg, &pRsp); + } + + if (pRsp != NULL) { + pRsp->ahandle = pMsg->ahandle; + rpcSendResponse(pRsp); + free(pRsp); + } else { + if (code != 0) code = terrno; + SRpcMsg rpcRsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code}; + rpcSendResponse(&rpcRsp); + } + + rpcFreeCont(pMsg->pCont); + taosFreeQitem(pMsg); +} + +static void dndWriteSnodeMsgToWorker(SDnode *pDnode, SDnodeWorker *pWorker, SRpcMsg *pMsg) { + int32_t code = TSDB_CODE_DND_SNODE_NOT_DEPLOYED; + + SSnode *pSnode = dndAcquireSnode(pDnode); + if (pSnode != NULL) { + code = dndWriteMsgToWorker(pWorker, pMsg, sizeof(SRpcMsg)); + } + dndReleaseSnode(pDnode, pSnode); + + if (code != 0) { + if (pMsg->msgType & 1u) { + SRpcMsg rsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code}; + rpcSendResponse(&rsp); + } + rpcFreeCont(pMsg->pCont); + } +} + +void dndProcessSnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { + dndWriteSnodeMsgToWorker(pDnode, &pDnode->smgmt.writeWorker, pMsg); +} + +int32_t dndInitSnode(SDnode *pDnode) { + SSnodeMgmt *pMgmt = &pDnode->smgmt; + taosInitRWLatch(&pMgmt->latch); + + if (dndReadSnodeFile(pDnode) != 0) { + return -1; + } + + if (pMgmt->dropped) return 0; + if (!pMgmt->deployed) return 0; + + return dndOpenSnode(pDnode); +} + +void dndCleanupSnode(SDnode *pDnode) { + SSnodeMgmt *pMgmt = &pDnode->smgmt; + if (pMgmt->pSnode) { + dndStopSnodeWorker(pDnode); + sndClose(pMgmt->pSnode); + pMgmt->pSnode = NULL; + } +} diff --git a/source/dnode/mgmt/impl/src/dndWorker.c b/source/dnode/mgmt/impl/src/dndWorker.c index da0e3a9319..c421437e4d 100644 --- a/source/dnode/mgmt/impl/src/dndWorker.c +++ b/source/dnode/mgmt/impl/src/dndWorker.c @@ -16,9 +16,9 @@ #define _DEFAULT_SOURCE #include "dndWorker.h" -int32_t dndInitWorker(SDnode *pDnode, SDnodeWorker *pWorker, EDndWorkerType type, const char *name, int32_t minNum, - int32_t maxNum, FProcessItem fp) { - if (pDnode == NULL || pWorker == NULL || name == NULL || minNum < 0 || maxNum <= 0 || fp == NULL) { +int32_t dndInitWorker(SDnode *pDnode, SDnodeWorker *pWorker, EWorkerType type, const char *name, int32_t minNum, + int32_t maxNum, void *queueFp) { + if (pDnode == NULL || pWorker == NULL || name == NULL || minNum < 0 || maxNum <= 0 || queueFp == NULL) { terrno = TSDB_CODE_INVALID_PARA; return -1; } @@ -27,19 +27,32 @@ int32_t dndInitWorker(SDnode *pDnode, SDnodeWorker *pWorker, EDndWorkerType type pWorker->name = name; pWorker->minNum = minNum; pWorker->maxNum = maxNum; - pWorker->fp = fp; + pWorker->queueFp = queueFp; pWorker->pDnode = pDnode; if (pWorker->type == DND_WORKER_SINGLE) { SWorkerPool *pPool = &pWorker->pool; + pPool->name = name; pPool->min = minNum; pPool->max = maxNum; if (tWorkerInit(pPool) != 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - - pWorker->queue = tWorkerAllocQueue(&pPool, pDnode, fp); + pWorker->queue = tWorkerAllocQueue(pPool, pDnode, (FProcessItem)queueFp); + if (pWorker->queue == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + } else if (pWorker->type == DND_WORKER_MULTI) { + SMWorkerPool *pPool = &pWorker->mpool; + pPool->name = name; + pPool->max = maxNum; + if (tMWorkerInit(pPool) != 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pWorker->queue = tMWorkerAllocQueue(pPool, pDnode, (FProcessItems)queueFp); if (pWorker->queue == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -52,12 +65,17 @@ int32_t dndInitWorker(SDnode *pDnode, SDnodeWorker *pWorker, EDndWorkerType type } void dndCleanupWorker(SDnodeWorker *pWorker) { + while (!taosQueueEmpty(pWorker->queue)) { + taosMsleep(10); + } + if (pWorker->type == DND_WORKER_SINGLE) { - while (!taosQueueEmpty(pWorker->queue)) { - taosMsleep(10); - } tWorkerCleanup(&pWorker->pool); tWorkerFreeQueue(&pWorker->pool, pWorker->queue); + } else if (pWorker->type == DND_WORKER_MULTI) { + tWorkerCleanup(&pWorker->mpool); + tMWorkerFreeQueue(&pWorker->mpool, pWorker->queue); + } else { } } diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 3423ce41e2..7ae4d49059 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -15,7 +15,7 @@ #include "sndInt.h" -SSnode *sndOpen(const SSnodeOpt *pOption) { +SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) { SSnode *pSnode = calloc(1, sizeof(SSnode)); return pSnode; } @@ -28,3 +28,5 @@ int32_t sndProcessWriteMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { *pRsp = NULL; return 0; } + +void sndDestroy(const char *path) {} \ No newline at end of file From 34b74ff0a412d47449c4fa3055935132f6fc411e Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 28 Dec 2021 16:49:46 +0800 Subject: [PATCH 16/21] catalog code refactor --- source/client/src/clientEnv.c | 2 +- source/libs/catalog/inc/catalogInt.h | 1 + source/libs/catalog/src/catalog.c | 38 +++++++++++++--------------- 3 files changed, 19 insertions(+), 22 deletions(-) diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 917ee1dee1..db1ea435f1 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -227,7 +227,7 @@ void taos_init_imp(void) { rpcInit(); - SCatalogCfg cfg = {.enableVgroupCache = true, .maxDBCacheNum = 100, .maxTblCacheNum = 100}; + SCatalogCfg cfg = {.maxDBCacheNum = 100, .maxTblCacheNum = 100}; catalogInit(&cfg); tscDebug("starting to initialize TAOS driver, local ep: %s", tsLocalEp); diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 7f2ddb5723..548b25efc8 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -67,6 +67,7 @@ typedef uint32_t (*tableNameHashFp)(const char *, uint32_t); #define ctgTrace(...) do { if (ctgDebugFlag & DEBUG_TRACE) { taosPrintLog("CTG ", ctgDebugFlag, __VA_ARGS__); }} while(0) #define ctgDebugL(...) do { if (ctgDebugFlag & DEBUG_DEBUG) { taosPrintLongString("CTG ", ctgDebugFlag, __VA_ARGS__); }} while(0) +#define CTG_CACHE_ENABLED(cfg) ((cfg)->maxDBCacheNum > 0 || (cfg)->maxTblCacheNum > 0) #define CTG_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) #define CTG_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 3f9c469ef5..fda3cddf70 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -343,7 +343,9 @@ int32_t ctgUpdateTableMetaCache(struct SCatalog *pCatalog, STableMetaOutput *out ctgError("init hash[%d] for tablemeta cache failed", ctgMgmt.cfg.maxTblCacheNum); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } + } + if (NULL == pCatalog->tableCache.stableCache) { pCatalog->tableCache.stableCache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_ENTRY_LOCK); if (NULL == pCatalog->tableCache.stableCache) { ctgError("init hash[%d] for stablemeta cache failed", ctgMgmt.cfg.maxTblCacheNum); @@ -354,55 +356,51 @@ int32_t ctgUpdateTableMetaCache(struct SCatalog *pCatalog, STableMetaOutput *out if (output->metaNum == 2) { if (taosHashPut(pCatalog->tableCache.cache, output->ctbFname, strlen(output->ctbFname), &output->ctbMeta, sizeof(output->ctbMeta)) != 0) { ctgError("push ctable[%s] to table cache failed", output->ctbFname); - goto error_exit; + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } if (TSDB_SUPER_TABLE != output->tbMeta->tableType) { ctgError("table type[%d] error, expected:%d", output->tbMeta->tableType, TSDB_SUPER_TABLE); - goto error_exit; + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } } int32_t tbSize = sizeof(*output->tbMeta) + sizeof(SSchema) * (output->tbMeta->tableInfo.numOfColumns + output->tbMeta->tableInfo.numOfTags); if (taosHashPut(pCatalog->tableCache.cache, output->tbFname, strlen(output->tbFname), output->tbMeta, tbSize) != 0) { ctgError("push table[%s] to table cache failed", output->tbFname); - goto error_exit; + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } if (TSDB_SUPER_TABLE == output->tbMeta->tableType) { if (taosHashPut(pCatalog->tableCache.stableCache, &output->tbMeta->suid, sizeof(output->tbMeta->suid), &output->tbMeta, POINTER_BYTES) != 0) { ctgError("push suid[%"PRIu64"] to stable cache failed", output->tbMeta->suid); - goto error_exit; + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } } return TSDB_CODE_SUCCESS; - -error_exit: - if (pCatalog->vgroupCache.cache) { - taosHashCleanup(pCatalog->vgroupCache.cache); - pCatalog->vgroupCache.cache = NULL; - } - - pCatalog->vgroupCache.vgroupVersion = CTG_DEFAULT_INVALID_VERSION; - - CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } int32_t catalogInit(SCatalogCfg *cfg) { - ctgMgmt.pCluster = taosHashInit(CTG_DEFAULT_CACHE_CLUSTER_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); - if (NULL == ctgMgmt.pCluster) { - CTG_ERR_LRET(TSDB_CODE_CTG_INTERNAL_ERROR, "init %d cluster cache failed", CTG_DEFAULT_CACHE_CLUSTER_NUMBER); + if (ctgMgmt.pCluster) { + ctgError("catalog already init"); + CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } if (cfg) { memcpy(&ctgMgmt.cfg, cfg, sizeof(*cfg)); } else { - ctgMgmt.cfg.enableVgroupCache = true; ctgMgmt.cfg.maxDBCacheNum = CTG_DEFAULT_CACHE_DB_NUMBER; ctgMgmt.cfg.maxTblCacheNum = CTG_DEFAULT_CACHE_TABLEMETA_NUMBER; } + if (CTG_CACHE_ENABLED(&ctgMgmt.cfg)) { + ctgMgmt.pCluster = taosHashInit(CTG_DEFAULT_CACHE_CLUSTER_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + if (NULL == ctgMgmt.pCluster) { + CTG_ERR_LRET(TSDB_CODE_CTG_INTERNAL_ERROR, "init %d cluster cache failed", CTG_DEFAULT_CACHE_CLUSTER_NUMBER); + } + } + return TSDB_CODE_SUCCESS; } @@ -430,8 +428,6 @@ int32_t catalogGetHandle(const char* clusterId , struct SCatalog** catalogHandle CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } - clusterCtg->vgroupCache.vgroupVersion = CTG_DEFAULT_INVALID_VERSION; - if (taosHashPut(ctgMgmt.pCluster, clusterId, clen, &clusterCtg, POINTER_BYTES)) { ctgError("put cluster %s cache to hash failed", clusterId); tfree(clusterCtg); @@ -479,7 +475,7 @@ int32_t catalogUpdateDBVgroupCache(struct SCatalog* pCatalog, const char* dbName } if (NULL == pCatalog->dbCache.cache) { - pCatalog->dbCache.cache = taosHashInit(CTG_DEFAULT_CACHE_DB_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + pCatalog->dbCache.cache = taosHashInit(ctgMgmt.cfg.maxDBCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); if (NULL == pCatalog->dbCache.cache) { ctgError("init hash[%d] for db cache failed", CTG_DEFAULT_CACHE_DB_NUMBER); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); From c9f38d2be294544308154059da53d6023e6941d3 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 28 Dec 2021 17:01:09 +0800 Subject: [PATCH 17/21] send create topic req --- include/common/tmsg.h | 21 +++++++++++++ include/libs/parser/parser.h | 4 +-- source/client/src/clientImpl.c | 56 +++++++++++++++++++++++++++++++++- 3 files changed, 78 insertions(+), 3 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 6727dd3289..25002a9f92 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1063,6 +1063,27 @@ typedef struct STaskDropRsp { int32_t code; } STaskDropRsp; +typedef struct { + int8_t igExists; + char* name; + char* phyPlan; +} SCMCreateTopicReq; + +static FORCE_INLINE int tSerializeSCMCreateTopicReq(void** buf, const SCMCreateTopicReq* pReq) { + int tlen = 0; + tlen += taosEncodeString(buf, pReq->name); + tlen += taosEncodeFixedI8(buf, pReq->igExists); + tlen += taosEncodeString(buf, pReq->phyPlan); + return tlen; +} + +static FORCE_INLINE void* tDeserializeSCMCreateTopicReq(void* buf, SCMCreateTopicReq* pReq) { + buf = taosDecodeFixedI8(buf, &(pReq->igExists)); + buf = taosDecodeString(buf, &(pReq->name)); + buf = taosDecodeString(buf, &(pReq->phyPlan)); + return buf; +} + typedef struct { char name[TSDB_TOPIC_FNAME_LEN]; int8_t igExists; diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index f2f3fcd49b..c7d637ee8a 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -46,7 +46,7 @@ int32_t qParseQuerySql(SParseContext* pContext, SQueryNode** pQuery); bool qIsDdlQuery(const SQueryNode* pQuery); -void qDestoryQuery(SQueryNode* pQuery); +void qDestroyQuery(SQueryNode* pQuery); /** * Convert a normal sql statement to only query tags information to enable that the subscribe client can be aware quickly of the true vgroup ids that @@ -86,4 +86,4 @@ void addIntoSourceParam(SSourceParam* pSourceParam, tExprNode* pNode, SColumn* p } #endif -#endif /*_TD_PARSER_H_*/ \ No newline at end of file +#endif /*_TD_PARSER_H_*/ diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 0ee99f77aa..6220d7eb50 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -212,6 +212,60 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag, void** pJob) { return scheduleAsyncExecJob(pRequest->pTscObj->pTransporter, NULL/*todo appInfo.xxx*/, pDag, pJob); } +TAOS_RES *tmq_create_topic(TAOS* taos, const char* name, const char* sql, int sqlLen) { + STscObj* pTscObj = (STscObj*)taos; + SRequestObj* pRequest = NULL; + SQueryNode* pQuery = NULL; + SQueryDag* pDag = NULL; + char *dagStr = NULL; + + //parse sql to logical plan and physical plan + //send topic name and plans to mnode + + terrno = TSDB_CODE_SUCCESS; + + CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return); + CHECK_CODE_GOTO(parseSql(pRequest, &pQuery), _return); + //TODO: check sql valid + + CHECK_CODE_GOTO(qCreateQueryDag(pQuery, &pDag), _return); + + dagStr = qDagToString(pDag); + if(dagStr == NULL) { + //TODO + } + + SCMCreateTopicReq req = { + .name = (char*)name, + .igExists = 0, + .phyPlan = dagStr, + }; + + void* buf = NULL; + int tlen = tSerializeSCMCreateTopicReq(&buf, &req); + + pRequest->body.requestMsg = (SDataBuf){ .pData = buf, .len = tlen }; + + SMsgSendInfo* body = buildSendMsgInfoImpl(pRequest); + SEpSet* pEpSet = &pTscObj->pAppInfo->mgmtEp.epSet; + + int64_t transporterId = 0; + asyncSendMsgToServer(pTscObj->pTransporter, pEpSet, &transporterId, body); + + tsem_wait(&pRequest->body.rspSem); + + destroySendMsgInfo(body); + +_return: + qDestroyQuery(pQuery); + qDestroyQueryDag(pDag); + destroySendMsgInfo(body); + if (pRequest != NULL && terrno != TSDB_CODE_SUCCESS) { + pRequest->code = terrno; + } + return pRequest; +} + TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) { STscObj *pTscObj = (STscObj *)taos; if (sqlLen > (size_t) tsMaxSQLStringLen) { @@ -239,7 +293,7 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) { } _return: - qDestoryQuery(pQuery); + qDestroyQuery(pQuery); qDestroyQueryDag(pDag); if (NULL != pRequest && TSDB_CODE_SUCCESS != terrno) { pRequest->code = terrno; From 2340bbd7fa7a8b4010508cecf9f1a0278ea9c041 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 28 Dec 2021 17:25:44 +0800 Subject: [PATCH 18/21] catalog --- source/libs/catalog/inc/catalogInt.h | 1 - source/libs/catalog/src/catalog.c | 10 +++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 548b25efc8..d2799cbe65 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -46,7 +46,6 @@ typedef struct STableMetaCache { } STableMetaCache; typedef struct SCatalog { - SVgroupListCache vgroupCache; SDBVgroupCache dbCache; STableMetaCache tableCache; } SCatalog; diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index fda3cddf70..49f4fdbfb7 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -415,16 +415,16 @@ int32_t catalogGetHandle(const char* clusterId , struct SCatalog** catalogHandle } size_t clen = strlen(clusterId); - SCatalog *clusterCtg = (SCatalog *)taosHashGet(ctgMgmt.pCluster, clusterId, clen); + SCatalog **ctg = (SCatalog **)taosHashGet(ctgMgmt.pCluster, clusterId, clen); - if (clusterCtg) { - *catalogHandle = clusterCtg; + if (ctg && (*ctg)) { + *catalogHandle = *ctg; return TSDB_CODE_SUCCESS; } - clusterCtg = calloc(1, sizeof(*clusterCtg)); + SCatalog *clusterCtg = calloc(1, sizeof(SCatalog)); if (NULL == clusterCtg) { - ctgError("calloc %d failed", (int32_t)sizeof(*clusterCtg)); + ctgError("calloc %d failed", (int32_t)sizeof(SCatalog)); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } From cfc899245713637a21fabd44cdc2c7f1983755d4 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 28 Dec 2021 17:32:32 +0800 Subject: [PATCH 19/21] catalog --- source/libs/catalog/inc/catalogInt.h | 2 +- source/libs/catalog/src/catalog.c | 2 +- source/libs/parser/src/dCDAstProcess.c | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index d2799cbe65..820bcdfa3f 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -66,7 +66,7 @@ typedef uint32_t (*tableNameHashFp)(const char *, uint32_t); #define ctgTrace(...) do { if (ctgDebugFlag & DEBUG_TRACE) { taosPrintLog("CTG ", ctgDebugFlag, __VA_ARGS__); }} while(0) #define ctgDebugL(...) do { if (ctgDebugFlag & DEBUG_DEBUG) { taosPrintLongString("CTG ", ctgDebugFlag, __VA_ARGS__); }} while(0) -#define CTG_CACHE_ENABLED(cfg) ((cfg)->maxDBCacheNum > 0 || (cfg)->maxTblCacheNum > 0) +#define CTG_CACHE_ENABLED() (ctgMgmt.cfg.maxDBCacheNum > 0 || ctgMgmt.cfg.maxTblCacheNum > 0) #define CTG_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) #define CTG_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 49f4fdbfb7..a65f471cfd 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -394,7 +394,7 @@ int32_t catalogInit(SCatalogCfg *cfg) { ctgMgmt.cfg.maxTblCacheNum = CTG_DEFAULT_CACHE_TABLEMETA_NUMBER; } - if (CTG_CACHE_ENABLED(&ctgMgmt.cfg)) { + if (CTG_CACHE_ENABLED()) { ctgMgmt.pCluster = taosHashInit(CTG_DEFAULT_CACHE_CLUSTER_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); if (NULL == ctgMgmt.pCluster) { CTG_ERR_LRET(TSDB_CODE_CTG_INTERNAL_ERROR, "init %d cluster cache failed", CTG_DEFAULT_CACHE_CLUSTER_NUMBER); diff --git a/source/libs/parser/src/dCDAstProcess.c b/source/libs/parser/src/dCDAstProcess.c index 7ff9596045..d1604d99d6 100644 --- a/source/libs/parser/src/dCDAstProcess.c +++ b/source/libs/parser/src/dCDAstProcess.c @@ -313,7 +313,7 @@ int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx *pCtx, SMsgBuf* p return code; } - code = tNameGetTableName(&name, pCreateTableInfo->tagdata.name); + const char* pSTableName = tNameGetTableName(&name); SArray* pValList = pCreateTableInfo->pTagVals; if (code != TSDB_CODE_SUCCESS) { @@ -326,7 +326,7 @@ int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx *pCtx, SMsgBuf* p char dbName[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(&name, dbName); - catalogGetTableMeta(pCtx->pCatalog, pCtx->pTransporter, &pCtx->mgmtEpSet, dbName, pCreateTableInfo->tagdata.name, &pSuperTableMeta); + catalogGetTableMeta(pCtx->pCatalog, pCtx->pTransporter, &pCtx->mgmtEpSet, dbName, pSTableName, &pSuperTableMeta); // too long tag values will return invalid sql, not be truncated automatically SSchema *pTagSchema = getTableTagSchema(pSuperTableMeta); From 041e6716010b2e641e076c2ed2b566200392bc38 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 28 Dec 2021 01:38:15 -0800 Subject: [PATCH 20/21] refact mnode queue --- include/dnode/mnode/mnode.h | 20 +- include/dnode/snode/snode.h | 2 +- source/dnode/mgmt/impl/inc/dndInt.h | 32 +- source/dnode/mgmt/impl/src/dndBnode.c | 22 +- source/dnode/mgmt/impl/src/dndMnode.c | 368 ++++---------------- source/dnode/mgmt/impl/src/dndQnode.c | 18 +- source/dnode/mgmt/impl/src/dndSnode.c | 22 +- source/dnode/mgmt/impl/src/dndWorker.c | 19 +- source/dnode/mgmt/impl/test/dnode/dnode.cpp | 2 +- source/dnode/mnode/impl/src/mndDnode.c | 2 +- source/dnode/mnode/impl/src/mnode.c | 8 +- source/libs/parser/src/astToMsg.c | 1 + tests/script/unique/dnode/basic1.sim | 11 + 13 files changed, 160 insertions(+), 367 deletions(-) diff --git a/include/dnode/mnode/mnode.h b/include/dnode/mnode/mnode.h index e0619b2133..a288e3e630 100644 --- a/include/dnode/mnode/mnode.h +++ b/include/dnode/mnode/mnode.h @@ -147,28 +147,12 @@ void mndCleanupMsg(SMnodeMsg *pMsg); void mndSendRsp(SMnodeMsg *pMsg, int32_t code); /** - * @brief Process the read request. + * @brief Process the read, write, sync request. * * @param pMsg The request msg. * @return int32_t 0 for success, -1 for failure. */ -void mndProcessReadMsg(SMnodeMsg *pMsg); - -/** - * @brief Process the write request. - * - * @param pMsg The request msg. - * @return int32_t 0 for success, -1 for failure. - */ -void mndProcessWriteMsg(SMnodeMsg *pMsg); - -/** - * @brief Process the sync request. - * - * @param pMsg The request msg. - * @return int32_t 0 for success, -1 for failure. - */ -void mndProcessSyncMsg(SMnodeMsg *pMsg); +void mndProcessMsg(SMnodeMsg *pMsg); #ifdef __cplusplus } diff --git a/include/dnode/snode/snode.h b/include/dnode/snode/snode.h index 97069437f2..4913d2572f 100644 --- a/include/dnode/snode/snode.h +++ b/include/dnode/snode/snode.h @@ -79,7 +79,7 @@ int32_t sndGetLoad(SSnode *pSnode, SSnodeLoad *pLoad); * @param pRsp The response message * @return int32_t 0 for success, -1 for failure */ -int32_t sndProcessWriteMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg **pRsp); +int32_t sndProcessMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg **pRsp); /** * @brief Drop a snode. diff --git a/source/dnode/mgmt/impl/inc/dndInt.h b/source/dnode/mgmt/impl/inc/dndInt.h index ff96b7cfdf..954e21aefa 100644 --- a/source/dnode/mgmt/impl/inc/dndInt.h +++ b/source/dnode/mgmt/impl/inc/dndInt.h @@ -65,8 +65,10 @@ typedef struct { void *queueFp; SDnode *pDnode; taos_queue queue; - SWorkerPool pool; - SMWorkerPool mpool; + union { + SWorkerPool pool; + SMWorkerPool mpool; + }; } SDnodeWorker; typedef struct { @@ -95,21 +97,17 @@ typedef struct { } SDnodeMgmt; typedef struct { - int32_t refCount; - int8_t deployed; - int8_t dropped; - int8_t replica; - int8_t selfIndex; - SReplica replicas[TSDB_MAX_REPLICA]; - char *file; - SMnode *pMnode; - SRWLatch latch; - taos_queue pReadQ; - taos_queue pWriteQ; - taos_queue pSyncQ; - SWorkerPool readPool; - SWorkerPool writePool; - SWorkerPool syncPool; + int32_t refCount; + int8_t deployed; + int8_t dropped; + SMnode *pMnode; + SRWLatch latch; + SDnodeWorker readWorker; + SDnodeWorker writeWorker; + SDnodeWorker syncWorker; + int8_t replica; + int8_t selfIndex; + SReplica replicas[TSDB_MAX_REPLICA]; } SMnodeMgmt; typedef struct { diff --git a/source/dnode/mgmt/impl/src/dndBnode.c b/source/dnode/mgmt/impl/src/dndBnode.c index b978c1102f..992f6ac0a1 100644 --- a/source/dnode/mgmt/impl/src/dndBnode.c +++ b/source/dnode/mgmt/impl/src/dndBnode.c @@ -140,20 +140,22 @@ static int32_t dndWriteBnodeFile(SDnode *pDnode) { fclose(fp); free(content); - if (taosRenameFile(file, file) != 0) { + char realfile[PATH_MAX + 20]; + snprintf(realfile, PATH_MAX + 20, "%s/bnode.json", pDnode->dir.dnode); + + if (taosRenameFile(file, realfile) != 0) { terrno = TSDB_CODE_DND_BNODE_WRITE_FILE_ERROR; dError("failed to rename %s since %s", file, terrstr()); return -1; } - dInfo("successed to write %s, deployed:%d dropped:%d", file, pMgmt->deployed, pMgmt->dropped); + dInfo("successed to write %s, deployed:%d dropped:%d", realfile, pMgmt->deployed, pMgmt->dropped); return 0; } static int32_t dndStartBnodeWorker(SDnode *pDnode) { SBnodeMgmt *pMgmt = &pDnode->bmgmt; - if (dndInitWorker(pDnode, &pMgmt->writeWorker, DND_WORKER_SINGLE, "bnode-write", 0, 1, - (FProcessItem)dndProcessBnodeQueue) != 0) { + if (dndInitWorker(pDnode, &pMgmt->writeWorker, DND_WORKER_MULTI, "bnode-write", 0, 1, dndProcessBnodeQueue) != 0) { dError("failed to start bnode write worker since %s", terrstr()); return -1; } @@ -202,7 +204,9 @@ static int32_t dndOpenBnode(SDnode *pDnode) { return -1; } + pMgmt->deployed = 1; if (dndWriteBnodeFile(pDnode) != 0) { + pMgmt->deployed = 0; dError("failed to write bnode file since %s", terrstr()); dndStopBnodeWorker(pDnode); bndClose(pBnode); @@ -211,7 +215,6 @@ static int32_t dndOpenBnode(SDnode *pDnode) { taosWLockLatch(&pMgmt->latch); pMgmt->pBnode = pBnode; - pMgmt->deployed = 1; taosWUnLockLatch(&pMgmt->latch); dInfo("bnode open successfully"); @@ -243,6 +246,8 @@ static int32_t dndDropBnode(SDnode *pDnode) { dndReleaseBnode(pDnode, pBnode); dndStopBnodeWorker(pDnode); + pMgmt->deployed = 0; + dndWriteBnodeFile(pDnode); bndClose(pBnode); pMgmt->pBnode = NULL; bndDestroy(pDnode->dir.bnode); @@ -353,7 +358,12 @@ int32_t dndInitBnode(SDnode *pDnode) { return -1; } - if (pMgmt->dropped) return 0; + if (pMgmt->dropped) { + dInfo("bnode has been deployed and needs to be deleted"); + bndDestroy(pDnode->dir.bnode); + return 0; + } + if (!pMgmt->deployed) return 0; return dndOpenBnode(pDnode); diff --git a/source/dnode/mgmt/impl/src/dndMnode.c b/source/dnode/mgmt/impl/src/dndMnode.c index 59f809489e..8fb95c0b75 100644 --- a/source/dnode/mgmt/impl/src/dndMnode.c +++ b/source/dnode/mgmt/impl/src/dndMnode.c @@ -17,42 +17,9 @@ #include "dndMnode.h" #include "dndDnode.h" #include "dndTransport.h" +#include "dndWorker.h" -static int32_t dndInitMnodeReadWorker(SDnode *pDnode); -static int32_t dndInitMnodeWriteWorker(SDnode *pDnode); -static int32_t dndInitMnodeSyncWorker(SDnode *pDnode); -static void dndCleanupMnodeReadWorker(SDnode *pDnode); -static void dndCleanupMnodeWriteWorker(SDnode *pDnode); -static void dndCleanupMnodeSyncWorker(SDnode *pDnode); -static void dndCleanupMnodeMgmtWorker(SDnode *pDnode); -static int32_t dndAllocMnodeReadQueue(SDnode *pDnode); -static int32_t dndAllocMnodeWriteQueue(SDnode *pDnode); -static int32_t dndAllocMnodeSyncQueue(SDnode *pDnode); -static void dndFreeMnodeReadQueue(SDnode *pDnode); -static void dndFreeMnodeWriteQueue(SDnode *pDnode); -static void dndFreeMnodeSyncQueue(SDnode *pDnode); -static void dndFreeMnodeMgmtQueue(SDnode *pDnode); - -static void dndProcessMnodeReadQueue(SDnode *pDnode, SMnodeMsg *pMsg); -static void dndProcessMnodeWriteQueue(SDnode *pDnode, SMnodeMsg *pMsg); -static void dndProcessMnodeSyncQueue(SDnode *pDnode, SMnodeMsg *pMsg); -static int32_t dndWriteMnodeMsgToQueue(SMnode *pMnode, taos_queue pQueue, SRpcMsg *pRpcMsg); -void dndProcessMnodeReadMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -void dndProcessMnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -void dndProcessMnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); - -static int32_t dndStartMnodeWorker(SDnode *pDnode); -static void dndStopMnodeWorker(SDnode *pDnode); - -static SMnode *dndAcquireMnode(SDnode *pDnode); -static void dndReleaseMnode(SDnode *pDnode, SMnode *pMnode); - -static int32_t dndReadMnodeFile(SDnode *pDnode); -static int32_t dndWriteMnodeFile(SDnode *pDnode); - -static int32_t dndOpenMnode(SDnode *pDnode, SMnodeOpt *pOption); -static int32_t dndAlterMnode(SDnode *pDnode, SMnodeOpt *pOption); -static int32_t dndDropMnode(SDnode *pDnode); +static void dndProcessMnodeQueue(SDnode *pDnode, SMnodeMsg *pMsg); static SMnode *dndAcquireMnode(SDnode *pDnode) { SMnodeMgmt *pMgmt = &pDnode->mmgmt; @@ -97,49 +64,52 @@ static int32_t dndReadMnodeFile(SDnode *pDnode) { char *content = calloc(1, maxLen + 1); cJSON *root = NULL; - FILE *fp = fopen(pMgmt->file, "r"); + char file[PATH_MAX + 20]; + snprintf(file, PATH_MAX + 20, "%s/mnode.json", pDnode->dir.dnode); + + FILE *fp = fopen(file, "r"); if (fp == NULL) { - dDebug("file %s not exist", pMgmt->file); + dDebug("file %s not exist", file); code = 0; goto PRASE_MNODE_OVER; } len = (int32_t)fread(content, 1, maxLen, fp); if (len <= 0) { - dError("failed to read %s since content is null", pMgmt->file); + dError("failed to read %s since content is null", file); goto PRASE_MNODE_OVER; } content[len] = 0; root = cJSON_Parse(content); if (root == NULL) { - dError("failed to read %s since invalid json format", pMgmt->file); + dError("failed to read %s since invalid json format", file); goto PRASE_MNODE_OVER; } cJSON *deployed = cJSON_GetObjectItem(root, "deployed"); if (!deployed || deployed->type != cJSON_Number) { - dError("failed to read %s since deployed not found", pMgmt->file); + dError("failed to read %s since deployed not found", file); goto PRASE_MNODE_OVER; } pMgmt->deployed = deployed->valueint; cJSON *dropped = cJSON_GetObjectItem(root, "dropped"); if (!dropped || dropped->type != cJSON_Number) { - dError("failed to read %s since dropped not found", pMgmt->file); + dError("failed to read %s since dropped not found", file); goto PRASE_MNODE_OVER; } pMgmt->dropped = dropped->valueint; cJSON *mnodes = cJSON_GetObjectItem(root, "mnodes"); if (!mnodes || mnodes->type != cJSON_Array) { - dError("failed to read %s since nodes not found", pMgmt->file); + dError("failed to read %s since nodes not found", file); goto PRASE_MNODE_OVER; } pMgmt->replica = cJSON_GetArraySize(mnodes); if (pMgmt->replica <= 0 || pMgmt->replica > TSDB_MAX_REPLICA) { - dError("failed to read %s since mnodes size %d invalid", pMgmt->file, pMgmt->replica); + dError("failed to read %s since mnodes size %d invalid", file, pMgmt->replica); goto PRASE_MNODE_OVER; } @@ -151,28 +121,28 @@ static int32_t dndReadMnodeFile(SDnode *pDnode) { cJSON *id = cJSON_GetObjectItem(node, "id"); if (!id || id->type != cJSON_Number) { - dError("failed to read %s since id not found", pMgmt->file); + dError("failed to read %s since id not found", file); goto PRASE_MNODE_OVER; } pReplica->id = id->valueint; cJSON *fqdn = cJSON_GetObjectItem(node, "fqdn"); if (!fqdn || fqdn->type != cJSON_String || fqdn->valuestring == NULL) { - dError("failed to read %s since fqdn not found", pMgmt->file); + dError("failed to read %s since fqdn not found", file); goto PRASE_MNODE_OVER; } tstrncpy(pReplica->fqdn, fqdn->valuestring, TSDB_FQDN_LEN); cJSON *port = cJSON_GetObjectItem(node, "port"); if (!port || port->type != cJSON_Number) { - dError("failed to read %s since port not found", pMgmt->file); + dError("failed to read %s since port not found", file); goto PRASE_MNODE_OVER; } pReplica->port = port->valueint; } code = 0; - dDebug("succcessed to read file %s, deployed:%d dropped:%d", pMgmt->file, pMgmt->deployed, pMgmt->dropped); + dDebug("succcessed to read file %s, deployed:%d dropped:%d", file, pMgmt->deployed, pMgmt->dropped); PRASE_MNODE_OVER: if (content != NULL) free(content); @@ -186,8 +156,8 @@ PRASE_MNODE_OVER: static int32_t dndWriteMnodeFile(SDnode *pDnode) { SMnodeMgmt *pMgmt = &pDnode->mmgmt; - char file[PATH_MAX + 20] = {0}; - snprintf(file, sizeof(file), "%s.bak", pMgmt->file); + char file[PATH_MAX + 20]; + snprintf(file, PATH_MAX + 20, "%s/mnode.json.bak", pDnode->dir.dnode); FILE *fp = fopen(file, "w"); if (fp == NULL) { @@ -223,47 +193,36 @@ static int32_t dndWriteMnodeFile(SDnode *pDnode) { fclose(fp); free(content); - if (taosRenameFile(file, pMgmt->file) != 0) { + char realfile[PATH_MAX + 20]; + snprintf(realfile, PATH_MAX + 20, "%s/mnode.json", pDnode->dir.dnode); + + if (taosRenameFile(file, realfile) != 0) { terrno = TSDB_CODE_DND_MNODE_WRITE_FILE_ERROR; - dError("failed to rename %s since %s", pMgmt->file, terrstr()); + dError("failed to rename %s since %s", file, terrstr()); return -1; } - dInfo("successed to write %s, deployed:%d dropped:%d", pMgmt->file, pMgmt->deployed, pMgmt->dropped); + dInfo("successed to write %s, deployed:%d dropped:%d", realfile, pMgmt->deployed, pMgmt->dropped); return 0; } static int32_t dndStartMnodeWorker(SDnode *pDnode) { - if (dndInitMnodeReadWorker(pDnode) != 0) { + SMnodeMgmt *pMgmt = &pDnode->mmgmt; + if (dndInitWorker(pDnode, &pMgmt->readWorker, DND_WORKER_SINGLE, "mnode-read", 0, 1, dndProcessMnodeQueue) != 0) { dError("failed to start mnode read worker since %s", terrstr()); return -1; } - if (dndInitMnodeWriteWorker(pDnode) != 0) { + if (dndInitWorker(pDnode, &pMgmt->writeWorker, DND_WORKER_SINGLE, "mnode-write", 0, 1, dndProcessMnodeQueue) != 0) { dError("failed to start mnode write worker since %s", terrstr()); return -1; } - if (dndInitMnodeSyncWorker(pDnode) != 0) { + if (dndInitWorker(pDnode, &pMgmt->syncWorker, DND_WORKER_SINGLE, "mnode-sync", 0, 1, dndProcessMnodeQueue) != 0) { dError("failed to start mnode sync worker since %s", terrstr()); return -1; } - if (dndAllocMnodeReadQueue(pDnode) != 0) { - dError("failed to alloc mnode read queue since %s", terrstr()); - return -1; - } - - if (dndAllocMnodeWriteQueue(pDnode) != 0) { - dError("failed to alloc mnode write queue since %s", terrstr()); - return -1; - } - - if (dndAllocMnodeSyncQueue(pDnode) != 0) { - dError("failed to alloc mnode sync queue since %s", terrstr()); - return -1; - } - return 0; } @@ -274,18 +233,13 @@ static void dndStopMnodeWorker(SDnode *pDnode) { pMgmt->deployed = 0; taosWUnLockLatch(&pMgmt->latch); - while (pMgmt->refCount > 1) taosMsleep(10); - while (!taosQueueEmpty(pMgmt->pReadQ)) taosMsleep(10); - while (!taosQueueEmpty(pMgmt->pWriteQ)) taosMsleep(10); - while (!taosQueueEmpty(pMgmt->pSyncQ)) taosMsleep(10); + while (pMgmt->refCount > 1) { + taosMsleep(10); + } - dndCleanupMnodeReadWorker(pDnode); - dndCleanupMnodeWriteWorker(pDnode); - dndCleanupMnodeSyncWorker(pDnode); - - dndFreeMnodeReadQueue(pDnode); - dndFreeMnodeWriteQueue(pDnode); - dndFreeMnodeSyncQueue(pDnode); + dndCleanupWorker(&pMgmt->readWorker); + dndCleanupWorker(&pMgmt->writeWorker); + dndCleanupWorker(&pMgmt->syncWorker); } static bool dndNeedDeployMnode(SDnode *pDnode) { @@ -383,28 +337,21 @@ static int32_t dndOpenMnode(SDnode *pDnode, SMnodeOpt *pOption) { dError("failed to open mnode since %s", terrstr()); return -1; } - pMgmt->deployed = 1; - int32_t code = dndWriteMnodeFile(pDnode); - if (code != 0) { - dError("failed to write mnode file since %s", terrstr()); - code = terrno; - pMgmt->deployed = 0; + if (dndStartMnodeWorker(pDnode) != 0) { + dError("failed to start mnode worker since %s", terrstr()); mndClose(pMnode); mndDestroy(pDnode->dir.mnode); - terrno = code; return -1; } - code = dndStartMnodeWorker(pDnode); - if (code != 0) { - dError("failed to start mnode worker since %s", terrstr()); - code = terrno; + pMgmt->deployed = 1; + if (dndWriteMnodeFile(pDnode) != 0) { + dError("failed to write mnode file since %s", terrstr()); pMgmt->deployed = 0; dndStopMnodeWorker(pDnode); mndClose(pMnode); mndDestroy(pDnode->dir.mnode); - terrno = code; return -1; } @@ -461,6 +408,7 @@ static int32_t dndDropMnode(SDnode *pDnode) { dndReleaseMnode(pDnode, pMnode); dndStopMnodeWorker(pDnode); + pMgmt->deployed = 0; dndWriteMnodeFile(pDnode); mndClose(pMnode); pMgmt->pMnode = NULL; @@ -528,13 +476,12 @@ int32_t dndProcessDropMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { } } - -static void dndProcessMnodeReadQueue(SDnode *pDnode, SMnodeMsg *pMsg) { +static void dndProcessMnodeQueue(SDnode *pDnode, SMnodeMsg *pMsg) { SMnodeMgmt *pMgmt = &pDnode->mmgmt; SMnode *pMnode = dndAcquireMnode(pDnode); if (pMnode != NULL) { - mndProcessReadMsg(pMsg); + mndProcessMsg(pMsg); dndReleaseMnode(pDnode, pMnode); } else { mndSendRsp(pMsg, terrno); @@ -543,208 +490,43 @@ static void dndProcessMnodeReadQueue(SDnode *pDnode, SMnodeMsg *pMsg) { mndCleanupMsg(pMsg); } -static void dndProcessMnodeWriteQueue(SDnode *pDnode, SMnodeMsg *pMsg) { - SMnodeMgmt *pMgmt = &pDnode->mmgmt; +static void dndWriteMnodeMsgToWorker(SDnode *pDnode, SDnodeWorker *pWorker, SRpcMsg *pRpcMsg) { + int32_t code = TSDB_CODE_DND_MNODE_NOT_DEPLOYED; SMnode *pMnode = dndAcquireMnode(pDnode); if (pMnode != NULL) { - mndProcessWriteMsg(pMsg); - dndReleaseMnode(pDnode, pMnode); - } else { - mndSendRsp(pMsg, terrno); + SMnodeMsg *pMsg = mndInitMsg(pMnode, pRpcMsg); + if (pMsg == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + } else { + code = dndWriteMsgToWorker(pWorker, pMsg, 0); + } + + if (code != 0) { + mndCleanupMsg(pMsg); + } } + dndReleaseMnode(pDnode, pMnode); - mndCleanupMsg(pMsg); -} - -static void dndProcessMnodeSyncQueue(SDnode *pDnode, SMnodeMsg *pMsg) { - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - - SMnode *pMnode = dndAcquireMnode(pDnode); - if (pMnode != NULL) { - mndProcessSyncMsg(pMsg); - dndReleaseMnode(pDnode, pMnode); - } else { - mndSendRsp(pMsg, terrno); + if (code != 0) { + if (pRpcMsg->msgType & 1u) { + SRpcMsg rsp = {.handle = pRpcMsg->handle, .ahandle = pRpcMsg->ahandle, .code = code}; + rpcSendResponse(&rsp); + } + rpcFreeCont(pRpcMsg->pCont); } - - mndCleanupMsg(pMsg); -} - -static int32_t dndWriteMnodeMsgToQueue(SMnode *pMnode, taos_queue pQueue, SRpcMsg *pRpcMsg) { - SMnodeMsg *pMsg = mndInitMsg(pMnode, pRpcMsg); - if (pMsg == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - if (taosWriteQitem(pQueue, pMsg) != 0) { - mndCleanupMsg(pMsg); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - return 0; } void dndProcessMnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - SMnode *pMnode = dndAcquireMnode(pDnode); - if (pMnode == NULL || dndWriteMnodeMsgToQueue(pMnode, pMgmt->pWriteQ, pMsg) != 0) { - if (pMsg->msgType & 1u) { - SRpcMsg rsp = {.handle = pMsg->handle, .code = terrno}; - rpcSendResponse(&rsp); - } - rpcFreeCont(pMsg->pCont); - pMsg->pCont = NULL; - } - - dndReleaseMnode(pDnode, pMnode); + dndWriteMnodeMsgToWorker(pDnode, &pDnode->mmgmt.writeWorker, pMsg); } void dndProcessMnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - SMnode *pMnode = dndAcquireMnode(pDnode); - if (pMnode == NULL || dndWriteMnodeMsgToQueue(pMnode, pMgmt->pSyncQ, pMsg) != 0) { - if (pMsg->msgType & 1u) { - SRpcMsg rsp = {.handle = pMsg->handle, .code = terrno}; - rpcSendResponse(&rsp); - } - rpcFreeCont(pMsg->pCont); - pMsg->pCont = NULL; - } - - dndReleaseMnode(pDnode, pMnode); + dndWriteMnodeMsgToWorker(pDnode, &pDnode->mmgmt.syncWorker, pMsg); } void dndProcessMnodeReadMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - SMnode *pMnode = dndAcquireMnode(pDnode); - if (pMnode == NULL || dndWriteMnodeMsgToQueue(pMnode, pMgmt->pReadQ, pMsg) != 0) { - if (pMsg->msgType & 1u) { - SRpcMsg rsp = {.handle = pMsg->handle, .code = terrno}; - rpcSendResponse(&rsp); - } - rpcFreeCont(pMsg->pCont); - pMsg->pCont = NULL; - } - - dndReleaseMnode(pDnode, pMnode); -} - - -static int32_t dndAllocMnodeReadQueue(SDnode *pDnode) { - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - pMgmt->pReadQ = tWorkerAllocQueue(&pMgmt->readPool, pDnode, (FProcessItem)dndProcessMnodeReadQueue); - if (pMgmt->pReadQ == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - return 0; -} - -static void dndFreeMnodeReadQueue(SDnode *pDnode) { - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - tWorkerFreeQueue(&pMgmt->readPool, pMgmt->pReadQ); - pMgmt->pReadQ = NULL; -} - -static int32_t dndInitMnodeReadWorker(SDnode *pDnode) { - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - SWorkerPool *pPool = &pMgmt->readPool; - pPool->name = "mnode-read"; - pPool->min = 0; - pPool->max = 1; - if (tWorkerInit(pPool) != 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - dDebug("mnode read worker is initialized"); - return 0; -} - -static void dndCleanupMnodeReadWorker(SDnode *pDnode) { - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - tWorkerCleanup(&pMgmt->readPool); - dDebug("mnode read worker is closed"); -} - -static int32_t dndAllocMnodeWriteQueue(SDnode *pDnode) { - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - pMgmt->pWriteQ = tWorkerAllocQueue(&pMgmt->writePool, pDnode, (FProcessItem)dndProcessMnodeWriteQueue); - if (pMgmt->pWriteQ == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - return 0; -} - -static void dndFreeMnodeWriteQueue(SDnode *pDnode) { - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - tWorkerFreeQueue(&pMgmt->writePool, pMgmt->pWriteQ); - pMgmt->pWriteQ = NULL; -} - -static int32_t dndInitMnodeWriteWorker(SDnode *pDnode) { - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - SWorkerPool *pPool = &pMgmt->writePool; - pPool->name = "mnode-write"; - pPool->min = 0; - pPool->max = 1; - if (tWorkerInit(pPool) != 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - dDebug("mnode write worker is initialized"); - return 0; -} - -static void dndCleanupMnodeWriteWorker(SDnode *pDnode) { - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - tWorkerCleanup(&pMgmt->writePool); - dDebug("mnode write worker is closed"); -} - -static int32_t dndAllocMnodeSyncQueue(SDnode *pDnode) { - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - pMgmt->pSyncQ = tWorkerAllocQueue(&pMgmt->syncPool, pDnode, (FProcessItem)dndProcessMnodeSyncQueue); - if (pMgmt->pSyncQ == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - return 0; -} - -static void dndFreeMnodeSyncQueue(SDnode *pDnode) { - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - tWorkerFreeQueue(&pMgmt->syncPool, pMgmt->pSyncQ); - pMgmt->pSyncQ = NULL; -} - -static int32_t dndInitMnodeSyncWorker(SDnode *pDnode) { - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - SWorkerPool *pPool = &pMgmt->syncPool; - pPool->name = "mnode-sync"; - pPool->min = 0; - pPool->max = 1; - if (tWorkerInit(pPool) != 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - dDebug("mnode sync worker is initialized"); - return 0; -} - -static void dndCleanupMnodeSyncWorker(SDnode *pDnode) { - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - tWorkerCleanup(&pMgmt->syncPool); - dDebug("mnode sync worker is closed"); + dndWriteMnodeMsgToWorker(pDnode, &pDnode->mmgmt.readWorker, pMsg); } int32_t dndInitMnode(SDnode *pDnode) { @@ -752,14 +534,6 @@ int32_t dndInitMnode(SDnode *pDnode) { SMnodeMgmt *pMgmt = &pDnode->mmgmt; taosInitRWLatch(&pMgmt->latch); - char path[PATH_MAX]; - snprintf(path, PATH_MAX, "%s/mnode.json", pDnode->dir.dnode); - pMgmt->file = strdup(path); - if (pMgmt->file == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - if (dndReadMnodeFile(pDnode) != 0) { return -1; } @@ -790,13 +564,13 @@ int32_t dndInitMnode(SDnode *pDnode) { } void dndCleanupMnode(SDnode *pDnode) { - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - dInfo("dnode-mnode start to clean up"); - if (pMgmt->pMnode) dndStopMnodeWorker(pDnode); - tfree(pMgmt->file); - mndClose(pMgmt->pMnode); - pMgmt->pMnode = NULL; + SMnodeMgmt *pMgmt = &pDnode->mmgmt; + if (pMgmt->pMnode) { + dndStopMnodeWorker(pDnode); + mndClose(pMgmt->pMnode); + pMgmt->pMnode = NULL; + } dInfo("dnode-mnode is cleaned up"); } diff --git a/source/dnode/mgmt/impl/src/dndQnode.c b/source/dnode/mgmt/impl/src/dndQnode.c index 0b92de81c1..5d04a4f449 100644 --- a/source/dnode/mgmt/impl/src/dndQnode.c +++ b/source/dnode/mgmt/impl/src/dndQnode.c @@ -140,26 +140,27 @@ static int32_t dndWriteQnodeFile(SDnode *pDnode) { fclose(fp); free(content); - if (taosRenameFile(file, file) != 0) { + char realfile[PATH_MAX + 20]; + snprintf(realfile, PATH_MAX + 20, "%s/qnode.json", pDnode->dir.dnode); + + if (taosRenameFile(file, realfile) != 0) { terrno = TSDB_CODE_DND_QNODE_WRITE_FILE_ERROR; dError("failed to rename %s since %s", file, terrstr()); return -1; } - dInfo("successed to write %s, deployed:%d dropped:%d", file, pMgmt->deployed, pMgmt->dropped); + dInfo("successed to write %s, deployed:%d dropped:%d", realfile, pMgmt->deployed, pMgmt->dropped); return 0; } static int32_t dndStartQnodeWorker(SDnode *pDnode) { SQnodeMgmt *pMgmt = &pDnode->qmgmt; - if (dndInitWorker(pDnode, &pMgmt->queryWorker, DND_WORKER_SINGLE, "qnode-query", 0, 1, - (FProcessItem)dndProcessQnodeQueue) != 0) { + if (dndInitWorker(pDnode, &pMgmt->queryWorker, DND_WORKER_SINGLE, "qnode-query", 0, 1, dndProcessQnodeQueue) != 0) { dError("failed to start qnode query worker since %s", terrstr()); return -1; } - if (dndInitWorker(pDnode, &pMgmt->fetchWorker, DND_WORKER_SINGLE, "qnode-fetch", 0, 1, - (FProcessItem)dndProcessQnodeQueue) != 0) { + if (dndInitWorker(pDnode, &pMgmt->fetchWorker, DND_WORKER_SINGLE, "qnode-fetch", 0, 1, dndProcessQnodeQueue) != 0) { dError("failed to start qnode fetch worker since %s", terrstr()); return -1; } @@ -209,7 +210,9 @@ static int32_t dndOpenQnode(SDnode *pDnode) { return -1; } + pMgmt->deployed = 1; if (dndWriteQnodeFile(pDnode) != 0) { + pMgmt->deployed = 0; dError("failed to write qnode file since %s", terrstr()); dndStopQnodeWorker(pDnode); qndClose(pQnode); @@ -218,7 +221,6 @@ static int32_t dndOpenQnode(SDnode *pDnode) { taosWLockLatch(&pMgmt->latch); pMgmt->pQnode = pQnode; - pMgmt->deployed = 1; taosWUnLockLatch(&pMgmt->latch); dInfo("qnode open successfully"); @@ -250,6 +252,8 @@ static int32_t dndDropQnode(SDnode *pDnode) { dndReleaseQnode(pDnode, pQnode); dndStopQnodeWorker(pDnode); + pMgmt->deployed = 0; + dndWriteQnodeFile(pDnode); qndClose(pQnode); pMgmt->pQnode = NULL; diff --git a/source/dnode/mgmt/impl/src/dndSnode.c b/source/dnode/mgmt/impl/src/dndSnode.c index c1eb347350..151fc7e6a1 100644 --- a/source/dnode/mgmt/impl/src/dndSnode.c +++ b/source/dnode/mgmt/impl/src/dndSnode.c @@ -140,20 +140,22 @@ static int32_t dndWriteSnodeFile(SDnode *pDnode) { fclose(fp); free(content); - if (taosRenameFile(file, file) != 0) { + char realfile[PATH_MAX + 20]; + snprintf(realfile, PATH_MAX + 20, "%s/snode.json", pDnode->dir.dnode); + + if (taosRenameFile(file, realfile) != 0) { terrno = TSDB_CODE_DND_SNODE_WRITE_FILE_ERROR; dError("failed to rename %s since %s", file, terrstr()); return -1; } - dInfo("successed to write %s, deployed:%d dropped:%d", file, pMgmt->deployed, pMgmt->dropped); + dInfo("successed to write %s, deployed:%d dropped:%d", realfile, pMgmt->deployed, pMgmt->dropped); return 0; } static int32_t dndStartSnodeWorker(SDnode *pDnode) { SSnodeMgmt *pMgmt = &pDnode->smgmt; - if (dndInitWorker(pDnode, &pMgmt->writeWorker, DND_WORKER_SINGLE, "snode-write", 0, 1, - (FProcessItem)dndProcessSnodeQueue) != 0) { + if (dndInitWorker(pDnode, &pMgmt->writeWorker, DND_WORKER_SINGLE, "snode-write", 0, 1, dndProcessSnodeQueue) != 0) { dError("failed to start snode write worker since %s", terrstr()); return -1; } @@ -202,7 +204,9 @@ static int32_t dndOpenSnode(SDnode *pDnode) { return -1; } + pMgmt->deployed = 1; if (dndWriteSnodeFile(pDnode) != 0) { + pMgmt->deployed = 0; dError("failed to write snode file since %s", terrstr()); dndStopSnodeWorker(pDnode); sndClose(pSnode); @@ -211,7 +215,6 @@ static int32_t dndOpenSnode(SDnode *pDnode) { taosWLockLatch(&pMgmt->latch); pMgmt->pSnode = pSnode; - pMgmt->deployed = 1; taosWUnLockLatch(&pMgmt->latch); dInfo("snode open successfully"); @@ -243,6 +246,8 @@ static int32_t dndDropSnode(SDnode *pDnode) { dndReleaseSnode(pDnode, pSnode); dndStopSnodeWorker(pDnode); + pMgmt->deployed = 0; + dndWriteSnodeFile(pDnode); sndClose(pSnode); pMgmt->pSnode = NULL; sndDestroy(pDnode->dir.snode); @@ -328,7 +333,12 @@ int32_t dndInitSnode(SDnode *pDnode) { return -1; } - if (pMgmt->dropped) return 0; + if (pMgmt->dropped) { + dInfo("snode has been deployed and needs to be deleted"); + sndDestroy(pDnode->dir.snode); + return 0; + } + if (!pMgmt->deployed) return 0; return dndOpenSnode(pDnode); diff --git a/source/dnode/mgmt/impl/src/dndWorker.c b/source/dnode/mgmt/impl/src/dndWorker.c index c421437e4d..b1107fd185 100644 --- a/source/dnode/mgmt/impl/src/dndWorker.c +++ b/source/dnode/mgmt/impl/src/dndWorker.c @@ -73,7 +73,7 @@ void dndCleanupWorker(SDnodeWorker *pWorker) { tWorkerCleanup(&pWorker->pool); tWorkerFreeQueue(&pWorker->pool, pWorker->queue); } else if (pWorker->type == DND_WORKER_MULTI) { - tWorkerCleanup(&pWorker->mpool); + tMWorkerCleanup(&pWorker->mpool); tMWorkerFreeQueue(&pWorker->mpool, pWorker->queue); } else { } @@ -85,16 +85,23 @@ int32_t dndWriteMsgToWorker(SDnodeWorker *pWorker, void *pCont, int32_t contLen) return -1; } - void *pMsg = taosAllocateQitem(contLen); + void *pMsg = NULL; + if (contLen != 0) { + pMsg = taosAllocateQitem(contLen); + if (pMsg != NULL) { + memcpy(pMsg, pCont, contLen); + } + } else { + pMsg = pCont; + } + if (pMsg == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - memcpy(pMsg, pCont, contLen); - - if (taosWriteQitem(pWorker, pMsg) != 0) { - taosFreeItem(pMsg); + if (taosWriteQitem(pWorker->queue, pMsg) != 0) { + taosFreeQitem(pMsg); terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } diff --git a/source/dnode/mgmt/impl/test/dnode/dnode.cpp b/source/dnode/mgmt/impl/test/dnode/dnode.cpp index dc352c5a3f..ec2c2d9a44 100644 --- a/source/dnode/mgmt/impl/test/dnode/dnode.cpp +++ b/source/dnode/mgmt/impl/test/dnode/dnode.cpp @@ -162,7 +162,7 @@ TEST_F(DndTestDnode, 03_Create_Drop_Restart_Dnode) { SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); strcpy(pReq->fqdn, "localhost"); - pReq->port = htonl(904); + pReq->port = htonl(9044); SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_DNODE, pReq, contLen); ASSERT_NE(pMsg, nullptr); diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 56559cbea1..2d236906e1 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -388,7 +388,7 @@ static int32_t mndCreateDnode(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDnodeMsg * dnodeObj.updateTime = dnodeObj.createdTime; dnodeObj.port = pCreate->port; memcpy(dnodeObj.fqdn, pCreate->fqdn, TSDB_FQDN_LEN); - snprintf(dnodeObj.ep, "%s:%u", dnodeObj.fqdn, dnodeObj.port); + snprintf(dnodeObj.ep, TSDB_EP_LEN, "%s:%u", dnodeObj.fqdn, dnodeObj.port); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pMsg->rpcMsg); if (pTrans == NULL) { diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 64ea85044a..9281e46f4f 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -390,7 +390,7 @@ void mndSendRsp(SMnodeMsg *pMsg, int32_t code) { rpcSendResponse(&rpcRsp); } -static void mndProcessRpcMsg(SMnodeMsg *pMsg) { +void mndProcessMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; int32_t code = 0; tmsg_t msgType = pMsg->rpcMsg.msgType; @@ -451,12 +451,6 @@ void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp) { } } -void mndProcessReadMsg(SMnodeMsg *pMsg) { mndProcessRpcMsg(pMsg); } - -void mndProcessWriteMsg(SMnodeMsg *pMsg) { mndProcessRpcMsg(pMsg); } - -void mndProcessSyncMsg(SMnodeMsg *pMsg) { mndProcessRpcMsg(pMsg); } - uint64_t mndGenerateUid(char *name, int32_t len) { int64_t us = taosGetTimestampUs(); int32_t hashval = MurmurHash3_32(name, len); diff --git a/source/libs/parser/src/astToMsg.c b/source/libs/parser/src/astToMsg.c index 2f80af225a..6c99411b71 100644 --- a/source/libs/parser/src/astToMsg.c +++ b/source/libs/parser/src/astToMsg.c @@ -428,6 +428,7 @@ SDropDnodeMsg *buildDropDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMsgBuf char* end = NULL; SDropDnodeMsg * pDrop = (SDropDnodeMsg *)calloc(1, sizeof(SDropDnodeMsg)); pDrop->dnodeId = strtoll(pzName->z, &end, 10); + pDrop->dnodeId = htonl(pDrop->dnodeId); *len = sizeof(SDropDnodeMsg); if (end - pzName->z != pzName->n) { diff --git a/tests/script/unique/dnode/basic1.sim b/tests/script/unique/dnode/basic1.sim index 730864ef26..49b29a4ac8 100644 --- a/tests/script/unique/dnode/basic1.sim +++ b/tests/script/unique/dnode/basic1.sim @@ -94,5 +94,16 @@ if $rows != 2 then return -1 endi +print =============== drop dnode +sql drop dnode 2; +sql show dnodes; +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT \ No newline at end of file From 97f334c9dd05bbcc0ca9efbf42828a97886a50a6 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 28 Dec 2021 17:51:13 +0800 Subject: [PATCH 21/21] send create topic req --- include/client/taos.h | 3 +++ include/dnode/vnode/tq/tq.h | 2 ++ source/client/src/clientImpl.c | 3 --- source/client/test/clientTests.cpp | 35 ++++++++++++++++++++++++++++++ source/libs/parser/src/parser.c | 2 +- 5 files changed, 41 insertions(+), 4 deletions(-) diff --git a/include/client/taos.h b/include/client/taos.h index 7357478555..4669ca51f7 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -193,6 +193,9 @@ DLL_EXPORT void taos_close_stream(TAOS_STREAM *tstr); DLL_EXPORT int taos_load_table_info(TAOS *taos, const char* tableNameList); DLL_EXPORT TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision); + +DLL_EXPORT TAOS_RES *tmq_create_topic(TAOS* taos, const char* name, const char* sql, int sqlLen); + #ifdef __cplusplus } #endif diff --git a/include/dnode/vnode/tq/tq.h b/include/dnode/vnode/tq/tq.h index b6cb96a57b..5774131377 100644 --- a/include/dnode/vnode/tq/tq.h +++ b/include/dnode/vnode/tq/tq.h @@ -251,6 +251,8 @@ typedef struct STqMetaStore { STqMetaList* bucket[TQ_BUCKET_SIZE]; // a table head STqMetaList* unpersistHead; + // topics that are not connectted + STqMetaList* unconnectTopic; // TODO:temporaral use, to be replaced by unified tfile int fileFd; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index c3c2f70b33..b40c718d51 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -237,9 +237,6 @@ TAOS_RES *tmq_create_topic(TAOS* taos, const char* name, const char* sql, int sq SQueryDag* pDag = NULL; char *dagStr = NULL; - //parse sql to logical plan and physical plan - //send topic name and plans to mnode - terrno = TSDB_CODE_SUCCESS; CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return); diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 83d0e61eb3..de494cb031 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -399,6 +399,41 @@ TEST(testCase, drop_stable_Test) { taos_close(pConn); } +TEST(testCase, create_topic_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "create database abc1"); + if (taos_errno(pRes) != 0) { + printf("error in create db, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("error in use db, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create stable st1(ts timestamp, k int) tags(a int)"); + if (taos_errno(pRes) != 0) { + printf("error in create stable, reason:%s\n", taos_errstr(pRes)); + } + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + ASSERT_TRUE(pFields == NULL); + + int32_t numOfFields = taos_num_fields(pRes); + ASSERT_EQ(numOfFields, 0); + + taos_free_result(pRes); + + char* sql = "select * from st1"; + tmq_create_topic(pConn, "test_topic_1", sql, strlen(sql)); + taos_close(pConn); +} + + //TEST(testCase, show_table_Test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // assert(pConn != NULL); diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 5c9a48e52f..2ccd76723b 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -229,6 +229,6 @@ void qParserClearupMetaRequestInfo(SCatalogReq* pMetaReq) { taosArrayDestroy(pMetaReq->pUdf); } -void qDestoryQuery(SQueryNode* pQuery) { +void qDestroyQuery(SQueryNode* pQuery) { // todo }