From e6f810190899bc009c72487f3d65823a6b2ca4eb Mon Sep 17 00:00:00 2001 From: dengyihao Date: Tue, 7 Jul 2020 11:41:32 +0800 Subject: [PATCH 01/12] support cancel query --- src/client/src/tscServer.c | 5 +++-- src/client/src/tscSql.c | 9 ++++----- src/vnode/src/vnodeRead.c | 10 ++++------ 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 584e70ba6c..b2482a90ca 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -438,8 +438,9 @@ void tscKillSTableQuery(SSqlObj *pSql) { * here, we cannot set the command = TSDB_SQL_KILL_QUERY. Otherwise, it may cause * sub-queries not correctly released and master sql object of super table query reaches an abnormal state. */ - pSql->pSubs[i]->res.code = TSDB_CODE_TSC_QUERY_CANCELLED; - rpcCancelRequest(pSql->pSubs[i]->pRpcCtx); + rpcCancelRequest(pSub->pRpcCtx); + pSub->res.code = TSDB_CODE_TSC_QUERY_CANCELLED; + tscQueueAsyncRes(pSub); } /* diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 8fbc1f0109..f9f93b3f89 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -617,19 +617,18 @@ void taos_stop_query(TAOS_RES *res) { if (pSql->signature != pSql) return; tscDebug("%p start to cancel query", res); - pSql->res.code = TSDB_CODE_TSC_QUERY_CANCELLED; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) { tscKillSTableQuery(pSql); - return; } - if (pSql->cmd.command >= TSDB_SQL_LOCAL) { - return; + if (pSql->cmd.command < TSDB_SQL_LOCAL) { + rpcCancelRequest(pSql->pRpcCtx); } + pSql->res.code = TSDB_CODE_TSC_QUERY_CANCELLED; + tscQueueAsyncRes(pSql); - rpcCancelRequest(pSql->pRpcCtx); tscDebug("%p query is cancelled", res); } diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index c8a6a4ce4e..8665aa01cb 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -135,7 +135,9 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { } else { assert(pQInfo == NULL); } - + if (handle != NULL) { + dnodePutItemIntoReadQueue(pVnode, handle); + } vDebug("vgId:%d, QInfo:%p, dnode query msg disposed", vgId, pQInfo); } else { assert(pCont != NULL); @@ -146,14 +148,10 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { } else { vDebug("vgId:%d, QInfo:%p, dnode query msg in progress", pVnode->vgId, *(void**) pCont); code = TSDB_CODE_VND_ACTION_IN_PROGRESS; + qTableQuery(*handle); // do execute query } - } - - if (handle != NULL) { - qTableQuery(*handle); // do execute query qReleaseQInfo(pVnode->qMgmt, (void**) &handle, false); } - return code; } From fbf91959d11d417130e4d1d7cb15120b7ff5c0d3 Mon Sep 17 00:00:00 2001 From: dengyihao Date: Tue, 7 Jul 2020 15:30:30 +0800 Subject: [PATCH 02/12] bugfix --- src/vnode/src/vnodeRead.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index 8665aa01cb..fa65a56c81 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -137,6 +137,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { } if (handle != NULL) { dnodePutItemIntoReadQueue(pVnode, handle); + qReleaseQInfo(pVnode->qMgmt, (void**) &handle, false); } vDebug("vgId:%d, QInfo:%p, dnode query msg disposed", vgId, pQInfo); } else { From a54a425c753719e3366ce9e579745d44aea00f5e Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 8 Jul 2020 10:33:11 +0800 Subject: [PATCH 03/12] failed to create table in tsdb while sync failed --- src/mnode/src/mnodeTable.c | 26 +++++++++++++++++++------- src/mnode/src/mnodeVgroup.c | 18 ++++++++++++------ 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 47add8f7a3..5c32130925 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -783,9 +783,15 @@ static int32_t mnodeProcessTableMetaMsg(SMnodeMsg *pMsg) { static int32_t mnodeCreateSuperTableCb(SMnodeMsg *pMsg, int32_t code) { SSuperTableObj *pTable = (SSuperTableObj *)pMsg->pTable; - if (pTable != NULL) { - mLInfo("app:%p:%p, stable:%s, is created in sdb, result:%s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, - tstrerror(code)); + assert(pTable); + + if (code == TSDB_CODE_SUCCESS) { + mLInfo("stable:%s, is created in sdb", pTable->info.tableId); + } else { + mError("app:%p:%p, stable:%s, failed to create in sdb, reason:%s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, + tstrerror(code)); + SSdbOper desc = {.type = SDB_OPER_GLOBAL, .pObj = pTable, .table = tsSuperTableSdb}; + sdbDeleteRow(&desc); } return code; @@ -1561,10 +1567,16 @@ static int32_t mnodeDoCreateChildTableCb(SMnodeMsg *pMsg, int32_t code) { SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable; assert(pTable); - mDebug("app:%p:%p, table:%s, create table in id:%d, uid:%" PRIu64 ", result:%s", pMsg->rpcMsg.ahandle, pMsg, - pTable->info.tableId, pTable->sid, pTable->uid, tstrerror(code)); - - if (code != TSDB_CODE_SUCCESS) return code; + if (code == TSDB_CODE_SUCCESS) { + mDebug("app:%p:%p, table:%s, create table in sid:%d, uid:%" PRIu64, pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, + pTable->sid, pTable->uid); + } else { + mError("app:%p:%p, table:%s, failed to create table sid:%d, uid:%" PRIu64 ", reason:%s", pMsg->rpcMsg.ahandle, pMsg, + pTable->info.tableId, pTable->sid, pTable->uid, tstrerror(code)); + SSdbOper desc = {.type = SDB_OPER_GLOBAL, .pObj = pTable, .table = tsChildTableSdb}; + sdbDeleteRow(&desc); + return code; + } SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont; SMDCreateTableMsg *pMDCreate = mnodeBuildCreateChildTableMsg(pCreate, pTable); diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index 3a9076335a..8fb1b749c2 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -348,17 +348,23 @@ void *mnodeGetNextVgroup(void *pIter, SVgObj **pVgroup) { } static int32_t mnodeCreateVgroupCb(SMnodeMsg *pMsg, int32_t code) { + SVgObj *pVgroup = pMsg->pVgroup; + SDbObj *pDb = pMsg->pDb; + assert(pVgroup); + if (code != TSDB_CODE_SUCCESS) { - pMsg->pVgroup = NULL; + mError("app:%p:%p, vgId:%d, failed to create in sdb, reason:%s", pMsg->rpcMsg.ahandle, pMsg, pVgroup->vgId, + tstrerror(code)); + SSdbOper desc = {.type = SDB_OPER_GLOBAL, .pObj = pVgroup, .table = tsVgroupSdb}; + sdbDeleteRow(&desc); return code; } - SVgObj *pVgroup = pMsg->pVgroup; - SDbObj *pDb = pMsg->pDb; - - mInfo("vgId:%d, is created in mnode, db:%s replica:%d", pVgroup->vgId, pDb->name, pVgroup->numOfVnodes); + mInfo("app:%p:%p, vgId:%d, is created in mnode, db:%s replica:%d", pMsg->rpcMsg.ahandle, pMsg, pVgroup->vgId, + pDb->name, pVgroup->numOfVnodes); for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { - mInfo("vgId:%d, index:%d, dnode:%d", pVgroup->vgId, i, pVgroup->vnodeGid[i].dnodeId); + mInfo("app:%p:%p, vgId:%d, index:%d, dnode:%d", pMsg->rpcMsg.ahandle, pMsg, pVgroup->vgId, i, + pVgroup->vnodeGid[i].dnodeId); } mnodeIncVgroupRef(pVgroup); From 354dd93d2bbd2fc724e11aa63dda5c2ef75dd09c Mon Sep 17 00:00:00 2001 From: Hui Li Date: Wed, 8 Jul 2020 10:44:41 +0800 Subject: [PATCH 04/12] [add cluster sim cases] --- tests/script/unique/cluster/client1_0.sim | 16 + tests/script/unique/cluster/cluster_main.sim | 494 +++++++++++++++++++ 2 files changed, 510 insertions(+) create mode 100644 tests/script/unique/cluster/client1_0.sim create mode 100644 tests/script/unique/cluster/cluster_main.sim diff --git a/tests/script/unique/cluster/client1_0.sim b/tests/script/unique/cluster/client1_0.sim new file mode 100644 index 0000000000..f7ed46436e --- /dev/null +++ b/tests/script/unique/cluster/client1_0.sim @@ -0,0 +1,16 @@ +sql connect + +$db = db1 +$stb = stb1 +print =============== client1_0: + +sql use $db + +$tblNum = 1000 + +$i = 1 +while $i < $tblNum + $tb = tb . $i + sql create table $tb using $stb tags ($i, 'abcd') + $i = $i + 1 +endw diff --git a/tests/script/unique/cluster/cluster_main.sim b/tests/script/unique/cluster/cluster_main.sim new file mode 100644 index 0000000000..5b356948ea --- /dev/null +++ b/tests/script/unique/cluster/cluster_main.sim @@ -0,0 +1,494 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 + +system sh/cfg.sh -n dnode1 -c numOfMnodes -v 3 +system sh/cfg.sh -n dnode2 -c numOfMnodes -v 3 +system sh/cfg.sh -n dnode3 -c numOfMnodes -v 3 +system sh/cfg.sh -n dnode4 -c numOfMnodes -v 3 + +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/cfg.sh -n dnode2 -c walLevel -v 1 +system sh/cfg.sh -n dnode3 -c walLevel -v 1 +system sh/cfg.sh -n dnode4 -c walLevel -v 1 + +system sh/cfg.sh -n dnode1 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode4 -c balanceInterval -v 10 + +system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4 +system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 4 +system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4 +system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4 + +system sh/cfg.sh -n dnode1 -c alternativeRole -v 0 +system sh/cfg.sh -n dnode2 -c alternativeRole -v 0 +system sh/cfg.sh -n dnode3 -c alternativeRole -v 0 +system sh/cfg.sh -n dnode4 -c alternativeRole -v 0 + +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 1000 +system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 1000 +system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 1000 +system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 1000 + +system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator +system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator +system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator +system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator + +print ============== step0: start tarbitrator +system sh/exec_tarbitrator.sh -s start + +print ============== step1: start dnode1/dnode2/dnode3 +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +sleep 3000 +sql connect +sql create dnode $hostname2 +sql create dnode $hostname3 +sleep 3000 + +print ============== step2: create db1 with replica 3 +$db = db1 +print create database $db replica 3 +#sql create database $db replica 3 maxTables $totalTableNum +sql create database $db replica 3 +sql use $db + +print ============== step3: create stable stb1 +$stb = stb1 +sql create table $stb (ts timestamp, c1 int, c2 int) tags(t1 int, t2 binary(8)) + +print ============== step4: start 10 client1/ 10 client2/ 10 client3/ 10 client4/ 1 client5 +run_back unique/cluster/client1_0.sim +#run_back unique/cluster/client1_1.sim +#run_back unique/big_cluster/client1_2.sim +#run_back unique/big_cluster/client1_3.sim +#run_back unique/big_cluster/client1_4.sim +#run_back unique/big_cluster/client1_5.sim +#run_back unique/big_cluster/client1_6.sim +#run_back unique/big_cluster/client1_7.sim +#run_back unique/big_cluster/client1_8.sim +#run_back unique/big_cluster/client1_9.sim + + +print wait for a while to let clients start insert data +sleep 5000 + +$loop_cnt = 0 +loop_cluster_do: +print **** **** **** START loop cluster do **** **** **** **** +print ============== step5: start dnode4 and add into cluster, then wait dnode4 ready +system sh/exec.sh -n dnode4 -s start +sql create dnode $hostname4 + +wait_dnode4_ready_0: +$cnt = $cnt + 1 +if $cnt == 10 then + return -1 +endi +sql show dnodes +if $rows != 4 then + sleep 2000 + goto wait_dnode4_ready_0 +endi +print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 +print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 +print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 +print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 +$dnode1Status = $data4_1 +$dnode2Status = $data4_2 +$dnode3Status = $data4_3 +#$dnode4Status = $data4_4 + +if $loop_cnt == 0 then + $dnode4Status = $data4_4 +elif $loop_cnt == 1 then + $dnode4Status = $data4_6 +elif $loop_cnt == 2 then + $dnode4Status = $data4_8 +else then + print **** **** **** END loop cluster do 2**** **** **** **** + return +endi + +if $dnode4Status != ready then + sleep 2000 + goto wait_dnode4_ready_0 +endi + + +print ============== step6: stop and drop dnode1, then remove data dir of dnode1 +system sh/exec.sh -n dnode1 -s stop -x SIGINT + +$cnt = 0 +wait_dnode1_offline_0: +$cnt = $cnt + 1 +if $cnt == 10 then + return -1 +endi +sql show dnodes +if $rows != 4 then + sleep 2000 + goto wait_dnode1_offline_0 +endi +print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 +print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 +print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 +print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 + +$dnode2Status = $data4_2 +$dnode3Status = $data4_3 +$dnode4Status = $data4_4 + +if $loop_cnt == 0 then + $dnode1Status = $data4_1 +elif $loop_cnt == 1 then + $dnode1Status = $data4_5 +elif $loop_cnt == 2 then + $dnode1Status = $data4_7 +elif $loop_cnt == 3 then + $dnode1Status = $data4_9 +else then + print **** **** **** END loop cluster do 1**** **** **** **** + return +endi + +if $dnode1Status != offline then + sleep 2000 + goto wait_dnode1_offline_0 +endi + +sql drop dnode $hostname1 +system rm -rf ../../../sim/dnode1 + + +print ============== step7: stop dnode2, because mnodes < 50%, so clusert don't provide services +system sh/exec.sh -n dnode2 -s stop -x SIGINT + +sql show dnodes -x wait_dnode2_offline_0 +if $rows != 3 then + sleep 2000 + goto wait_dnode2_offline_0 +endi +wait_dnode2_offline_0: + +#$cnt = 0 +#wait_dnode2_offline_0: +#$cnt = $cnt + 1 +#if $cnt == 10 then +# return -1 +#endi +#sql show dnodes -x wait_dnode2_offline_0 +#if $rows != 3 then +# sleep 2000 +# goto wait_dnode2_offline_0 +#endi +#print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 +#print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 +#print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 +#print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 +#$dnode1Status = $data4_1 +#$dnode2Status = $data4_2 +#$dnode3Status = $data4_3 +#$dnode4Status = $data4_4 +# +#if $dnode2Status != offline then +# sleep 2000 +# goto wait_dnode1_offline_0 +#endi + +print ============== step8: restart dnode2, then wait sync end +system sh/exec.sh -n dnode2 -s start + +$cnt = 0 +wait_dnode2_ready_0: +$cnt = $cnt + 1 +if $cnt == 10 then + return -1 +endi +sql show dnodes +if $rows != 3 then + sleep 2000 + goto wait_dnode2_ready_0 +endi +print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 +print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 +print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 +print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 +$dnode1Status = $data4_1 +$dnode2Status = $data4_2 +$dnode3Status = $data4_3 +$dnode4Status = $data4_4 + +if $dnode2Status != ready then + sleep 2000 + goto wait_dnode2_ready_0 +endi + + +print ============== step9: stop dnode3, then wait sync end +system sh/exec.sh -n dnode3 -s stop -x SIGINT + +$cnt = 0 +wait_dnode3_offline_0: +$cnt = $cnt + 1 +if $cnt == 10 then + return -1 +endi +sql show dnodes +if $rows != 3 then + sleep 2000 + goto wait_dnode3_offline_0 +endi +print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 +print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 +print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 +print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 +$dnode1Status = $data4_1 +$dnode2Status = $data4_2 +$dnode3Status = $data4_3 +$dnode4Status = $data4_4 + +if $dnode3Status != offline then + sleep 2000 + goto wait_dnode3_offline_0 +endi + +print ============== step10: restart dnode3, then wait sync end +system sh/exec.sh -n dnode3 -s start + +$cnt = 0 +wait_dnode3_ready_0: +$cnt = $cnt + 1 +if $cnt == 10 then + return -1 +endi +sql show dnodes +if $rows != 3 then + sleep 2000 + goto wait_dnode3_ready_0 +endi +print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 +print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 +print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 +print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 +$dnode1Status = $data4_1 +$dnode2Status = $data4_2 +$dnode3Status = $data4_3 +$dnode4Status = $data4_4 + +if $dnode3Status != ready then + sleep 2000 + goto wait_dnode3_ready_0 +endi + +print ============== step11: stop dnode4, then wait sync end +system sh/exec.sh -n dnode4 -s stop -x SIGINT + +$cnt = 0 +wait_dnode4_offline_0: +$cnt = $cnt + 1 +if $cnt == 10 then + return -1 +endi +sql show dnodes +if $rows != 3 then + sleep 2000 + goto wait_dnode4_offline_0 +endi +print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 +print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 +print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 +print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 +$dnode1Status = $data4_1 +$dnode2Status = $data4_2 +$dnode3Status = $data4_3 +#$dnode4Status = $data4_4 + +if $loop_cnt == 0 then + $dnode4Status = $data4_4 +elif $loop_cnt == 1 then + $dnode4Status = $data4_6 +elif $loop_cnt == 2 then + $dnode4Status = $data4_8 +else then + print **** **** **** END loop cluster do 2**** **** **** **** + return +endi + +if $dnode4Status != offline then + sleep 2000 + goto wait_dnode4_offline_0 +endi + +print ============== step12: restart dnode4, then wait sync end +system sh/exec.sh -n dnode4 -s start + +$cnt = 0 +wait_dnode4_ready_0: +$cnt = $cnt + 1 +if $cnt == 10 then + return -1 +endi +sql show dnodes +if $rows != 3 then + sleep 2000 + goto wait_dnode4_ready_0 +endi +print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 +print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 +print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 +print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 +$dnode1Status = $data4_1 +$dnode2Status = $data4_2 +$dnode3Status = $data4_3 +#$dnode4Status = $data4_4 + +if $loop_cnt == 0 then + $dnode4Status = $data4_4 +elif $loop_cnt == 1 then + $dnode4Status = $data4_6 +elif $loop_cnt == 2 then + $dnode4Status = $data4_8 +else then + print **** **** **** END loop cluster do 2**** **** **** **** + return +endi + +if $dnode4Status != ready then + sleep 2000 + goto wait_dnode4_ready_0 +endi + +print ============== step13: alter replica 2 +sql alter database $db replica 2 +sql show database +print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 + +if $data0_5 != 2 then + print rplica is not modify to 2, error!!!!!! + return +endi + +print ============== step14: stop and drop dnode4, then remove data dir of dnode4 +system sh/exec.sh -n dnode4 -s stop -x SIGINT + +$cnt = 0 +wait_dnode4_offline_1: +$cnt = $cnt + 1 +if $cnt == 10 then + return -1 +endi +sql show dnodes +if $rows != 3 then + sleep 2000 + goto wait_dnode4_offline_1 +endi +print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 +print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 +print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 +print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 + +$dnode2Status = $data4_2 +$dnode3Status = $data4_3 +#$dnode4Status = $data4_4 + +if $loop_cnt == 0 then + $dnode4Status = $data4_4 +elif $loop_cnt == 1 then + $dnode4Status = $data4_6 +elif $loop_cnt == 2 then + $dnode4Status = $data4_8 +else then + print **** **** **** END loop cluster do 2**** **** **** **** + return +endi + +if $dnode4Status != offline then + sleep 2000 + goto wait_dnode4_offline_1 +endi + +sql drop dnode $hostname4 +system rm -rf ../../../sim/dnode4 + + +print ============== step15: alter replica 1 +sql alter database $db replica 1 +sql show database +print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 + +if $data0_5 != 1 then + print rplica is not modify to 1, error!!!!!! + return +endi + + +print ============== step16: alter replica 2 +sql alter database $db replica 1 +sql show database +print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 + +if $data0_5 != 2 then + print rplica is not modify to 2, error!!!!!! + return +endi + +print ============== step17: start dnode1 and add into cluster, then wait dnode1 ready +system sh/exec.sh -n dnode1 -s start +sql create dnode $hostname1 + +wait_dnode1_ready_0: +$cnt = $cnt + 1 +if $cnt == 10 then + return -1 +endi +sql show dnodes +if $rows != 3 then + sleep 2000 + goto wait_dnode1_ready_0 +endi +print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 +print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 +print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 +print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 +#$dnode1Status = $data4_1 +$dnode2Status = $data4_2 +$dnode3Status = $data4_3 +$dnode4Status = $data4_4 + +if $loop_cnt == 0 then + $dnode1Status = $data4_1 +elif $loop_cnt == 1 then + $dnode1Status = $data4_5 +elif $loop_cnt == 2 then + $dnode1Status = $data4_7 +elif $loop_cnt == 3 then + $dnode1Status = $data4_9 +else then + print **** **** **** END loop cluster do 3**** **** **** **** + return +endi + +if $dnode1Status != ready then + sleep 2000 + goto wait_dnode1_ready_0 +endi + +print ============== step18: alter replica 3 +sql alter database $db replica 3 +sql show database +print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 + +if $data0_5 != 3 then + print rplica is not modify to 3, error!!!!!! + return +endi + +$loop_cnt = $loop_cnt + 1 +goto loop_cluster_do From fb55414c64961698a43d1ae213ab65bee2a6ecd4 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Wed, 8 Jul 2020 11:48:01 +0800 Subject: [PATCH 05/12] [coverity scan] --- src/kit/taosdemo/taosdemo.c | 10 + src/kit/taosdump/taosdump.c | 1950 ++++++++++++++------- src/kit/taosmigrate/taosmigrate.c | 1 + src/kit/taosmigrate/taosmigrateMnodeWal.c | 6 + src/kit/taosmigrate/taosmigrateVnodeCfg.c | 2 + 5 files changed, 1356 insertions(+), 613 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 6ba420514b..317da97fd3 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -870,6 +870,11 @@ void *readTable(void *sarg) { int64_t sTime = rinfo->start_time; char *tb_prefix = rinfo->tb_prefix; FILE *fp = fopen(rinfo->fp, "a"); + if (NULL == fp) { + printf("fopen %s fail, reason:%s.\n", rinfo->fp, strerror(errno)); + return NULL; + } + int num_of_DPT = rinfo->nrecords_per_table; int num_of_tables = rinfo->end_table_id - rinfo->start_table_id + 1; int totalData = num_of_DPT * num_of_tables; @@ -925,6 +930,11 @@ void *readMetric(void *sarg) { TAOS *taos = rinfo->taos; char command[BUFFER_SIZE] = "\0"; FILE *fp = fopen(rinfo->fp, "a"); + if (NULL == fp) { + printf("fopen %s fail, reason:%s.\n", rinfo->fp, strerror(errno)); + return NULL; + } + int num_of_DPT = rinfo->nrecords_per_table; int num_of_tables = rinfo->end_table_id - rinfo->start_table_id + 1; int totalData = num_of_DPT * num_of_tables; diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index 0b1890c5ab..70f18b32bd 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -27,19 +27,18 @@ #include #include #include +#include #include "taos.h" -#include "taosmsg.h" -#include "tsclient.h" #include "taosdef.h" +#include "taosmsg.h" +#include "tglobal.h" +#include "tsclient.h" +#include "tsdb.h" #include "tutil.h" -#include "tglobal.h" - #define COMMAND_SIZE 65536 -#define DEFAULT_DUMP_FILE "taosdump.sql" - -#define MAX_DBS 100 +//#define DEFAULT_DUMP_FILE "taosdump.sql" int converStringToReadable(char *str, int size, char *buf, int bufsize); int convertNCharToReadable(char *str, int size, char *buf, int bufsize); @@ -90,21 +89,21 @@ enum _describe_table_index { }; typedef struct { - char field[TSDB_COL_NAME_LEN]; + char field[TSDB_COL_NAME_LEN + 1]; char type[16]; int length; char note[128]; } SColDes; typedef struct { - char name[TSDB_COL_NAME_LEN]; + char name[TSDB_COL_NAME_LEN + 1]; SColDes cols[]; } STableDef; extern char version[]; typedef struct { - char name[TSDB_DB_NAME_LEN]; + char name[TSDB_DB_NAME_LEN + 1]; int32_t replica; int32_t days; int32_t keep; @@ -119,8 +118,8 @@ typedef struct { } SDbInfo; typedef struct { - char name[TSDB_TABLE_NAME_LEN]; - char metric[TSDB_TABLE_NAME_LEN]; + char name[TSDB_TABLE_NAME_LEN + 1]; + char metric[TSDB_TABLE_NAME_LEN + 1]; } STableRecord; typedef struct { @@ -128,6 +127,16 @@ typedef struct { STableRecord tableRecord; } STableRecordInfo; +typedef struct { + pthread_t threadID; + int32_t threadIndex; + int32_t totalThreads; + char dbName[TSDB_TABLE_NAME_LEN + 1]; + void *taosCon; +} SThreadParaObj; + +static int64_t totalDumpOutRows = 0; + SDbInfo **dbInfos = NULL; const char *argp_program_version = version; @@ -142,7 +151,7 @@ static char doc[] = ""; /* to force a line-break, e.g.\n<-- here."; */ /* A description of the arguments we accept. */ -static char args_doc[] = "dbname [tbname ...]\n--databases dbname ...\n--all-databases\n-i input_file"; +static char args_doc[] = "dbname [tbname ...]\n--databases dbname ...\n--all-databases\n-i inpath\n-o outpath"; /* Keys for options without short-options. */ #define OPT_ABORT 1 /* –abort */ @@ -150,60 +159,68 @@ static char args_doc[] = "dbname [tbname ...]\n--databases dbname ...\n--all-dat /* The options we understand. */ static struct argp_option options[] = { // connection option - {"host", 'h', "HOST", 0, "Server host dumping data from. Default is localhost.", 0}, - {"user", 'u', "USER", 0, "User name used to connect to server. Default is root.", 0}, - {"password", 'p', "PASSWORD", 0, "User password to connect to server. Default is taosdata.", 0}, - {"port", 'P', "PORT", 0, "Port to connect", 0}, + {"host", 'h', "HOST", 0, "Server host dumping data from. Default is localhost.", 0}, + {"user", 'u', "USER", 0, "User name used to connect to server. Default is root.", 0}, + {"password", 'p', "PASSWORD", 0, "User password to connect to server. Default is taosdata.", 0}, + {"port", 'P', "PORT", 0, "Port to connect", 0}, + {"cversion", 'v', "CVERION", 0, "client version", 0}, + {"mysqlFlag", 'q', "MYSQLFLAG", 0, "mysqlFlag, Default is 0", 0}, // input/output file - {"output", 'o', "OUTPUT", 0, "Output file name.", 1}, - {"input", 'i', "INPUT", 0, "Input file name.", 1}, - {"config", 'c', "CONFIG_DIR", 0, "Configure directory. Default is /etc/taos/taos.cfg.", 1}, - {"encode", 'e', "ENCODE", 0, "Input file encoding.", 1}, + {"outpath", 'o', "OUTPATH", 0, "Output file path.", 1}, + {"inpath", 'i', "INPATH", 0, "Input file path.", 1}, + {"config", 'c', "CONFIG_DIR", 0, "Configure directory. Default is /etc/taos/taos.cfg.", 1}, + {"encode", 'e', "ENCODE", 0, "Input file encoding.", 1}, // dump unit options - {"all-databases", 'A', 0, 0, "Dump all databases.", 2}, - {"databases", 'B', 0, 0, "Dump assigned databases", 2}, + {"all-databases", 'A', 0, 0, "Dump all databases.", 2}, + {"databases", 'B', 0, 0, "Dump assigned databases", 2}, // dump format options - {"schemaonly", 's', 0, 0, "Only dump schema.", 3}, - {"with-property", 'M', 0, 0, "Dump schema with properties.", 3}, - {"start-time", 'S', "START_TIME", 0, "Start time to dump.", 3}, - {"end-time", 'E', "END_TIME", 0, "End time to dump.", 3}, - {"data-batch", 'N', "DATA_BATCH", 0, "Number of data point per insert statement. Default is 1.", 3}, - {"allow-sys", 'a', 0, 0, "Allow to dump sys database", 3}, + {"schemaonly", 's', 0, 0, "Only dump schema.", 3}, + {"with-property", 'M', 0, 0, "Dump schema with properties.", 3}, + {"start-time", 'S', "START_TIME", 0, "Start time to dump.", 3}, + {"end-time", 'E', "END_TIME", 0, "End time to dump.", 3}, + {"data-batch", 'N', "DATA_BATCH", 0, "Number of data point per insert statement. Default is 1.", 3}, + {"table-batch", 'T', "TABLE_BATCH", 0, "Number of table dumpout into one output file. Default is 1.", 3}, + {"thread_num", 't', "THREAD_NUM", 0, "Number of thread for dump in file. Default is 5.", 3}, + {"allow-sys", 'a', 0, 0, "Allow to dump sys database", 3}, {0}}; /* Used by main to communicate with parse_opt. */ -typedef struct SDumpArguments { +struct arguments { // connection option - char *host; - char *user; - char *password; - uint16_t port; + char *host; + char *user; + char *password; + uint16_t port; + char cversion[TSDB_FILENAME_LEN+1]; + uint16_t mysqlFlag; // output file - char output[TSDB_FILENAME_LEN]; - char input[TSDB_FILENAME_LEN]; - char *encode; + char outpath[TSDB_FILENAME_LEN+1]; + char inpath[TSDB_FILENAME_LEN+1]; + char *encode; // dump unit option - bool all_databases; - bool databases; + bool all_databases; + bool databases; // dump format option - bool schemaonly; - bool with_property; - int64_t start_time; - int64_t end_time; - int data_batch; - bool allow_sys; + bool schemaonly; + bool with_property; + int64_t start_time; + int64_t end_time; + int32_t data_batch; + int32_t table_batch; // num of table which will be dump into one output file. + bool allow_sys; // other options - int abort; - char **arg_list; - int arg_list_len; - bool isDumpIn; -} SDumpArguments; + int32_t thread_num; + int abort; + char **arg_list; + int arg_list_len; + bool isDumpIn; +}; /* Parse a single option. */ static error_t parse_opt(int key, char *arg, struct argp_state *state) { /* Get the input argument from argp_parse, which we know is a pointer to our arguments structure. */ - SDumpArguments *arguments = state->input; + struct arguments *arguments = state->input; wordexp_t full_path; switch (key) { @@ -223,13 +240,24 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { case 'P': arguments->port = atoi(arg); break; - // output file + case 'q': + arguments->mysqlFlag = atoi(arg); + break; + case 'v': + if (wordexp(arg, &full_path, 0) != 0) { + fprintf(stderr, "Invalid client vesion %s\n", arg); + return -1; + } + strcpy(arguments->cversion, full_path.we_wordv[0]); + wordfree(&full_path); + break; + // output file path case 'o': if (wordexp(arg, &full_path, 0) != 0) { fprintf(stderr, "Invalid path %s\n", arg); return -1; } - tstrncpy(arguments->output, full_path.we_wordv[0], TSDB_FILENAME_LEN); + strcpy(arguments->outpath, full_path.we_wordv[0]); wordfree(&full_path); break; case 'i': @@ -238,7 +266,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { fprintf(stderr, "Invalid path %s\n", arg); return -1; } - tstrncpy(arguments->input, full_path.we_wordv[0], TSDB_FILENAME_LEN); + strcpy(arguments->inpath, full_path.we_wordv[0]); wordfree(&full_path); break; case 'c': @@ -246,7 +274,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { fprintf(stderr, "Invalid path %s\n", arg); return -1; } - tstrncpy(configDir, full_path.we_wordv[0], TSDB_FILENAME_LEN); + strcpy(configDir, full_path.we_wordv[0]); wordfree(&full_path); break; case 'e': @@ -276,13 +304,19 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { case 'N': arguments->data_batch = atoi(arg); break; + case 'T': + arguments->table_batch = atoi(arg); + break; + case 't': + arguments->thread_num = atoi(arg); + break; case OPT_ABORT: arguments->abort = 1; break; case ARGP_KEY_ARG: - arguments->arg_list = &state->argv[state->next - 1]; + arguments->arg_list = &state->argv[state->next - 1]; arguments->arg_list_len = state->argc - state->next + 1; - state->next = state->argc; + state->next = state->argc; break; default: @@ -294,52 +328,70 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { /* Our argp parser. */ static struct argp argp = {options, parse_opt, args_doc, doc}; -TAOS *taos = NULL; -char *command = NULL; -char *lcommand = NULL; -char *buffer = NULL; - -int taosDumpOut(SDumpArguments *arguments); - -int taosDumpIn(SDumpArguments *arguments); - +int taosDumpOut(struct arguments *arguments); +int taosDumpIn(struct arguments *arguments); void taosDumpCreateDbClause(SDbInfo *dbInfo, bool isDumpProperty, FILE *fp); - -int taosDumpDb(SDbInfo *dbInfo, SDumpArguments *arguments, FILE *fp); - -void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, SDumpArguments *arguments, FILE *fp); - -void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols, SDumpArguments *arguments, - FILE *fp); - -int32_t taosDumpTable(char *table, char *metric, SDumpArguments *arguments, FILE *fp); - -int32_t taosDumpMetric(char *metric, SDumpArguments *arguments, FILE *fp); - -int taosDumpTableData(FILE *fp, char *tbname, SDumpArguments *arguments); - -int taosCheckParam(SDumpArguments *arguments); - +int taosDumpDb(SDbInfo *dbInfo, struct arguments *arguments, FILE *fp, TAOS *taosCon); +int32_t taosDumpStable(char *table, FILE *fp, TAOS* taosCon); +void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, FILE *fp); +void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols, FILE *fp); +int32_t taosDumpTable(char *table, char *metric, struct arguments *arguments, FILE *fp, TAOS* taosCon); +int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* taosCon); +int taosCheckParam(struct arguments *arguments); void taosFreeDbInfos(); +static void taosStartDumpOutWorkThreads(struct arguments* args, int32_t numOfThread, char *dbName); + +struct arguments tsArguments = { + // connection option + NULL, + "root", + "taosdata", + 0, + "", + 0, + // outpath and inpath + "", + "", + NULL, + // dump unit option + false, + false, + // dump format option + false, + false, + 0, + INT64_MAX, + 1, + 1, + false, + // other options + 5, + 0, + NULL, + 0, + false +}; + +int queryDB(TAOS *taos, char *command) { + TAOS_RES *pSql = NULL; + int32_t code = -1; + + pSql = taos_query(taos, command); + code = taos_errno(pSql); + if (code) { + fprintf(stderr, "sql error: %s, reason:%s\n", command, taos_errstr(pSql)); + } + taos_free_result(pSql); + return code; +} int main(int argc, char *argv[]) { - SDumpArguments arguments = { - // connection option - NULL, TSDB_DEFAULT_USER, TSDB_DEFAULT_PASS, 0, - // output file - DEFAULT_DUMP_FILE, DEFAULT_DUMP_FILE, NULL, - // dump unit option - false, false, - // dump format option - false, false, 0, INT64_MAX, 1, false, - // other options - 0, NULL, 0, false}; /* Parse our arguments; every option seen by parse_opt will be reflected in arguments. */ - argp_parse(&argp, argc, argv, 0, 0, &arguments); + argp_parse(&argp, argc, argv, 0, 0, &tsArguments); - if (arguments.abort) { + if (tsArguments.abort) { #ifndef _ALPINE error(10, 0, "ABORTED"); #else @@ -347,14 +399,48 @@ int main(int argc, char *argv[]) { #endif } - if (taosCheckParam(&arguments) < 0) { + printf("====== arguments config ======\n"); + { + printf("host: %s\n", tsArguments.host); + printf("user: %s\n", tsArguments.user); + printf("password: %s\n", tsArguments.password); + printf("port: %u\n", tsArguments.port); + printf("cversion: %s\n", tsArguments.cversion); + printf("mysqlFlag: %d", tsArguments.mysqlFlag); + printf("outpath: %s\n", tsArguments.outpath); + printf("inpath: %s\n", tsArguments.inpath); + printf("encode: %s\n", tsArguments.encode); + printf("all_databases: %d\n", tsArguments.all_databases); + printf("databases: %d\n", tsArguments.databases); + printf("schemaonly: %d\n", tsArguments.schemaonly); + printf("with_property: %d\n", tsArguments.with_property); + printf("start_time: %" PRId64 "\n", tsArguments.start_time); + printf("end_time: %" PRId64 "\n", tsArguments.end_time); + printf("data_batch: %d\n", tsArguments.data_batch); + printf("table_batch: %d\n", tsArguments.table_batch); + printf("allow_sys: %d\n", tsArguments.allow_sys); + printf("abort: %d\n", tsArguments.abort); + printf("isDumpIn: %d\n", tsArguments.isDumpIn); + printf("arg_list_len: %d\n", tsArguments.arg_list_len); + + for (int32_t i = 0; i < tsArguments.arg_list_len; i++) { + printf("arg_list[%d]: %s\n", i, tsArguments.arg_list[i]); + } + } + printf("==============================\n"); + + if (tsArguments.cversion[0] != 0){ + strcpy(version, tsArguments.cversion); + } + + if (taosCheckParam(&tsArguments) < 0) { exit(EXIT_FAILURE); } - if (arguments.isDumpIn) { - if (taosDumpIn(&arguments) < 0) return -1; + if (tsArguments.isDumpIn) { + if (taosDumpIn(&tsArguments) < 0) return -1; } else { - if (taosDumpOut(&arguments) < 0) return -1; + if (taosDumpOut(&tsArguments) < 0) return -1; } return 0; @@ -362,96 +448,214 @@ int main(int argc, char *argv[]) { void taosFreeDbInfos() { if (dbInfos == NULL) return; - for (int i = 0; i < MAX_DBS; i++) tfree(dbInfos[i]); + for (int i = 0; i < 128; i++) tfree(dbInfos[i]); tfree(dbInfos); } -int taosGetTableRecordInfo(char *table, STableRecordInfo *pTableRecordInfo) { +// check table is normal table or super table +int taosGetTableRecordInfo(char *table, STableRecordInfo *pTableRecordInfo, TAOS *taosCon) { TAOS_ROW row = NULL; bool isSet = false; + TAOS_RES *result = NULL; memset(pTableRecordInfo, 0, sizeof(STableRecordInfo)); - sprintf(command, "show tables like %s", table); - TAOS_RES *result = taos_query(taos, command);\ - int32_t code = taos_errno(result); + char* tempCommand = (char *)malloc(COMMAND_SIZE); + if (tempCommand == NULL) { + fprintf(stderr, "failed to allocate memory\n"); + return -1; + } + sprintf(tempCommand, "show tables like %s", table); + + result = taos_query(taosCon, tempCommand); + int32_t code = taos_errno(result); + if (code != 0) { - fprintf(stderr, "failed to run command %s, error: %s\n", command, taos_errstr(result)); + fprintf(stderr, "failed to run command %s\n", tempCommand); + free(tempCommand); taos_free_result(result); return -1; } TAOS_FIELD *fields = taos_fetch_fields(result); - if ((row = taos_fetch_row(result)) != NULL) { + while ((row = taos_fetch_row(result)) != NULL) { isSet = true; pTableRecordInfo->isMetric = false; strncpy(pTableRecordInfo->tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes); strncpy(pTableRecordInfo->tableRecord.metric, (char *)row[TSDB_SHOW_TABLES_METRIC_INDEX], fields[TSDB_SHOW_TABLES_METRIC_INDEX].bytes); + break; } taos_free_result(result); result = NULL; - if (isSet) return 0; - - sprintf(command, "show stables like %s", table); - - result = taos_query(taos, command); + if (isSet) { + free(tempCommand); + return 0; + } + + sprintf(tempCommand, "show stables like %s", table); + + result = taos_query(taosCon, tempCommand); code = taos_errno(result); + if (code != 0) { - fprintf(stderr, "failed to run command %s, error: %s\n", command, taos_errstr(result)); + fprintf(stderr, "failed to run command %s\n", tempCommand); + free(tempCommand); taos_free_result(result); return -1; } - if ((row = taos_fetch_row(result)) != NULL) { + while ((row = taos_fetch_row(result)) != NULL) { isSet = true; pTableRecordInfo->isMetric = true; - tstrncpy(pTableRecordInfo->tableRecord.metric, table, TSDB_TABLE_NAME_LEN); + strcpy(pTableRecordInfo->tableRecord.metric, table); + break; } taos_free_result(result); result = NULL; - if (isSet) return 0; - + if (isSet) { + free(tempCommand); + return 0; + } fprintf(stderr, "invalid table/metric %s\n", table); + free(tempCommand); return -1; } -int taosDumpOut(SDumpArguments *arguments) { - TAOS_ROW row; - TAOS_RES* result = NULL; - char *temp = NULL; - FILE *fp = NULL; - int count = 0; - STableRecordInfo tableRecordInfo; - fp = fopen(arguments->output, "w"); - if (fp == NULL) { - fprintf(stderr, "failed to open file %s\n", arguments->output); +int32_t taosSaveAllNormalTableToTempFile(TAOS *taosCon, char*meter, char* metric, int* fd) { + STableRecord tableRecord; + + if (-1 == *fd) { + *fd = open(".tables.tmp.0", O_RDWR | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); + if (*fd == -1) { + fprintf(stderr, "failed to open temp file: .tables.tmp.0\n"); + return -1; + } + } + + memset(tableRecord.name, 0, sizeof(STableRecord)); + strcpy(tableRecord.name, meter); + strcpy(tableRecord.metric, metric); + + twrite(*fd, &tableRecord, sizeof(STableRecord)); + return 0; +} + + +int32_t taosSaveTableOfMetricToTempFile(TAOS *taosCon, char* metric, struct arguments *arguments, int32_t* totalNumOfThread) { + TAOS_ROW row; + int fd = -1; + STableRecord tableRecord; + + char* tmpCommand = (char *)malloc(COMMAND_SIZE); + if (tmpCommand == NULL) { + fprintf(stderr, "failed to allocate memory\n"); return -1; } - dbInfos = (SDbInfo **)calloc(MAX_DBS, sizeof(SDbInfo *)); + sprintf(tmpCommand, "select tbname from %s", metric); + + TAOS_RES *result = taos_query(taosCon, tmpCommand); + int32_t code = taos_errno(result); + if (code != 0) { + fprintf(stderr, "failed to run command %s\n", tmpCommand); + free(tmpCommand); + taos_free_result(result); + return -1; + } + + TAOS_FIELD *fields = taos_fetch_fields(result); + + int32_t numOfTable = 0; + int32_t numOfThread = *totalNumOfThread; + char tmpFileName[TSDB_FILENAME_LEN + 1]; + while ((row = taos_fetch_row(result)) != NULL) { + if (0 == numOfTable) { + memset(tmpFileName, 0, TSDB_FILENAME_LEN); + sprintf(tmpFileName, ".tables.tmp.%d", numOfThread); + fd = open(tmpFileName, O_RDWR | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); + if (fd == -1) { + fprintf(stderr, "failed to open temp file: %s\n", tmpFileName); + taos_free_result(result); + for (int32_t loopCnt = 0; loopCnt < numOfThread; loopCnt++) { + sprintf(tmpFileName, ".tables.tmp.%d", loopCnt); + remove(tmpFileName); + } + free(tmpCommand); + return -1; + } + + numOfThread++; + } + + memset(tableRecord.name, 0, sizeof(STableRecord)); + strncpy(tableRecord.name, (char *)row[0], fields[0].bytes); + strcpy(tableRecord.metric, metric); + + twrite(fd, &tableRecord, sizeof(STableRecord)); + + numOfTable++; + + if (numOfTable >= arguments->table_batch) { + numOfTable = 0; + tclose(fd); + fd = -1; + } + } + tclose(fd); + fd = -1; + taos_free_result(result); + + *totalNumOfThread = numOfThread; + + free(tmpCommand); + + return 0; +} + +int taosDumpOut(struct arguments *arguments) { + TAOS *taos = NULL; + TAOS_RES *result = NULL; + char *command = NULL; + + TAOS_ROW row; + FILE *fp = NULL; + int32_t count = 0; + STableRecordInfo tableRecordInfo; + + char tmpBuf[TSDB_FILENAME_LEN+9] = {0}; + if (arguments->outpath[0] != 0) { + sprintf(tmpBuf, "%s/dbs.sql", arguments->outpath); + } else { + sprintf(tmpBuf, "dbs.sql"); + } + + fp = fopen(tmpBuf, "w"); + if (fp == NULL) { + fprintf(stderr, "failed to open file %s\n", tmpBuf); + return -1; + } + + dbInfos = (SDbInfo **)calloc(128, sizeof(SDbInfo *)); if (dbInfos == NULL) { fprintf(stderr, "failed to allocate memory\n"); goto _exit_failure; } - temp = (char *)malloc(2 * COMMAND_SIZE); - if (temp == NULL) { + command = (char *)malloc(COMMAND_SIZE); + if (command == NULL) { fprintf(stderr, "failed to allocate memory\n"); goto _exit_failure; } - command = temp; - buffer = command + COMMAND_SIZE; - /* Connect to server */ taos = taos_connect(arguments->host, arguments->user, arguments->password, NULL, arguments->port); if (taos == NULL) { @@ -465,29 +669,30 @@ int taosDumpOut(SDumpArguments *arguments) { taosDumpCharset(fp); sprintf(command, "show databases"); - result = taos_query(taos, command); + result = taos_query(taos, command); int32_t code = taos_errno(result); + if (code != 0) { - fprintf(stderr, "failed to run command: %s, reason: %s\n", command, taos_errstr(result)); - taos_free_result(result); + fprintf(stderr, "failed to run command: %s, reason: %s\n", command, taos_errstr(taos)); goto _exit_failure; } TAOS_FIELD *fields = taos_fetch_fields(result); while ((row = taos_fetch_row(result)) != NULL) { + // sys database name : 'monitor', but subsequent version changed to 'log' if (strncasecmp(row[TSDB_SHOW_DB_NAME_INDEX], "monitor", fields[TSDB_SHOW_DB_NAME_INDEX].bytes) == 0 && (!arguments->allow_sys)) continue; - if (arguments->databases) { + if (arguments->databases) { // input multi dbs for (int i = 0; arguments->arg_list[i]; i++) { if (strncasecmp(arguments->arg_list[i], (char *)row[TSDB_SHOW_DB_NAME_INDEX], fields[TSDB_SHOW_DB_NAME_INDEX].bytes) == 0) goto _dump_db_point; } continue; - } else if (!arguments->all_databases) { + } else if (!arguments->all_databases) { // only input one db if (strncasecmp(arguments->arg_list[0], (char *)row[TSDB_SHOW_DB_NAME_INDEX], fields[TSDB_SHOW_DB_NAME_INDEX].bytes) == 0) goto _dump_db_point; @@ -504,19 +709,19 @@ int taosDumpOut(SDumpArguments *arguments) { } strncpy(dbInfos[count]->name, (char *)row[TSDB_SHOW_DB_NAME_INDEX], fields[TSDB_SHOW_DB_NAME_INDEX].bytes); - if (strcmp(arguments->user, TSDB_DEFAULT_USER) == 0) { - dbInfos[count]->replica = (int)(*((int16_t *)row[TSDB_SHOW_DB_REPLICA_INDEX])); - dbInfos[count]->days = (int)(*((int16_t *)row[TSDB_SHOW_DB_DAYS_INDEX])); - dbInfos[count]->keep = *((int *)row[TSDB_SHOW_DB_KEEP_INDEX]); - dbInfos[count]->tables = *((int *)row[TSDB_SHOW_DB_TABLES_INDEX]); - dbInfos[count]->rows = *((int *)row[TSDB_SHOW_DB_ROWS_INDEX]); - dbInfos[count]->cache = *((int *)row[TSDB_SHOW_DB_CACHE_INDEX]); - dbInfos[count]->ablocks = *((int *)row[TSDB_SHOW_DB_ABLOCKS_INDEX]); - dbInfos[count]->tblocks = (int)(*((int16_t *)row[TSDB_SHOW_DB_TBLOCKS_INDEX])); - dbInfos[count]->ctime = *((int *)row[TSDB_SHOW_DB_CTIME_INDEX]); - dbInfos[count]->clog = (int)(*((int8_t *)row[TSDB_SHOW_DB_CLOG_INDEX])); - dbInfos[count]->comp = (int)(*((int8_t *)row[TSDB_SHOW_DB_COMP_INDEX])); - } + #if 0 + dbInfos[count]->replica = (int)(*((int16_t *)row[TSDB_SHOW_DB_REPLICA_INDEX])); + dbInfos[count]->days = (int)(*((int16_t *)row[TSDB_SHOW_DB_DAYS_INDEX])); + dbInfos[count]->keep = *((int *)row[TSDB_SHOW_DB_KEEP_INDEX]); + dbInfos[count]->tables = *((int *)row[TSDB_SHOW_DB_TABLES_INDEX]); + dbInfos[count]->rows = *((int *)row[TSDB_SHOW_DB_ROWS_INDEX]); + dbInfos[count]->cache = *((int *)row[TSDB_SHOW_DB_CACHE_INDEX]); + dbInfos[count]->ablocks = *((int *)row[TSDB_SHOW_DB_ABLOCKS_INDEX]); + dbInfos[count]->tblocks = (int)(*((int16_t *)row[TSDB_SHOW_DB_TBLOCKS_INDEX])); + dbInfos[count]->ctime = *((int *)row[TSDB_SHOW_DB_CTIME_INDEX]); + dbInfos[count]->clog = (int)(*((int8_t *)row[TSDB_SHOW_DB_CLOG_INDEX])); + dbInfos[count]->comp = (int)(*((int8_t *)row[TSDB_SHOW_DB_COMP_INDEX])); +#endif count++; @@ -528,42 +733,71 @@ int taosDumpOut(SDumpArguments *arguments) { } } - // taos_free_result(result); - if (count == 0) { fprintf(stderr, "No databases valid to dump\n"); goto _exit_failure; } - if (arguments->databases || arguments->all_databases) { + if (arguments->databases || arguments->all_databases) { // case: taosdump --databases dbx dby ... OR taosdump --all-databases for (int i = 0; i < count; i++) { - (void)taosDumpDb(dbInfos[i], arguments, fp); + taosDumpDb(dbInfos[i], arguments, fp, taos); } } else { - if (arguments->arg_list_len == 1) { - (void)taosDumpDb(dbInfos[0], arguments, fp); - } else { + if (arguments->arg_list_len == 1) { // case: taosdump + taosDumpDb(dbInfos[0], arguments, fp, taos); + } else { // case: taosdump tablex tabley ... taosDumpCreateDbClause(dbInfos[0], arguments->with_property, fp); sprintf(command, "use %s", dbInfos[0]->name); - if (taos_query(taos, command) == NULL ) { + + result = taos_query(taos, command); + int32_t code = taos_errno(result); + if (code != 0) { fprintf(stderr, "invalid database %s\n", dbInfos[0]->name); goto _exit_failure; } fprintf(fp, "USE %s;\n\n", dbInfos[0]->name); + int32_t totalNumOfThread = 1; // 0: all normal talbe into .tables.tmp.0 + int normalTblFd = -1; + int32_t retCode; for (int i = 1; arguments->arg_list[i]; i++) { - if (taosGetTableRecordInfo(arguments->arg_list[i], &tableRecordInfo) < 0) { - fprintf(stderr, "invalide table %s\n", arguments->arg_list[i]); + if (taosGetTableRecordInfo(arguments->arg_list[i], &tableRecordInfo, taos) < 0) { + fprintf(stderr, "input the invalide table %s\n", arguments->arg_list[i]); continue; } - if (tableRecordInfo.isMetric) { // dump whole metric - (void)taosDumpMetric(tableRecordInfo.tableRecord.metric, arguments, fp); - } else { // dump MTable and NTable - (void)taosDumpTable(tableRecordInfo.tableRecord.name, tableRecordInfo.tableRecord.metric, arguments, fp); + if (tableRecordInfo.isMetric) { // dump all table of this metric + (void)taosDumpStable(tableRecordInfo.tableRecord.metric, fp, taos); + retCode = taosSaveTableOfMetricToTempFile(taos, tableRecordInfo.tableRecord.metric, arguments, &totalNumOfThread); + } else { + if (tableRecordInfo.tableRecord.metric[0] != '\0') { // dump this sub table and it's metric + (void)taosDumpStable(tableRecordInfo.tableRecord.metric, fp, taos); + } + retCode = taosSaveAllNormalTableToTempFile(taos, tableRecordInfo.tableRecord.name, tableRecordInfo.tableRecord.metric, &normalTblFd); } + + if (retCode < 0) { + if (-1 != normalTblFd){ + tclose(normalTblFd); + } + goto _clean_tmp_file; + } + } + + if (-1 != normalTblFd){ + tclose(normalTblFd); + } + + // start multi threads to dumpout + taosStartDumpOutWorkThreads(arguments, totalNumOfThread, dbInfos[0]->name); + + char tmpFileName[TSDB_FILENAME_LEN + 1]; + _clean_tmp_file: + for (int loopCnt = 0; loopCnt < totalNumOfThread; loopCnt++) { + sprintf(tmpFileName, ".tables.tmp.%d", loopCnt); + remove(tmpFileName); } } } @@ -572,21 +806,120 @@ int taosDumpOut(SDumpArguments *arguments) { fclose(fp); taos_close(taos); taos_free_result(result); - tfree(temp); - taosFreeDbInfos(); + tfree(command); + taosFreeDbInfos(); + fprintf(stderr, "dump out rows: %" PRId64 "\n", totalDumpOutRows); return 0; _exit_failure: fclose(fp); taos_close(taos); taos_free_result(result); - tfree(temp); + tfree(command); taosFreeDbInfos(); + fprintf(stderr, "dump out rows: %" PRId64 "\n", totalDumpOutRows); return -1; } +int taosGetTableDes(char *table, STableDef *tableDes, TAOS* taosCon) { + TAOS_ROW row = NULL; + TAOS_RES *tmpResult = NULL; + int count = 0; + + char* tempCommand = (char *)malloc(COMMAND_SIZE); + if (tempCommand == NULL) { + fprintf(stderr, "failed to allocate memory\n"); + return -1; + } + + sprintf(tempCommand, "describe %s", table); + + tmpResult = taos_query(taosCon, tempCommand); + int32_t code = taos_errno(tmpResult); + if (code != 0) { + fprintf(stderr, "failed to run command %s\n", tempCommand); + free(tempCommand); + taos_free_result(tmpResult); + return -1; + } + + TAOS_FIELD *fields = taos_fetch_fields(tmpResult); + + strcpy(tableDes->name, table); + + while ((row = taos_fetch_row(tmpResult)) != NULL) { + strncpy(tableDes->cols[count].field, (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX], + fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes); + strncpy(tableDes->cols[count].type, (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], + fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes); + tableDes->cols[count].length = *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]); + strncpy(tableDes->cols[count].note, (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], + fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes); + + count++; + } + + taos_free_result(tmpResult); + tmpResult = NULL; + + free(tempCommand); + + return count; +} + +int32_t taosDumpTable(char *table, char *metric, struct arguments *arguments, FILE *fp, TAOS* taosCon) { + int count = 0; + + STableDef *tableDes = (STableDef *)calloc(1, sizeof(STableDef) + sizeof(SColDes) * TSDB_MAX_COLUMNS); + + if (metric != NULL && metric[0] != '\0') { // dump table schema which is created by using super table + /* + count = taosGetTableDes(metric, tableDes, taosCon); + + if (count < 0) { + free(tableDes); + return -1; + } + + taosDumpCreateTableClause(tableDes, count, fp); + + memset(tableDes, 0, sizeof(STableDef) + sizeof(SColDes) * TSDB_MAX_COLUMNS); + */ + + count = taosGetTableDes(table, tableDes, taosCon); + + if (count < 0) { + free(tableDes); + return -1; + } + + taosDumpCreateMTableClause(tableDes, metric, count, fp); + + } else { // dump table definition + count = taosGetTableDes(table, tableDes, taosCon); + + if (count < 0) { + free(tableDes); + return -1; + } + + taosDumpCreateTableClause(tableDes, count, fp); + } + + free(tableDes); + + return taosDumpTableData(fp, table, arguments, taosCon); +} + void taosDumpCreateDbClause(SDbInfo *dbInfo, bool isDumpProperty, FILE *fp) { - char *pstr = buffer; + + char* tmpCommand = (char *)malloc(COMMAND_SIZE); + if (tmpCommand == NULL) { + fprintf(stderr, "failed to allocate memory\n"); + return; + } + + char *pstr = tmpCommand; pstr += sprintf(pstr, "CREATE DATABASE IF NOT EXISTS %s", dbInfo->name); if (isDumpProperty) { @@ -596,78 +929,304 @@ void taosDumpCreateDbClause(SDbInfo *dbInfo, bool isDumpProperty, FILE *fp) { dbInfo->ablocks, dbInfo->tblocks, dbInfo->ctime, dbInfo->clog, dbInfo->comp); } - fprintf(fp, "%s\n\n", buffer); + pstr += sprintf(pstr, ";"); + + fprintf(fp, "%s\n\n", tmpCommand); + free(tmpCommand); } -int taosDumpDb(SDbInfo *dbInfo, SDumpArguments *arguments, FILE *fp) { +void* taosDumpOutWorkThreadFp(void *arg) +{ + SThreadParaObj *pThread = (SThreadParaObj*)arg; + STableRecord tableRecord; + int fd; + + char tmpFileName[TSDB_FILENAME_LEN*4] = {0}; + sprintf(tmpFileName, ".tables.tmp.%d", pThread->threadIndex); + fd = open(tmpFileName, O_RDWR | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); + if (fd == -1) { + fprintf(stderr, "taosDumpTableFp() failed to open temp file: %s\n", tmpFileName); + return NULL; + } + + FILE *fp = NULL; + memset(tmpFileName, 0, TSDB_FILENAME_LEN + 128); + + if (tsArguments.outpath[0] != 0) { + sprintf(tmpFileName, "%s/%s.tables.%d.sql", tsArguments.outpath, pThread->dbName, pThread->threadIndex); + } else { + sprintf(tmpFileName, "%s.tables.%d.sql", pThread->dbName, pThread->threadIndex); + } + + fp = fopen(tmpFileName, "w"); + if (fp == NULL) { + fprintf(stderr, "failed to open file %s\n", tmpFileName); + return NULL; + } + + memset(tmpFileName, 0, TSDB_FILENAME_LEN); + sprintf(tmpFileName, "use %s", pThread->dbName); + + TAOS_RES* tmpResult = taos_query(pThread->taosCon, tmpFileName); + int32_t code = taos_errno(tmpResult); + if (code != 0) { + fprintf(stderr, "invalid database %s\n", pThread->dbName); + taos_free_result(tmpResult); + return NULL; + } + + fprintf(fp, "USE %s\n\n", pThread->dbName); + while (read(fd, &tableRecord, sizeof(STableRecord)) > 0) { + taosDumpTable(tableRecord.name, tableRecord.metric, &tsArguments, fp, pThread->taosCon); + } + + taos_free_result(tmpResult); + tclose(fd); + fclose(fp); + + return NULL; +} + +static void taosStartDumpOutWorkThreads(struct arguments* args, int32_t numOfThread, char *dbName) +{ + pthread_attr_t thattr; + SThreadParaObj *threadObj = (SThreadParaObj *)calloc(numOfThread, sizeof(SThreadParaObj)); + for (int t = 0; t < numOfThread; ++t) { + SThreadParaObj *pThread = threadObj + t; + pThread->threadIndex = t; + pThread->totalThreads = numOfThread; + strcpy(pThread->dbName, dbName); + pThread->taosCon = taos_connect(args->host, args->user, args->password, NULL, args->port); + + if (pThread->taosCon == NULL) { + fprintf(stderr, "ERROR: thread:%d failed connect to TDengine, error:%s\n", pThread->threadIndex, taos_errstr(pThread->taosCon)); + exit(0); + } + + pthread_attr_init(&thattr); + pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + + if (pthread_create(&(pThread->threadID), &thattr, taosDumpOutWorkThreadFp, (void*)pThread) != 0) { + fprintf(stderr, "ERROR: thread:%d failed to start\n", pThread->threadIndex); + exit(0); + } + } + + for (int32_t t = 0; t < numOfThread; ++t) { + pthread_join(threadObj[t].threadID, NULL); + } + + for (int32_t t = 0; t < numOfThread; ++t) { + taos_close(threadObj[t].taosCon); + } + free(threadObj); +} + + + +int32_t taosDumpStable(char *table, FILE *fp, TAOS* taosCon) { + int count = 0; + + STableDef *tableDes = (STableDef *)calloc(1, sizeof(STableDef) + sizeof(SColDes) * TSDB_MAX_COLUMNS); + if (NULL == tableDes) { + fprintf(stderr, "failed to allocate memory\n"); + exit(-1); + } + + count = taosGetTableDes(table, tableDes, taosCon); + + if (count < 0) { + free(tableDes); + fprintf(stderr, "failed to get stable schema\n"); + exit(-1); + } + + taosDumpCreateTableClause(tableDes, count, fp); + + free(tableDes); + return 0; +} + + +int32_t taosDumpCreateSuperTableClause(TAOS* taosCon, char* dbName, FILE *fp) +{ + TAOS_ROW row; + int fd = -1; + STableRecord tableRecord; + + char* tmpCommand = (char *)malloc(COMMAND_SIZE); + if (tmpCommand == NULL) { + fprintf(stderr, "failed to allocate memory\n"); + exit(-1); + } + + sprintf(tmpCommand, "use %s", dbName); + + TAOS_RES* tmpResult = taos_query(taosCon, tmpCommand); + int32_t code = taos_errno(tmpResult); + if (code != 0) { + fprintf(stderr, "invalid database %s, error: %s\n", dbName, taos_errstr(taosCon)); + free(tmpCommand); + taos_free_result(tmpResult); + exit(-1); + } + + taos_free_result(tmpResult); + + sprintf(tmpCommand, "show stables"); + + tmpResult = taos_query(taosCon, tmpCommand); + code = taos_errno(tmpResult); + if (code != 0) { + fprintf(stderr, "failed to run command %s, error: %s\n", tmpCommand, taos_errstr(taosCon)); + free(tmpCommand); + taos_free_result(tmpResult); + exit(-1); + } + taos_free_result(tmpResult); + + TAOS_FIELD *fields = taos_fetch_fields(tmpResult); + + char tmpFileName[TSDB_FILENAME_LEN + 1]; + memset(tmpFileName, 0, TSDB_FILENAME_LEN); + sprintf(tmpFileName, ".stables.tmp"); + fd = open(tmpFileName, O_RDWR | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); + if (fd == -1) { + fprintf(stderr, "failed to open temp file: %s\n", tmpFileName); + taos_free_result(tmpResult); + free(tmpCommand); + remove(".stables.tmp"); + exit(-1); + } + + while ((row = taos_fetch_row(tmpResult)) != NULL) { + memset(&tableRecord, 0, sizeof(STableRecord)); + strncpy(tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes); + twrite(fd, &tableRecord, sizeof(STableRecord)); + } + + taos_free_result(tmpResult); + lseek(fd, 0, SEEK_SET); + + while (read(fd, &tableRecord, sizeof(STableRecord)) > 0) { + (void)taosDumpStable(tableRecord.name, fp, taosCon); + } + + tclose(fd); + remove(".stables.tmp"); + + free(tmpCommand); + return 0; +} + + +int taosDumpDb(SDbInfo *dbInfo, struct arguments *arguments, FILE *fp, TAOS *taosCon) { TAOS_ROW row; int fd = -1; STableRecord tableRecord; taosDumpCreateDbClause(dbInfo, arguments->with_property, fp); - sprintf(command, "use %s", dbInfo->name); - if (taos_errno(taos_query(taos, command)) != 0) { - fprintf(stderr, "invalid database %s\n", dbInfo->name); + char* tmpCommand = (char *)malloc(COMMAND_SIZE); + if (tmpCommand == NULL) { + fprintf(stderr, "failed to allocate memory\n"); return -1; } + sprintf(tmpCommand, "use %s", dbInfo->name); + + TAOS_RES* tmpResult = taos_query(taosCon, tmpCommand); + int32_t code = taos_errno(tmpResult); + if (code != 0) { + fprintf(stderr, "invalid database %s\n", dbInfo->name); + free(tmpCommand); + taos_free_result(tmpResult); + return -1; + } + taos_free_result(tmpResult); + fprintf(fp, "USE %s\n\n", dbInfo->name); + + (void)taosDumpCreateSuperTableClause(taosCon, dbInfo->name, fp); - sprintf(command, "show tables"); - TAOS_RES* result = taos_query(taos,command); - int32_t code = taos_errno(result); + sprintf(tmpCommand, "show tables"); + + tmpResult = taos_query(taosCon, tmpCommand); + code = taos_errno(tmpResult); if (code != 0) { - fprintf(stderr, "failed to run command %s, error: %s\n", command, taos_errstr(result)); - taos_free_result(result); + fprintf(stderr, "failed to run command %s\n", tmpCommand); + free(tmpCommand); + taos_free_result(tmpResult); return -1; } - TAOS_FIELD *fields = taos_fetch_fields(result); + TAOS_FIELD *fields = taos_fetch_fields(tmpResult); - fd = open(".table.tmp", O_RDWR | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); - if (fd == -1) { - fprintf(stderr, "failed to open temp file\n"); - taos_free_result(result); - return -1; - } + int32_t numOfTable = 0; + int32_t numOfThread = 0; + char tmpFileName[TSDB_FILENAME_LEN + 1]; + while ((row = taos_fetch_row(tmpResult)) != NULL) { + if (0 == numOfTable) { + memset(tmpFileName, 0, TSDB_FILENAME_LEN); + sprintf(tmpFileName, ".tables.tmp.%d", numOfThread); + fd = open(tmpFileName, O_RDWR | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); + if (fd == -1) { + fprintf(stderr, "failed to open temp file: %s\n", tmpFileName); + taos_free_result(tmpResult); + for (int32_t loopCnt = 0; loopCnt < numOfThread; loopCnt++) { + sprintf(tmpFileName, ".tables.tmp.%d", loopCnt); + remove(tmpFileName); + } + free(tmpCommand); + return -1; + } - while ((row = taos_fetch_row(result)) != NULL) { + numOfThread++; + } + memset(&tableRecord, 0, sizeof(STableRecord)); strncpy(tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes); strncpy(tableRecord.metric, (char *)row[TSDB_SHOW_TABLES_METRIC_INDEX], fields[TSDB_SHOW_TABLES_METRIC_INDEX].bytes); twrite(fd, &tableRecord, sizeof(STableRecord)); + + numOfTable++; + + if (numOfTable >= arguments->table_batch) { + numOfTable = 0; + tclose(fd); + fd = -1; + } } + tclose(fd); + fd = -1; + taos_free_result(tmpResult); - taos_free_result(result); - - (void)lseek(fd, 0, SEEK_SET); - - STableRecord tableInfo; - while (1) { - memset(&tableInfo, 0, sizeof(STableRecord)); - ssize_t ret = read(fd, &tableInfo, sizeof(STableRecord)); - if (ret <= 0) break; - - tableInfo.name[sizeof(tableInfo.name) - 1] = 0; - tableInfo.metric[sizeof(tableInfo.metric) - 1] = 0; - taosDumpTable(tableInfo.name, tableInfo.metric, arguments, fp); + // start multi threads to dumpout + taosStartDumpOutWorkThreads(arguments, numOfThread, dbInfo->name); + for (int loopCnt = 0; loopCnt < numOfThread; loopCnt++) { + sprintf(tmpFileName, ".tables.tmp.%d", loopCnt); + remove(tmpFileName); } - - close(fd); - (void)remove(".table.tmp"); + + free(tmpCommand); return 0; } -void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, SDumpArguments *arguments, FILE *fp) { - char *pstr = NULL; - pstr = buffer; +void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, FILE *fp) { int counter = 0; int count_temp = 0; - pstr += sprintf(buffer, "CREATE TABLE IF NOT EXISTS %s", tableDes->name); + char* tmpBuf = (char *)malloc(COMMAND_SIZE); + if (tmpBuf == NULL) { + fprintf(stderr, "failed to allocate memory\n"); + return; + } + + char* pstr = tmpBuf; + + pstr += sprintf(tmpBuf, "CREATE TABLE IF NOT EXISTS %s", tableDes->name); for (; counter < numOfCols; counter++) { if (tableDes->cols[counter].note[0] != '\0') break; @@ -699,19 +1258,27 @@ void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, SDumpArgument } } - pstr += sprintf(pstr, ")"); + pstr += sprintf(pstr, ");"); - fprintf(fp, "%s\n\n", buffer); + fprintf(fp, "%s\n", tmpBuf); + + free(tmpBuf); } -void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols, SDumpArguments *arguments, - FILE *fp) { - char *pstr = NULL; - pstr = buffer; +void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols, FILE *fp) { int counter = 0; int count_temp = 0; - pstr += sprintf(buffer, "CREATE TABLE IF NOT EXISTS %s USING %s TAGS (", tableDes->name, metric); + char* tmpBuf = (char *)malloc(COMMAND_SIZE); + if (tmpBuf == NULL) { + fprintf(stderr, "failed to allocate memory\n"); + return; + } + + char *pstr = NULL; + pstr = tmpBuf; + + pstr += sprintf(tmpBuf, "CREATE TABLE IF NOT EXISTS %s USING %s TAGS (", tableDes->name, metric); for (; counter < numOfCols; counter++) { if (tableDes->cols[counter].note[0] != '\0') break; @@ -721,54 +1288,6 @@ void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols count_temp = counter; for (; counter < numOfCols; counter++) { - TAOS_ROW row = NULL; - - sprintf(command, "select %s from %s limit 1", tableDes->cols[counter].field, tableDes->name); - - TAOS_RES* result = taos_query(taos, command); - int32_t code = taos_errno(result); - if (code != 0) { - fprintf(stderr, "failed to run command %s, error: %s\n", command, taos_errstr(result)); - return; - } - - TAOS_FIELD *fields = taos_fetch_fields(result); - - row = taos_fetch_row(result); - switch (fields[0].type) { - case TSDB_DATA_TYPE_BOOL: - sprintf(tableDes->cols[counter].note, "%d", ((((int)(*((char *)row[0]))) == 1) ? 1 : 0)); - break; - case TSDB_DATA_TYPE_TINYINT: - sprintf(tableDes->cols[counter].note, "%d", (int)(*((char *)row[0]))); - break; - case TSDB_DATA_TYPE_SMALLINT: - sprintf(tableDes->cols[counter].note, "%d", (int)(*((short *)row[0]))); - break; - case TSDB_DATA_TYPE_INT: - sprintf(tableDes->cols[counter].note, "%d", *((int *)row[0])); - break; - case TSDB_DATA_TYPE_BIGINT: - sprintf(tableDes->cols[counter].note, "%" PRId64 "", *((int64_t *)row[0])); - break; - case TSDB_DATA_TYPE_FLOAT: - sprintf(tableDes->cols[counter].note, "%f", GET_FLOAT_VAL(row[0])); - break; - case TSDB_DATA_TYPE_DOUBLE: - sprintf(tableDes->cols[counter].note, "%f", GET_DOUBLE_VAL(row[0])); - break; - case TSDB_DATA_TYPE_TIMESTAMP: - sprintf(tableDes->cols[counter].note, "%" PRId64 "", *(int64_t *)row[0]); - break; - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: - default: - strncpy(tableDes->cols[counter].note, (char *)row[0], fields[0].bytes); - break; - } - - taos_free_result(result); - if (counter != count_temp) { if (strcasecmp(tableDes->cols[counter].type, "binary") == 0 || strcasecmp(tableDes->cols[counter].type, "nchar") == 0) { @@ -792,193 +1311,95 @@ void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols /* } */ } - pstr += sprintf(pstr, ")"); + pstr += sprintf(pstr, ");"); - fprintf(fp, "%s\n\n", buffer); + fprintf(fp, "%s\n", tmpBuf); + free(tmpBuf); } -int taosGetTableDes(char *table, STableDef *tableDes) { - TAOS_ROW row = NULL; - int count = 0; - - sprintf(command, "describe %s", table); - - TAOS_RES* result = taos_query(taos, command); - int32_t code = taos_errno(result); - if (code != 0) { - fprintf(stderr, "failed to run command %s, error: %s\n", command, taos_errstr(result)); - taos_free_result(result); - return -1; - } - - TAOS_FIELD *fields = taos_fetch_fields(result); - - tstrncpy(tableDes->name, table, TSDB_COL_NAME_LEN); - - while ((row = taos_fetch_row(result)) != NULL) { - strncpy(tableDes->cols[count].field, (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX], - fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes); - strncpy(tableDes->cols[count].type, (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], - fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes); - tableDes->cols[count].length = *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]); - strncpy(tableDes->cols[count].note, (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], - fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes); - - count++; - } - - taos_free_result(result); - result = NULL; - - return count; -} - -int32_t taosDumpTable(char *table, char *metric, SDumpArguments *arguments, FILE *fp) { - int count = 0; - - STableDef *tableDes = (STableDef *)calloc(1, sizeof(STableDef) + sizeof(SColDes) * TSDB_MAX_COLUMNS); - - if (metric != NULL && metric[0] != '\0') { // dump metric definition - count = taosGetTableDes(metric, tableDes); - - if (count < 0) { - free(tableDes); - return -1; - } - - taosDumpCreateTableClause(tableDes, count, arguments, fp); - - memset(tableDes, 0, sizeof(STableDef) + sizeof(SColDes) * TSDB_MAX_COLUMNS); - - count = taosGetTableDes(table, tableDes); - - if (count < 0) { - free(tableDes); - return -1; - } - - taosDumpCreateMTableClause(tableDes, metric, count, arguments, fp); - - } else { // dump table definition - count = taosGetTableDes(table, tableDes); - - if (count < 0) { - free(tableDes); - return -1; - } - - taosDumpCreateTableClause(tableDes, count, arguments, fp); - } - - free(tableDes); - - return taosDumpTableData(fp, table, arguments); -} - -int32_t taosDumpMetric(char *metric, SDumpArguments *arguments, FILE *fp) { - TAOS_ROW row = NULL; - int fd = -1; - STableRecord tableRecord; - - //tstrncpy(tableRecord.metric, metric, TSDB_TABLE_NAME_LEN); - - sprintf(command, "select tbname from %s", metric); - TAOS_RES* result = taos_query(taos, command); - int32_t code = taos_errno(result); - if (code != 0) { - fprintf(stderr, "failed to run command %s, error: %s\n", command, taos_errstr(result)); - taos_free_result(result); - return -1; - } - - fd = open(".table.tmp", O_RDWR | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); - if (fd < 0) { - fprintf(stderr, "failed to open temp file"); - return -1; - } - - TAOS_FIELD *fields = taos_fetch_fields(result); - - while ((row = taos_fetch_row(result)) != NULL) { - memset(&tableRecord, 0, sizeof(STableRecord)); - tstrncpy(tableRecord.name, (char *)row[0], fields[0].bytes); - tstrncpy(tableRecord.metric, metric, TSDB_TABLE_NAME_LEN); - twrite(fd, &tableRecord, sizeof(STableRecord)); - } - - taos_free_result(result); - result = NULL; - - (void)lseek(fd, 0, SEEK_SET); - - //STableRecord tableInfo; - char tableName[TSDB_TABLE_NAME_LEN] ; - char metricName[TSDB_TABLE_NAME_LEN]; - ssize_t ret; - while (1) { - //memset(&tableInfo, 0, sizeof(STableRecord)); - memset(tableName, 0, TSDB_TABLE_NAME_LEN); - memset(metricName, 0, TSDB_TABLE_NAME_LEN); - //ssize_t ret = read(fd, &tableInfo, sizeof(STableRecord)); - //if (ret <= 0) break; - ret = read(fd, tableName, TSDB_TABLE_NAME_LEN); - if (ret <= 0) break; - - ret = read(fd, metricName, TSDB_TABLE_NAME_LEN); - if (ret <= 0) break; - - //tableInfo.name[sizeof(tableInfo.name) - 1] = 0; - //tableInfo.metric[sizeof(tableInfo.metric) - 1] = 0; - //taosDumpTable(tableInfo.name, tableInfo.metric, arguments, fp); - //tstrncpy(tableName, tableInfo.name, TSDB_TABLE_NAME_LEN-1); - //tstrncpy(metricName, tableInfo.metric, TSDB_TABLE_NAME_LEN-1); - taosDumpTable(tableName, metricName, arguments, fp); - } - - close(fd); - (void)remove(".table.tmp"); - - return 0; -} - -int taosDumpTableData(FILE *fp, char *tbname, SDumpArguments *arguments) { +int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* taosCon) { /* char temp[MAX_COMMAND_SIZE] = "\0"; */ + int64_t totalRows = 0; int count = 0; char *pstr = NULL; TAOS_ROW row = NULL; int numFields = 0; char *tbuf = NULL; - if (arguments->schemaonly) return 0; - - sprintf(command, "select * from %s where _c0 >= %" PRId64 " and _c0 <= %" PRId64 " order by _c0 asc", tbname, arguments->start_time, - arguments->end_time); - - TAOS_RES* result = taos_query(taos, command); - int32_t code = taos_errno(result); - if (code != 0) { - fprintf(stderr, "failed to run command %s, reason: %s\n", command, taos_errstr(result)); - taos_free_result(result); + char* tmpCommand = (char *)calloc(1, COMMAND_SIZE); + if (tmpCommand == NULL) { + fprintf(stderr, "failed to allocate memory\n"); return -1; } - numFields = taos_field_count(result); + char* tmpBuffer = (char *)calloc(1, COMMAND_SIZE); + if (tmpBuffer == NULL) { + fprintf(stderr, "failed to allocate memory\n"); + free(tmpCommand); + return -1; + } + + pstr = tmpBuffer; + + if (arguments->schemaonly) { + free(tmpCommand); + free(tmpBuffer); + return 0; + } + + sprintf(tmpCommand, + "select * from %s where _c0 >= %" PRId64 " and _c0 <= %" PRId64 " order by _c0 asc", + tbname, + arguments->start_time, + arguments->end_time); + + TAOS_RES* tmpResult = taos_query(taosCon, tmpCommand); + int32_t code = taos_errno(tmpResult); + if (code != 0) { + fprintf(stderr, "failed to run command %s, reason: %s\n", tmpCommand, taos_errstr(taosCon)); + free(tmpCommand); + free(tmpBuffer); + taos_free_result(tmpResult); + return -1; + } + + numFields = taos_field_count(taosCon); assert(numFields > 0); - TAOS_FIELD *fields = taos_fetch_fields(result); + TAOS_FIELD *fields = taos_fetch_fields(tmpResult); tbuf = (char *)malloc(COMMAND_SIZE); if (tbuf == NULL) { fprintf(stderr, "No enough memory\n"); + free(tmpCommand); + free(tmpBuffer); + taos_free_result(tmpResult); return -1; } + char sqlStr[8] = "\0"; + if (arguments->mysqlFlag) { + sprintf(sqlStr, "INSERT"); + } else { + sprintf(sqlStr, "IMPORT"); + } + + int rowFlag = 0; count = 0; - while ((row = taos_fetch_row(result)) != NULL) { - pstr = buffer; + while ((row = taos_fetch_row(tmpResult)) != NULL) { + pstr = tmpBuffer; if (count == 0) { - pstr += sprintf(pstr, "INSERT INTO %s VALUES (", tbname); - } else { - pstr += sprintf(pstr, "("); + pstr += sprintf(pstr, "%s INTO %s VALUES (", sqlStr, tbname); + } else { + if (arguments->mysqlFlag) { + if (0 == rowFlag) { + pstr += sprintf(pstr, "("); + rowFlag++; + } else { + pstr += sprintf(pstr, ", ("); + } + } else { + pstr += sprintf(pstr, "("); + } } for (int col = 0; col < numFields; col++) { @@ -1003,7 +1424,7 @@ int taosDumpTableData(FILE *fp, char *tbname, SDumpArguments *arguments) { pstr += sprintf(pstr, "%d", *((int *)row[col])); break; case TSDB_DATA_TYPE_BIGINT: - pstr += sprintf(pstr, "%" PRId64, *((int64_t *)row[col])); + pstr += sprintf(pstr, "%" PRId64 "", *((int64_t *)row[col])); break; case TSDB_DATA_TYPE_FLOAT: pstr += sprintf(pstr, "%f", GET_FLOAT_VAL(row[col])); @@ -1022,34 +1443,52 @@ int taosDumpTableData(FILE *fp, char *tbname, SDumpArguments *arguments) { pstr += sprintf(pstr, "\'%s\'", tbuf); break; case TSDB_DATA_TYPE_TIMESTAMP: - pstr += sprintf(pstr, "%" PRId64, *(int64_t *)row[col]); + if (!arguments->mysqlFlag) { + pstr += sprintf(pstr, "%" PRId64 "", *(int64_t *)row[col]); + } else { + char buf[64] = "\0"; + int64_t ts = *((int64_t *)row[col]); + time_t tt = (time_t)(ts / 1000); + struct tm *ptm = localtime(&tt); + strftime(buf, 64, "%y-%m-%d %H:%M:%S", ptm); + pstr += sprintf(pstr, "\'%s.%03d\'", buf, (int)(ts % 1000)); + } break; default: break; } } - sprintf(pstr, ")"); + pstr += sprintf(pstr, ") "); + + totalRows++; count++; - fprintf(fp, "%s", buffer); + fprintf(fp, "%s", tmpBuffer); if (count >= arguments->data_batch) { - fprintf(fp, "\n"); + fprintf(fp, ";\n"); count = 0; - } else { - fprintf(fp, "\\\n"); - } + } //else { + //fprintf(fp, "\\\n"); + //} } + atomic_add_fetch_64(&totalDumpOutRows, totalRows); + fprintf(fp, "\n"); - if (tbuf) free(tbuf); - taos_free_result(result); - result = NULL; + if (tbuf) { + free(tbuf); + } + + taos_free_result(tmpResult); + tmpResult = NULL; + free(tmpCommand); + free(tmpBuffer); return 0; } -int taosCheckParam(SDumpArguments *arguments) { +int taosCheckParam(struct arguments *arguments) { if (arguments->all_databases && arguments->databases) { fprintf(stderr, "conflict option --all-databases and --databases\n"); return -1; @@ -1059,19 +1498,25 @@ int taosCheckParam(SDumpArguments *arguments) { fprintf(stderr, "start time is larger than end time\n"); return -1; } + if (arguments->arg_list_len == 0) { if ((!arguments->all_databases) && (!arguments->isDumpIn)) { fprintf(stderr, "taosdump requires parameters\n"); return -1; } } - - if (arguments->isDumpIn && (strcmp(arguments->output, DEFAULT_DUMP_FILE) != 0)) { - fprintf(stderr, "duplicate parameter input and output file\n"); +/* + if (arguments->isDumpIn && (strcmp(arguments->outpath, DEFAULT_DUMP_FILE) != 0)) { + fprintf(stderr, "duplicate parameter input and output file path\n"); + return -1; + } +*/ + if (!arguments->isDumpIn && arguments->encode != NULL) { + fprintf(stderr, "invalid option in dump out\n"); return -1; } - if (!arguments->isDumpIn && arguments->encode != NULL) { + if (arguments->table_batch <= 0) { fprintf(stderr, "invalid option in dump out\n"); return -1; } @@ -1134,177 +1579,6 @@ void taosReplaceCtrlChar(char *str) { *pstr = '\0'; } -int taosDumpIn(SDumpArguments *arguments) { - assert(arguments->isDumpIn); - - int tsize = 0; - FILE * fp = NULL; - char * line = NULL; - _Bool isRun = true; - size_t line_size = 0; - char * pstr = NULL, *lstr = NULL; - iconv_t cd = (iconv_t)-1; - size_t inbytesleft = 0; - size_t outbytesleft = COMMAND_SIZE; - char fcharset[64]; - char * tcommand = NULL; - - fp = fopen(arguments->input, "r"); - if (fp == NULL) { - fprintf(stderr, "failed to open input file %s\n", arguments->input); - return -1; - } - - taosLoadFileCharset(fp, fcharset); - - taos = taos_connect(arguments->host, arguments->user, arguments->password, NULL, arguments->port); - if (taos == NULL) { - fprintf(stderr, "failed to connect to TDengine server\n"); - goto _dumpin_exit_failure; - } - - command = (char *)malloc(COMMAND_SIZE); - lcommand = (char *)malloc(COMMAND_SIZE); - if (command == NULL || lcommand == NULL) { - fprintf(stderr, "failed to connect to allocate memory\n"); - goto _dumpin_exit_failure; - } - - // Resolve locale - if (*fcharset != '\0') { - arguments->encode = fcharset; - } - - if (arguments->encode != NULL && strcasecmp(tsCharset, arguments->encode) != 0) { - cd = iconv_open(tsCharset, arguments->encode); - if (cd == (iconv_t)-1) { - fprintf(stderr, "Failed to open iconv handle\n"); - goto _dumpin_exit_failure; - } - } - - pstr = command; - int64_t linenu = 0; - while (1) { - ssize_t size = getline(&line, &line_size, fp); - linenu++; - if (size <= 0) break; - if (size == 1) { - if (pstr != command) { - inbytesleft = pstr - command; - memset(lcommand, 0, COMMAND_SIZE); - pstr = command; - lstr = lcommand; - outbytesleft = COMMAND_SIZE; - if (cd != (iconv_t)-1) { - iconv(cd, &pstr, &inbytesleft, &lstr, &outbytesleft); - tcommand = lcommand; - } else { - tcommand = command; - } - taosReplaceCtrlChar(tcommand); - - TAOS_RES* result = taos_query(taos, tcommand); - if (taos_errno(result) != 0){ - fprintf(stderr, "linenu: %" PRId64 " failed to run command %s reason:%s \ncontinue...\n", linenu, command, - taos_errstr(result)); - taos_free_result(result); - } - - pstr = command; - pstr[0] = '\0'; - tsize = 0; - isRun = true; - } - - continue; - } - - /* if (line[0] == '-' && line[1] == '-') continue; */ - - line[size - 1] = 0; - - if (tsize + size - 1 > COMMAND_SIZE) { - fprintf(stderr, "command is too long\n"); - goto _dumpin_exit_failure; - } - - if (line[size - 2] == '\\') { - line[size - 2] = ' '; - isRun = false; - } else { - isRun = true; - } - - memcpy(pstr, line, size - 1); - pstr += (size - 1); - *pstr = '\0'; - - if (!isRun) continue; - - if (command != pstr && !isEmptyCommand(command)) { - inbytesleft = pstr - command; - memset(lcommand, 0, COMMAND_SIZE); - pstr = command; - lstr = lcommand; - outbytesleft = COMMAND_SIZE; - if (cd != (iconv_t)-1) { - iconv(cd, &pstr, &inbytesleft, &lstr, &outbytesleft); - tcommand = lcommand; - } else { - tcommand = command; - } - taosReplaceCtrlChar(tcommand); - TAOS_RES* result = taos_query(taos, tcommand); - int32_t code = taos_errno(result); - if (code != 0) - { - fprintf(stderr, "linenu:%" PRId64 " failed to run command %s reason: %s \ncontinue...\n", linenu, command, - taos_errstr(result)); - } - taos_free_result(result); - } - - pstr = command; - *pstr = '\0'; - tsize = 0; - } - - if (pstr != command) { - inbytesleft = pstr - command; - memset(lcommand, 0, COMMAND_SIZE); - pstr = command; - lstr = lcommand; - outbytesleft = COMMAND_SIZE; - if (cd != (iconv_t)-1) { - iconv(cd, &pstr, &inbytesleft, &lstr, &outbytesleft); - tcommand = lcommand; - } else { - tcommand = command; - } - taosReplaceCtrlChar(lcommand); - if (taos_query(taos, tcommand) == NULL) - fprintf(stderr, "linenu:%" PRId64 " failed to run command %s reason:%s \ncontinue...\n", linenu, command, - taos_errstr(taos)); - } - - if (cd != (iconv_t)-1) iconv_close(cd); - tfree(line); - tfree(command); - tfree(lcommand); - taos_close(taos); - fclose(fp); - return 0; - -_dumpin_exit_failure: - if (cd != (iconv_t)-1) iconv_close(cd); - tfree(command); - tfree(lcommand); - taos_close(taos); - fclose(fp); - return -1; -} - char *ascii_literal_list[] = { "\\x00", "\\x01", "\\x02", "\\x03", "\\x04", "\\x05", "\\x06", "\\x07", "\\x08", "\\t", "\\n", "\\x0b", "\\x0c", "\\r", "\\x0e", "\\x0f", "\\x10", "\\x11", "\\x12", "\\x13", "\\x14", "\\x15", "\\x16", "\\x17", "\\x18", "\\x19", @@ -1351,7 +1625,7 @@ int convertNCharToReadable(char *str, int size, char *buf, int bufsize) { if ((int)wc < 256) { pbuf = stpcpy(pbuf, ascii_literal_list[(int)wc]); - } else if (byte_width > 0) { + } else { memcpy(pbuf, pstr, byte_width); pbuf += byte_width; } @@ -1399,3 +1673,453 @@ _exit_no_charset: tfree(line); return; } + +// ======== dumpIn support multi threads functions ================================// + +static char **tsDumpInSqlFiles = NULL; +static int32_t tsSqlFileNum = 0; +static char tsDbSqlFile[TSDB_FILENAME_LEN] = {0}; +static char tsfCharset[64] = {0}; +static int taosGetFilesNum(const char *directoryName, const char *prefix) +{ + char cmd[1024] = { 0 }; + sprintf(cmd, "ls %s/*.%s | wc -l ", directoryName, prefix); + + FILE *fp = popen(cmd, "r"); + if (fp == NULL) { + fprintf(stderr, "ERROR: failed to execute:%s, error:%s\n", cmd, strerror(errno)); + exit(0); + } + + int fileNum = 0; + if (fscanf(fp, "%d", &fileNum) != 1) { + fprintf(stderr, "ERROR: failed to execute:%s, parse result error\n", cmd); + exit(0); + } + + if (fileNum <= 0) { + fprintf(stderr, "ERROR: directory:%s is empry\n", directoryName); + exit(0); + } + + pclose(fp); + return fileNum; +} + +static void taosParseDirectory(const char *directoryName, const char *prefix, char **fileArray, int totalFiles) +{ + char cmd[1024] = { 0 }; + sprintf(cmd, "ls %s/*.%s | sort", directoryName, prefix); + + FILE *fp = popen(cmd, "r"); + if (fp == NULL) { + fprintf(stderr, "ERROR: failed to execute:%s, error:%s\n", cmd, strerror(errno)); + exit(0); + } + + int fileNum = 0; + while (fscanf(fp, "%s", fileArray[fileNum++])) { + if (strcmp(fileArray[fileNum-1], tsDbSqlFile) == 0) { + fileNum--; + } + if (fileNum >= totalFiles) { + break; + } + } + + if (fileNum != totalFiles) { + fprintf(stderr, "ERROR: directory:%s changed while read\n", directoryName); + exit(0); + } + + pclose(fp); +} + +static void taosCheckTablesSQLFile(const char *directoryName) +{ + char cmd[1024] = { 0 }; + sprintf(cmd, "ls %s/dbs.sql", directoryName); + + FILE *fp = popen(cmd, "r"); + if (fp == NULL) { + fprintf(stderr, "ERROR: failed to execute:%s, error:%s\n", cmd, strerror(errno)); + exit(0); + } + + while (fscanf(fp, "%s", tsDbSqlFile)) { + break; + } + + pclose(fp); +} + +static void taosMallocSQLFiles() +{ + tsDumpInSqlFiles = (char**)calloc(tsSqlFileNum, sizeof(char*)); + for (int i = 0; i < tsSqlFileNum; i++) { + tsDumpInSqlFiles[i] = calloc(1, TSDB_FILENAME_LEN); + } +} + +static void taosFreeSQLFiles() +{ + for (int i = 0; i < tsSqlFileNum; i++) { + tfree(tsDumpInSqlFiles[i]); + } + tfree(tsDumpInSqlFiles); +} + +static void taosGetDirectoryFileList(char *inputDir) +{ + struct stat fileStat; + if (stat(inputDir, &fileStat) < 0) { + fprintf(stderr, "ERROR: %s not exist\n", inputDir); + exit(0); + } + + if (fileStat.st_mode & S_IFDIR) { + taosCheckTablesSQLFile(inputDir); + tsSqlFileNum = taosGetFilesNum(inputDir, "sql"); + int totalSQLFileNum = tsSqlFileNum; + if (tsDbSqlFile[0] != 0) { + tsSqlFileNum--; + } + taosMallocSQLFiles(); + taosParseDirectory(inputDir, "sql", tsDumpInSqlFiles, tsSqlFileNum); + fprintf(stdout, "\nstart to dispose %d files in %s\n", totalSQLFileNum, inputDir); + } + else { + fprintf(stderr, "ERROR: %s is not a directory\n", inputDir); + exit(0); + } +} + +static FILE* taosOpenDumpInFile(char *fptr) { + wordexp_t full_path; + + if (wordexp(fptr, &full_path, 0) != 0) { + fprintf(stderr, "ERROR: illegal file name: %s\n", fptr); + return NULL; + } + + char *fname = full_path.we_wordv[0]; + + if (access(fname, F_OK) != 0) { + fprintf(stderr, "ERROR: file %s is not exist\n", fptr); + + wordfree(&full_path); + return NULL; + } + + if (access(fname, R_OK) != 0) { + fprintf(stderr, "ERROR: file %s is not readable\n", fptr); + + wordfree(&full_path); + return NULL; + } + + FILE *f = fopen(fname, "r"); + if (f == NULL) { + fprintf(stderr, "ERROR: failed to open file %s\n", fname); + wordfree(&full_path); + return NULL; + } + + wordfree(&full_path); + + return f; +} + +int taosDumpInOneFile_old(TAOS * taos, FILE* fp, char* fcharset, char* encode) { + char *command = NULL; + char *lcommand = NULL; + int tsize = 0; + char *line = NULL; + _Bool isRun = true; + size_t line_size = 0; + char *pstr = NULL; + char *lstr = NULL; + size_t inbytesleft = 0; + size_t outbytesleft = COMMAND_SIZE; + char *tcommand = NULL; + char *charsetOfFile = NULL; + iconv_t cd = (iconv_t)(-1); + + command = (char *)malloc(COMMAND_SIZE); + lcommand = (char *)malloc(COMMAND_SIZE); + if (command == NULL || lcommand == NULL) { + fprintf(stderr, "failed to connect to allocate memory\n"); + goto _dumpin_exit_failure; + } + + // Resolve locale + if (*fcharset != '\0') { + charsetOfFile = fcharset; + } else { + charsetOfFile = encode; + } + + if (charsetOfFile != NULL && strcasecmp(tsCharset, charsetOfFile) != 0) { + cd = iconv_open(tsCharset, charsetOfFile); + if (cd == ((iconv_t)(-1))) { + fprintf(stderr, "Failed to open iconv handle\n"); + goto _dumpin_exit_failure; + } + } + + pstr = command; + int64_t linenu = 0; + while (1) { + ssize_t size = getline(&line, &line_size, fp); + linenu++; + if (size <= 0) break; + if (size == 1) { + if (pstr != command) { + inbytesleft = pstr - command; + memset(lcommand, 0, COMMAND_SIZE); + pstr = command; + lstr = lcommand; + outbytesleft = COMMAND_SIZE; + if (cd != ((iconv_t)(-1))) { + iconv(cd, &pstr, &inbytesleft, &lstr, &outbytesleft); + tcommand = lcommand; + } else { + tcommand = command; + } + + taosReplaceCtrlChar(tcommand); + + if (queryDB(taos, tcommand) != 0) { + fprintf(stderr, "error sql: linenu: %" PRId64 " failed\n", linenu); + exit(0); + } + + pstr = command; + pstr[0] = '\0'; + tsize = 0; + isRun = true; + } + + continue; + } + + /* if (line[0] == '-' && line[1] == '-') continue; */ + + line[size - 1] = 0; + + if (tsize + size - 1 > COMMAND_SIZE) { + fprintf(stderr, "command is too long\n"); + goto _dumpin_exit_failure; + } + + if (line[size - 2] == '\\') { + line[size - 2] = ' '; + isRun = false; + } else { + isRun = true; + } + + memcpy(pstr, line, size - 1); + pstr += (size - 1); + *pstr = '\0'; + + if (!isRun) continue; + + if (command != pstr && !isEmptyCommand(command)) { + inbytesleft = pstr - command; + memset(lcommand, 0, COMMAND_SIZE); + pstr = command; + lstr = lcommand; + outbytesleft = COMMAND_SIZE; + if (cd != ((iconv_t)(-1))) { + iconv(cd, &pstr, &inbytesleft, &lstr, &outbytesleft); + tcommand = lcommand; + } else { + tcommand = command; + } + taosReplaceCtrlChar(tcommand); + if (queryDB(taos, tcommand) != 0) { + fprintf(stderr, "error sql: linenu:%" PRId64 " failed\n", linenu); + exit(0); + } + } + + pstr = command; + *pstr = '\0'; + tsize = 0; + } + + if (pstr != command) { + inbytesleft = pstr - command; + memset(lcommand, 0, COMMAND_SIZE); + pstr = command; + lstr = lcommand; + outbytesleft = COMMAND_SIZE; + if (cd != ((iconv_t)(-1))) { + iconv(cd, &pstr, &inbytesleft, &lstr, &outbytesleft); + tcommand = lcommand; + } else { + tcommand = command; + } + taosReplaceCtrlChar(lcommand); + if (queryDB(taos, tcommand) != 0) + fprintf(stderr, "error sql: linenu:%" PRId64 " failed \n", linenu); + } + + if (cd != ((iconv_t)(-1))) iconv_close(cd); + tfree(line); + tfree(command); + tfree(lcommand); + taos_close(taos); + fclose(fp); + return 0; + +_dumpin_exit_failure: + if (cd != ((iconv_t)(-1))) iconv_close(cd); + tfree(command); + tfree(lcommand); + taos_close(taos); + fclose(fp); + return -1; +} + +int taosDumpInOneFile(TAOS * taos, FILE* fp, char* fcharset, char* encode, char* fileName) { + int read_len = 0; + char * cmd = NULL; + size_t cmd_len = 0; + char * line = NULL; + size_t line_len = 0; + + cmd = (char *)malloc(COMMAND_SIZE); + if (cmd == NULL) { + fprintf(stderr, "failed to allocate memory\n"); + return -1; + } + + int lineNo = 0; + while ((read_len = getline(&line, &line_len, fp)) != -1) { + ++lineNo; + if (read_len >= COMMAND_SIZE) continue; + line[--read_len] = '\0'; + + //if (read_len == 0 || isCommentLine(line)) { // line starts with # + if (read_len == 0 ) { + continue; + } + + if (line[read_len - 1] == '\\') { + line[read_len - 1] = ' '; + memcpy(cmd + cmd_len, line, read_len); + cmd_len += read_len; + continue; + } + + memcpy(cmd + cmd_len, line, read_len); + if (queryDB(taos, cmd)) { + fprintf(stderr, "error sql: linenu:%d, file:%s\n", lineNo, fileName); + } + + memset(cmd, 0, COMMAND_SIZE); + cmd_len = 0; + } + + tfree(cmd); + tfree(line); + fclose(fp); + return 0; +} + +void* taosDumpInWorkThreadFp(void *arg) +{ + SThreadParaObj *pThread = (SThreadParaObj*)arg; + for (int32_t f = 0; f < tsSqlFileNum; ++f) { + if (f % pThread->totalThreads == pThread->threadIndex) { + char *SQLFileName = tsDumpInSqlFiles[f]; + FILE* fp = taosOpenDumpInFile(SQLFileName); + if (NULL == fp) { + continue; + } + fprintf(stderr, "Success Open input file: %s\n", SQLFileName); + taosDumpInOneFile(pThread->taosCon, fp, tsfCharset, tsArguments.encode, SQLFileName); + } + } + + return NULL; +} + +static void taosStartDumpInWorkThreads(struct arguments *args) +{ + pthread_attr_t thattr; + SThreadParaObj *pThread; + int32_t totalThreads = args->thread_num; + + if (totalThreads > tsSqlFileNum) { + totalThreads = tsSqlFileNum; + } + + SThreadParaObj *threadObj = (SThreadParaObj *)calloc(totalThreads, sizeof(SThreadParaObj)); + for (int32_t t = 0; t < totalThreads; ++t) { + pThread = threadObj + t; + pThread->threadIndex = t; + pThread->totalThreads = totalThreads; + pThread->taosCon = taos_connect(args->host, args->user, args->password, NULL, args->port); + if (pThread->taosCon == NULL) { + fprintf(stderr, "ERROR: thread:%d failed connect to TDengine, error:%s\n", pThread->threadIndex, taos_errstr(pThread->taosCon)); + exit(0); + } + + pthread_attr_init(&thattr); + pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + + if (pthread_create(&(pThread->threadID), &thattr, taosDumpInWorkThreadFp, (void*)pThread) != 0) { + fprintf(stderr, "ERROR: thread:%d failed to start\n", pThread->threadIndex); + exit(0); + } + } + + for (int t = 0; t < totalThreads; ++t) { + pthread_join(threadObj[t].threadID, NULL); + } + + for (int t = 0; t < totalThreads; ++t) { + taos_close(threadObj[t].taosCon); + } + free(threadObj); +} + + +int taosDumpIn(struct arguments *arguments) { + assert(arguments->isDumpIn); + + TAOS *taos = NULL; + FILE *fp = NULL; + + taos = taos_connect(arguments->host, arguments->user, arguments->password, NULL, arguments->port); + if (taos == NULL) { + fprintf(stderr, "failed to connect to TDengine server\n"); + return -1; + } + + taosGetDirectoryFileList(arguments->inpath); + + if (tsDbSqlFile[0] != 0) { + fp = taosOpenDumpInFile(tsDbSqlFile); + if (NULL == fp) { + fprintf(stderr, "failed to open input file %s\n", tsDbSqlFile); + return -1; + } + fprintf(stderr, "Success Open input file: %s\n", tsDbSqlFile); + + taosLoadFileCharset(fp, tsfCharset); + + taosDumpInOneFile(taos, fp, tsfCharset, arguments->encode, tsDbSqlFile); + } + + taosStartDumpInWorkThreads(arguments); + + taos_close(taos); + taosFreeSQLFiles(); + return 0; +} + + diff --git a/src/kit/taosmigrate/taosmigrate.c b/src/kit/taosmigrate/taosmigrate.c index b7bf6fc1ba..599cc49334 100644 --- a/src/kit/taosmigrate/taosmigrate.c +++ b/src/kit/taosmigrate/taosmigrate.c @@ -51,6 +51,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { break; case 'f': arguments->fqdn = arg; + break; case 'g': arguments->dnodeGroups = arg; break; diff --git a/src/kit/taosmigrate/taosmigrateMnodeWal.c b/src/kit/taosmigrate/taosmigrateMnodeWal.c index 6315ff99f7..28e2b7772b 100644 --- a/src/kit/taosmigrate/taosmigrateMnodeWal.c +++ b/src/kit/taosmigrate/taosmigrateMnodeWal.c @@ -96,6 +96,7 @@ void walModWalFile(char* walfile) { if (wfd < 0) { printf("wal:%s, failed to open(%s)\n", newWalFile, strerror(errno)); free(buffer); + close(rfd); return ; } @@ -116,6 +117,11 @@ void walModWalFile(char* walfile) { break; } + if (pHead->len >= 1024000 - sizeof(SWalHead)) { + printf("wal:%s, SWalHead.len(%d) overflow, skip the rest of file\n", walfile, pHead->len); + break; + } + ret = read(rfd, pHead->cont, pHead->len); if ( ret != pHead->len) { printf("wal:%s, failed to read body, skip, len:%d ret:%d\n", walfile, pHead->len, ret); diff --git a/src/kit/taosmigrate/taosmigrateVnodeCfg.c b/src/kit/taosmigrate/taosmigrateVnodeCfg.c index 1cb2fee353..b925fb10aa 100644 --- a/src/kit/taosmigrate/taosmigrateVnodeCfg.c +++ b/src/kit/taosmigrate/taosmigrateVnodeCfg.c @@ -99,6 +99,8 @@ static int32_t readVnodeCfg(SVnodeObj *pVnode, char* cfgFile) goto PARSE_OVER; } + content[maxLen] = (char)0; + root = cJSON_Parse(content); if (root == NULL) { printf("failed to json parse %s, invalid json format\n", cfgFile); From 43600966541d4662f1ebb6f8e23c66a02cae58c1 Mon Sep 17 00:00:00 2001 From: Steven Li Date: Wed, 8 Jul 2020 06:42:37 +0000 Subject: [PATCH 06/12] Now crash_gen is able to manage the start/stop/restart of taosd --- tests/pytest/crash_gen.py | 203 +++++++++++++++++++++++++++++++------- 1 file changed, 167 insertions(+), 36 deletions(-) diff --git a/tests/pytest/crash_gen.py b/tests/pytest/crash_gen.py index 49c428b7f1..afe61128a5 100755 --- a/tests/pytest/crash_gen.py +++ b/tests/pytest/crash_gen.py @@ -1629,73 +1629,204 @@ class MyLoggingAdapter(logging.LoggerAdapter): # return '[%s] %s' % (self.extra['connid'], msg), kwargs class SvcManager: - + MAX_QUEUE_SIZE = 30 + def __init__(self): print("Starting service manager") signal.signal(signal.SIGTERM, self.sigIntHandler) signal.signal(signal.SIGINT, self.sigIntHandler) + signal.signal(signal.SIGUSR1, self.sigUsrHandler) self.ioThread = None self.subProcess = None self.shouldStop = False - self.status = MainExec.STATUS_RUNNING + # self.status = MainExec.STATUS_RUNNING # set inside _startTaosService() def svcOutputReader(self, out: IO, queue): - # print("This is the svcOutput Reader...") - for line in out : # iter(out.readline, b''): + # Important Reference: https://stackoverflow.com/questions/375427/non-blocking-read-on-a-subprocess-pipe-in-python + print("This is the svcOutput Reader...") + # for line in out : + for line in iter(out.readline, b''): # print("Finished reading a line: {}".format(line)) - queue.put(line.rstrip()) # get rid of new lines - print("No more output from incoming IO") # meaning sub process must have died + # print("Adding item to queue...") + line = line.decode("utf-8").rstrip() + queue.put(line) # This might block, and then causing "out" buffer to block + print("_i", end="", flush=True) + + # Trim the queue if necessary + oneTenthQSize = self.MAX_QUEUE_SIZE // 10 + if (queue.qsize() >= (self.MAX_QUEUE_SIZE - oneTenthQSize) ) : # 90% full? + print("Triming IPC queue by: {}".format(oneTenthQSize)) + for i in range(0, oneTenthQSize) : + try: + queue.get_nowait() + except Empty: + break # break out of for loop, no more trimming + + if self.shouldStop : + print("Stopping to read output from sub process") + break + + # queue.put(line) + print("No more output (most likely) from IO thread managing TDengine service") # meaning sub process must have died out.close() - def sigIntHandler(self, signalNumber, frame): - if self.status != MainExec.STATUS_RUNNING : - print("Ignoring repeated SIGINT...") - return # do nothing if it's already not running - self.status = MainExec.STATUS_STOPPING # immediately set our status + def _doMenu(self): + choice = "" + while True: + print("\nInterrupting Service Program, Choose an Action: ") + print("1: Resume") + print("2: Terminate") + print("3: Restart") + # Remember to update the if range below + # print("Enter Choice: ", end="", flush=True) + while choice == "": + choice = input("Enter Choice: ") + if choice != "": + break # done with reading repeated input + if choice in ["1", "2", "3"]: + break # we are done with whole method + print("Invalid choice, please try again.") + choice = "" # reset + return choice - print("Terminating program...") - self.subProcess.send_signal(signal.SIGINT) - self.shouldStop = True - self.joinIoThread() + def sigUsrHandler(self, signalNumber, frame) : + print("Interrupting main thread execution upon SIGUSR1") + if self.status != MainExec.STATUS_RUNNING : + print("Ignoring repeated SIG...") + return # do nothing if it's already not running + self.status = MainExec.STATUS_STOPPING + + choice = self._doMenu() + if choice == "1" : + self.sigHandlerResume() # TODO: can the sub-process be blocked due to us not reading from queue? + elif choice == "2" : + self._stopTaosService() + elif choice == "3" : + self._stopTaosService() + self._startTaosService() + else: + raise RuntimeError("Invalid menu choice: {}".format(choice)) + + def sigIntHandler(self, signalNumber, frame): + print("Sig INT Handler starting...") + if self.status != MainExec.STATUS_RUNNING : + print("Ignoring repeated SIG_INT...") + return + + self.status = MainExec.STATUS_STOPPING # immediately set our status + self._stopTaosService() + print("INT signal handler returning...") + + def sigHandlerResume(self) : + print("Resuming TDengine service manager thread (main thread)...\n\n") + self.status = MainExec.STATUS_RUNNING def joinIoThread(self): if self.ioThread : self.ioThread.join() - self.ioThread = None + self.ioThread = None + else : + print("Joining empty thread, doing nothing") + + def _procIpcBatch(self): + # Process all the output generated by the underlying sub process, managed by IO thread + while True : + try: + line = self.ipcQueue.get_nowait() # getting output at fast speed + print("_o", end="", flush=True) + except Empty: + # time.sleep(2.3) # wait only if there's no output + # no more output + return # we are done with THIS BATCH + else: # got line + print(line) - def run(self): + def _procIpcAll(self): + while True : + print("<<", end="", flush=True) + self._procIpcBatch() # process one batch + + # check if the ioThread is still running + if (not self.ioThread) or (not self.ioThread.is_alive()): + print("IO Thread (with subprocess) has ended, main thread now exiting...") + self._stopTaosService() + self._procIpcBatch() # one more batch + return # TODO: maybe one last batch? + + # Maybe handler says we should exit now + if self.shouldStop: + print("Main thread ending all IPC processing with IOThread/SubProcess") + self._procIpcBatch() # one more batch + return + + print(">>", end="", flush=True) + time.sleep(0.5) + + def _startTaosService(self): ON_POSIX = 'posix' in sys.builtin_module_names svcCmd = ['../../build/build/bin/taosd', '-c', '../../build/test/cfg'] # svcCmd = ['vmstat', '1'] - self.subProcess = subprocess.Popen(svcCmd, stdout=subprocess.PIPE, bufsize=1, close_fds=ON_POSIX, text=True) - q = Queue() - self.ioThread = threading.Thread(target=self.svcOutputReader, args=(self.subProcess.stdout, q)) + if self.subProcess : # already there + raise RuntimeError("Corrupt process state") + + self.subProcess = subprocess.Popen( + svcCmd, + stdout=subprocess.PIPE, + # bufsize=1, # not supported in binary mode + close_fds=ON_POSIX) # had text=True, which interferred with reading EOF + self.ipcQueue = Queue() + + if self.ioThread : + raise RuntimeError("Corrupt thread state") + self.ioThread = threading.Thread(target=self.svcOutputReader, args=(self.subProcess.stdout, self.ipcQueue)) self.ioThread.daemon = True # thread dies with the program self.ioThread.start() + self.shouldStop = False # don't let the main loop stop + self.status = MainExec.STATUS_RUNNING + + def _stopTaosService(self): + # can be called from both main thread or signal handler + print("Terminating TDengine service running as the sub process...") + # Linux will send Control-C generated SIGINT to the TDengine process already, ref: https://unix.stackexchange.com/questions/176235/fork-and-how-signals-are-delivered-to-processes + if not self.subProcess : + print("Process already stopped") + return + + retCode = self.subProcess.poll() + if retCode : # valid return code, process ended + self.subProcess = None + else: # process still alive, let's interrupt it + print("Sub process still running, sending SIG_INT and waiting for it to stop...") + self.subProcess.send_signal(signal.SIGINT) # sub process should end, then IPC queue should end, causing IO thread to end + try : + self.subProcess.wait(10) + except subprocess.TimeoutExpired as err: + print("Time out waiting for TDengine service process to exit") + else: + print("Sub process has ended") + self.subProcess = None + + if self.subProcess and (not self.subProcess.poll()): + print("Sub process is still running... pid = {}".format(self.subProcess.pid)) + + self.shouldStop = True + self.joinIoThread() + + def run(self): + self._startTaosService() + # proc = subprocess.Popen(['echo', '"to stdout"'], # stdout=subprocess.PIPE, # ) # stdout_value = proc.communicate()[0] # print('\tstdout: {}'.format(repr(stdout_value))) - while True : - try: - line = q.get_nowait() # getting output at fast speed - except Empty: - # print('no output yet') - time.sleep(2.3) # wait only if there's no output - else: # got line - print(line) - # print("----end of iteration----") - if self.shouldStop: - print("Ending main Svc thread") - break + self._procIpcAll() - print("end of loop") - - self.joinIoThread() - print("Finished") + print("End of loop reading from IPC queue") + self.joinIoThread() # should have started already + print("SvcManager Run Finished") class ClientManager: def __init__(self): From 85ddff090a4c4ad55d060825e941d9a962fe3097 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 8 Jul 2020 15:23:37 +0800 Subject: [PATCH 07/12] refactor pathfinding in util/dnodes.py --- tests/pytest/util/dnodes.py | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 370af1ba13..788f26a262 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -190,32 +190,31 @@ class TDDnode: "dnode:%d is deployed and configured by %s" % (self.index, self.cfgPath)) - def start(self): + def getBuildPath(self): selfPath = os.path.dirname(os.path.realpath(__file__)) - binPath = "" if ("community" in selfPath): - projPath = selfPath + "/../../../../" - - for root, dirs, files in os.walk(projPath): - if ("taosd" in files): - rootRealPath = os.path.dirname(os.path.realpath(root)) - if ("packaging" not in rootRealPath): - binPath = os.path.join(root, "taosd") - break + projPath = selfPath[:selfPath.find("community")] else: - projPath = selfPath + "/../../../" - for root, dirs, files in os.walk(projPath): - if ("taosd" in files): - rootRealPath = os.path.dirname(os.path.realpath(root)) - if ("packaging" not in rootRealPath): - binPath = os.path.join(root, "taosd") - break + projPath = selfPath[:selfPath.find("tests")] - if (binPath == ""): + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:root.find("build")] + break + return buildPath + + def start(self): + buildPath = self.getBuildPath() + + if (buildPath == ""): tdLog.exit("taosd not found!") else: - tdLog.info("taosd found in %s" % rootRealPath) + tdLog.info("taosd found in %s" % buildPath) + + binPath = buildPath + "/build/bin/taosd" if self.deployed == 0: tdLog.exit("dnode:%d is not deployed" % (self.index)) From 44bc7056da3be24b6ab4c7d16c95452f6234db6f Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 8 Jul 2020 15:55:10 +0800 Subject: [PATCH 08/12] turn on output for debug. --- tests/test-all.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test-all.sh b/tests/test-all.sh index 275c6b1677..26005c8bc4 100755 --- a/tests/test-all.sh +++ b/tests/test-all.sh @@ -27,7 +27,7 @@ function runPyCaseOneByOne { if [[ $line != *sleep* ]]; then case=`echo $line|awk '{print $NF}'` start_time=`date +%s` - $line > /dev/null 2>&1 && \ + $line && \ echo -e "${GREEN}$case success${NC}" | tee -a pytest-out.log || \ echo -e "${RED}$case failed${NC}" | tee -a pytest-out.log end_time=`date +%s` From d2b805f3a5424b63a08defd0ec33f003bfdc7598 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 8 Jul 2020 16:19:01 +0800 Subject: [PATCH 09/12] fix pathfinding for CI --- tests/pytest/util/dnodes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 788f26a262..aa865ff9b4 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -202,7 +202,7 @@ class TDDnode: if ("taosd" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): - buildPath = root[:root.find("build")] + buildPath = root[:len(root)-len("/build/bin")] break return buildPath From 095d5a68bf5f457039483a5b08179a70dc26937a Mon Sep 17 00:00:00 2001 From: Steven Li Date: Wed, 8 Jul 2020 08:27:59 +0000 Subject: [PATCH 10/12] Added the -a option to crash_gen tool to automatically start/stop TDengine service --- tests/pytest/crash_gen.py | 58 +++++++++++++++++++++++++++------------ 1 file changed, 41 insertions(+), 17 deletions(-) diff --git a/tests/pytest/crash_gen.py b/tests/pytest/crash_gen.py index afe61128a5..e882780b8f 100755 --- a/tests/pytest/crash_gen.py +++ b/tests/pytest/crash_gen.py @@ -1629,7 +1629,7 @@ class MyLoggingAdapter(logging.LoggerAdapter): # return '[%s] %s' % (self.extra['connid'], msg), kwargs class SvcManager: - MAX_QUEUE_SIZE = 30 + MAX_QUEUE_SIZE = 10000 def __init__(self): print("Starting service manager") @@ -1667,7 +1667,7 @@ class SvcManager: break # queue.put(line) - print("No more output (most likely) from IO thread managing TDengine service") # meaning sub process must have died + print("\nNo more output (most likely) from IO thread managing TDengine service") # meaning sub process must have died out.close() def _doMenu(self): @@ -1700,10 +1700,10 @@ class SvcManager: if choice == "1" : self.sigHandlerResume() # TODO: can the sub-process be blocked due to us not reading from queue? elif choice == "2" : - self._stopTaosService() + self.stopTaosService() elif choice == "3" : - self._stopTaosService() - self._startTaosService() + self.stopTaosService() + self.startTaosService() else: raise RuntimeError("Invalid menu choice: {}".format(choice)) @@ -1714,7 +1714,7 @@ class SvcManager: return self.status = MainExec.STATUS_STOPPING # immediately set our status - self._stopTaosService() + self.stopTaosService() print("INT signal handler returning...") def sigHandlerResume(self) : @@ -1728,12 +1728,17 @@ class SvcManager: else : print("Joining empty thread, doing nothing") + TD_READY_MSG = "TDengine is initialized successfully" def _procIpcBatch(self): # Process all the output generated by the underlying sub process, managed by IO thread while True : try: line = self.ipcQueue.get_nowait() # getting output at fast speed - print("_o", end="", flush=True) + print("_o", end="", flush=True) + if self.status == MainExec.STATUS_STARTING : # we are starting, let's see if we have started + if line.find(self.TD_READY_MSG) != -1 : # found + self.status = MainExec.STATUS_RUNNING + except Empty: # time.sleep(2.3) # wait only if there's no output # no more output @@ -1743,13 +1748,13 @@ class SvcManager: def _procIpcAll(self): while True : - print("<<", end="", flush=True) + print("<", end="", flush=True) self._procIpcBatch() # process one batch # check if the ioThread is still running if (not self.ioThread) or (not self.ioThread.is_alive()): print("IO Thread (with subprocess) has ended, main thread now exiting...") - self._stopTaosService() + self.stopTaosService() self._procIpcBatch() # one more batch return # TODO: maybe one last batch? @@ -1759,10 +1764,10 @@ class SvcManager: self._procIpcBatch() # one more batch return - print(">>", end="", flush=True) + print(">", end="", flush=True) time.sleep(0.5) - def _startTaosService(self): + def startTaosService(self): ON_POSIX = 'posix' in sys.builtin_module_names svcCmd = ['../../build/build/bin/taosd', '-c', '../../build/test/cfg'] # svcCmd = ['vmstat', '1'] @@ -1783,9 +1788,19 @@ class SvcManager: self.ioThread.start() self.shouldStop = False # don't let the main loop stop - self.status = MainExec.STATUS_RUNNING + self.status = MainExec.STATUS_STARTING - def _stopTaosService(self): + # wait for service to start + for i in range(0, 10) : + time.sleep(1.0) + self._procIpcBatch() # pump messages + print("_zz_", end="", flush=True) + if self.status == MainExec.STATUS_RUNNING : + print("TDengine service READY to process requests") + return # now we've started + raise RuntimeError("TDengine service did not start successfully") # TODO: handle this better? + + def stopTaosService(self): # can be called from both main thread or signal handler print("Terminating TDengine service running as the sub process...") # Linux will send Control-C generated SIGINT to the TDengine process already, ref: https://unix.stackexchange.com/questions/176235/fork-and-how-signals-are-delivered-to-processes @@ -1804,7 +1819,7 @@ class SvcManager: except subprocess.TimeoutExpired as err: print("Time out waiting for TDengine service process to exit") else: - print("Sub process has ended") + print("TDengine service process terminated successfully from SIG_INT") self.subProcess = None if self.subProcess and (not self.subProcess.poll()): @@ -1814,7 +1829,7 @@ class SvcManager: self.joinIoThread() def run(self): - self._startTaosService() + self.startTaosService() # proc = subprocess.Popen(['echo', '"to stdout"'], # stdout=subprocess.PIPE, @@ -1878,6 +1893,10 @@ class ClientManager: self._printLastNumbers() def run(self): + if gConfig.auto_start_service : + svcMgr = SvcManager() + svcMgr.startTaosService() + self._printLastNumbers() dbManager = DbManager() # Regular function @@ -1889,6 +1908,8 @@ class ClientManager: # print("exec stats: {}".format(self.tc.getExecStats())) # print("TC failed = {}".format(self.tc.isFailed())) self.conclude() + if gConfig.auto_start_service : + svcMgr.stopTaosService() # print("TC failed (2) = {}".format(self.tc.isFailed())) return 1 if self.tc.isFailed() else 0 # Linux return code: ref https://shapeshed.com/unix-exit-codes/ @@ -1898,8 +1919,9 @@ class ClientManager: class MainExec: - STATUS_RUNNING = 1 - STATUS_STOPPING = 2 + STATUS_STARTING = 1 + STATUS_RUNNING = 2 + STATUS_STOPPING = 3 # STATUS_STOPPED = 3 # Not used yet @classmethod @@ -1968,6 +1990,8 @@ def main(): ''')) + parser.add_argument('-a', '--auto-start-service', action='store_true', + help='Automatically start/stop the TDengine service (default: false)') parser.add_argument('-c', '--connector-type', action='store', default='native', type=str, help='Connector type to use: native, rest, or mixed (default: 10)') parser.add_argument('-d', '--debug', action='store_true', From f13e3d6adf4a404ea7632117a0a028e21c0bae28 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 8 Jul 2020 16:33:40 +0800 Subject: [PATCH 11/12] turn off output. --- tests/test-all.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test-all.sh b/tests/test-all.sh index 26005c8bc4..275c6b1677 100755 --- a/tests/test-all.sh +++ b/tests/test-all.sh @@ -27,7 +27,7 @@ function runPyCaseOneByOne { if [[ $line != *sleep* ]]; then case=`echo $line|awk '{print $NF}'` start_time=`date +%s` - $line && \ + $line > /dev/null 2>&1 && \ echo -e "${GREEN}$case success${NC}" | tee -a pytest-out.log || \ echo -e "${RED}$case failed${NC}" | tee -a pytest-out.log end_time=`date +%s` From f0bbd1d32e4c4096567556612303ecb2186f07f4 Mon Sep 17 00:00:00 2001 From: Steven Li Date: Wed, 8 Jul 2020 08:49:51 +0000 Subject: [PATCH 12/12] Allow crash_gen tool to tolerate 0x5 and 0x510 error codes --- tests/pytest/crash_gen.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/pytest/crash_gen.py b/tests/pytest/crash_gen.py index e882780b8f..0688bb828e 100755 --- a/tests/pytest/crash_gen.py +++ b/tests/pytest/crash_gen.py @@ -1213,7 +1213,11 @@ class Task(): self._executeInternal(te, wt) # TODO: no return value? except taos.error.ProgrammingError as err: errno2 = err.errno if (err.errno > 0) else 0x80000000 + err.errno # correct error scheme - if ( errno2 in [0x200, 0x360, 0x362, 0x36A, 0x36B, 0x36D, 0x381, 0x380, 0x383, 0x503, 0x600, + if ( errno2 in [ + 0x05, # TSDB_CODE_RPC_NOT_READY + 0x200, 0x360, 0x362, 0x36A, 0x36B, 0x36D, 0x381, 0x380, 0x383, 0x503, + 0x510, # vnode not in ready state + 0x600, 1000 # REST catch-all error ]) : # allowed errors self.logDebug("[=] Acceptable Taos library exception: errno=0x{:X}, msg: {}, SQL: {}".format(errno2, err, self._lastSql))