From 5d068661b309b5c12f82264761b9e9b707b79784 Mon Sep 17 00:00:00 2001 From: SunShine Chan Date: Mon, 28 Jun 2021 07:39:52 +0800 Subject: [PATCH 01/26] CHANGE keepColumnName to 1 --- packaging/cfg/taos.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/cfg/taos.cfg b/packaging/cfg/taos.cfg index d3bd7510a3..9ad06cd1bd 100644 --- a/packaging/cfg/taos.cfg +++ b/packaging/cfg/taos.cfg @@ -40,7 +40,7 @@ # ratioOfQueryCores 1.0 # the last_row/first/last aggregator will not change the original column name in the result fields -# keepColumnName 0 +keepColumnName 1 # number of management nodes in the system # numOfMnodes 3 From 5ec984c03f4ce3e15e3fe188740f61e664b37f25 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 1 Jul 2021 11:27:59 +0800 Subject: [PATCH 02/26] [TD-4947] handle mixed group by with top/bottom corrupt --- src/client/src/tscSQLParser.c | 15 +++++++++++++-- src/query/src/qExecutor.c | 15 +++++++++++++-- tests/script/fullGeneralSuite.sim | 1 + 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 658fdf06bb..a2dd266db5 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -5283,12 +5283,23 @@ int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq assert(pExpr->base.functionId == TSDB_FUNC_TS); pExpr = tscExprGet(pQueryInfo, 1); - if (pExpr->base.colInfo.colIndex != index.columnIndex && index.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX) { + //if (pExpr->base.colInfo.colIndex != index.columnIndex && index.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX) { + // return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); + //} + + bool validOrder = false; + SArray *columnInfo = pQueryInfo->groupbyExpr.columnInfo; + if (columnInfo != NULL && taosArrayGetSize(columnInfo) > 0) { + SColIndex* pColIndex = taosArrayGet(columnInfo, 0); + validOrder = (pColIndex->colIndex == index.columnIndex); + } + if (!validOrder) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } + } tVariantListItem* pItem = taosArrayGet(pSqlNode->pSortOrder, 0); pQueryInfo->order.order = pItem->sortOrder; + pQueryInfo->order.orderColId = pSchema[index.columnIndex].colId; return TSDB_CODE_SUCCESS; } diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 5989bd810b..1d717bc84a 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -240,6 +240,8 @@ static void sortGroupResByOrderList(SGroupResInfo *pGroupResInfo, SQueryRuntimeE if (taosArrayGetSize(columnOrderList) <= 0) { return; } + taosArrayDestroy(columnOrderList); + int32_t orderId = pRuntimeEnv->pQueryAttr->order.orderColId; if (orderId <= 0) { return; @@ -1740,8 +1742,7 @@ static SQLFunctionCtx* createSQLFunctionCtx(SQueryRuntimeEnv* pRuntimeEnv, SExpr } for(int32_t i = 1; i < numOfOutput; ++i) { - (*rowCellInfoOffset)[i] = (int32_t)((*rowCellInfoOffset)[i - 1] + sizeof(SResultRowCellInfo) + - pExpr[i - 1].base.interBytes * GET_ROW_PARAM_FOR_MULTIOUTPUT(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery)); + (*rowCellInfoOffset)[i] = (int32_t)((*rowCellInfoOffset)[i - 1] + sizeof(SResultRowCellInfo) + pExpr[i - 1].base.interBytes); } setCtxTagColumnInfo(pFuncCtx, numOfOutput); @@ -3713,6 +3714,9 @@ static int32_t doCopyToSDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* } int32_t numOfRowsToCopy = pRow->numOfRows; + if (numOfResult + numOfRowsToCopy >= pRuntimeEnv->resultInfo.capacity) { + break; + } pGroupResInfo->index += 1; @@ -6100,7 +6104,14 @@ SOperatorInfo* createGroupbyOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperato SGroupbyOperatorInfo* pInfo = calloc(1, sizeof(SGroupbyOperatorInfo)); pInfo->colIndex = -1; // group by column index + pInfo->binfo.pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); + + SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; + + pQueryAttr->resultRowSize = (pQueryAttr->resultRowSize * + GET_ROW_PARAM_FOR_MULTIOUTPUT(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery)); + pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); initResultRowInfo(&pInfo->binfo.resultRowInfo, 8, TSDB_DATA_TYPE_INT); diff --git a/tests/script/fullGeneralSuite.sim b/tests/script/fullGeneralSuite.sim index 9372657533..c820dd3bf5 100644 --- a/tests/script/fullGeneralSuite.sim +++ b/tests/script/fullGeneralSuite.sim @@ -131,6 +131,7 @@ run general/parser/join.sim run general/parser/join_multivnode.sim run general/parser/select_with_tags.sim run general/parser/groupby.sim +run general/parser/top_groupby.sim run general/parser/tags_dynamically_specifiy.sim run general/parser/set_tag_vals.sim #unsupport run general/parser/repeatAlter.sim From 42e02636ad633197498fc33c8c8c465fc3c64ce4 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 1 Jul 2021 11:50:21 +0800 Subject: [PATCH 03/26] [TD-4947] handle mixed group by with top/bottom corrupt --- src/query/src/qExecutor.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 1d717bc84a..fbdc28c875 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -6110,7 +6110,7 @@ SOperatorInfo* createGroupbyOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperato SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; pQueryAttr->resultRowSize = (pQueryAttr->resultRowSize * - GET_ROW_PARAM_FOR_MULTIOUTPUT(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery)); + (int32_t)(GET_ROW_PARAM_FOR_MULTIOUTPUT(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery))); pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); initResultRowInfo(&pInfo->binfo.resultRowInfo, 8, TSDB_DATA_TYPE_INT); From d0a59870bb9d5be866af7a984cd2c5def259757a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 1 Jul 2021 14:08:27 +0800 Subject: [PATCH 04/26] [TD-4947] handle mixed group by with top/bottom corrupt --- tests/script/general/parser/top_groupby.sim | 52 +++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 tests/script/general/parser/top_groupby.sim diff --git a/tests/script/general/parser/top_groupby.sim b/tests/script/general/parser/top_groupby.sim new file mode 100644 index 0000000000..5709f4d1d7 --- /dev/null +++ b/tests/script/general/parser/top_groupby.sim @@ -0,0 +1,52 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 2 +system sh/exec.sh -n dnode1 -s start + +sleep 100 +sql connect +print ======================== dnode1 start + +$db = testdb + +sql create database $db +sql use $db + +sql create stable st2 (ts timestamp, f1 int, f2 float, f3 double, f4 bigint, f5 smallint, f6 tinyint, f7 bool, f8 binary(10), f9 nchar(10)) tags (id1 int, id2 float, id3 nchar(10), id4 double, id5 smallint, id6 bigint, id7 binary(10)) + +sql create table tb1 using st2 tags (1,1.0,"1",1.0,1,1,"1"); + +sql insert into tb1 values (now-200s,1,1.0,1.0,1,1,1,true,"1","1") +sql insert into tb1 values (now-100s,2,2.0,2.0,2,2,2,true,"2","2") +sql insert into tb1 values (now,3,3.0,3.0,3,3,3,true,"3","3") +sql insert into tb1 values (now+100s,4,4.0,4.0,4,4,4,true,"4","4") +sql insert into tb1 values (now+200s,4,4.0,4.0,4,4,4,true,"4","4") +sql insert into tb1 values (now+300s,4,4.0,4.0,4,4,4,true,"4","4") +sql insert into tb1 values (now+400s,4,4.0,4.0,4,4,4,true,"4","4") +sql insert into tb1 values (now+500s,4,4.0,4.0,4,4,4,true,"4","4") + +sql select top(f1, 2) from tb1 group by f1; + +if $rows != 5 then + return -1 +endi + +sql select bottom(f1, 2) from tb1 group by f1; + +if $rows != 5 then + return -1 +endi + +sql select top(f1, 100) from tb1 group by f1; + +if $rows != 8 then + return -1 +endi + +sql select bottom(f1, 100) from tb1 group by f1; + +if $rows != 8 then + return -1 +endi + From 64f01133db701369cb52ed7f4e96b8c260f28554 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 1 Jul 2021 16:12:21 +0800 Subject: [PATCH 05/26] [TD-4947] handle mixed group by with top/bottom corrupt --- src/client/src/tscSQLParser.c | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index a2dd266db5..0387d916ea 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -5278,21 +5278,23 @@ int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq } if (isTopBottomQuery(pQueryInfo)) { - /* order of top/bottom query in interval is not valid */ - SExprInfo* pExpr = tscExprGet(pQueryInfo, 0); - assert(pExpr->base.functionId == TSDB_FUNC_TS); - - pExpr = tscExprGet(pQueryInfo, 1); - //if (pExpr->base.colInfo.colIndex != index.columnIndex && index.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX) { - // return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - //} - bool validOrder = false; SArray *columnInfo = pQueryInfo->groupbyExpr.columnInfo; if (columnInfo != NULL && taosArrayGetSize(columnInfo) > 0) { SColIndex* pColIndex = taosArrayGet(columnInfo, 0); validOrder = (pColIndex->colIndex == index.columnIndex); - } + } else { + /* order of top/bottom query in interval is not valid */ + SExprInfo* pExpr = tscExprGet(pQueryInfo, 0); + assert(pExpr->base.functionId == TSDB_FUNC_TS); + + pExpr = tscExprGet(pQueryInfo, 1); + if (pExpr->base.colInfo.colIndex != index.columnIndex && index.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX) { + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); + } + validOrder = true; + } + if (!validOrder) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } From d59c5809eff34fcb4de878c6dba9bc082c2a4f40 Mon Sep 17 00:00:00 2001 From: Jun Li Date: Sun, 4 Jul 2021 00:10:44 -0700 Subject: [PATCH 06/26] Make taosHashGetSize thread-safe and rename function --- src/client/src/tscLocal.c | 2 +- src/client/src/tscPrepare.c | 2 +- src/client/src/tscServer.c | 4 ++-- src/tsdb/src/tsdbFS.c | 2 +- src/util/inc/hash.h | 2 +- src/util/src/hash.c | 31 ++++++++++++++++++++++++------- 6 files changed, 30 insertions(+), 13 deletions(-) diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index 668a9e9406..d1a325be35 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -920,7 +920,7 @@ int tscProcessLocalCmd(SSqlObj *pSql) { } else if (pCmd->command == TSDB_SQL_SHOW_CREATE_DATABASE) { pRes->code = tscProcessShowCreateDatabase(pSql); } else if (pCmd->command == TSDB_SQL_RESET_CACHE) { - taosHashEmpty(tscTableMetaInfo); + taosHashClear(tscTableMetaInfo); pRes->code = TSDB_CODE_SUCCESS; } else if (pCmd->command == TSDB_SQL_SERV_VERSION) { pRes->code = tscProcessServerVer(pSql); diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index 8bb776ffee..956f1c661a 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -1159,7 +1159,7 @@ static void insertBatchClean(STscStmt* pStmt) { pCmd->insertParam.pDataBlocks = tscDestroyBlockArrayList(pCmd->insertParam.pDataBlocks); pCmd->insertParam.numOfTables = 0; - taosHashEmpty(pCmd->insertParam.pTableBlockHashList); + taosHashClear(pCmd->insertParam.pTableBlockHashList); tscFreeSqlResult(pSql); tscFreeSubobj(pSql); tfree(pSql->pSubs); diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 7cd11cfe4b..d92c3d88c2 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -2309,7 +2309,7 @@ int tscProcessDropDbRsp(SSqlObj *pSql) { //TODO LOCK DB WHEN MODIFY IT //pSql->pTscObj->db[0] = 0; - taosHashEmpty(tscTableMetaInfo); + taosHashClear(tscTableMetaInfo); return 0; } @@ -2340,7 +2340,7 @@ int tscProcessAlterTableMsgRsp(SSqlObj *pSql) { tfree(pTableMetaInfo->pTableMeta); if (isSuperTable) { // if it is a super table, iterate the hashTable and remove all the childTableMeta - taosHashEmpty(tscTableMetaInfo); + taosHashClear(tscTableMetaInfo); } return 0; diff --git a/src/tsdb/src/tsdbFS.c b/src/tsdb/src/tsdbFS.c index 54372ae8c2..e53d2826c7 100644 --- a/src/tsdb/src/tsdbFS.c +++ b/src/tsdb/src/tsdbFS.c @@ -771,7 +771,7 @@ int tsdbLoadMetaCache(STsdbRepo *pRepo, bool recoverMeta) { int64_t maxBufSize = 0; SMFInfo minfo; - taosHashEmpty(pfs->metaCache); + taosHashClear(pfs->metaCache); // No meta file, just return if (pfs->cstatus->pmf == NULL) return 0; diff --git a/src/util/inc/hash.h b/src/util/inc/hash.h index c37426069c..616b844c13 100644 --- a/src/util/inc/hash.h +++ b/src/util/inc/hash.h @@ -140,7 +140,7 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), void *param); -void taosHashEmpty(SHashObj *pHashObj); +void taosHashClear(SHashObj *pHashObj); /** * clean up hash table diff --git a/src/util/src/hash.c b/src/util/src/hash.c index 5b3218382f..d7bee9b67c 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -144,6 +144,14 @@ static FORCE_INLINE SHashNode *doUpdateHashNode(SHashObj *pHashObj, SHashEntry* */ static void pushfrontNodeInEntryList(SHashEntry *pEntry, SHashNode *pNode); +/** + * Check whether the hash table is empty or not. + * + * @param pHashObj the hash table object + * @return if the hash table is empty or not + */ +static FORCE_INLINE bool taosHashTableEmpty(const SHashObj *pHashObj); + /** * Get the next element in hash table for iterator * @param pIter @@ -195,7 +203,16 @@ void taosHashSetEqualFp(SHashObj *pHashObj, _equal_fn_t fp) { } } -int32_t taosHashGetSize(const SHashObj *pHashObj) { return (int32_t)((pHashObj == NULL) ? 0 : pHashObj->size); } +int32_t taosHashGetSize(const SHashObj *pHashObj) { + if (!pHashObj) { + return 0; + } + return (int32_t)atomic_load_64(&pHashObj->size); +} + +static FORCE_INLINE bool taosHashTableEmpty(const SHashObj *pHashObj) { + return taosHashGetSize(pHashObj) == 0; +} int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size) { uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen); @@ -281,7 +298,7 @@ void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen) { } void* taosHashGetClone(SHashObj *pHashObj, const void *key, size_t keyLen, void (*fp)(void *), void* d, size_t dsize) { - if (pHashObj->size <= 0 || keyLen == 0 || key == NULL) { + if (taosHashTableEmpty(pHashObj) || keyLen == 0 || key == NULL) { return NULL; } @@ -338,7 +355,7 @@ int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) { } int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t dsize) { - if (pHashObj == NULL || pHashObj->size <= 0) { + if (pHashObj == NULL || taosHashTableEmpty(pHashObj)) { return -1; } @@ -405,7 +422,7 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe } int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), void *param) { - if (pHashObj == NULL || pHashObj->size == 0) { + if (pHashObj == NULL || taosHashTableEmpty(pHashObj)) { return 0; } @@ -478,7 +495,7 @@ int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), voi return 0; } -void taosHashEmpty(SHashObj *pHashObj) { +void taosHashClear(SHashObj *pHashObj) { if (pHashObj == NULL) { return; } @@ -517,7 +534,7 @@ void taosHashCleanup(SHashObj *pHashObj) { return; } - taosHashEmpty(pHashObj); + taosHashClear(pHashObj); tfree(pHashObj->hashList); // destroy mem block @@ -535,7 +552,7 @@ void taosHashCleanup(SHashObj *pHashObj) { // for profile only int32_t taosHashGetMaxOverflowLinkLength(const SHashObj *pHashObj) { - if (pHashObj == NULL || pHashObj->size == 0) { + if (pHashObj == NULL || taosHashTableEmpty(pHashObj)) { return 0; } From 7f2e888bbc0f74e491cc2c7d4a04a363e06f94b9 Mon Sep 17 00:00:00 2001 From: Zhiyu Yang <69311263+zyyang-taosdata@users.noreply.github.com> Date: Mon, 5 Jul 2021 11:15:42 +0800 Subject: [PATCH 07/26] [TD-4939]: add a Proxool Connection Pool Demo (#6657) * [TD-4939]: add a Proxool Connection Pool Demo * change --- tests/examples/JDBC/connectionPools/pom.xml | 53 +++++++++++++----- .../com/taosdata/example/ProxoolDemo.java | 56 +++++++++++++++++++ .../src/main/resources/proxool.xml | 27 +++++++++ 3 files changed, 123 insertions(+), 13 deletions(-) create mode 100644 tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/ProxoolDemo.java create mode 100644 tests/examples/JDBC/connectionPools/src/main/resources/proxool.xml diff --git a/tests/examples/JDBC/connectionPools/pom.xml b/tests/examples/JDBC/connectionPools/pom.xml index 84467003f9..045e9d336c 100644 --- a/tests/examples/JDBC/connectionPools/pom.xml +++ b/tests/examples/JDBC/connectionPools/pom.xml @@ -9,12 +9,12 @@ 1.0-SNAPSHOT + com.taosdata.jdbc taos-jdbcdriver 2.0.18 - com.alibaba @@ -50,6 +50,12 @@ log4j 1.2.17 + + + com.cloudhopper.proxool + proxool + 0.9.1 + @@ -57,25 +63,46 @@ org.apache.maven.plugins maven-assembly-plugin - 3.1.0 - - - - com.taosdata.example.ConnectionPoolDemo - - - - jar-with-dependencies - - + 3.3.0 - make-assembly + ConnectionPoolDemo + + ConnectionPoolDemo + + + com.taosdata.example.ConnectionPoolDemo + + + + jar-with-dependencies + + package single + + + ProxoolDemo + + ProxoolDemo + + + com.taosdata.example.ProxoolDemo + + + + jar-with-dependencies + + + package + + single + + + diff --git a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/ProxoolDemo.java b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/ProxoolDemo.java new file mode 100644 index 0000000000..632ad8c9bf --- /dev/null +++ b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/ProxoolDemo.java @@ -0,0 +1,56 @@ +package com.taosdata.example; + +import org.logicalcobwebs.proxool.ProxoolException; +import org.logicalcobwebs.proxool.configuration.JAXPConfigurator; + +import java.sql.*; + +public class ProxoolDemo { + + + public static void main(String[] args) { + + String xml = parseConfigurationXml(args); + if (xml == null) { + printHelp(); + System.exit(0); + } + + try { + JAXPConfigurator.configure(xml, false); + Class.forName("org.logicalcobwebs.proxool.ProxoolDriver"); + Connection connection = DriverManager.getConnection("proxool.ds"); + + Statement stmt = connection.createStatement(); + + ResultSet rs = stmt.executeQuery("show databases"); + ResultSetMetaData metaData = rs.getMetaData(); + while (rs.next()) { + for (int i = 1; i <= metaData.getColumnCount(); i++) { + System.out.print(metaData.getColumnLabel(i) + ": " + rs.getString(i)); + } + System.out.println(); + } + + stmt.close(); + + } catch (ClassNotFoundException | SQLException | ProxoolException e) { + e.printStackTrace(); + } + } + + private static String parseConfigurationXml(String[] args) { + String host = null; + for (int i = 0; i < args.length; i++) { + if ("--xml".equalsIgnoreCase(args[i]) && i < args.length - 1) { + host = args[++i]; + } + } + return host; + } + + private static void printHelp() { + System.out.println("Usage: java -jar ProxoolDemo.jar --xml [xml]"); + } + +} diff --git a/tests/examples/JDBC/connectionPools/src/main/resources/proxool.xml b/tests/examples/JDBC/connectionPools/src/main/resources/proxool.xml new file mode 100644 index 0000000000..67baa1c393 --- /dev/null +++ b/tests/examples/JDBC/connectionPools/src/main/resources/proxool.xml @@ -0,0 +1,27 @@ + + + + ds + + jdbc:TAOS-RS://127.0.0.1:6041/log + + com.taosdata.jdbc.rs.RestfulDriver + + + + + + + 100 + + 100 + + 1 + + 5 + + 30000 + + select server_status() + + \ No newline at end of file From 4e9433020c5112aabcb70c46e1d82186c0a695af Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 5 Jul 2021 13:21:05 +0800 Subject: [PATCH 08/26] [TD-5018]: taosdemo prompt if the batch is too large to insert data. (#6737) --- src/kit/taosdemo/taosdemo.c | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 8f505c53eb..17e417d3f7 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -4834,7 +4834,7 @@ static int64_t generateStbRowData( dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "%"PRId64",", rand_bigint()); } else { - errorPrint( "No support data type: %s\n", stbInfo->columns[i].dataType); + errorPrint( "Not support data type: %s\n", stbInfo->columns[i].dataType); return -1; } } @@ -4842,6 +4842,7 @@ static int64_t generateStbRowData( dataLen -= 1; dataLen += snprintf(pstr + dataLen, maxLen - dataLen, ")"); + verbosePrint("%s() LN%d, dataLen:%"PRId64"\n", __func__, __LINE__, dataLen); verbosePrint("%s() LN%d, recBuf:\n\t%s\n", __func__, __LINE__, recBuf); return strlen(recBuf); @@ -5094,24 +5095,25 @@ static int32_t generateStbDataTail( } else { tsRand = false; } - verbosePrint("%s() LN%d batch=%u\n", __func__, __LINE__, batch); + verbosePrint("%s() LN%d batch=%u buflen=%"PRId64"\n", + __func__, __LINE__, batch, remainderBufLen); int32_t k = 0; for (k = 0; k < batch;) { char data[MAX_DATA_SIZE]; memset(data, 0, MAX_DATA_SIZE); - int64_t retLen = 0; + int64_t lenOfRow = 0; if (tsRand) { - retLen = generateStbRowData(superTblInfo, data, + lenOfRow = generateStbRowData(superTblInfo, data, startTime + getTSRandTail( superTblInfo->timeStampStep, k, superTblInfo->disorderRatio, superTblInfo->disorderRange) ); } else { - retLen = getRowDataFromSample( + lenOfRow = getRowDataFromSample( data, remainderBufLen < MAX_DATA_SIZE ? remainderBufLen : MAX_DATA_SIZE, startTime + superTblInfo->timeStampStep * k, @@ -5119,14 +5121,14 @@ static int32_t generateStbDataTail( pSamplePos); } - if (retLen > remainderBufLen) { + if (lenOfRow > remainderBufLen) { break; } - pstr += snprintf(pstr , retLen + 1, "%s", data); + pstr += snprintf(pstr , lenOfRow + 1, "%s", data); k++; - len += retLen; - remainderBufLen -= retLen; + len += lenOfRow; + remainderBufLen -= lenOfRow; verbosePrint("%s() LN%d len=%"PRIu64" k=%u \nbuffer=%s\n", __func__, __LINE__, len, k, buffer); @@ -5915,11 +5917,14 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { startTs = taosGetTimestampMs(); if (recOfBatch == 0) { - errorPrint("[%d] %s() LN%d try inserting records of batch is %d\n", - pThreadInfo->threadID, __func__, __LINE__, - recOfBatch); - errorPrint("%s\n", "\tPlease check if the batch or the buffer length is proper value!\n"); - goto free_of_interlace; + errorPrint("[%d] %s() LN%d Failed to insert records of batch %d\n", + pThreadInfo->threadID, __func__, __LINE__, + batchPerTbl); + errorPrint("\tIf the batch is %d, the length of the SQL to insert a row must be less then %"PRId64"\n", + batchPerTbl, maxSqlLen / batchPerTbl); + errorPrint("\tPlease check if the buffer length(%"PRId64") or batch(%d) is set with proper value!\n", + maxSqlLen, batchPerTbl); + goto free_of_interlace; } int64_t affectedRows = execInsert(pThreadInfo, recOfBatch); From 238925ddbc52c0f822f83e178235c38e1ea3a0cd Mon Sep 17 00:00:00 2001 From: zhaoyanggh Date: Mon, 5 Jul 2021 13:26:49 +0800 Subject: [PATCH 09/26] add manual test case of subscription with no result file defined --- .../tools/taosdemoAllTest/sub_no_result.json | 25 ++++++ .../taosdemoAllTest/subscribeNoResult.py | 82 +++++++++++++++++++ 2 files changed, 107 insertions(+) create mode 100644 tests/pytest/tools/taosdemoAllTest/sub_no_result.json create mode 100644 tests/pytest/tools/taosdemoAllTest/subscribeNoResult.py diff --git a/tests/pytest/tools/taosdemoAllTest/sub_no_result.json b/tests/pytest/tools/taosdemoAllTest/sub_no_result.json new file mode 100644 index 0000000000..cdf7c2314e --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/sub_no_result.json @@ -0,0 +1,25 @@ +{ + "filetype": "subscribe", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "databases": "db", + "confirm_parameter_prompt": "no", + "specified_table_query": + { + "concurrent":1, + "mode":"sync", + "interval": 0, + "restart":"yes", + "keepProgress":"no", + "endAfterConsume": 1100000, + "sqls": [ + { + "sql": "select * from st;", + "result": "" + }] + } + } + \ No newline at end of file diff --git a/tests/pytest/tools/taosdemoAllTest/subscribeNoResult.py b/tests/pytest/tools/taosdemoAllTest/subscribeNoResult.py new file mode 100644 index 0000000000..270eea17cb --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/subscribeNoResult.py @@ -0,0 +1,82 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +import time +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * +import _thread + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.ts = 1601481600000 + self.numberOfRecords = 1100000 + + def execCmdAndGetOutput(self, cmd): + r = os.popen(cmd) + text = r.read() + r.close() + return text + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + def run(self): + buildPath = self.getBuildPath() + tdSql.prepare() + tdSql.execute("create table st(ts timestamp, c1 timestamp, c2 int, c3 bigint, c4 float, c5 double, c6 binary(8), c7 smallint, c8 tinyint, c9 bool, c10 nchar(8)) tags(t1 int)") + tdSql.execute("create table t1 using st tags(0)") + currts = self.ts + finish = 0 + while(finish < self.numberOfRecords): + sql = "insert into t1 values" + for i in range(finish, self.numberOfRecords): + sql += "(%d, 1019774612, 29931, 1442173978, 165092.468750, 1128.643179, 'MOCq1pTu', 18405, 82, 0, 'g0A6S0Fu')" % (currts + i) + finish = i + 1 + if (1048576 - len(sql)) < 16384: + break + tdSql.execute(sql) + + binPath = buildPath+ "/build/bin/" + + os.system("%staosdemo -f tools/taosdemoAllTest/sub_no_result.json -g 2>&1 | tee sub_no_result.log" % binPath) + test_line = int(self.execCmdAndGetOutput("cat sub_no_result.log | wc -l")) + if(test_line < 1100024): + tdLog.exit("failed test subscribeNoResult: %d != expected(1100024)" % test_line) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file From 18df0e379251c3823b62e929fad398b884272b76 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Mon, 5 Jul 2021 14:35:24 +0800 Subject: [PATCH 10/26] [TD-5000]: add test case for nested query --- tests/pytest/fulltest.sh | 1 + .../query/nestedQuery/queryWithOrderLimit.py | 79 +++++++++++++++++++ tests/pytest/query/queryPerformance.py | 12 ++- 3 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 tests/pytest/query/nestedQuery/queryWithOrderLimit.py diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 17d0bbe190..2cbc3747f6 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -238,6 +238,7 @@ python3 ./test.py -f query/queryTsisNull.py python3 ./test.py -f query/subqueryFilter.py # python3 ./test.py -f query/nestedQuery/queryInterval.py python3 ./test.py -f query/queryStateWindow.py +python3 ./test.py -f query/nestedQuery/queryWithOrderLimit.py #stream diff --git a/tests/pytest/query/nestedQuery/queryWithOrderLimit.py b/tests/pytest/query/nestedQuery/queryWithOrderLimit.py new file mode 100644 index 0000000000..ba66b6ecb6 --- /dev/null +++ b/tests/pytest/query/nestedQuery/queryWithOrderLimit.py @@ -0,0 +1,79 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql +from util.dnodes import tdDnodes +import random + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.ts = 1593548685000 + self.tables = 10 + self.rowsPerTable = 100 + + + def run(self): + # tdSql.execute("drop database db ") + tdSql.prepare() + tdSql.execute("create table st (ts timestamp, num int, value int) tags (loc nchar(30))") + for i in range(self.tables): + for j in range(self.rowsPerTable): + args1=(i, i, self.ts + i * self.rowsPerTable + j * 10000, i, random.randint(1, 100)) + tdSql.execute("insert into t%d using st tags('beijing%d') values(%d, %d, %d)" % args1) + + tdSql.query("select * from (select * from st)") + tdSql.checkRows(self.tables * self.rowsPerTable) + + tdSql.query("select * from (select * from st limit 10)") + tdSql.checkRows(10) + + tdSql.query("select * from (select * from st order by ts desc limit 10)") + tdSql.checkRows(10) + + # bug: https://jira.taosdata.com:18080/browse/TD-5043 + # tdSql.query("select * from (select * from st order by ts desc limit 10 offset 1000)") + # tdSql.checkRows(0) + + tdSql.query("select avg(value), sum(value) from st group by tbname") + tdSql.checkRows(self.tables) + + tdSql.query("select * from (select avg(value), sum(value) from st group by tbname)") + tdSql.checkRows(self.tables) + + tdSql.query("select avg(value), sum(value) from st group by tbname slimit 5") + tdSql.checkRows(5) + + tdSql.query("select * from (select avg(value), sum(value) from st group by tbname slimit 5)") + tdSql.checkRows(5) + + tdSql.query("select avg(value), sum(value) from st group by tbname slimit 5 soffset 7") + tdSql.checkRows(3) + + tdSql.query("select * from (select avg(value), sum(value) from st group by tbname slimit 5 soffset 7)") + tdSql.checkRows(3) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/query/queryPerformance.py b/tests/pytest/query/queryPerformance.py index 720ae745cb..742a3c2cd1 100644 --- a/tests/pytest/query/queryPerformance.py +++ b/tests/pytest/query/queryPerformance.py @@ -45,28 +45,38 @@ class taosdemoQueryPerformace: sql = "select count(*) from test.meters" tableid = 1 cursor.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql)) + sql = "select avg(f1), max(f2), min(f3) from test.meters" tableid = 2 cursor.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql)) + sql = "select count(*) from test.meters where loc='beijing'" tableid = 3 cursor.execute("create table if not exists %s%d using %s tags(%d, \"%s\")" % (self.tbPerfix, tableid, self.stbName, tableid, sql)) + sql = "select avg(f1), max(f2), min(f3) from test.meters where areaid=10" tableid = 4 cursor.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql)) + sql = "select avg(f1), max(f2), min(f3) from test.t10 interval(10s)" tableid = 5 cursor.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql)) + sql = "select last_row(*) from meters" tableid = 6 cursor.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql)) + sql = "select * from meters" tableid = 7 cursor.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql)) + sql = "select avg(f1), max(f2), min(f3) from meters where ts <= '2017-07-15 10:40:01.000' and ts <= '2017-07-15 14:00:40.000'" tableid = 8 cursor.execute("create table if not exists %s%d using %s tags(%d, \"%s\")" % (self.tbPerfix, tableid, self.stbName, tableid, sql)) - + + sql = "select last(*) from meters" + tableid = 9 + cursor.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql)) cursor.close() def query(self): From 17817986bb204deb3bb21dfa9e45b667fe6434ef Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Mon, 5 Jul 2021 15:29:46 +0800 Subject: [PATCH 11/26] [TD-4657] : describe short version of "LIMIT/SLIMIT OFFSET" clause. --- documentation20/cn/12.taos-sql/docs.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation20/cn/12.taos-sql/docs.md b/documentation20/cn/12.taos-sql/docs.md index 764560dbeb..e5570ed3be 100644 --- a/documentation20/cn/12.taos-sql/docs.md +++ b/documentation20/cn/12.taos-sql/docs.md @@ -696,9 +696,9 @@ Query OK, 1 row(s) in set (0.001091s) * 暂不支持含列名的四则运算表达式作为 SQL 函数的应用对象（例如，不支持 `select min(2*a) from t;`，但可以写 `select 2*min(a) from t;`）。 - WHERE 语句可以使用各种逻辑判断来过滤数字值，或使用通配符来过滤字符串。 - 输出结果缺省按首列时间戳升序排序，但可以指定按降序排序( _c0 指首列时间戳)。使用 ORDER BY 对其他字段进行排序为非法操作。 -- 参数 LIMIT 控制输出条数，OFFSET 指定从第几条开始输出。LIMIT/OFFSET 对结果集的执行顺序在 ORDER BY 之后。 +- 参数 LIMIT 控制输出条数，OFFSET 指定从第几条开始输出。LIMIT/OFFSET 对结果集的执行顺序在 ORDER BY 之后。且 `LIMIT 5 OFFSET 2` 可以简写为 `LIMIT 2, 5`。 * 在有 GROUP BY 子句的情况下，LIMIT 参数控制的是每个分组中至多允许输出的条数。 -- 参数 SLIMIT 控制由 GROUP BY 指令划分的分组中，至多允许输出几个分组的数据。 +- 参数 SLIMIT 控制由 GROUP BY 指令划分的分组中，至多允许输出几个分组的数据。且 `SLIMIT 5 OFFSET 2` 可以简写为 `SLIMIT 2, 5`。 - 通过 “>>” 输出结果可以导出到指定文件。 ### 支持的条件过滤操作 From 9debc4992d07d7ce80d8eb9a65f9131f93e230c8 Mon Sep 17 00:00:00 2001 From: wpan Date: Mon, 5 Jul 2021 16:06:43 +0800 Subject: [PATCH 12/26] add diff... group by check --- src/client/src/tscSQLParser.c | 10 ++++++++++ src/query/src/qExecutor.c | 3 ++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 4d04d4184f..eb9756f2ee 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -6520,6 +6520,7 @@ int32_t doFunctionsCompatibleCheck(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, char* const char* msg3 = "group by/session/state_window not allowed on projection query"; const char* msg4 = "retrieve tags not compatible with group by or interval query"; const char* msg5 = "functions can not be mixed up"; + const char* msg6 = "TWA/Diff/Derivative/Irate only support group by tbname"; // only retrieve tags, group by is not supportted if (tscQueryTags(pQueryInfo)) { @@ -6571,6 +6572,15 @@ int32_t doFunctionsCompatibleCheck(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, char* } } + if (f == TSDB_FUNC_DIFF || f == TSDB_FUNC_DERIVATIVE || f == TSDB_FUNC_TWA || f == TSDB_FUNC_IRATE) { + for (int32_t j = 0; j < pQueryInfo->groupbyExpr.numOfGroupCols; ++j) { + SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, j); + if (pColIndex->colIndex != TSDB_TBNAME_COLUMN_INDEX) { + return invalidOperationMsg(msg, msg6); + } + } + } + if (IS_MULTIOUTPUT(aAggs[f].status) && f != TSDB_FUNC_TOP && f != TSDB_FUNC_BOTTOM && f != TSDB_FUNC_DIFF && f != TSDB_FUNC_DERIVATIVE && f != TSDB_FUNC_TAGPRJ && f != TSDB_FUNC_PRJ) { return invalidOperationMsg(msg, msg1); diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 6bd5e03377..c218c2b03e 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -1408,8 +1408,9 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SGroupbyOperatorInfo *pIn } doApplyFunctions(pRuntimeEnv, pInfo->binfo.pCtx, &w, pSDataBlock->info.rows - num, num, tsList, pSDataBlock->info.rows, pOperator->numOfOutput); - tfree(pInfo->prevData); } + + tfree(pInfo->prevData); } static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSWindowOperatorInfo *pInfo, SSDataBlock *pSDataBlock) { From 70f9acbfde7136d52d0fb08ccc8e31894c8332bb Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 5 Jul 2021 18:02:32 +0800 Subject: [PATCH 13/26] [TD-4947] handle mixed group by with top/bottom corrupt --- src/query/src/qExecutor.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 34d0108cbf..a50c17b8db 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -242,9 +242,6 @@ static void sortGroupResByOrderList(SGroupResInfo *pGroupResInfo, SQueryRuntimeE if (size <= 0) { return; } - - taosArrayDestroy(columnOrderList); - int32_t orderId = pRuntimeEnv->pQueryAttr->order.orderColId; if (orderId <= 0) { return; From 752db3477a05a46abe3848d207f2d29cbb506deb Mon Sep 17 00:00:00 2001 From: wpan Date: Mon, 5 Jul 2021 18:18:24 +0800 Subject: [PATCH 14/26] add interval value check --- src/client/src/tscSQLParser.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 4d04d4184f..f0889d20b0 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -916,7 +916,8 @@ static int32_t checkInvalidExprForTimeWindow(SSqlCmd* pCmd, SQueryInfo* pQueryIn int32_t validateIntervalNode(SSqlObj* pSql, SQueryInfo* pQueryInfo, SSqlNode* pSqlNode) { const char* msg1 = "sliding cannot be used without interval"; - const char* msg2 = "interval cannot be less than 10 ms"; + const char* msg2 = "interval cannot be less than 1 us"; + const char* msg3 = "interval value is too small"; SSqlCmd* pCmd = &pSql->cmd; @@ -943,6 +944,10 @@ int32_t validateIntervalNode(SSqlObj* pSql, SQueryInfo* pQueryInfo, SSqlNode* pS return TSDB_CODE_TSC_INVALID_OPERATION; } + if (pQueryInfo->interval.interval <= 0) { + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); + } + if (pQueryInfo->interval.intervalUnit != 'n' && pQueryInfo->interval.intervalUnit != 'y') { // interval cannot be less than 10 milliseconds From d390ad7a8e80568831bc207aaf21d313512c9b91 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Mon, 5 Jul 2021 18:27:32 +0800 Subject: [PATCH 15/26] [TD-4983]: add test case for interval boundry --- tests/pytest/dbmgmt/nanoSecondCheck.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/pytest/dbmgmt/nanoSecondCheck.py b/tests/pytest/dbmgmt/nanoSecondCheck.py index 27050a2213..a5e9adacee 100644 --- a/tests/pytest/dbmgmt/nanoSecondCheck.py +++ b/tests/pytest/dbmgmt/nanoSecondCheck.py @@ -99,6 +99,15 @@ class TDTestCase: tdSql.query('select avg(speed) from tb interval(100000000b)') tdSql.checkRows(4) + tdSql.error('select avg(speed) from tb interval(1b);') + tdSql.error('select avg(speed) from tb interval(999b);') + + tdSql.query('select avg(speed) from tb interval(1000b);') + tdSql.checkRows(5) + + tdSql.query('select avg(speed) from tb interval(1u);') + tdSql.checkRows(5) + tdSql.query('select avg(speed) from tb interval(100000000b) sliding (100000000b);') tdSql.checkRows(4) From 3fad48199559cadf1ae3e05caeabcda9e5d16302 Mon Sep 17 00:00:00 2001 From: wpan Date: Tue, 6 Jul 2021 09:06:36 +0800 Subject: [PATCH 16/26] fix offset issue --- src/client/src/tscUtil.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 982b96eecc..838416fa2e 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -2947,6 +2947,7 @@ int32_t tscQueryInfoCopy(SQueryInfo* pQueryInfo, const SQueryInfo* pSrc) { pQueryInfo->fillType = pSrc->fillType; pQueryInfo->fillVal = NULL; pQueryInfo->clauseLimit = pSrc->clauseLimit; + pQueryInfo->prjOffset = pSrc->prjOffset; pQueryInfo->numOfTables = 0; pQueryInfo->window = pSrc->window; pQueryInfo->sessionWindow = pSrc->sessionWindow; @@ -3326,6 +3327,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, __async_cb_func_t pNewQueryInfo->fillType = pQueryInfo->fillType; pNewQueryInfo->fillVal = NULL; pNewQueryInfo->clauseLimit = pQueryInfo->clauseLimit; + pNewQueryInfo->prjOffset = pQueryInfo->prjOffset; pNewQueryInfo->numOfTables = 0; pNewQueryInfo->pTableMetaInfo = NULL; pNewQueryInfo->bufLen = pQueryInfo->bufLen; From a657f4529be76758ccab21dbc070304af7c16ca8 Mon Sep 17 00:00:00 2001 From: Zhiyu Yang <69311263+zyyang-taosdata@users.noreply.github.com> Date: Tue, 6 Jul 2021 09:23:05 +0800 Subject: [PATCH 17/26] add tq release packagine scripts (#6739) * change version number * change packaging scripts * add more scripts * change scripts * change * change * change Co-authored-by: plum-lihui --- cmake/define.inc | 4 + cmake/input.inc | 3 + cmake/version.inc | 2 +- packaging/release.sh | 14 +- packaging/tools/install_arbi_tq.sh | 298 +++++++++ packaging/tools/install_client_tq.sh | 251 ++++++++ packaging/tools/install_tq.sh | 918 +++++++++++++++++++++++++++ packaging/tools/makearbi_tq.sh | 75 +++ packaging/tools/makeclient_tq.sh | 203 ++++++ packaging/tools/makepkg_tq.sh | 224 +++++++ packaging/tools/remove_arbi_tq.sh | 130 ++++ packaging/tools/remove_client_tq.sh | 85 +++ packaging/tools/remove_tq.sh | 227 +++++++ packaging/tools/startPre.sh | 0 src/inc/taosdef.h | 2 + src/kit/shell/src/shellEngine.c | 7 + src/kit/taosdemo/taosdemo.c | 7 + src/kit/taosdump/taosdump.c | 6 + src/os/src/linux/linuxEnv.c | 7 + src/os/src/windows/wEnv.c | 9 +- src/util/src/tconfig.c | 3 + src/util/src/tlog.c | 2 + 22 files changed, 2470 insertions(+), 7 deletions(-) create mode 100755 packaging/tools/install_arbi_tq.sh create mode 100755 packaging/tools/install_client_tq.sh create mode 100755 packaging/tools/install_tq.sh create mode 100755 packaging/tools/makearbi_tq.sh create mode 100755 packaging/tools/makeclient_tq.sh create mode 100755 packaging/tools/makepkg_tq.sh create mode 100755 packaging/tools/remove_arbi_tq.sh create mode 100755 packaging/tools/remove_client_tq.sh create mode 100755 packaging/tools/remove_tq.sh mode change 100644 => 100755 packaging/tools/startPre.sh diff --git a/cmake/define.inc b/cmake/define.inc index 598957baa8..c4f9396cf1 100755 --- a/cmake/define.inc +++ b/cmake/define.inc @@ -41,6 +41,10 @@ IF (TD_POWER) ADD_DEFINITIONS(-D_TD_POWER_) ENDIF () +IF (TD_TQ) + ADD_DEFINITIONS(-D_TD_TQ_) +ENDIF () + IF (TD_MEM_CHECK) ADD_DEFINITIONS(-DTAOS_MEM_CHECK) ENDIF () diff --git a/cmake/input.inc b/cmake/input.inc index c073bbbc03..04160b4775 100755 --- a/cmake/input.inc +++ b/cmake/input.inc @@ -46,6 +46,9 @@ ENDIF () IF (${DBNAME} MATCHES "power") SET(TD_POWER TRUE) MESSAGE(STATUS "power is true") +ELSEIF (${DBNAME} MATCHES "tq") + SET(TD_TQ TRUE) + MESSAGE(STATUS "tq is true") ENDIF () IF (${DLLTYPE} MATCHES "go") diff --git a/cmake/version.inc b/cmake/version.inc index 134f09f179..93bad45fe8 100755 --- a/cmake/version.inc +++ b/cmake/version.inc @@ -10,7 +10,7 @@ ENDIF () IF (DEFINED VERCOMPATIBLE) SET(TD_VER_COMPATIBLE ${VERCOMPATIBLE}) ELSE () - SET(TD_VER_COMPATIBLE "2.0.0.0") + SET(TD_VER_COMPATIBLE "1.0.0.0") ENDIF () find_program(HAVE_GIT NAMES git) diff --git a/packaging/release.sh b/packaging/release.sh index 1d81f818a9..34fda44243 100755 --- a/packaging/release.sh +++ b/packaging/release.sh @@ -11,7 +11,7 @@ set -e # -V [stable | beta] # -l [full | lite] # -s [static | dynamic] -# -d [taos | power] +# -d [taos | power | tq ] # -n [2.0.0.3] # -m [2.0.0.0] @@ -22,10 +22,10 @@ cpuType=x64 # [aarch32 | aarch64 | x64 | x86 | mips64 ...] osType=Linux # [Linux | Kylin | Alpine | Raspberrypi | Darwin | Windows | Ningsi60 | Ningsi80 |...] pagMode=full # [full | lite] soMode=dynamic # [static | dynamic] +dbName=taos # [taos | power | tq] allocator=glibc # [glibc | jemalloc] -dbName=taos # [taos | power] verNumber="" -verNumberComp="2.0.0.0" +verNumberComp="1.0.0.0" while getopts "hv:V:c:o:l:s:d:a:n:m:" arg do @@ -78,7 +78,7 @@ do echo " -l [full | lite] " echo " -a [glibc | jemalloc] " echo " -s [static | dynamic] " - echo " -d [taos | power] " + echo " -d [taos | power | tq ] " echo " -n [version number] " echo " -m [compatible version number] " exit 0 @@ -204,7 +204,7 @@ else exit 1 fi -make +make -j8 cd ${curr_dir} @@ -249,6 +249,10 @@ if [ "$osType" != "Darwin" ]; then ${csudo} ./makepkg.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${csudo} ./makeclient.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${csudo} ./makearbi.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} + elif [[ "$dbName" == "tq" ]]; then + ${csudo} ./makepkg_tq.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${dbName} + ${csudo} ./makeclient_tq.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${dbName} + ${csudo} ./makearbi_tq.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} else ${csudo} ./makepkg_power.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${dbName} ${csudo} ./makeclient_power.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${dbName} diff --git a/packaging/tools/install_arbi_tq.sh b/packaging/tools/install_arbi_tq.sh new file mode 100755 index 0000000000..bd852dd0ad --- /dev/null +++ b/packaging/tools/install_arbi_tq.sh @@ -0,0 +1,298 @@ +#!/bin/bash +# +# This file is used to install database on linux systems. The operating system +# is required to use systemd to manage services at boot + +set -e +#set -x + +# -----------------------Variables definition--------------------- +script_dir=$(dirname $(readlink -f "$0")) + +bin_link_dir="/usr/bin" +#inc_link_dir="/usr/include" + +#install main path +install_main_dir="/usr/local/tarbitrator" + +# old bin dir +bin_dir="/usr/local/tarbitrator/bin" + +service_config_dir="/etc/systemd/system" + +# Color setting +RED='\033[0;31m' +GREEN='\033[1;32m' +GREEN_DARK='\033[0;32m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' + +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo" +fi + +update_flag=0 + +initd_mod=0 +service_mod=2 +if pidof systemd &> /dev/null; then + service_mod=0 +elif $(which service &> /dev/null); then + service_mod=1 + service_config_dir="/etc/init.d" + if $(which chkconfig &> /dev/null); then + initd_mod=1 + elif $(which insserv &> /dev/null); then + initd_mod=2 + elif $(which update-rc.d &> /dev/null); then + initd_mod=3 + else + service_mod=2 + fi +else + service_mod=2 +fi + + +# get the operating system type for using the corresponding init file +# ubuntu/debian(deb), centos/fedora(rpm), others: opensuse, redhat, ..., no verification +#osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release) +if [[ -e /etc/os-release ]]; then + osinfo=$(cat /etc/os-release | grep "NAME" | cut -d '"' -f2) ||: +else + osinfo="" +fi +#echo "osinfo: ${osinfo}" +os_type=0 +if echo $osinfo | grep -qwi "ubuntu" ; then +# echo "This is ubuntu system" + os_type=1 +elif echo $osinfo | grep -qwi "debian" ; then +# echo "This is debian system" + os_type=1 +elif echo $osinfo | grep -qwi "Kylin" ; then +# echo "This is Kylin system" + os_type=1 +elif echo $osinfo | grep -qwi "centos" ; then +# echo "This is centos system" + os_type=2 +elif echo $osinfo | grep -qwi "fedora" ; then +# echo "This is fedora system" + os_type=2 +else + echo " osinfo: ${osinfo}" + echo " This is an officially unverified linux system," + echo " if there are any problems with the installation and operation, " + echo " please feel free to contact taosdata.com for support." + os_type=1 +fi + +function kill_tarbitrator() { + pid=$(ps -ef | grep "tarbitrator" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo} kill -9 $pid || : + fi +} + +function install_main_path() { + #create install main dir and all sub dir + ${csudo} rm -rf ${install_main_dir} || : + ${csudo} mkdir -p ${install_main_dir} + ${csudo} mkdir -p ${install_main_dir}/bin + #${csudo} mkdir -p ${install_main_dir}/include + ${csudo} mkdir -p ${install_main_dir}/init.d +} + +function install_bin() { + # Remove links + ${csudo} rm -f ${bin_link_dir}/rmtarbitrator || : + ${csudo} rm -f ${bin_link_dir}/tarbitrator || : + ${csudo} cp -r ${script_dir}/bin/* ${install_main_dir}/bin && ${csudo} chmod 0555 ${install_main_dir}/bin/* + + #Make link + [ -x ${install_main_dir}/bin/remove_arbi_tq.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove_arbi_tq.sh ${bin_link_dir}/rmtarbitrator || : + [ -x ${install_main_dir}/bin/tarbitrator ] && ${csudo} ln -s ${install_main_dir}/bin/tarbitrator ${bin_link_dir}/tarbitrator || : +} + +function install_header() { + ${csudo} rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taoserror.h || : + ${csudo} cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo} chmod 644 ${install_main_dir}/include/* + ${csudo} ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h + ${csudo} ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h +} + +function clean_service_on_sysvinit() { + #restart_config_str="taos:2345:respawn:${service_config_dir}/taosd start" + #${csudo} sed -i "\|${restart_config_str}|d" /etc/inittab || : + + if pidof tarbitrator &> /dev/null; then + ${csudo} service tarbitratord stop || : + fi + + if ((${initd_mod}==1)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} chkconfig --del tarbitratord || : + fi + elif ((${initd_mod}==2)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} insserv -r tarbitratord || : + fi + elif ((${initd_mod}==3)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} update-rc.d -f tarbitratord remove || : + fi + fi + + ${csudo} rm -f ${service_config_dir}/tarbitratord || : + + if $(which init &> /dev/null); then + ${csudo} init q || : + fi +} + +function install_service_on_sysvinit() { + clean_service_on_sysvinit + sleep 1 + + # Install tqd service + + if ((${os_type}==1)); then + ${csudo} cp -f ${script_dir}/init.d/tarbitratord.deb ${install_main_dir}/init.d/tarbitratord + ${csudo} cp ${script_dir}/init.d/tarbitratord.deb ${service_config_dir}/tarbitratord && ${csudo} chmod a+x ${service_config_dir}/tarbitratord + elif ((${os_type}==2)); then + ${csudo} cp -f ${script_dir}/init.d/tarbitratord.rpm ${install_main_dir}/init.d/tarbitratord + ${csudo} cp ${script_dir}/init.d/tarbitratord.rpm ${service_config_dir}/tarbitratord && ${csudo} chmod a+x ${service_config_dir}/tarbitratord + fi + + #restart_config_str="tq:2345:respawn:${service_config_dir}/tqd start" + #${csudo} grep -q -F "$restart_config_str" /etc/inittab || ${csudo} bash -c "echo '${restart_config_str}' >> /etc/inittab" + + if ((${initd_mod}==1)); then + ${csudo} chkconfig --add tarbitratord || : + ${csudo} chkconfig --level 2345 tarbitratord on || : + elif ((${initd_mod}==2)); then + ${csudo} insserv tarbitratord || : + ${csudo} insserv -d tarbitratord || : + elif ((${initd_mod}==3)); then + ${csudo} update-rc.d tarbitratord defaults || : + fi +} + +function clean_service_on_systemd() { + tarbitratord_service_config="${service_config_dir}/tarbitratord.service" + if systemctl is-active --quiet tarbitratord; then + echo "tarbitrator is running, stopping it..." + ${csudo} systemctl stop tarbitratord &> /dev/null || echo &> /dev/null + fi + ${csudo} systemctl disable tarbitratord &> /dev/null || echo &> /dev/null + + ${csudo} rm -f ${tarbitratord_service_config} +} + +# tq:2345:respawn:/etc/init.d/tarbitratord start + +function install_service_on_systemd() { + clean_service_on_systemd + + tarbitratord_service_config="${service_config_dir}/tarbitratord.service" + + ${csudo} bash -c "echo '[Unit]' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Description=TQ arbitrator service' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'After=network-online.target' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Wants=network-online.target' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo '[Service]' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Type=simple' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'ExecStart=/usr/bin/tarbitrator' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'TimeoutStopSec=1000000s' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'LimitCORE=infinity' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'TimeoutStartSec=0' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'StandardOutput=null' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Restart=always' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'StartLimitBurst=3' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'StartLimitInterval=60s' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo '[Install]' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'WantedBy=multi-user.target' >> ${tarbitratord_service_config}" + ${csudo} systemctl enable tarbitratord +} + +function install_service() { + if ((${service_mod}==0)); then + install_service_on_systemd + elif ((${service_mod}==1)); then + install_service_on_sysvinit + else + # must manual stop taosd + kill_tarbitrator + fi +} + +function update_tq() { + # Start to update + echo -e "${GREEN}Start to update TQ's arbitrator ...${NC}" + # Stop the service if running + if pidof tarbitrator &> /dev/null; then + if ((${service_mod}==0)); then + ${csudo} systemctl stop tarbitratord || : + elif ((${service_mod}==1)); then + ${csudo} service tarbitratord stop || : + else + kill_tarbitrator + fi + sleep 1 + fi + + install_main_path + #install_header + install_bin + install_service + + echo + #echo -e "${GREEN_DARK}To configure TQ ${NC}: edit /etc/taos/taos.cfg" + if ((${service_mod}==0)); then + echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo} systemctl start tarbitratord${NC}" + elif ((${service_mod}==1)); then + echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo} service tarbitratord start${NC}" + else + echo -e "${GREEN_DARK}To start arbitrator ${NC}: ./tarbitrator${NC}" + fi + echo + echo -e "\033[44;32;1mTQ's arbitrator is updated successfully!${NC}" +} + +function install_tq() { + # Start to install + echo -e "${GREEN}Start to install TQ's arbitrator ...${NC}" + + install_main_path + #install_header + install_bin + install_service + echo + #echo -e "${GREEN_DARK}To configure TQ ${NC}: edit /etc/taos/taos.cfg" + if ((${service_mod}==0)); then + echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo} systemctl start tarbitratord${NC}" + elif ((${service_mod}==1)); then + echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo} service tarbitratord start${NC}" + else + echo -e "${GREEN_DARK}To start arbitrator ${NC}: tarbitrator${NC}" + fi + + echo -e "\033[44;32;1mTQ's arbitrator is installed successfully!${NC}" + echo +} + + +## ==============================Main program starts from here============================ +# Install server and client +if [ -x ${bin_dir}/tarbitrator ]; then + update_flag=1 + update_tq +else + install_tq +fi + diff --git a/packaging/tools/install_client_tq.sh b/packaging/tools/install_client_tq.sh new file mode 100755 index 0000000000..2537442ee2 --- /dev/null +++ b/packaging/tools/install_client_tq.sh @@ -0,0 +1,251 @@ +#!/bin/bash +# +# This file is used to install TQ client on linux systems. The operating system +# is required to use systemd to manage services at boot + +set -e +#set -x + +# -----------------------Variables definition--------------------- + +osType=Linux +pagMode=full + +if [ "$osType" != "Darwin" ]; then + script_dir=$(dirname $(readlink -f "$0")) + # Dynamic directory + data_dir="/var/lib/tq" + log_dir="/var/log/tq" +else + script_dir=`dirname $0` + cd ${script_dir} + script_dir="$(pwd)" + data_dir="/var/lib/tq" + log_dir="~/TQLog" +fi + +log_link_dir="/usr/local/tq/log" + +cfg_install_dir="/etc/tq" + +if [ "$osType" != "Darwin" ]; then + bin_link_dir="/usr/bin" + lib_link_dir="/usr/lib" + lib64_link_dir="/usr/lib64" + inc_link_dir="/usr/include" +else + bin_link_dir="/usr/local/bin" + lib_link_dir="/usr/local/lib" + inc_link_dir="/usr/local/include" +fi + +#install main path +install_main_dir="/usr/local/tq" + +# old bin dir +bin_dir="/usr/local/tq/bin" + +# v1.5 jar dir +#v15_java_app_dir="/usr/local/lib/tq" + +# Color setting +RED='\033[0;31m' +GREEN='\033[1;32m' +GREEN_DARK='\033[0;32m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' + +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo" +fi + +update_flag=0 + +function kill_client() { + pid=$(ps -ef | grep "tq" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo} kill -9 $pid || : + fi +} + +function install_main_path() { + #create install main dir and all sub dir + ${csudo} rm -rf ${install_main_dir} || : + ${csudo} mkdir -p ${install_main_dir} + ${csudo} mkdir -p ${install_main_dir}/cfg + ${csudo} mkdir -p ${install_main_dir}/bin + ${csudo} mkdir -p ${install_main_dir}/connector + ${csudo} mkdir -p ${install_main_dir}/driver + ${csudo} mkdir -p ${install_main_dir}/examples + ${csudo} mkdir -p ${install_main_dir}/include +} + +function install_bin() { + # Remove links + ${csudo} rm -f ${bin_link_dir}/tq || : + if [ "$osType" != "Darwin" ]; then + ${csudo} rm -f ${bin_link_dir}/tqdemo || : + ${csudo} rm -f ${bin_link_dir}/tqdump || : + fi + ${csudo} rm -f ${bin_link_dir}/rmtq || : + ${csudo} rm -f ${bin_link_dir}/set_core || : + + ${csudo} cp -r ${script_dir}/bin/* ${install_main_dir}/bin && ${csudo} chmod 0555 ${install_main_dir}/bin/* + + #Make link + [ -x ${install_main_dir}/bin/tq ] && ${csudo} ln -s ${install_main_dir}/bin/tq ${bin_link_dir}/tq || : + if [ "$osType" != "Darwin" ]; then + [ -x ${install_main_dir}/bin/tqdemo ] && ${csudo} ln -s ${install_main_dir}/bin/tqdemo ${bin_link_dir}/tqdemo || : + [ -x ${install_main_dir}/bin/tqdump ] && ${csudo} ln -s ${install_main_dir}/bin/tqdump ${bin_link_dir}/tqdump || : + fi + [ -x ${install_main_dir}/bin/remove_client_tq.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove_client_tq.sh ${bin_link_dir}/rmtq || : + [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo} ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : +} + +function clean_lib() { + sudo rm -f /usr/lib/libtaos.* || : + sudo rm -rf ${lib_dir} || : +} + +function install_lib() { + # Remove links + ${csudo} rm -f ${lib_link_dir}/libtaos.* || : + ${csudo} rm -f ${lib64_link_dir}/libtaos.* || : + #${csudo} rm -rf ${v15_java_app_dir} || : + + ${csudo} cp -rf ${script_dir}/driver/* ${install_main_dir}/driver && ${csudo} chmod 777 ${install_main_dir}/driver/* + + if [ "$osType" != "Darwin" ]; then + ${csudo} ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1 + ${csudo} ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so + + if [ -d "${lib64_link_dir}" ]; then + ${csudo} ln -s ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : + ${csudo} ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : + fi + else + ${csudo} ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.1.dylib + ${csudo} ln -s ${lib_link_dir}/libtaos.1.dylib ${lib_link_dir}/libtaos.dylib + fi + + ${csudo} ldconfig +} + +function install_header() { + ${csudo} rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taoserror.h || : + ${csudo} cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo} chmod 644 ${install_main_dir}/include/* + ${csudo} ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h + ${csudo} ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h +} + +function install_config() { + #${csudo} rm -f ${install_main_dir}/cfg/taos.cfg || : + + if [ ! -f ${cfg_install_dir}/taos.cfg ]; then + ${csudo} mkdir -p ${cfg_install_dir} + [ -f ${script_dir}/cfg/taos.cfg ] && ${csudo} cp ${script_dir}/cfg/taos.cfg ${cfg_install_dir} + ${csudo} chmod 644 ${cfg_install_dir}/* + fi + + ${csudo} cp -f ${script_dir}/cfg/taos.cfg ${install_main_dir}/cfg/taos.cfg.org + ${csudo} ln -s ${cfg_install_dir}/taos.cfg ${install_main_dir}/cfg +} + + +function install_log() { + ${csudo} rm -rf ${log_dir} || : + + if [ "$osType" != "Darwin" ]; then + ${csudo} mkdir -p ${log_dir} && ${csudo} chmod 777 ${log_dir} + else + mkdir -p ${log_dir} && ${csudo} chmod 777 ${log_dir} + fi + ${csudo} ln -s ${log_dir} ${install_main_dir}/log +} + +function install_connector() { + ${csudo} cp -rf ${script_dir}/connector/* ${install_main_dir}/connector +} + +function install_examples() { + if [ -d ${script_dir}/examples ]; then + ${csudo} cp -rf ${script_dir}/examples/* ${install_main_dir}/examples + fi +} + +function update_tq() { + # Start to update + if [ ! -e tq.tar.gz ]; then + echo "File tq.tar.gz does not exist" + exit 1 + fi + tar -zxf tq.tar.gz + + echo -e "${GREEN}Start to update TQ client...${NC}" + # Stop the client shell if running + if pidof tq &> /dev/null; then + kill_client + sleep 1 + fi + + install_main_path + + install_log + install_header + install_lib + if [ "$pagMode" != "lite" ]; then + install_connector + fi + install_examples + install_bin + install_config + + echo + echo -e "\033[44;32;1mTQ client is updated successfully!${NC}" + + rm -rf $(tar -tf tq.tar.gz) +} + +function install_tq() { + # Start to install + if [ ! -e tq.tar.gz ]; then + echo "File tq.tar.gz does not exist" + exit 1 + fi + tar -zxf tq.tar.gz + + echo -e "${GREEN}Start to install TQ client...${NC}" + + install_main_path + install_log + install_header + install_lib + if [ "$pagMode" != "lite" ]; then + install_connector + fi + install_examples + install_bin + install_config + + echo + echo -e "\033[44;32;1mTQ client is installed successfully!${NC}" + + rm -rf $(tar -tf tq.tar.gz) +} + + +## ==============================Main program starts from here============================ +# Install or updata client and client +# if server is already install, don't install client + if [ -e ${bin_dir}/tqd ]; then + echo -e "\033[44;32;1mThere are already installed TQ server, so don't need install client!${NC}" + exit 0 + fi + + if [ -x ${bin_dir}/tq ]; then + update_flag=1 + update_tq + else + install_tq + fi diff --git a/packaging/tools/install_tq.sh b/packaging/tools/install_tq.sh new file mode 100755 index 0000000000..6579592fdf --- /dev/null +++ b/packaging/tools/install_tq.sh @@ -0,0 +1,918 @@ +#!/bin/bash +# +# This file is used to install database on linux systems. The operating system +# is required to use systemd to manage services at boot + +set -e +#set -x + +verMode=edge +pagMode=full + +iplist="" +serverFqdn="" +# -----------------------Variables definition--------------------- +script_dir=$(dirname $(readlink -f "$0")) +# Dynamic directory +data_dir="/var/lib/tq" +log_dir="/var/log/tq" + +data_link_dir="/usr/local/tq/data" +log_link_dir="/usr/local/tq/log" + +cfg_install_dir="/etc/tq" + +bin_link_dir="/usr/bin" +lib_link_dir="/usr/lib" +lib64_link_dir="/usr/lib64" +inc_link_dir="/usr/include" + +#install main path +install_main_dir="/usr/local/tq" + +# old bin dir +bin_dir="/usr/local/tq/bin" + +# v1.5 jar dir +#v15_java_app_dir="/usr/local/lib/tq" + +service_config_dir="/etc/systemd/system" +nginx_port=6060 +nginx_dir="/usr/local/nginxd" + +# Color setting +RED='\033[0;31m' +GREEN='\033[1;32m' +GREEN_DARK='\033[0;32m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' + +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo" +fi + +update_flag=0 + +initd_mod=0 +service_mod=2 +if pidof systemd &> /dev/null; then + service_mod=0 +elif $(which service &> /dev/null); then + service_mod=1 + service_config_dir="/etc/init.d" + if $(which chkconfig &> /dev/null); then + initd_mod=1 + elif $(which insserv &> /dev/null); then + initd_mod=2 + elif $(which update-rc.d &> /dev/null); then + initd_mod=3 + else + service_mod=2 + fi +else + service_mod=2 +fi + + +# get the operating system type for using the corresponding init file +# ubuntu/debian(deb), centos/fedora(rpm), others: opensuse, redhat, ..., no verification +#osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release) +if [[ -e /etc/os-release ]]; then + osinfo=$(cat /etc/os-release | grep "NAME" | cut -d '"' -f2) ||: +else + osinfo="" +fi +#echo "osinfo: ${osinfo}" +os_type=0 +if echo $osinfo | grep -qwi "ubuntu" ; then +# echo "This is ubuntu system" + os_type=1 +elif echo $osinfo | grep -qwi "debian" ; then +# echo "This is debian system" + os_type=1 +elif echo $osinfo | grep -qwi "Kylin" ; then +# echo "This is Kylin system" + os_type=1 +elif echo $osinfo | grep -qwi "centos" ; then +# echo "This is centos system" + os_type=2 +elif echo $osinfo | grep -qwi "fedora" ; then +# echo "This is fedora system" + os_type=2 +else + echo " osinfo: ${osinfo}" + echo " This is an officially unverified linux system," + echo " if there are any problems with the installation and operation, " + echo " please feel free to contact taosdata.com for support." + os_type=1 +fi + + +# ============================= get input parameters ================================================= + +# install.sh -v [server | client] -e [yes | no] -i [systemd | service | ...] + +# set parameters by default value +interactiveFqdn=yes # [yes | no] +verType=server # [server | client] +initType=systemd # [systemd | service | ...] + +while getopts "hv:e:i:" arg +do + case $arg in + e) + #echo "interactiveFqdn=$OPTARG" + interactiveFqdn=$( echo $OPTARG ) + ;; + v) + #echo "verType=$OPTARG" + verType=$(echo $OPTARG) + ;; + i) + #echo "initType=$OPTARG" + initType=$(echo $OPTARG) + ;; + h) + echo "Usage: `basename $0` -v [server | client] -e [yes | no]" + exit 0 + ;; + ?) #unknow option + echo "unkonw argument" + exit 1 + ;; + esac +done + +#echo "verType=${verType} interactiveFqdn=${interactiveFqdn}" + +function kill_process() { + pid=$(ps -ef | grep "$1" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo} kill -9 $pid || : + fi +} + +function install_main_path() { + #create install main dir and all sub dir + ${csudo} rm -rf ${install_main_dir} || : + ${csudo} mkdir -p ${install_main_dir} + ${csudo} mkdir -p ${install_main_dir}/cfg + ${csudo} mkdir -p ${install_main_dir}/bin + ${csudo} mkdir -p ${install_main_dir}/connector + ${csudo} mkdir -p ${install_main_dir}/driver + ${csudo} mkdir -p ${install_main_dir}/examples + ${csudo} mkdir -p ${install_main_dir}/include + ${csudo} mkdir -p ${install_main_dir}/init.d + if [ "$verMode" == "cluster" ]; then + ${csudo} mkdir -p ${nginx_dir} + fi +} + +function install_bin() { + # Remove links + ${csudo} rm -f ${bin_link_dir}/tq || : + ${csudo} rm -f ${bin_link_dir}/tqd || : + ${csudo} rm -f ${bin_link_dir}/tqdemo || : + ${csudo} rm -f ${bin_link_dir}/rmtq || : + ${csudo} rm -f ${bin_link_dir}/tarbitrator || : + ${csudo} rm -f ${bin_link_dir}/set_core || : + + ${csudo} cp -r ${script_dir}/bin/* ${install_main_dir}/bin && ${csudo} chmod 0555 ${install_main_dir}/bin/* + + #Make link + [ -x ${install_main_dir}/bin/tq ] && ${csudo} ln -s ${install_main_dir}/bin/tq ${bin_link_dir}/tq || : + [ -x ${install_main_dir}/bin/tqd ] && ${csudo} ln -s ${install_main_dir}/bin/tqd ${bin_link_dir}/tqd || : + [ -x ${install_main_dir}/bin/tqdemo ] && ${csudo} ln -s ${install_main_dir}/bin/tqdemo ${bin_link_dir}/tqdemo || : + [ -x ${install_main_dir}/bin/remove_tq.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove_tq.sh ${bin_link_dir}/rmtq || : + [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo} ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : + [ -x ${install_main_dir}/bin/tarbitrator ] && ${csudo} ln -s ${install_main_dir}/bin/tarbitrator ${bin_link_dir}/tarbitrator || : + + if [ "$verMode" == "cluster" ]; then + ${csudo} cp -r ${script_dir}/nginxd/* ${nginx_dir} && ${csudo} chmod 0555 ${nginx_dir}/* + ${csudo} mkdir -p ${nginx_dir}/logs + ${csudo} chmod 777 ${nginx_dir}/sbin/nginx + fi +} + +function install_lib() { + # Remove links + ${csudo} rm -f ${lib_link_dir}/libtaos.* || : + ${csudo} rm -f ${lib64_link_dir}/libtaos.* || : + #${csudo} rm -rf ${v15_java_app_dir} || : + ${csudo} cp -rf ${script_dir}/driver/* ${install_main_dir}/driver && ${csudo} chmod 777 ${install_main_dir}/driver/* + + ${csudo} ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1 + ${csudo} ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so + + if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then + ${csudo} ln -s ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : + ${csudo} ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : + fi + + #if [ "$verMode" == "cluster" ]; then + # # Compatible with version 1.5 + # ${csudo} mkdir -p ${v15_java_app_dir} + # ${csudo} ln -s ${install_main_dir}/connector/taos-jdbcdriver-1.0.2-dist.jar ${v15_java_app_dir}/JDBCDriver-1.0.2-dist.jar + # ${csudo} chmod 777 ${v15_java_app_dir} || : + #fi + + ${csudo} ldconfig +} + +function install_header() { + ${csudo} rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taoserror.h || : + ${csudo} cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo} chmod 644 ${install_main_dir}/include/* + ${csudo} ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h + ${csudo} ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h +} + +function add_newHostname_to_hosts() { + localIp="127.0.0.1" + OLD_IFS="$IFS" + IFS=" " + iphost=$(cat /etc/hosts | grep $1 | awk '{print $1}') + arr=($iphost) + IFS="$OLD_IFS" + for s in ${arr[@]} + do + if [[ "$s" == "$localIp" ]]; then + return + fi + done + ${csudo} echo "127.0.0.1 $1" >> /etc/hosts ||: +} + +function set_hostname() { + echo -e -n "${GREEN}Please enter one hostname(must not be 'localhost')${NC}:" + read newHostname + while true; do + if [[ ! -z "$newHostname" && "$newHostname" != "localhost" ]]; then + break + else + read -p "Please enter one hostname(must not be 'localhost'):" newHostname + fi + done + + ${csudo} hostname $newHostname ||: + retval=`echo $?` + if [[ $retval != 0 ]]; then + echo + echo "set hostname fail!" + return + fi + #echo -e -n "$(hostnamectl status --static)" + #echo -e -n "$(hostnamectl status --transient)" + #echo -e -n "$(hostnamectl status --pretty)" + + #ubuntu/centos /etc/hostname + if [[ -e /etc/hostname ]]; then + ${csudo} echo $newHostname > /etc/hostname ||: + fi + + #debian: #HOSTNAME=yourname + if [[ -e /etc/sysconfig/network ]]; then + ${csudo} sed -i -r "s/#*\s*(HOSTNAME=\s*).*/\1$newHostname/" /etc/sysconfig/network ||: + fi + + ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$newHostname/" ${cfg_install_dir}/taos.cfg + serverFqdn=$newHostname + + if [[ -e /etc/hosts ]]; then + add_newHostname_to_hosts $newHostname + fi +} + +function is_correct_ipaddr() { + newIp=$1 + OLD_IFS="$IFS" + IFS=" " + arr=($iplist) + IFS="$OLD_IFS" + for s in ${arr[@]} + do + if [[ "$s" == "$newIp" ]]; then + return 0 + fi + done + + return 1 +} + +function set_ipAsFqdn() { + iplist=$(ip address |grep inet |grep -v inet6 |grep -v 127.0.0.1 |awk '{print $2}' |awk -F "/" '{print $1}') ||: + if [ -z "$iplist" ]; then + iplist=$(ifconfig |grep inet |grep -v inet6 |grep -v 127.0.0.1 |awk '{print $2}' |awk -F ":" '{print $2}') ||: + fi + + if [ -z "$iplist" ]; then + echo + echo -e -n "${GREEN}Unable to get local ip, use 127.0.0.1${NC}" + localFqdn="127.0.0.1" + # Write the local FQDN to configuration file + ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg + serverFqdn=$localFqdn + echo + return + fi + + echo -e -n "${GREEN}Please choose an IP from local IP list${NC}:" + echo + echo -e -n "${GREEN}$iplist${NC}" + echo + echo + echo -e -n "${GREEN}Notes: if IP is used as the node name, data can NOT be migrated to other machine directly${NC}:" + read localFqdn + while true; do + if [ ! -z "$localFqdn" ]; then + # Check if correct ip address + is_correct_ipaddr $localFqdn + retval=`echo $?` + if [[ $retval != 0 ]]; then + read -p "Please choose an IP from local IP list:" localFqdn + else + # Write the local FQDN to configuration file + ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg + serverFqdn=$localFqdn + break + fi + else + read -p "Please choose an IP from local IP list:" localFqdn + fi + done +} + +function local_fqdn_check() { + #serverFqdn=$(hostname) + echo + echo -e -n "System hostname is: ${GREEN}$serverFqdn${NC}" + echo + if [[ "$serverFqdn" == "" ]] || [[ "$serverFqdn" == "localhost" ]]; then + echo -e -n "${GREEN}It is strongly recommended to configure a hostname for this machine ${NC}" + echo + + while true + do + read -r -p "Set hostname now? [Y/n] " input + if [ ! -n "$input" ]; then + set_hostname + break + else + case $input in + [yY][eE][sS]|[yY]) + set_hostname + break + ;; + + [nN][oO]|[nN]) + set_ipAsFqdn + break + ;; + + *) + echo "Invalid input..." + ;; + esac + fi + done + fi +} + +function install_config() { + #${csudo} rm -f ${install_main_dir}/cfg/taos.cfg || : + + if [ ! -f ${cfg_install_dir}/taos.cfg ]; then + ${csudo} mkdir -p ${cfg_install_dir} + [ -f ${script_dir}/cfg/taos.cfg ] && ${csudo} cp ${script_dir}/cfg/taos.cfg ${cfg_install_dir} + ${csudo} chmod 644 ${cfg_install_dir}/* + fi + + ${csudo} cp -f ${script_dir}/cfg/taos.cfg ${install_main_dir}/cfg/taos.cfg.org + ${csudo} ln -s ${cfg_install_dir}/taos.cfg ${install_main_dir}/cfg + + [ ! -z $1 ] && return 0 || : # only install client + + if ((${update_flag}==1)); then + return 0 + fi + + if [ "$interactiveFqdn" == "no" ]; then + return 0 + fi + + local_fqdn_check + + #FQDN_FORMAT="(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)" + #FQDN_FORMAT="(:[1-6][0-9][0-9][0-9][0-9]$)" + #PORT_FORMAT="(/[1-6][0-9][0-9][0-9][0-9]?/)" + #FQDN_PATTERN=":[0-9]{1,5}$" + + # first full-qualified domain name (FQDN) for TQ cluster system + echo + echo -e -n "${GREEN}Enter FQDN:port (like h1.taosdata.com:6030) of an existing TQ cluster node to join${NC}" + echo + echo -e -n "${GREEN}OR leave it blank to build one${NC}:" + read firstEp + while true; do + if [ ! -z "$firstEp" ]; then + # check the format of the firstEp + #if [[ $firstEp == $FQDN_PATTERN ]]; then + # Write the first FQDN to configuration file + ${csudo} sed -i -r "s/#*\s*(firstEp\s*).*/\1$firstEp/" ${cfg_install_dir}/taos.cfg + break + #else + # read -p "Please enter the correct FQDN:port: " firstEp + #fi + else + break + fi + done +} + + +function install_log() { + ${csudo} rm -rf ${log_dir} || : + ${csudo} mkdir -p ${log_dir} && ${csudo} chmod 777 ${log_dir} + + ${csudo} ln -s ${log_dir} ${install_main_dir}/log +} + +function install_data() { + ${csudo} mkdir -p ${data_dir} + + ${csudo} ln -s ${data_dir} ${install_main_dir}/data +} + +function install_connector() { + ${csudo} cp -rf ${script_dir}/connector/* ${install_main_dir}/connector +} + +function install_examples() { + if [ -d ${script_dir}/examples ]; then + ${csudo} cp -rf ${script_dir}/examples/* ${install_main_dir}/examples + fi +} + +function clean_service_on_sysvinit() { + #restart_config_str="tq:2345:respawn:${service_config_dir}/tqd start" + #${csudo} sed -i "\|${restart_config_str}|d" /etc/inittab || : + + if pidof tqd &> /dev/null; then + ${csudo} service tqd stop || : + fi + + if pidof tarbitrator &> /dev/null; then + ${csudo} service tarbitratord stop || : + fi + + if ((${initd_mod}==1)); then + if [ -e ${service_config_dir}/tqd ]; then + ${csudo} chkconfig --del tqd || : + fi + + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} chkconfig --del tarbitratord || : + fi + elif ((${initd_mod}==2)); then + if [ -e ${service_config_dir}/tqd ]; then + ${csudo} insserv -r tqd || : + fi + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} insserv -r tarbitratord || : + fi + elif ((${initd_mod}==3)); then + if [ -e ${service_config_dir}/tqd ]; then + ${csudo} update-rc.d -f tqd remove || : + fi + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} update-rc.d -f tarbitratord remove || : + fi + fi + + ${csudo} rm -f ${service_config_dir}/tqd || : + ${csudo} rm -f ${service_config_dir}/tarbitratord || : + + if $(which init &> /dev/null); then + ${csudo} init q || : + fi +} + +function install_service_on_sysvinit() { + clean_service_on_sysvinit + sleep 1 + + # Install tqd service + + if ((${os_type}==1)); then + ${csudo} cp -f ${script_dir}/init.d/tqd.deb ${install_main_dir}/init.d/tqd + ${csudo} cp ${script_dir}/init.d/tqd.deb ${service_config_dir}/tqd && ${csudo} chmod a+x ${service_config_dir}/tqd + ${csudo} cp -f ${script_dir}/init.d/tarbitratord.deb ${install_main_dir}/init.d/tarbitratord + ${csudo} cp ${script_dir}/init.d/tarbitratord.deb ${service_config_dir}/tarbitratord && ${csudo} chmod a+x ${service_config_dir}/tarbitratord + elif ((${os_type}==2)); then + ${csudo} cp -f ${script_dir}/init.d/tqd.rpm ${install_main_dir}/init.d/tqd + ${csudo} cp ${script_dir}/init.d/tqd.rpm ${service_config_dir}/tqd && ${csudo} chmod a+x ${service_config_dir}/tqd + ${csudo} cp -f ${script_dir}/init.d/tarbitratord.rpm ${install_main_dir}/init.d/tarbitratord + ${csudo} cp ${script_dir}/init.d/tarbitratord.rpm ${service_config_dir}/tarbitratord && ${csudo} chmod a+x ${service_config_dir}/tarbitratord + fi + + #restart_config_str="tq:2345:respawn:${service_config_dir}/tqd start" + #${csudo} grep -q -F "$restart_config_str" /etc/inittab || ${csudo} bash -c "echo '${restart_config_str}' >> /etc/inittab" + + if ((${initd_mod}==1)); then + ${csudo} chkconfig --add tqd || : + ${csudo} chkconfig --level 2345 tqd on || : + ${csudo} chkconfig --add tarbitratord || : + ${csudo} chkconfig --level 2345 tarbitratord on || : + elif ((${initd_mod}==2)); then + ${csudo} insserv tqd || : + ${csudo} insserv -d tqd || : + ${csudo} insserv tarbitratord || : + ${csudo} insserv -d tarbitratord || : + elif ((${initd_mod}==3)); then + ${csudo} update-rc.d tqd defaults || : + ${csudo} update-rc.d tarbitratord defaults || : + fi +} + +function clean_service_on_systemd() { + tqd_service_config="${service_config_dir}/tqd.service" + if systemctl is-active --quiet tqd; then + echo "TQ is running, stopping it..." + ${csudo} systemctl stop tqd &> /dev/null || echo &> /dev/null + fi + ${csudo} systemctl disable tqd &> /dev/null || echo &> /dev/null + ${csudo} rm -f ${tqd_service_config} + + tarbitratord_service_config="${service_config_dir}/tarbitratord.service" + if systemctl is-active --quiet tarbitratord; then + echo "tarbitrator is running, stopping it..." + ${csudo} systemctl stop tarbitratord &> /dev/null || echo &> /dev/null + fi + ${csudo} systemctl disable tarbitratord &> /dev/null || echo &> /dev/null + ${csudo} rm -f ${tarbitratord_service_config} + + if [ "$verMode" == "cluster" ]; then + nginx_service_config="${service_config_dir}/nginxd.service" + if systemctl is-active --quiet nginxd; then + echo "Nginx for TDengine is running, stopping it..." + ${csudo} systemctl stop nginxd &> /dev/null || echo &> /dev/null + fi + ${csudo} systemctl disable nginxd &> /dev/null || echo &> /dev/null + ${csudo} rm -f ${nginx_service_config} + fi +} + +# tq:2345:respawn:/etc/init.d/tqd start + +function install_service_on_systemd() { + clean_service_on_systemd + + tqd_service_config="${service_config_dir}/tqd.service" + ${csudo} bash -c "echo '[Unit]' >> ${tqd_service_config}" + ${csudo} bash -c "echo 'Description=TQ server service' >> ${tqd_service_config}" + ${csudo} bash -c "echo 'After=network-online.target' >> ${tqd_service_config}" + ${csudo} bash -c "echo 'Wants=network-online.target' >> ${tqd_service_config}" + ${csudo} bash -c "echo >> ${tqd_service_config}" + ${csudo} bash -c "echo '[Service]' >> ${tqd_service_config}" + ${csudo} bash -c "echo 'Type=simple' >> ${tqd_service_config}" + ${csudo} bash -c "echo 'ExecStart=/usr/bin/tqd' >> ${tqd_service_config}" + ${csudo} bash -c "echo 'ExecStartPre=/usr/local/tq/bin/startPre.sh' >> ${tqd_service_config}" + ${csudo} bash -c "echo 'TimeoutStopSec=1000000s' >> ${tqd_service_config}" + ${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${tqd_service_config}" + ${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${tqd_service_config}" + ${csudo} bash -c "echo 'LimitCORE=infinity' >> ${tqd_service_config}" + ${csudo} bash -c "echo 'TimeoutStartSec=0' >> ${tqd_service_config}" + ${csudo} bash -c "echo 'StandardOutput=null' >> ${tqd_service_config}" + ${csudo} bash -c "echo 'Restart=always' >> ${tqd_service_config}" + ${csudo} bash -c "echo 'StartLimitBurst=3' >> ${tqd_service_config}" + ${csudo} bash -c "echo 'StartLimitInterval=60s' >> ${tqd_service_config}" + ${csudo} bash -c "echo >> ${tqd_service_config}" + ${csudo} bash -c "echo '[Install]' >> ${tqd_service_config}" + ${csudo} bash -c "echo 'WantedBy=multi-user.target' >> ${tqd_service_config}" + ${csudo} systemctl enable tqd + + tarbitratord_service_config="${service_config_dir}/tarbitratord.service" + ${csudo} bash -c "echo '[Unit]' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Description=TDengine arbitrator service' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'After=network-online.target' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Wants=network-online.target' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo '[Service]' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Type=simple' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'ExecStart=/usr/bin/tarbitrator' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'TimeoutStopSec=1000000s' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'LimitCORE=infinity' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'TimeoutStartSec=0' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'StandardOutput=null' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Restart=always' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'StartLimitBurst=3' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'StartLimitInterval=60s' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo '[Install]' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'WantedBy=multi-user.target' >> ${tarbitratord_service_config}" + #${csudo} systemctl enable tarbitratord + + if [ "$verMode" == "cluster" ]; then + nginx_service_config="${service_config_dir}/nginxd.service" + ${csudo} bash -c "echo '[Unit]' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'Description=Nginx For PowrDB Service' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'After=network-online.target' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'Wants=network-online.target' >> ${nginx_service_config}" + ${csudo} bash -c "echo >> ${nginx_service_config}" + ${csudo} bash -c "echo '[Service]' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'Type=forking' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'PIDFile=/usr/local/nginxd/logs/nginx.pid' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'ExecStart=/usr/local/nginxd/sbin/nginx' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'ExecStop=/usr/local/nginxd/sbin/nginx -s stop' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'TimeoutStopSec=1000000s' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'LimitCORE=infinity' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'TimeoutStartSec=0' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'StandardOutput=null' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'Restart=always' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'StartLimitBurst=3' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'StartLimitInterval=60s' >> ${nginx_service_config}" + ${csudo} bash -c "echo >> ${nginx_service_config}" + ${csudo} bash -c "echo '[Install]' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'WantedBy=multi-user.target' >> ${nginx_service_config}" + if ! ${csudo} systemctl enable nginxd &> /dev/null; then + ${csudo} systemctl daemon-reexec + ${csudo} systemctl enable nginxd + fi + ${csudo} systemctl start nginxd + fi +} + +function install_service() { + if ((${service_mod}==0)); then + install_service_on_systemd + elif ((${service_mod}==1)); then + install_service_on_sysvinit + else + # must manual stop tqd + kill_process tqd + fi +} + +vercomp () { + if [[ $1 == $2 ]]; then + return 0 + fi + local IFS=. + local i ver1=($1) ver2=($2) + # fill empty fields in ver1 with zeros + for ((i=${#ver1[@]}; i<${#ver2[@]}; i++)); do + ver1[i]=0 + done + + for ((i=0; i<${#ver1[@]}; i++)); do + if [[ -z ${ver2[i]} ]] + then + # fill empty fields in ver2 with zeros + ver2[i]=0 + fi + if ((10#${ver1[i]} > 10#${ver2[i]})) + then + return 1 + fi + if ((10#${ver1[i]} < 10#${ver2[i]})) + then + return 2 + fi + done + return 0 +} + +function is_version_compatible() { + + curr_version=$(${bin_dir}/tqd -V | head -1 | cut -d ' ' -f 3) + + min_compatible_version=$(${script_dir}/bin/tqd -V | head -1 | cut -d ' ' -f 5) + + vercomp $curr_version $min_compatible_version + case $? in + 0) return 0;; + 1) return 0;; + 2) return 1;; + esac +} + +function update_tq() { + # Start to update + if [ ! -e tq.tar.gz ]; then + echo "File tq.tar.gz does not exist" + exit 1 + fi + tar -zxf tq.tar.gz + + # Check if version compatible + if ! is_version_compatible; then + echo -e "${RED}Version incompatible${NC}" + return 1 + fi + + echo -e "${GREEN}Start to update TQ...${NC}" + # Stop the service if running + if pidof tqd &> /dev/null; then + if ((${service_mod}==0)); then + ${csudo} systemctl stop tqd || : + elif ((${service_mod}==1)); then + ${csudo} service tqd stop || : + else + kill_process tqd + fi + sleep 1 + fi + if [ "$verMode" == "cluster" ]; then + if pidof nginx &> /dev/null; then + if ((${service_mod}==0)); then + ${csudo} systemctl stop nginxd || : + elif ((${service_mod}==1)); then + ${csudo} service nginxd stop || : + else + kill_process nginx + fi + sleep 1 + fi + fi + + install_main_path + + install_log + install_header + install_lib + if [ "$pagMode" != "lite" ]; then + install_connector + fi + install_examples + if [ -z $1 ]; then + install_bin + install_service + install_config + + openresty_work=false + if [ "$verMode" == "cluster" ]; then + # Check if openresty is installed + # Check if nginx is installed successfully + if type curl &> /dev/null; then + if curl -sSf http://127.0.0.1:${nginx_port} &> /dev/null; then + echo -e "\033[44;32;1mNginx for TQ is updated successfully!${NC}" + openresty_work=true + else + echo -e "\033[44;31;5mNginx for TQ does not work! Please try again!\033[0m" + fi + fi + fi + + #echo + #echo -e "\033[44;32;1mTQ is updated successfully!${NC}" + echo + echo -e "${GREEN_DARK}To configure TQ ${NC}: edit /etc/tq/taos.cfg" + if ((${service_mod}==0)); then + echo -e "${GREEN_DARK}To start TQ ${NC}: ${csudo} systemctl start tqd${NC}" + elif ((${service_mod}==1)); then + echo -e "${GREEN_DARK}To start TQ ${NC}: ${csudo} service tqd start${NC}" + else + echo -e "${GREEN_DARK}To start TQ ${NC}: ./tqd${NC}" + fi + + if [ ${openresty_work} = 'true' ]; then + echo -e "${GREEN_DARK}To access TQ ${NC}: use ${GREEN_UNDERLINE}tq -h $serverFqdn${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${nginx_port}${NC}" + else + echo -e "${GREEN_DARK}To access TQ ${NC}: use ${GREEN_UNDERLINE}tq -h $serverFqdn${NC} in shell${NC}" + fi + + echo + echo -e "\033[44;32;1mTQ is updated successfully!${NC}" + else + install_bin + install_config + + echo + echo -e "\033[44;32;1mTQ client is updated successfully!${NC}" + fi + + rm -rf $(tar -tf tq.tar.gz) +} + +function install_tq() { + # Start to install + if [ ! -e tq.tar.gz ]; then + echo "File tq.tar.gz does not exist" + exit 1 + fi + tar -zxf tq.tar.gz + + echo -e "${GREEN}Start to install TQ...${NC}" + + install_main_path + + if [ -z $1 ]; then + install_data + fi + + install_log + install_header + install_lib + if [ "$pagMode" != "lite" ]; then + install_connector + fi + install_examples + + if [ -z $1 ]; then # install service and client + # For installing new + install_bin + install_service + + openresty_work=false + if [ "$verMode" == "cluster" ]; then + # Check if nginx is installed successfully + if type curl &> /dev/null; then + if curl -sSf http://127.0.0.1:${nginx_port} &> /dev/null; then + echo -e "\033[44;32;1mNginx for TQ is installed successfully!${NC}" + openresty_work=true + else + echo -e "\033[44;31;5mNginx for TQ does not work! Please try again!\033[0m" + fi + fi + fi + + install_config + + # Ask if to start the service + #echo + #echo -e "\033[44;32;1mTQ is installed successfully!${NC}" + echo + echo -e "${GREEN_DARK}To configure TQ ${NC}: edit /etc/tq/taos.cfg" + if ((${service_mod}==0)); then + echo -e "${GREEN_DARK}To start TQ ${NC}: ${csudo} systemctl start tqd${NC}" + elif ((${service_mod}==1)); then + echo -e "${GREEN_DARK}To start TQ ${NC}: ${csudo} service tqd start${NC}" + else + echo -e "${GREEN_DARK}To start TQ ${NC}: tqd${NC}" + fi + + #if [ ${openresty_work} = 'true' ]; then + # echo -e "${GREEN_DARK}To access TQ ${NC}: use ${GREEN_UNDERLINE}tq${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${nginx_port}${NC}" + #else + # echo -e "${GREEN_DARK}To access TQ ${NC}: use ${GREEN_UNDERLINE}tq${NC} in shell${NC}" + #fi + + if [ ! -z "$firstEp" ]; then + tmpFqdn=${firstEp%%:*} + substr=":" + if [[ $firstEp =~ $substr ]];then + tmpPort=${firstEp#*:} + else + tmpPort="" + fi + if [[ "$tmpPort" != "" ]];then + echo -e "${GREEN_DARK}To access TQ ${NC}: tq -h $tmpFqdn -P $tmpPort${GREEN_DARK} to login into cluster, then${NC}" + else + echo -e "${GREEN_DARK}To access TQ ${NC}: tq -h $tmpFqdn${GREEN_DARK} to login into cluster, then${NC}" + fi + echo -e "${GREEN_DARK}execute ${NC}: create dnode 'newDnodeFQDN:port'; ${GREEN_DARK}to add this new node${NC}" + echo + elif [ ! -z "$serverFqdn" ]; then + echo -e "${GREEN_DARK}To access TQ ${NC}: tq -h $serverFqdn${GREEN_DARK} to login into TQ server${NC}" + echo + fi + echo -e "\033[44;32;1mTQ is installed successfully!${NC}" + echo + else # Only install client + install_bin + install_config + + echo + echo -e "\033[44;32;1mTQ client is installed successfully!${NC}" + fi + + rm -rf $(tar -tf tq.tar.gz) +} + + +## ==============================Main program starts from here============================ +serverFqdn=$(hostname) +if [ "$verType" == "server" ]; then + # Install server and client + if [ -x ${bin_dir}/tqd ]; then + update_flag=1 + update_tq + else + install_tq + fi +elif [ "$verType" == "client" ]; then + interactiveFqdn=no + # Only install client + if [ -x ${bin_dir}/tq ]; then + update_flag=1 + update_tq client + else + install_tq client + fi +else + echo "please input correct verType" +fi diff --git a/packaging/tools/makearbi_tq.sh b/packaging/tools/makearbi_tq.sh new file mode 100755 index 0000000000..c10dfec255 --- /dev/null +++ b/packaging/tools/makearbi_tq.sh @@ -0,0 +1,75 @@ +#!/bin/bash +# +# Generate arbitrator's tar.gz setup package for all os system + +set -e +#set -x + +curr_dir=$(pwd) +compile_dir=$1 +version=$2 +build_time=$3 +cpuType=$4 +osType=$5 +verMode=$6 +verType=$7 +pagMode=$8 + +script_dir="$(dirname $(readlink -f $0))" +top_dir="$(readlink -f ${script_dir}/../..)" + +# create compressed install file. +build_dir="${compile_dir}/build" +code_dir="${top_dir}/src" +release_dir="${top_dir}/release" + +#package_name='linux' +if [ "$verMode" == "cluster" ]; then + install_dir="${release_dir}/TQ-enterprise-arbitrator-${version}" +else + install_dir="${release_dir}/TQ-arbitrator-${version}" +fi + +# Directories and files. +bin_files="${build_dir}/bin/tarbitrator ${script_dir}/remove_arbi_tq.sh" +install_files="${script_dir}/install_arbi_tq.sh" + +#header_files="${code_dir}/inc/taos.h ${code_dir}/inc/taoserror.h" +init_file_tarbitrator_deb=${script_dir}/../deb/tarbitratord +init_file_tarbitrator_rpm=${script_dir}/../rpm/tarbitratord + +# make directories. +mkdir -p ${install_dir} && cp ${install_files} ${install_dir} && chmod a+x ${install_dir}/install_arbi_tq.sh || : +#mkdir -p ${install_dir}/inc && cp ${header_files} ${install_dir}/inc || : +mkdir -p ${install_dir}/bin && cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || : +mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_deb} ${install_dir}/init.d/tarbitratord.deb || : +mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_rpm} ${install_dir}/init.d/tarbitratord.rpm || : + +cd ${release_dir} + +if [ "$verMode" == "cluster" ]; then + pkg_name=${install_dir}-${osType}-${cpuType} +elif [ "$verMode" == "edge" ]; then + pkg_name=${install_dir}-${osType}-${cpuType} +else + echo "unknow verMode, nor cluster or edge" + exit 1 +fi + +if [ "$verType" == "beta" ]; then + pkg_name=${pkg_name}-${verType} +elif [ "$verType" == "stable" ]; then + pkg_name=${pkg_name} +else + echo "unknow verType, nor stabel or beta" + exit 1 +fi + +tar -zcv -f "$(basename ${pkg_name}).tar.gz" $(basename ${install_dir}) --remove-files || : +exitcode=$? +if [ "$exitcode" != "0" ]; then + echo "tar ${pkg_name}.tar.gz error !!!" + exit $exitcode +fi + +cd ${curr_dir} diff --git a/packaging/tools/makeclient_tq.sh b/packaging/tools/makeclient_tq.sh new file mode 100755 index 0000000000..51fd064e1b --- /dev/null +++ b/packaging/tools/makeclient_tq.sh @@ -0,0 +1,203 @@ +#!/bin/bash +# +# Generate tar.gz package for linux client in all os system +set -e +#set -x + +curr_dir=$(pwd) +compile_dir=$1 +version=$2 +build_time=$3 +cpuType=$4 +osType=$5 +verMode=$6 +verType=$7 +pagMode=$8 + +if [ "$osType" != "Darwin" ]; then + script_dir="$(dirname $(readlink -f $0))" + top_dir="$(readlink -f ${script_dir}/../..)" +else + script_dir=`dirname $0` + cd ${script_dir} + script_dir="$(pwd)" + top_dir=${script_dir}/../.. +fi + +# create compressed install file. +build_dir="${compile_dir}/build" +code_dir="${top_dir}/src" +release_dir="${top_dir}/release" + +#package_name='linux' + +if [ "$verMode" == "cluster" ]; then + install_dir="${release_dir}/TQ-enterprise-client-${version}" +else + install_dir="${release_dir}/TQ-client-${version}" +fi + +# Directories and files. + +if [ "$osType" != "Darwin" ]; then +# if [ "$pagMode" == "lite" ]; then +# strip ${build_dir}/bin/tqd +# strip ${build_dir}/bin/tq +# bin_files="${build_dir}/bin/tq ${script_dir}/remove_client_tq.sh" +# else +# bin_files="${build_dir}/bin/tq ${build_dir}/bin/tqdemo ${script_dir}/remove_client_tq.sh ${script_dir}/set_core.sh" +# fi + lib_files="${build_dir}/lib/libtaos.so.${version}" +else + bin_files="${build_dir}/bin/tq ${script_dir}/remove_client_tq.sh" + lib_files="${build_dir}/lib/libtaos.${version}.dylib" +fi + +header_files="${code_dir}/inc/taos.h ${code_dir}/inc/taoserror.h" +if [ "$verMode" == "cluster" ]; then + cfg_dir="${top_dir}/../enterprise/packaging/cfg" +else + cfg_dir="${top_dir}/packaging/cfg" +fi + +install_files="${script_dir}/install_client_tq.sh" + +# make directories. +mkdir -p ${install_dir} +mkdir -p ${install_dir}/inc && cp ${header_files} ${install_dir}/inc +mkdir -p ${install_dir}/cfg && cp ${cfg_dir}/taos.cfg ${install_dir}/cfg/taos.cfg + +sed -i '/dataDir/ {s/taos/tq/g}' ${install_dir}/cfg/taos.cfg +sed -i '/logDir/ {s/taos/tq/g}' ${install_dir}/cfg/taos.cfg +sed -i "s/TDengine/TQ/g" ${install_dir}/cfg/taos.cfg + +mkdir -p ${install_dir}/bin +if [ "$osType" != "Darwin" ]; then + if [ "$pagMode" == "lite" ]; then + strip ${build_dir}/bin/taos + cp ${build_dir}/bin/taos ${install_dir}/bin/tq + cp ${script_dir}/remove_tq.sh ${install_dir}/bin + else + cp ${build_dir}/bin/taos ${install_dir}/bin/tq + cp ${script_dir}/remove_tq.sh ${install_dir}/bin + cp ${build_dir}/bin/taosdemo ${install_dir}/bin/tqdemo + cp ${build_dir}/bin/taosdump ${install_dir}/bin/tqdump + cp ${script_dir}/set_core.sh ${install_dir}/bin + cp ${script_dir}/get_client.sh ${install_dir}/bin + cp ${script_dir}/taosd-dump-cfg.gdb ${install_dir}/bin + fi +else + cp ${bin_files} ${install_dir}/bin +fi +chmod a+x ${install_dir}/bin/* || : + +cd ${install_dir} + +if [ "$osType" != "Darwin" ]; then + tar -zcv -f tq.tar.gz * --remove-files || : +else + tar -zcv -f tq.tar.gz * || : + mv tq.tar.gz .. + rm -rf ./* + mv ../tq.tar.gz . +fi + +cd ${curr_dir} +cp ${install_files} ${install_dir} +if [ "$osType" == "Darwin" ]; then + sed 's/osType=Linux/osType=Darwin/g' ${install_dir}/install_client_tq.sh >> install_client_tq_temp.sh + mv install_client_tq_temp.sh ${install_dir}/install_client_tq.sh +fi +if [ "$pagMode" == "lite" ]; then + sed 's/pagMode=full/pagMode=lite/g' ${install_dir}/install_client_tq.sh >> install_client_tq_temp.sh + mv install_client_tq_temp.sh ${install_dir}/install_client_tq.sh +fi +chmod a+x ${install_dir}/install_client_tq.sh + +# Copy example code +mkdir -p ${install_dir}/examples +examples_dir="${top_dir}/tests/examples" +cp -r ${examples_dir}/c ${install_dir}/examples +sed -i '/passwd/ {s/taosdata/tqueue/g}' ${install_dir}/examples/c/*.c +sed -i '/root/ {s/taosdata/tqueue/g}' ${install_dir}/examples/c/*.c + +if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then + cp -r ${examples_dir}/JDBC ${install_dir}/examples + cp -r ${examples_dir}/matlab ${install_dir}/examples + sed -i '/password/ {s/taosdata/tqueue/g}' ${install_dir}/examples/matlab/TDengineDemo.m + cp -r ${examples_dir}/python ${install_dir}/examples + sed -i '/password/ {s/taosdata/tqueue/g}' ${install_dir}/examples/python/read_example.py + cp -r ${examples_dir}/R ${install_dir}/examples + sed -i '/password/ {s/taosdata/tqueue/g}' ${install_dir}/examples/R/command.txt + cp -r ${examples_dir}/go ${install_dir}/examples + sed -i '/root/ {s/taosdata/tqueue/g}' ${install_dir}/examples/go/taosdemo.go +fi +# Copy driver +mkdir -p ${install_dir}/driver +cp ${lib_files} ${install_dir}/driver + +# Copy connector +connector_dir="${code_dir}/connector" +mkdir -p ${install_dir}/connector + +if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then + if [ "$osType" != "Darwin" ]; then + cp ${build_dir}/lib/*.jar ${install_dir}/connector ||: + fi + if [ -d "${connector_dir}/grafanaplugin/dist" ]; then + cp -r ${connector_dir}/grafanaplugin/dist ${install_dir}/connector/grafanaplugin + else + echo "WARNING: grafanaplugin bunlded dir not found, please check if want to use it!" + fi + if find ${connector_dir}/go -mindepth 1 -maxdepth 1 | read; then + cp -r ${connector_dir}/go ${install_dir}/connector + else + echo "WARNING: go connector not found, please check if want to use it!" + fi + cp -r ${connector_dir}/python ${install_dir}/connector + + sed -i '/password/ {s/taosdata/tqueue/g}' ${install_dir}/connector/python/taos/cinterface.py + + sed -i '/password/ {s/taosdata/tqueue/g}' ${install_dir}/connector/python/taos/subscription.py + + sed -i '/self._password/ {s/taosdata/tqueue/g}' ${install_dir}/connector/python/taos/connection.py +fi +# Copy release note +# cp ${script_dir}/release_note ${install_dir} + +# exit 1 + +cd ${release_dir} + +if [ "$verMode" == "cluster" ]; then + pkg_name=${install_dir}-${osType}-${cpuType} +elif [ "$verMode" == "edge" ]; then + pkg_name=${install_dir}-${osType}-${cpuType} +else + echo "unknow verMode, nor cluster or edge" + exit 1 +fi + +if [ "$pagMode" == "lite" ]; then + pkg_name=${pkg_name}-Lite +fi + +if [ "$verType" == "beta" ]; then + pkg_name=${pkg_name}-${verType} +elif [ "$verType" == "stable" ]; then + pkg_name=${pkg_name} +else + echo "unknow verType, nor stable or beta" + exit 1 +fi + +if [ "$osType" != "Darwin" ]; then + tar -zcv -f "$(basename ${pkg_name}).tar.gz" $(basename ${install_dir}) --remove-files || : +else + tar -zcv -f "$(basename ${pkg_name}).tar.gz" $(basename ${install_dir}) || : + mv "$(basename ${pkg_name}).tar.gz" .. + rm -rf ./* + mv ../"$(basename ${pkg_name}).tar.gz" . +fi + +cd ${curr_dir} diff --git a/packaging/tools/makepkg_tq.sh b/packaging/tools/makepkg_tq.sh new file mode 100755 index 0000000000..086092ea33 --- /dev/null +++ b/packaging/tools/makepkg_tq.sh @@ -0,0 +1,224 @@ +#!/bin/bash +# +# Generate tar.gz package for all os system + +set -e +#set -x + +curr_dir=$(pwd) +compile_dir=$1 +version=$2 +build_time=$3 +cpuType=$4 +osType=$5 +verMode=$6 +verType=$7 +pagMode=$8 + +script_dir="$(dirname $(readlink -f $0))" +top_dir="$(readlink -f ${script_dir}/../..)" + +# create compressed install file. +build_dir="${compile_dir}/build" +code_dir="${top_dir}/src" +release_dir="${top_dir}/release" + +#package_name='linux' +if [ "$verMode" == "cluster" ]; then + install_dir="${release_dir}/TQ-enterprise-server-${version}" +else + install_dir="${release_dir}/TQ-server-${version}" +fi + +# Directories and files. +#if [ "$pagMode" == "lite" ]; then +# strip ${build_dir}/bin/taosd +# strip ${build_dir}/bin/taos +# bin_files="${build_dir}/bin/tqd ${build_dir}/bin/tq ${script_dir}/remove_tq.sh" +#else +# bin_files="${build_dir}/bin/tqd ${build_dir}/bin/tq ${build_dir}/bin/tqdemo ${build_dir}/bin/tarbitrator ${script_dir}/remove_tq.sh\ +# ${script_dir}/set_core.sh ${script_dir}/startPre.sh ${script_dir}/taosd-dump-cfg.gdb" +#fi + +lib_files="${build_dir}/lib/libtaos.so.${version}" +header_files="${code_dir}/inc/taos.h ${code_dir}/inc/taoserror.h" +if [ "$verMode" == "cluster" ]; then + cfg_dir="${top_dir}/../enterprise/packaging/cfg" +else + cfg_dir="${top_dir}/packaging/cfg" +fi +install_files="${script_dir}/install_tq.sh" +nginx_dir="${code_dir}/../../enterprise/src/plugins/web" + +# Init file +#init_dir=${script_dir}/deb +#if [ $package_type = "centos" ]; then +# init_dir=${script_dir}/rpm +#fi +#init_files=${init_dir}/tqd +# temp use rpm's tqd. TODO: later modify according to os type +#init_file_deb=${script_dir}/../deb/tqd +#init_file_rpm=${script_dir}/../rpm/tqd +#init_file_tarbitrator_deb=${script_dir}/../deb/tarbitratord +#init_file_tarbitrator_rpm=${script_dir}/../rpm/tarbitratord + +# make directories. +mkdir -p ${install_dir} +mkdir -p ${install_dir}/inc && cp ${header_files} ${install_dir}/inc +mkdir -p ${install_dir}/cfg && cp ${cfg_dir}/taos.cfg ${install_dir}/cfg/taos.cfg + +#mkdir -p ${install_dir}/bin && cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || : +mkdir -p ${install_dir}/bin +if [ "$pagMode" == "lite" ]; then + strip ${build_dir}/bin/taosd + strip ${build_dir}/bin/taos +# bin_files="${build_dir}/bin/tqd ${build_dir}/bin/tq ${script_dir}/remove_tq.sh" + cp ${build_dir}/bin/taos ${install_dir}/bin/tq + cp ${build_dir}/bin/taosd ${install_dir}/bin/tqd + cp ${script_dir}/remove_tq.sh ${install_dir}/bin +else +# bin_files="${build_dir}/bin/tqd ${build_dir}/bin/tq ${build_dir}/bin/tqdemo ${build_dir}/bin/tarbitrator ${script_dir}/remove_tq.sh ${script_dir}/set_core.sh" + cp ${build_dir}/bin/taos ${install_dir}/bin/tq + cp ${build_dir}/bin/taosd ${install_dir}/bin/tqd + cp ${script_dir}/remove_tq.sh ${install_dir}/bin + cp ${build_dir}/bin/taosdemo ${install_dir}/bin/tqdemo + cp ${build_dir}/bin/taosdump ${install_dir}/bin/tqdump + cp ${build_dir}/bin/tarbitrator ${install_dir}/bin + cp ${script_dir}/set_core.sh ${install_dir}/bin + cp ${script_dir}/get_client.sh ${install_dir}/bin + cp ${script_dir}/startPre.sh ${install_dir}/bin + cp ${script_dir}/taosd-dump-cfg.gdb ${install_dir}/bin +fi +chmod a+x ${install_dir}/bin/* || : + +#mkdir -p ${install_dir}/init.d && cp ${init_file_deb} ${install_dir}/init.d/tqd.deb +#mkdir -p ${install_dir}/init.d && cp ${init_file_rpm} ${install_dir}/init.d/tqd.rpm +#mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_deb} ${install_dir}/init.d/tarbitratord.deb || : +#mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_rpm} ${install_dir}/init.d/tarbitratord.rpm || : + +if [ "$verMode" == "cluster" ]; then + sed 's/verMode=edge/verMode=cluster/g' ${install_dir}/bin/remove_tq.sh >> remove_tq_temp.sh + mv remove_tq_temp.sh ${install_dir}/bin/remove_tq.sh + + mkdir -p ${install_dir}/nginxd && cp -r ${nginx_dir}/* ${install_dir}/nginxd + cp ${nginx_dir}/png/taos.png ${install_dir}/nginxd/admin/images/taos.png + rm -rf ${install_dir}/nginxd/png + + sed -i "s/TDengine/TQ/g" ${install_dir}/nginxd/admin/*.html + sed -i "s/TDengine/TQ/g" ${install_dir}/nginxd/admin/js/*.js + + sed -i '/dataDir/ {s/taos/tq/g}' ${install_dir}/cfg/taos.cfg + sed -i '/logDir/ {s/taos/tq/g}' ${install_dir}/cfg/taos.cfg + sed -i "s/TDengine/TQ/g" ${install_dir}/cfg/taos.cfg + + if [ "$cpuType" == "aarch64" ]; then + cp -f ${install_dir}/nginxd/sbin/arm/64bit/nginx ${install_dir}/nginxd/sbin/ + elif [ "$cpuType" == "aarch32" ]; then + cp -f ${install_dir}/nginxd/sbin/arm/32bit/nginx ${install_dir}/nginxd/sbin/ + fi + rm -rf ${install_dir}/nginxd/sbin/arm +fi + +cd ${install_dir} +tar -zcv -f tq.tar.gz * --remove-files || : +exitcode=$? +if [ "$exitcode" != "0" ]; then + echo "tar tq.tar.gz error !!!" + exit $exitcode +fi + +cd ${curr_dir} +cp ${install_files} ${install_dir} +if [ "$verMode" == "cluster" ]; then + sed 's/verMode=edge/verMode=cluster/g' ${install_dir}/install_tq.sh >> install_tq_temp.sh + mv install_tq_temp.sh ${install_dir}/install_tq.sh +fi +if [ "$pagMode" == "lite" ]; then + sed 's/pagMode=full/pagMode=lite/g' ${install_dir}/install.sh >> install_tq_temp.sh + mv install_tq_temp.sh ${install_dir}/install_tq.sh +fi +chmod a+x ${install_dir}/install_tq.sh + +# Copy example code +mkdir -p ${install_dir}/examples +examples_dir="${top_dir}/tests/examples" +cp -r ${examples_dir}/c ${install_dir}/examples +sed -i '/passwd/ {s/taosdata/tqueue/g}' ${install_dir}/examples/c/*.c +sed -i '/root/ {s/taosdata/tqueue/g}' ${install_dir}/examples/c/*.c + +if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then + cp -r ${examples_dir}/JDBC ${install_dir}/examples + cp -r ${examples_dir}/matlab ${install_dir}/examples + sed -i '/password/ {s/taosdata/tqueue/g}' ${install_dir}/examples/matlab/TDengineDemo.m + cp -r ${examples_dir}/python ${install_dir}/examples + sed -i '/password/ {s/taosdata/tqueue/g}' ${install_dir}/examples/python/read_example.py + cp -r ${examples_dir}/R ${install_dir}/examples + sed -i '/password/ {s/taosdata/tqueue/g}' ${install_dir}/examples/R/command.txt + cp -r ${examples_dir}/go ${install_dir}/examples + sed -i '/root/ {s/taosdata/tqueue/g}' ${install_dir}/examples/go/taosdemo.go +fi +# Copy driver +mkdir -p ${install_dir}/driver +cp ${lib_files} ${install_dir}/driver + +# Copy connector +connector_dir="${code_dir}/connector" +mkdir -p ${install_dir}/connector +if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then + cp ${build_dir}/lib/*.jar ${install_dir}/connector ||: + + if [ -d "${connector_dir}/grafanaplugin/dist" ]; then + cp -r ${connector_dir}/grafanaplugin/dist ${install_dir}/connector/grafanaplugin + else + echo "WARNING: grafanaplugin bundled dir not found, please check if want to use it!" + fi + if find ${connector_dir}/go -mindepth 1 -maxdepth 1 | read; then + cp -r ${connector_dir}/go ${install_dir}/connector + else + echo "WARNING: go connector not found, please check if want to use it!" + fi + cp -r ${connector_dir}/python ${install_dir}/connector/ + + sed -i '/password/ {s/taosdata/tqueue/g}' ${install_dir}/connector/python/taos/cinterface.py + + sed -i '/password/ {s/taosdata/tqueue/g}' ${install_dir}/connector/python/taos/subscription.py + + sed -i '/self._password/ {s/taosdata/tqueue/g}' ${install_dir}/connector/python/taos/connection.py +fi +# Copy release note +# cp ${script_dir}/release_note ${install_dir} + +# exit 1 + +cd ${release_dir} + +if [ "$verMode" == "cluster" ]; then + pkg_name=${install_dir}-${osType}-${cpuType} +elif [ "$verMode" == "edge" ]; then + pkg_name=${install_dir}-${osType}-${cpuType} +else + echo "unknow verMode, nor cluster or edge" + exit 1 +fi + +if [ "$pagMode" == "lite" ]; then + pkg_name=${pkg_name}-Lite +fi + +if [ "$verType" == "beta" ]; then + pkg_name=${pkg_name}-${verType} +elif [ "$verType" == "stable" ]; then + pkg_name=${pkg_name} +else + echo "unknow verType, nor stabel or beta" + exit 1 +fi + +tar -zcv -f "$(basename ${pkg_name}).tar.gz" $(basename ${install_dir}) --remove-files || : +exitcode=$? +if [ "$exitcode" != "0" ]; then + echo "tar ${pkg_name}.tar.gz error !!!" + exit $exitcode +fi + +cd ${curr_dir} diff --git a/packaging/tools/remove_arbi_tq.sh b/packaging/tools/remove_arbi_tq.sh new file mode 100755 index 0000000000..3d99b6d41a --- /dev/null +++ b/packaging/tools/remove_arbi_tq.sh @@ -0,0 +1,130 @@ +#!/bin/bash +# +# Script to stop the service and uninstall TQ's arbitrator + +set -e +#set -x + +verMode=edge + +RED='\033[0;31m' +GREEN='\033[1;32m' +NC='\033[0m' + +#install main path +install_main_dir="/usr/local/tarbitrator" +bin_link_dir="/usr/bin" +#inc_link_dir="/usr/include" + +service_config_dir="/etc/systemd/system" +tarbitrator_service_name="tarbitratord" +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo" +fi + +initd_mod=0 +service_mod=2 +if pidof systemd &> /dev/null; then + service_mod=0 +elif $(which service &> /dev/null); then + service_mod=1 + service_config_dir="/etc/init.d" + if $(which chkconfig &> /dev/null); then + initd_mod=1 + elif $(which insserv &> /dev/null); then + initd_mod=2 + elif $(which update-rc.d &> /dev/null); then + initd_mod=3 + else + service_mod=2 + fi +else + service_mod=2 +fi + +function kill_tarbitrator() { + pid=$(ps -ef | grep "tarbitrator" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo} kill -9 $pid || : + fi +} +function clean_bin() { + # Remove link + ${csudo} rm -f ${bin_link_dir}/tarbitrator || : +} + +function clean_header() { + # Remove link + ${csudo} rm -f ${inc_link_dir}/taos.h || : + ${csudo} rm -f ${inc_link_dir}/taoserror.h || : +} + +function clean_log() { + # Remove link + ${csudo} rm -rf /arbitrator.log || : +} + +function clean_service_on_systemd() { + tarbitratord_service_config="${service_config_dir}/${tarbitrator_service_name}.service" + + if systemctl is-active --quiet ${tarbitrator_service_name}; then + echo "TQ tarbitrator is running, stopping it..." + ${csudo} systemctl stop ${tarbitrator_service_name} &> /dev/null || echo &> /dev/null + fi + ${csudo} systemctl disable ${tarbitrator_service_name} &> /dev/null || echo &> /dev/null + + ${csudo} rm -f ${tarbitratord_service_config} +} + +function clean_service_on_sysvinit() { + if pidof tarbitrator &> /dev/null; then + echo "TQ's tarbitrator is running, stopping it..." + ${csudo} service tarbitratord stop || : + fi + + if ((${initd_mod}==1)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} chkconfig --del tarbitratord || : + fi + elif ((${initd_mod}==2)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} insserv -r tarbitratord || : + fi + elif ((${initd_mod}==3)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} update-rc.d -f tarbitratord remove || : + fi + fi + + ${csudo} rm -f ${service_config_dir}/tarbitratord || : + + if $(which init &> /dev/null); then + ${csudo} init q || : + fi +} + +function clean_service() { + if ((${service_mod}==0)); then + clean_service_on_systemd + elif ((${service_mod}==1)); then + clean_service_on_sysvinit + else + # must manual stop + kill_tarbitrator + fi +} + +# Stop service and disable booting start. +clean_service +# Remove binary file and links +clean_bin +# Remove header file. +##clean_header +# Remove log file +clean_log + +${csudo} rm -rf ${install_main_dir} + +echo -e "${GREEN}TQ's arbitrator is removed successfully!${NC}" +echo \ No newline at end of file diff --git a/packaging/tools/remove_client_tq.sh b/packaging/tools/remove_client_tq.sh new file mode 100755 index 0000000000..ad8056c18c --- /dev/null +++ b/packaging/tools/remove_client_tq.sh @@ -0,0 +1,85 @@ +#!/bin/bash +# +# Script to stop the client and uninstall database, but retain the config and log files. +set -e +# set -x + +RED='\033[0;31m' +GREEN='\033[1;32m' +NC='\033[0m' + +#install main path +install_main_dir="/usr/local/tq" + +log_link_dir="/usr/local/tq/log" +cfg_link_dir="/usr/local/tq/cfg" +bin_link_dir="/usr/bin" +lib_link_dir="/usr/lib" +lib64_link_dir="/usr/lib64" +inc_link_dir="/usr/include" + + +# v1.5 jar dir +#v15_java_app_dir="/usr/local/lib/tq" + +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo" +fi + +function kill_client() { + #pid=$(ps -ef | grep "tq" | grep -v "grep" | awk '{print $2}') + if [ -n "$(pidof tq)" ]; then + ${csudo} kill -9 $pid || : + fi +} + +function clean_bin() { + # Remove link + ${csudo} rm -f ${bin_link_dir}/tq || : + ${csudo} rm -f ${bin_link_dir}/tqdemo || : + ${csudo} rm -f ${bin_link_dir}/tqdump || : + ${csudo} rm -f ${bin_link_dir}/rmtq || : + ${csudo} rm -f ${bin_link_dir}/set_core || : +} + +function clean_lib() { + # Remove link + ${csudo} rm -f ${lib_link_dir}/libtaos.* || : + ${csudo} rm -f ${lib64_link_dir}/libtaos.* || : + #${csudo} rm -rf ${v15_java_app_dir} || : +} + +function clean_header() { + # Remove link + ${csudo} rm -f ${inc_link_dir}/taos.h || : + ${csudo} rm -f ${inc_link_dir}/taoserror.h || : +} + +function clean_config() { + # Remove link + ${csudo} rm -f ${cfg_link_dir}/* || : +} + +function clean_log() { + # Remove link + ${csudo} rm -rf ${log_link_dir} || : +} + +# Stop client. +kill_client +# Remove binary file and links +clean_bin +# Remove header file. +clean_header +# Remove lib file +clean_lib +# Remove link log directory +clean_log +# Remove link configuration file +clean_config + +${csudo} rm -rf ${install_main_dir} + +echo -e "${GREEN}TQ client is removed successfully!${NC}" +echo diff --git a/packaging/tools/remove_tq.sh b/packaging/tools/remove_tq.sh new file mode 100755 index 0000000000..211eed4dff --- /dev/null +++ b/packaging/tools/remove_tq.sh @@ -0,0 +1,227 @@ +#!/bin/bash +# +# Script to stop the service and uninstall TDengine, but retain the config, data and log files. + +set -e +#set -x + +verMode=edge + +RED='\033[0;31m' +GREEN='\033[1;32m' +NC='\033[0m' + +#install main path +install_main_dir="/usr/local/tq" +data_link_dir="/usr/local/tq/data" +log_link_dir="/usr/local/tq/log" +cfg_link_dir="/usr/local/tq/cfg" +bin_link_dir="/usr/bin" +lib_link_dir="/usr/lib" +lib64_link_dir="/usr/lib64" +inc_link_dir="/usr/include" +install_nginxd_dir="/usr/local/nginxd" + +# v1.5 jar dir +#v15_java_app_dir="/usr/local/lib/tq" + +service_config_dir="/etc/systemd/system" +tq_service_name="tqd" +tarbitrator_service_name="tarbitratord" +nginx_service_name="nginxd" +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo" +fi + +initd_mod=0 +service_mod=2 +if pidof systemd &> /dev/null; then + service_mod=0 +elif $(which service &> /dev/null); then + service_mod=1 + service_config_dir="/etc/init.d" + if $(which chkconfig &> /dev/null); then + initd_mod=1 + elif $(which insserv &> /dev/null); then + initd_mod=2 + elif $(which update-rc.d &> /dev/null); then + initd_mod=3 + else + service_mod=2 + fi +else + service_mod=2 +fi + +function kill_tqd() { + pid=$(ps -ef | grep "tqd" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo} kill -9 $pid || : + fi +} + +function kill_tarbitrator() { + pid=$(ps -ef | grep "tarbitrator" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo} kill -9 $pid || : + fi +} +function clean_bin() { + # Remove link + ${csudo} rm -f ${bin_link_dir}/tq || : + ${csudo} rm -f ${bin_link_dir}/tqd || : + ${csudo} rm -f ${bin_link_dir}/tqdemo || : + ${csudo} rm -f ${bin_link_dir}/tqdump || : + ${csudo} rm -f ${bin_link_dir}/rmtq || : + ${csudo} rm -f ${bin_link_dir}/tarbitrator || : + ${csudo} rm -f ${bin_link_dir}/set_core || : +} + +function clean_lib() { + # Remove link + ${csudo} rm -f ${lib_link_dir}/libtaos.* || : + ${csudo} rm -f ${lib64_link_dir}/libtaos.* || : + #${csudo} rm -rf ${v15_java_app_dir} || : +} + +function clean_header() { + # Remove link + ${csudo} rm -f ${inc_link_dir}/taos.h || : + ${csudo} rm -f ${inc_link_dir}/taoserror.h || : +} + +function clean_config() { + # Remove link + ${csudo} rm -f ${cfg_link_dir}/* || : +} + +function clean_log() { + # Remove link + ${csudo} rm -rf ${log_link_dir} || : +} + +function clean_service_on_systemd() { + tq_service_config="${service_config_dir}/${tq_service_name}.service" + if systemctl is-active --quiet ${tq_service_name}; then + echo "TQ tqd is running, stopping it..." + ${csudo} systemctl stop ${tq_service_name} &> /dev/null || echo &> /dev/null + fi + ${csudo} systemctl disable ${tq_service_name} &> /dev/null || echo &> /dev/null + ${csudo} rm -f ${tq_service_config} + + tarbitratord_service_config="${service_config_dir}/${tarbitrator_service_name}.service" + if systemctl is-active --quiet ${tarbitrator_service_name}; then + echo "TDengine tarbitrator is running, stopping it..." + ${csudo} systemctl stop ${tarbitrator_service_name} &> /dev/null || echo &> /dev/null + fi + ${csudo} systemctl disable ${tarbitrator_service_name} &> /dev/null || echo &> /dev/null + ${csudo} rm -f ${tarbitratord_service_config} + + if [ "$verMode" == "cluster" ]; then + nginx_service_config="${service_config_dir}/${nginx_service_name}.service" + if [ -d ${bin_dir}/web ]; then + if systemctl is-active --quiet ${nginx_service_name}; then + echo "Nginx for TDengine is running, stopping it..." + ${csudo} systemctl stop ${nginx_service_name} &> /dev/null || echo &> /dev/null + fi + ${csudo} systemctl disable ${nginx_service_name} &> /dev/null || echo &> /dev/null + + ${csudo} rm -f ${nginx_service_config} + fi + fi +} + +function clean_service_on_sysvinit() { + #restart_config_str="tq:2345:respawn:${service_config_dir}/tqd start" + #${csudo} sed -i "\|${restart_config_str}|d" /etc/inittab || : + + if pidof tqd &> /dev/null; then + echo "TQ tqd is running, stopping it..." + ${csudo} service tqd stop || : + fi + + if pidof tarbitrator &> /dev/null; then + echo "TQ tarbitrator is running, stopping it..." + ${csudo} service tarbitratord stop || : + fi + + if ((${initd_mod}==1)); then + if [ -e ${service_config_dir}/tqd ]; then + ${csudo} chkconfig --del tqd || : + fi + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} chkconfig --del tarbitratord || : + fi + elif ((${initd_mod}==2)); then + if [ -e ${service_config_dir}/tqd ]; then + ${csudo} insserv -r tqd || : + fi + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} insserv -r tarbitratord || : + fi + elif ((${initd_mod}==3)); then + if [ -e ${service_config_dir}/tqd ]; then + ${csudo} update-rc.d -f tqd remove || : + fi + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} update-rc.d -f tarbitratord remove || : + fi + fi + + ${csudo} rm -f ${service_config_dir}/tqd || : + ${csudo} rm -f ${service_config_dir}/tarbitratord || : + + if $(which init &> /dev/null); then + ${csudo} init q || : + fi +} + +function clean_service() { + if ((${service_mod}==0)); then + clean_service_on_systemd + elif ((${service_mod}==1)); then + clean_service_on_sysvinit + else + # must manual stop taosd + kill_tqd + kill_tarbitrator + fi +} + +# Stop service and disable booting start. +clean_service +# Remove binary file and links +clean_bin +# Remove header file. +clean_header +# Remove lib file +clean_lib +# Remove link log directory +clean_log +# Remove link configuration file +clean_config +# Remove data link directory +${csudo} rm -rf ${data_link_dir} || : + +${csudo} rm -rf ${install_main_dir} +${csudo} rm -rf ${install_nginxd_dir} +if [[ -e /etc/os-release ]]; then + osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release) +else + osinfo="" +fi + +#if echo $osinfo | grep -qwi "ubuntu" ; then +## echo "this is ubuntu system" +# ${csudo} rm -f /var/lib/dpkg/info/tdengine* || : +#elif echo $osinfo | grep -qwi "debian" ; then +## echo "this is debian system" +# ${csudo} rm -f /var/lib/dpkg/info/tdengine* || : +#elif echo $osinfo | grep -qwi "centos" ; then +## echo "this is centos system" +# ${csudo} rpm -e --noscripts tdengine || : +#fi + +echo -e "${GREEN}TQ is removed successfully!${NC}" +echo diff --git a/packaging/tools/startPre.sh b/packaging/tools/startPre.sh old mode 100644 new mode 100755 diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 365f24e126..d5c13885d5 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -81,6 +81,8 @@ extern const int32_t TYPE_BYTES[15]; #define TSDB_DEFAULT_USER "root" #ifdef _TD_POWER_ #define TSDB_DEFAULT_PASS "powerdb" +#elif (_TD_TQ_ == true) +#define TSDB_DEFAULT_PASS "tqueue" #else #define TSDB_DEFAULT_PASS "taosdata" #endif diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c index f6cb135dd1..58f4b7ff02 100644 --- a/src/kit/shell/src/shellEngine.c +++ b/src/kit/shell/src/shellEngine.c @@ -37,6 +37,13 @@ char PROMPT_HEADER[] = "power> "; char CONTINUE_PROMPT[] = " -> "; int prompt_size = 7; +#elif (_TD_TQ_ == true) +char CLIENT_VERSION[] = "Welcome to the TQ shell from %s, Client Version:%s\n" + "Copyright (c) 2020 by TQ, Inc. All rights reserved.\n\n"; +char PROMPT_HEADER[] = "tq> "; + +char CONTINUE_PROMPT[] = " -> "; +int prompt_size = 4; #else char CLIENT_VERSION[] = "Welcome to the TDengine shell from %s, Client Version:%s\n" "Copyright (c) 2020 by TAOS Data, Inc. All rights reserved.\n\n"; diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 17e417d3f7..6513f3e214 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -571,6 +571,8 @@ SArguments g_args = { "root", // user #ifdef _TD_POWER_ "powerdb", // password + #elif (_TD_TQ_ == true) + "tqueue", // password #else "taosdata", // password #endif @@ -681,6 +683,11 @@ static void printHelp() { "The password to use when connecting to the server. Default is 'powerdb'."); printf("%s%s%s%s\n", indent, "-c", indent, "Configuration directory. Default is '/etc/power/'."); +#elif (_TD_TQ_ == true) + printf("%s%s%s%s\n", indent, "-P", indent, + "The password to use when connecting to the server. Default is 'tqueue'."); + printf("%s%s%s%s\n", indent, "-c", indent, + "Configuration directory. Default is '/etc/tq/'."); #else printf("%s%s%s%s\n", indent, "-P", indent, "The password to use when connecting to the server. Default is 'taosdata'."); diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index d6feef4ccc..ac5c58c591 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -200,6 +200,8 @@ static struct argp_option options[] = { {"user", 'u', "USER", 0, "User name used to connect to server. Default is root.", 0}, #ifdef _TD_POWER_ {"password", 'p', "PASSWORD", 0, "User password to connect to server. Default is powerdb.", 0}, + #elif (_TD_TQ_ == true) + {"password", 'p', "PASSWORD", 0, "User password to connect to server. Default is tqueue.", 0}, #else {"password", 'p', "PASSWORD", 0, "User password to connect to server. Default is taosdata.", 0}, #endif @@ -212,6 +214,8 @@ static struct argp_option options[] = { {"resultFile", 'r', "RESULTFILE", 0, "DumpOut/In Result file path and name.", 1}, #ifdef _TD_POWER_ {"config", 'c', "CONFIG_DIR", 0, "Configure directory. Default is /etc/power/taos.cfg.", 1}, + #elif (_TD_TQ_ == true) + {"config", 'c', "CONFIG_DIR", 0, "Configure directory. Default is /etc/tq/taos.cfg.", 1}, #else {"config", 'c', "CONFIG_DIR", 0, "Configure directory. Default is /etc/taos/taos.cfg.", 1}, #endif @@ -425,6 +429,8 @@ struct arguments g_args = { "root", #ifdef _TD_POWER_ "powerdb", + #elif (_TD_TQ_ == true) + "tqueue", #else "taosdata", #endif diff --git a/src/os/src/linux/linuxEnv.c b/src/os/src/linux/linuxEnv.c index 417513314c..b7b268b19e 100644 --- a/src/os/src/linux/linuxEnv.c +++ b/src/os/src/linux/linuxEnv.c @@ -25,6 +25,13 @@ void osInit() { strcpy(tsDataDir, "/var/lib/power"); strcpy(tsLogDir, "/var/log/power"); strcpy(tsScriptDir, "/etc/power"); +#elif (_TD_TQ_ == true) + if (configDir[0] == 0) { + strcpy(configDir, "/etc/tq"); + } + strcpy(tsDataDir, "/var/lib/tq"); + strcpy(tsLogDir, "/var/log/tq"); + strcpy(tsScriptDir, "/etc/tq"); #else if (configDir[0] == 0) { strcpy(configDir, "/etc/taos"); diff --git a/src/os/src/windows/wEnv.c b/src/os/src/windows/wEnv.c index 21c6aa6642..b35cb8f040 100644 --- a/src/os/src/windows/wEnv.c +++ b/src/os/src/windows/wEnv.c @@ -31,7 +31,14 @@ void osInit() { strcpy(tsDataDir, "C:/PowerDB/data"); strcpy(tsLogDir, "C:/PowerDB/log"); strcpy(tsScriptDir, "C:/PowerDB/script"); - +#elif (_TD_TQ_ == true) + if (configDir[0] == 0) { + strcpy(configDir, "C:/TQ/cfg"); + } + strcpy(tsVnodeDir, "C:/TQ/data"); + strcpy(tsDataDir, "C:/TQ/data"); + strcpy(tsLogDir, "C:/TQ/log"); + strcpy(tsScriptDir, "C:/TQ/script"); #else if (configDir[0] == 0) { strcpy(configDir, "C:/TDengine/cfg"); diff --git a/src/util/src/tconfig.c b/src/util/src/tconfig.c index 30d53af4b9..80071986d6 100644 --- a/src/util/src/tconfig.c +++ b/src/util/src/tconfig.c @@ -312,6 +312,9 @@ void taosReadGlobalLogCfg() { #ifdef _TD_POWER_ printf("configDir:%s not there, use default value: /etc/power", configDir); strcpy(configDir, "/etc/power"); + #elif (_TD_TQ_ == true) + printf("configDir:%s not there, use default value: /etc/tq", configDir); + strcpy(configDir, "/etc/tq"); #else printf("configDir:%s not there, use default value: /etc/taos", configDir); strcpy(configDir, "/etc/taos"); diff --git a/src/util/src/tlog.c b/src/util/src/tlog.c index 7f127fc396..385dcac075 100644 --- a/src/util/src/tlog.c +++ b/src/util/src/tlog.c @@ -83,6 +83,8 @@ int64_t dbgWSize = 0; #ifdef _TD_POWER_ char tsLogDir[TSDB_FILENAME_LEN] = "/var/log/power"; +#elif (_TD_TQ_ == true) +char tsLogDir[TSDB_FILENAME_LEN] = "/var/log/tq"; #else char tsLogDir[TSDB_FILENAME_LEN] = "/var/log/taos"; #endif From 06f38b5b1c0f3f2da5b164192d28fcf7fe684b36 Mon Sep 17 00:00:00 2001 From: wpan Date: Tue, 6 Jul 2021 10:53:30 +0800 Subject: [PATCH 18/26] add group by condition check --- src/client/src/tscSQLParser.c | 10 ++++-- tests/script/general/parser/groupby.sim | 1 + tests/script/general/parser/having_child.sim | 34 ++------------------ 3 files changed, 10 insertions(+), 35 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index eb9756f2ee..97640ed410 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -6572,11 +6572,15 @@ int32_t doFunctionsCompatibleCheck(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, char* } } - if (f == TSDB_FUNC_DIFF || f == TSDB_FUNC_DERIVATIVE || f == TSDB_FUNC_TWA || f == TSDB_FUNC_IRATE) { + if ((!pQueryInfo->stateWindow) && (f == TSDB_FUNC_DIFF || f == TSDB_FUNC_DERIVATIVE || f == TSDB_FUNC_TWA || f == TSDB_FUNC_IRATE)) { for (int32_t j = 0; j < pQueryInfo->groupbyExpr.numOfGroupCols; ++j) { SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, j); - if (pColIndex->colIndex != TSDB_TBNAME_COLUMN_INDEX) { - return invalidOperationMsg(msg, msg6); + if (j == 0) { + if (pColIndex->colIndex != TSDB_TBNAME_COLUMN_INDEX) { + return invalidOperationMsg(msg, msg6); + } + } else if (!TSDB_COL_IS_TAG(pColIndex->flag)) { + return invalidOperationMsg(msg, msg6); } } } diff --git a/tests/script/general/parser/groupby.sim b/tests/script/general/parser/groupby.sim index e47af5588e..1fe19714bb 100644 --- a/tests/script/general/parser/groupby.sim +++ b/tests/script/general/parser/groupby.sim @@ -692,6 +692,7 @@ if $data31 != 4 then return -1 endi +sql_error select irate(c) from st where t1="1" and ts >= '2020-03-27 04:11:17.732' and ts < '2020-03-27 05:11:17.732' interval(1m) sliding(15s) group by tbname,c; sql select irate(c) from st where t1="1" and ts >= '2020-03-27 04:11:17.732' and ts < '2020-03-27 05:11:17.732' interval(1m) sliding(15s) group by tbname,t1,t2; if $rows != 40 then return -1 diff --git a/tests/script/general/parser/having_child.sim b/tests/script/general/parser/having_child.sim index a38db3fe44..0fe5448869 100644 --- a/tests/script/general/parser/having_child.sim +++ b/tests/script/general/parser/having_child.sim @@ -306,41 +306,11 @@ endi sql_error select avg(f1),count(f1),sum(f1),twa(f1) from tb1 group by tbname having twa(f1) > 0; -sql select avg(f1),count(f1),sum(f1),twa(f1) from tb1 group by f1 having twa(f1) > 3; -if $rows != 1 then - return -1 -endi -if $data00 != 4.000000000 then - return -1 -endi -if $data01 != 2 then - return -1 -endi -if $data02 != 8 then - return -1 -endi -if $data03 != 4.000000000 then - return -1 -endi +sql_error select avg(f1),count(f1),sum(f1),twa(f1) from tb1 group by f1 having twa(f1) > 3; sql_error select avg(f1),count(f1),sum(f1),twa(f1) from tb1 group by tbname having sum(f1) > 0; -sql select avg(f1),count(f1),sum(f1),twa(f1) from tb1 group by f1 having sum(f1) = 4; -if $rows != 1 then - return -1 -endi -if $data00 != 2.000000000 then - return -1 -endi -if $data01 != 2 then - return -1 -endi -if $data02 != 4 then - return -1 -endi -if $data03 != 2.000000000 then - return -1 -endi +sql_error select avg(f1),count(f1),sum(f1),twa(f1) from tb1 group by f1 having sum(f1) = 4; sql select avg(f1),count(f1),sum(f1) from tb1 group by f1 having sum(f1) > 0; if $rows != 4 then From beff12c7ba2b77a4cbf678f019908f9fdfa94767 Mon Sep 17 00:00:00 2001 From: wpan Date: Tue, 6 Jul 2021 11:47:18 +0800 Subject: [PATCH 19/26] fix bug issue --- tests/pytest/query/queryInterval.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/pytest/query/queryInterval.py b/tests/pytest/query/queryInterval.py index ce8d05ae50..d61e8cf288 100644 --- a/tests/pytest/query/queryInterval.py +++ b/tests/pytest/query/queryInterval.py @@ -114,8 +114,7 @@ class TDTestCase: tdSql.query("select first(ts),twa(c) from tb interval(14a)") tdSql.checkRows(6) - tdSql.query("select twa(c) from tb group by c") - tdSql.checkRows(4) + tdSql.error("select twa(c) from tb group by c") def stop(self): From 8868d447738d27c720a42f45781e5a8067999da5 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 6 Jul 2021 12:37:55 +0800 Subject: [PATCH 20/26] Feature/sangshuduo/td 3973 use jemalloc (#6752) * [TD-3973]: add jemalloc as submodule. * add macro definitions in cmake. * [TD-3973]: use jemalloc. build works as following instructions: cmake .. -DJEMALLOC_ENABLED=true make * fix jemalloc at tag 5.2.1 * link jemalloc works. * make install works. * support jemalloc in release.sh. * release script works. * fix a typo. * [TD-3937]: support jemalloc add install funtion to all scripts. Co-authored-by: Shuduo Sang --- packaging/tools/install.sh | 1 + packaging/tools/install_arbi.sh | 87 +++++-- packaging/tools/install_arbi_power.sh | 87 +++++-- packaging/tools/install_client.sh | 60 ++++- packaging/tools/install_client_power.sh | 60 ++++- packaging/tools/install_power.sh | 300 ++++++++++++++---------- 6 files changed, 419 insertions(+), 176 deletions(-) diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 325ac81053..3911b1bc72 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -896,6 +896,7 @@ function install_TDengine() { install_log install_header install_lib + install_jemalloc if [ "$pagMode" != "lite" ]; then install_connector fi diff --git a/packaging/tools/install_arbi.sh b/packaging/tools/install_arbi.sh index f47c3672cb..cdfeef0543 100755 --- a/packaging/tools/install_arbi.sh +++ b/packaging/tools/install_arbi.sh @@ -38,11 +38,11 @@ initd_mod=0 service_mod=2 if pidof systemd &> /dev/null; then service_mod=0 -elif $(which service &> /dev/null); then +elif $(which service &> /dev/null); then service_mod=1 - service_config_dir="/etc/init.d" + service_config_dir="/etc/init.d" if $(which chkconfig &> /dev/null); then - initd_mod=1 + initd_mod=1 elif $(which insserv &> /dev/null); then initd_mod=2 elif $(which update-rc.d &> /dev/null); then @@ -50,7 +50,7 @@ elif $(which service &> /dev/null); then else service_mod=2 fi -else +else service_mod=2 fi @@ -82,7 +82,7 @@ elif echo $osinfo | grep -qwi "fedora" ; then os_type=2 else echo " osinfo: ${osinfo}" - echo " This is an officially unverified linux system," + echo " This is an officially unverified linux system," echo " if there are any problems with the installation and operation, " echo " please feel free to contact taosdata.com for support." os_type=1 @@ -99,7 +99,7 @@ function install_main_path() { #create install main dir and all sub dir ${csudo} rm -rf ${install_main_dir} || : ${csudo} mkdir -p ${install_main_dir} - ${csudo} mkdir -p ${install_main_dir}/bin + ${csudo} mkdir -p ${install_main_dir}/bin #${csudo} mkdir -p ${install_main_dir}/include ${csudo} mkdir -p ${install_main_dir}/init.d } @@ -117,21 +117,67 @@ function install_bin() { function install_header() { ${csudo} rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taoserror.h || : - ${csudo} cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo} chmod 644 ${install_main_dir}/include/* + ${csudo} cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo} chmod 644 ${install_main_dir}/include/* ${csudo} ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h ${csudo} ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h } +function install_jemalloc() { + jemalloc_dir=${script_dir}/jemalloc + + if [ -d ${jemalloc_dir} ]; then + ${csudo} /usr/bin/install -c -d /usr/local/bin + + if [ -f ${jemalloc_dir}/bin/jemalloc-config ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc-config /usr/local/bin + fi + if [ -f ${jemalloc_dir}/bin/jemalloc.sh ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc.sh /usr/local/bin + fi + if [ -f ${jemalloc_dir}/bin/jeprof ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jeprof /usr/local/bin + fi + if [ -f ${jemalloc_dir}/include/jemalloc/jemalloc.h ]; then + ${csudo} /usr/bin/install -c -d /usr/local/include/jemalloc + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/include/jemalloc/jemalloc.h /usr/local/include/jemalloc + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc.so.2 ]; then + ${csudo} /usr/bin/install -c -d /usr/local/lib + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.so.2 /usr/local/lib + ${csudo} ln -sf libjemalloc.so.2 /usr/local/lib/libjemalloc.so + ${csudo} /usr/bin/install -c -d /usr/local/lib + if [ -f ${jemalloc_dir}/lib/libjemalloc.a ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.a /usr/local/lib + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc_pic.a /usr/local/lib + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then + ${csudo} /usr/bin/install -c -d /usr/local/lib/pkgconfig + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/lib/pkgconfig/jemalloc.pc /usr/local/lib/pkgconfig + fi + fi + if [ -f ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html ]; then + ${csudo} /usr/bin/install -c -d /usr/local/share/doc/jemalloc + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html /usr/local/share/doc/jemalloc + fi + if [ -f ${jemalloc_dir}/share/man/man3/jemalloc.3 ]; then + ${csudo} /usr/bin/install -c -d /usr/local/share/man/man3 + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/share/man/man3/jemalloc.3 /usr/local/share/man/man3 + fi + fi +} + function clean_service_on_sysvinit() { #restart_config_str="taos:2345:respawn:${service_config_dir}/taosd start" - #${csudo} sed -i "\|${restart_config_str}|d" /etc/inittab || : - + #${csudo} sed -i "\|${restart_config_str}|d" /etc/inittab || : + if pidof tarbitrator &> /dev/null; then ${csudo} service tarbitratord stop || : fi if ((${initd_mod}==1)); then - if [ -e ${service_config_dir}/tarbitratord ]; then + if [ -e ${service_config_dir}/tarbitratord ]; then ${csudo} chkconfig --del tarbitratord || : fi elif ((${initd_mod}==2)); then @@ -142,10 +188,10 @@ function clean_service_on_sysvinit() { if [ -e ${service_config_dir}/tarbitratord ]; then ${csudo} update-rc.d -f tarbitratord remove || : fi - fi + fi ${csudo} rm -f ${service_config_dir}/tarbitratord || : - + if $(which init &> /dev/null); then ${csudo} init q || : fi @@ -164,10 +210,10 @@ function install_service_on_sysvinit() { ${csudo} cp -f ${script_dir}/init.d/tarbitratord.rpm ${install_main_dir}/init.d/tarbitratord ${csudo} cp ${script_dir}/init.d/tarbitratord.rpm ${service_config_dir}/tarbitratord && ${csudo} chmod a+x ${service_config_dir}/tarbitratord fi - + #restart_config_str="taos:2345:respawn:${service_config_dir}/taosd start" #${csudo} grep -q -F "$restart_config_str" /etc/inittab || ${csudo} bash -c "echo '${restart_config_str}' >> /etc/inittab" - + if ((${initd_mod}==1)); then ${csudo} chkconfig --add tarbitratord || : ${csudo} chkconfig --level 2345 tarbitratord on || : @@ -245,12 +291,13 @@ function update_TDengine() { fi sleep 1 fi - + install_main_path #install_header install_bin install_service - + install_jemalloc + echo #echo -e "${GREEN_DARK}To configure TDengine ${NC}: edit /etc/taos/taos.cfg" if ((${service_mod}==0)); then @@ -259,7 +306,7 @@ function update_TDengine() { echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo} service tarbitratord start${NC}" else echo -e "${GREEN_DARK}To start arbitrator ${NC}: ./tarbitrator${NC}" - fi + fi echo echo -e "\033[44;32;1mTDengine's arbitrator is updated successfully!${NC}" } @@ -267,11 +314,13 @@ function update_TDengine() { function install_TDengine() { # Start to install echo -e "${GREEN}Start to install TDengine's arbitrator ...${NC}" - - install_main_path + + install_main_path #install_header install_bin install_service + install_jemalloc + echo #echo -e "${GREEN_DARK}To configure TDengine ${NC}: edit /etc/taos/taos.cfg" if ((${service_mod}==0)); then diff --git a/packaging/tools/install_arbi_power.sh b/packaging/tools/install_arbi_power.sh index 3f27175151..161b916b4c 100755 --- a/packaging/tools/install_arbi_power.sh +++ b/packaging/tools/install_arbi_power.sh @@ -38,11 +38,11 @@ initd_mod=0 service_mod=2 if pidof systemd &> /dev/null; then service_mod=0 -elif $(which service &> /dev/null); then +elif $(which service &> /dev/null); then service_mod=1 - service_config_dir="/etc/init.d" + service_config_dir="/etc/init.d" if $(which chkconfig &> /dev/null); then - initd_mod=1 + initd_mod=1 elif $(which insserv &> /dev/null); then initd_mod=2 elif $(which update-rc.d &> /dev/null); then @@ -50,7 +50,7 @@ elif $(which service &> /dev/null); then else service_mod=2 fi -else +else service_mod=2 fi @@ -82,7 +82,7 @@ elif echo $osinfo | grep -qwi "fedora" ; then os_type=2 else echo " osinfo: ${osinfo}" - echo " This is an officially unverified linux system," + echo " This is an officially unverified linux system," echo " if there are any problems with the installation and operation, " echo " please feel free to contact taosdata.com for support." os_type=1 @@ -99,7 +99,7 @@ function install_main_path() { #create install main dir and all sub dir ${csudo} rm -rf ${install_main_dir} || : ${csudo} mkdir -p ${install_main_dir} - ${csudo} mkdir -p ${install_main_dir}/bin + ${csudo} mkdir -p ${install_main_dir}/bin #${csudo} mkdir -p ${install_main_dir}/include ${csudo} mkdir -p ${install_main_dir}/init.d } @@ -115,23 +115,69 @@ function install_bin() { [ -x ${install_main_dir}/bin/tarbitrator ] && ${csudo} ln -s ${install_main_dir}/bin/tarbitrator ${bin_link_dir}/tarbitrator || : } +function install_jemalloc() { + jemalloc_dir=${script_dir}/jemalloc + + if [ -d ${jemalloc_dir} ]; then + ${csudo} /usr/bin/install -c -d /usr/local/bin + + if [ -f ${jemalloc_dir}/bin/jemalloc-config ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc-config /usr/local/bin + fi + if [ -f ${jemalloc_dir}/bin/jemalloc.sh ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc.sh /usr/local/bin + fi + if [ -f ${jemalloc_dir}/bin/jeprof ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jeprof /usr/local/bin + fi + if [ -f ${jemalloc_dir}/include/jemalloc/jemalloc.h ]; then + ${csudo} /usr/bin/install -c -d /usr/local/include/jemalloc + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/include/jemalloc/jemalloc.h /usr/local/include/jemalloc + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc.so.2 ]; then + ${csudo} /usr/bin/install -c -d /usr/local/lib + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.so.2 /usr/local/lib + ${csudo} ln -sf libjemalloc.so.2 /usr/local/lib/libjemalloc.so + ${csudo} /usr/bin/install -c -d /usr/local/lib + if [ -f ${jemalloc_dir}/lib/libjemalloc.a ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.a /usr/local/lib + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc_pic.a /usr/local/lib + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then + ${csudo} /usr/bin/install -c -d /usr/local/lib/pkgconfig + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/lib/pkgconfig/jemalloc.pc /usr/local/lib/pkgconfig + fi + fi + if [ -f ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html ]; then + ${csudo} /usr/bin/install -c -d /usr/local/share/doc/jemalloc + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html /usr/local/share/doc/jemalloc + fi + if [ -f ${jemalloc_dir}/share/man/man3/jemalloc.3 ]; then + ${csudo} /usr/bin/install -c -d /usr/local/share/man/man3 + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/share/man/man3/jemalloc.3 /usr/local/share/man/man3 + fi + fi +} + function install_header() { ${csudo} rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taoserror.h || : - ${csudo} cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo} chmod 644 ${install_main_dir}/include/* + ${csudo} cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo} chmod 644 ${install_main_dir}/include/* ${csudo} ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h ${csudo} ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h } function clean_service_on_sysvinit() { #restart_config_str="taos:2345:respawn:${service_config_dir}/taosd start" - #${csudo} sed -i "\|${restart_config_str}|d" /etc/inittab || : - + #${csudo} sed -i "\|${restart_config_str}|d" /etc/inittab || : + if pidof tarbitrator &> /dev/null; then ${csudo} service tarbitratord stop || : fi if ((${initd_mod}==1)); then - if [ -e ${service_config_dir}/tarbitratord ]; then + if [ -e ${service_config_dir}/tarbitratord ]; then ${csudo} chkconfig --del tarbitratord || : fi elif ((${initd_mod}==2)); then @@ -142,10 +188,10 @@ function clean_service_on_sysvinit() { if [ -e ${service_config_dir}/tarbitratord ]; then ${csudo} update-rc.d -f tarbitratord remove || : fi - fi + fi ${csudo} rm -f ${service_config_dir}/tarbitratord || : - + if $(which init &> /dev/null); then ${csudo} init q || : fi @@ -164,10 +210,10 @@ function install_service_on_sysvinit() { ${csudo} cp -f ${script_dir}/init.d/tarbitratord.rpm ${install_main_dir}/init.d/tarbitratord ${csudo} cp ${script_dir}/init.d/tarbitratord.rpm ${service_config_dir}/tarbitratord && ${csudo} chmod a+x ${service_config_dir}/tarbitratord fi - + #restart_config_str="power:2345:respawn:${service_config_dir}/powerd start" #${csudo} grep -q -F "$restart_config_str" /etc/inittab || ${csudo} bash -c "echo '${restart_config_str}' >> /etc/inittab" - + if ((${initd_mod}==1)); then ${csudo} chkconfig --add tarbitratord || : ${csudo} chkconfig --level 2345 tarbitratord on || : @@ -245,12 +291,13 @@ function update_PowerDB() { fi sleep 1 fi - + install_main_path #install_header install_bin install_service - + install_jemalloc + echo #echo -e "${GREEN_DARK}To configure PowerDB ${NC}: edit /etc/taos/taos.cfg" if ((${service_mod}==0)); then @@ -259,7 +306,7 @@ function update_PowerDB() { echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo} service tarbitratord start${NC}" else echo -e "${GREEN_DARK}To start arbitrator ${NC}: ./tarbitrator${NC}" - fi + fi echo echo -e "\033[44;32;1mPowerDB's arbitrator is updated successfully!${NC}" } @@ -267,11 +314,13 @@ function update_PowerDB() { function install_PowerDB() { # Start to install echo -e "${GREEN}Start to install PowerDB's arbitrator ...${NC}" - - install_main_path + + install_main_path #install_header install_bin install_service + install_jemalloc + echo #echo -e "${GREEN_DARK}To configure PowerDB ${NC}: edit /etc/taos/taos.cfg" if ((${service_mod}==0)); then diff --git a/packaging/tools/install_client.sh b/packaging/tools/install_client.sh index 0a0a6633e3..2969a3cc98 100755 --- a/packaging/tools/install_client.sh +++ b/packaging/tools/install_client.sh @@ -119,16 +119,16 @@ function install_lib() { if [ "$osType" != "Darwin" ]; then ${csudo} ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1 ${csudo} ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so - + if [ -d "${lib64_link_dir}" ]; then - ${csudo} ln -s ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : - ${csudo} ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : - fi + ${csudo} ln -s ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : + ${csudo} ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : + fi else ${csudo} ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.1.dylib ${csudo} ln -s ${lib_link_dir}/libtaos.1.dylib ${lib_link_dir}/libtaos.dylib fi - + ${csudo} ldconfig } @@ -139,6 +139,52 @@ function install_header() { ${csudo} ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h } +function install_jemalloc() { + jemalloc_dir=${script_dir}/jemalloc + + if [ -d ${jemalloc_dir} ]; then + ${csudo} /usr/bin/install -c -d /usr/local/bin + + if [ -f ${jemalloc_dir}/bin/jemalloc-config ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc-config /usr/local/bin + fi + if [ -f ${jemalloc_dir}/bin/jemalloc.sh ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc.sh /usr/local/bin + fi + if [ -f ${jemalloc_dir}/bin/jeprof ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jeprof /usr/local/bin + fi + if [ -f ${jemalloc_dir}/include/jemalloc/jemalloc.h ]; then + ${csudo} /usr/bin/install -c -d /usr/local/include/jemalloc + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/include/jemalloc/jemalloc.h /usr/local/include/jemalloc + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc.so.2 ]; then + ${csudo} /usr/bin/install -c -d /usr/local/lib + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.so.2 /usr/local/lib + ${csudo} ln -sf libjemalloc.so.2 /usr/local/lib/libjemalloc.so + ${csudo} /usr/bin/install -c -d /usr/local/lib + if [ -f ${jemalloc_dir}/lib/libjemalloc.a ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.a /usr/local/lib + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc_pic.a /usr/local/lib + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then + ${csudo} /usr/bin/install -c -d /usr/local/lib/pkgconfig + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/lib/pkgconfig/jemalloc.pc /usr/local/lib/pkgconfig + fi + fi + if [ -f ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html ]; then + ${csudo} /usr/bin/install -c -d /usr/local/share/doc/jemalloc + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html /usr/local/share/doc/jemalloc + fi + if [ -f ${jemalloc_dir}/share/man/man3/jemalloc.3 ]; then + ${csudo} /usr/bin/install -c -d /usr/local/share/man/man3 + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/share/man/man3/jemalloc.3 /usr/local/share/man/man3 + fi + fi +} + function install_config() { #${csudo} rm -f ${install_main_dir}/cfg/taos.cfg || : @@ -194,6 +240,7 @@ function update_TDengine() { install_log install_header install_lib + install_jemalloc if [ "$pagMode" != "lite" ]; then install_connector fi @@ -217,10 +264,11 @@ function install_TDengine() { echo -e "${GREEN}Start to install TDengine client...${NC}" - install_main_path + install_main_path install_log install_header install_lib + install_jemalloc if [ "$pagMode" != "lite" ]; then install_connector fi diff --git a/packaging/tools/install_client_power.sh b/packaging/tools/install_client_power.sh index 8d7463366f..d56527ecda 100755 --- a/packaging/tools/install_client_power.sh +++ b/packaging/tools/install_client_power.sh @@ -119,16 +119,16 @@ function install_lib() { if [ "$osType" != "Darwin" ]; then ${csudo} ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1 ${csudo} ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so - + if [ -d "${lib64_link_dir}" ]; then - ${csudo} ln -s ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : - ${csudo} ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : - fi + ${csudo} ln -s ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : + ${csudo} ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : + fi else ${csudo} ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.1.dylib ${csudo} ln -s ${lib_link_dir}/libtaos.1.dylib ${lib_link_dir}/libtaos.dylib fi - + ${csudo} ldconfig } @@ -139,6 +139,52 @@ function install_header() { ${csudo} ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h } +function install_jemalloc() { + jemalloc_dir=${script_dir}/jemalloc + + if [ -d ${jemalloc_dir} ]; then + ${csudo} /usr/bin/install -c -d /usr/local/bin + + if [ -f ${jemalloc_dir}/bin/jemalloc-config ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc-config /usr/local/bin + fi + if [ -f ${jemalloc_dir}/bin/jemalloc.sh ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc.sh /usr/local/bin + fi + if [ -f ${jemalloc_dir}/bin/jeprof ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jeprof /usr/local/bin + fi + if [ -f ${jemalloc_dir}/include/jemalloc/jemalloc.h ]; then + ${csudo} /usr/bin/install -c -d /usr/local/include/jemalloc + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/include/jemalloc/jemalloc.h /usr/local/include/jemalloc + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc.so.2 ]; then + ${csudo} /usr/bin/install -c -d /usr/local/lib + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.so.2 /usr/local/lib + ${csudo} ln -sf libjemalloc.so.2 /usr/local/lib/libjemalloc.so + ${csudo} /usr/bin/install -c -d /usr/local/lib + if [ -f ${jemalloc_dir}/lib/libjemalloc.a ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.a /usr/local/lib + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc_pic.a /usr/local/lib + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then + ${csudo} /usr/bin/install -c -d /usr/local/lib/pkgconfig + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/lib/pkgconfig/jemalloc.pc /usr/local/lib/pkgconfig + fi + fi + if [ -f ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html ]; then + ${csudo} /usr/bin/install -c -d /usr/local/share/doc/jemalloc + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html /usr/local/share/doc/jemalloc + fi + if [ -f ${jemalloc_dir}/share/man/man3/jemalloc.3 ]; then + ${csudo} /usr/bin/install -c -d /usr/local/share/man/man3 + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/share/man/man3/jemalloc.3 /usr/local/share/man/man3 + fi + fi +} + function install_config() { #${csudo} rm -f ${install_main_dir}/cfg/taos.cfg || : @@ -194,6 +240,7 @@ function update_PowerDB() { install_log install_header install_lib + install_jemalloc if [ "$pagMode" != "lite" ]; then install_connector fi @@ -217,10 +264,11 @@ function install_PowerDB() { echo -e "${GREEN}Start to install PowerDB client...${NC}" - install_main_path + install_main_path install_log install_header install_lib + install_jemalloc if [ "$pagMode" != "lite" ]; then install_connector fi diff --git a/packaging/tools/install_power.sh b/packaging/tools/install_power.sh index 9f28435cb5..b1c4167e94 100755 --- a/packaging/tools/install_power.sh +++ b/packaging/tools/install_power.sh @@ -58,11 +58,11 @@ initd_mod=0 service_mod=2 if pidof systemd &> /dev/null; then service_mod=0 -elif $(which service &> /dev/null); then +elif $(which service &> /dev/null); then service_mod=1 - service_config_dir="/etc/init.d" + service_config_dir="/etc/init.d" if $(which chkconfig &> /dev/null); then - initd_mod=1 + initd_mod=1 elif $(which insserv &> /dev/null); then initd_mod=2 elif $(which update-rc.d &> /dev/null); then @@ -70,7 +70,7 @@ elif $(which service &> /dev/null); then else service_mod=2 fi -else +else service_mod=2 fi @@ -102,7 +102,7 @@ elif echo $osinfo | grep -qwi "fedora" ; then os_type=2 else echo " osinfo: ${osinfo}" - echo " This is an officially unverified linux system," + echo " This is an officially unverified linux system," echo " if there are any problems with the installation and operation, " echo " please feel free to contact taosdata.com for support." os_type=1 @@ -137,7 +137,7 @@ do echo "Usage: `basename $0` -v [server | client] -e [yes | no]" exit 0 ;; - ?) #unknow option + ?) #unknow option echo "unkonw argument" exit 1 ;; @@ -156,9 +156,9 @@ function kill_process() { function install_main_path() { #create install main dir and all sub dir ${csudo} rm -rf ${install_main_dir} || : - ${csudo} mkdir -p ${install_main_dir} + ${csudo} mkdir -p ${install_main_dir} ${csudo} mkdir -p ${install_main_dir}/cfg - ${csudo} mkdir -p ${install_main_dir}/bin + ${csudo} mkdir -p ${install_main_dir}/bin ${csudo} mkdir -p ${install_main_dir}/connector ${csudo} mkdir -p ${install_main_dir}/driver ${csudo} mkdir -p ${install_main_dir}/examples @@ -200,29 +200,75 @@ function install_lib() { ${csudo} rm -f ${lib_link_dir}/libtaos.* || : ${csudo} rm -f ${lib64_link_dir}/libtaos.* || : #${csudo} rm -rf ${v15_java_app_dir} || : - ${csudo} cp -rf ${script_dir}/driver/* ${install_main_dir}/driver && ${csudo} chmod 777 ${install_main_dir}/driver/* - + ${csudo} cp -rf ${script_dir}/driver/* ${install_main_dir}/driver && ${csudo} chmod 777 ${install_main_dir}/driver/* + ${csudo} ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1 ${csudo} ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so - + if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then ${csudo} ln -s ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : ${csudo} ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : fi - - #if [ "$verMode" == "cluster" ]; then + + #if [ "$verMode" == "cluster" ]; then # # Compatible with version 1.5 # ${csudo} mkdir -p ${v15_java_app_dir} # ${csudo} ln -s ${install_main_dir}/connector/taos-jdbcdriver-1.0.2-dist.jar ${v15_java_app_dir}/JDBCDriver-1.0.2-dist.jar # ${csudo} chmod 777 ${v15_java_app_dir} || : #fi - + ${csudo} ldconfig } +function install_jemalloc() { + jemalloc_dir=${script_dir}/jemalloc + + if [ -d ${jemalloc_dir} ]; then + ${csudo} /usr/bin/install -c -d /usr/local/bin + + if [ -f ${jemalloc_dir}/bin/jemalloc-config ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc-config /usr/local/bin + fi + if [ -f ${jemalloc_dir}/bin/jemalloc.sh ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc.sh /usr/local/bin + fi + if [ -f ${jemalloc_dir}/bin/jeprof ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jeprof /usr/local/bin + fi + if [ -f ${jemalloc_dir}/include/jemalloc/jemalloc.h ]; then + ${csudo} /usr/bin/install -c -d /usr/local/include/jemalloc + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/include/jemalloc/jemalloc.h /usr/local/include/jemalloc + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc.so.2 ]; then + ${csudo} /usr/bin/install -c -d /usr/local/lib + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.so.2 /usr/local/lib + ${csudo} ln -sf libjemalloc.so.2 /usr/local/lib/libjemalloc.so + ${csudo} /usr/bin/install -c -d /usr/local/lib + if [ -f ${jemalloc_dir}/lib/libjemalloc.a ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.a /usr/local/lib + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc_pic.a /usr/local/lib + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then + ${csudo} /usr/bin/install -c -d /usr/local/lib/pkgconfig + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/lib/pkgconfig/jemalloc.pc /usr/local/lib/pkgconfig + fi + fi + if [ -f ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html ]; then + ${csudo} /usr/bin/install -c -d /usr/local/share/doc/jemalloc + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html /usr/local/share/doc/jemalloc + fi + if [ -f ${jemalloc_dir}/share/man/man3/jemalloc.3 ]; then + ${csudo} /usr/bin/install -c -d /usr/local/share/man/man3 + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/share/man/man3/jemalloc.3 /usr/local/share/man/man3 + fi + fi +} + function install_header() { ${csudo} rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taoserror.h || : - ${csudo} cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo} chmod 644 ${install_main_dir}/include/* + ${csudo} cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo} chmod 644 ${install_main_dir}/include/* ${csudo} ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h ${csudo} ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h } @@ -239,13 +285,13 @@ function add_newHostname_to_hosts() { if [[ "$s" == "$localIp" ]]; then return fi - done + done ${csudo} echo "127.0.0.1 $1" >> /etc/hosts ||: } function set_hostname() { echo -e -n "${GREEN}Please enter one hostname(must not be 'localhost')${NC}:" - read newHostname + read newHostname while true; do if [[ ! -z "$newHostname" && "$newHostname" != "localhost" ]]; then break @@ -259,25 +305,25 @@ function set_hostname() { if [[ $retval != 0 ]]; then echo echo "set hostname fail!" - return + return fi #echo -e -n "$(hostnamectl status --static)" #echo -e -n "$(hostnamectl status --transient)" #echo -e -n "$(hostnamectl status --pretty)" - + #ubuntu/centos /etc/hostname if [[ -e /etc/hostname ]]; then ${csudo} echo $newHostname > /etc/hostname ||: fi - + #debian: #HOSTNAME=yourname if [[ -e /etc/sysconfig/network ]]; then ${csudo} sed -i -r "s/#*\s*(HOSTNAME=\s*).*/\1$newHostname/" /etc/sysconfig/network ||: fi ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$newHostname/" ${cfg_install_dir}/taos.cfg - serverFqdn=$newHostname - + serverFqdn=$newHostname + if [[ -e /etc/hosts ]]; then add_newHostname_to_hosts $newHostname fi @@ -295,7 +341,7 @@ function is_correct_ipaddr() { return 0 fi done - + return 1 } @@ -309,13 +355,13 @@ function set_ipAsFqdn() { echo echo -e -n "${GREEN}Unable to get local ip, use 127.0.0.1${NC}" localFqdn="127.0.0.1" - # Write the local FQDN to configuration file - ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg + # Write the local FQDN to configuration file + ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg serverFqdn=$localFqdn echo return - fi - + fi + echo -e -n "${GREEN}Please choose an IP from local IP list${NC}:" echo echo -e -n "${GREEN}$iplist${NC}" @@ -324,15 +370,15 @@ function set_ipAsFqdn() { echo -e -n "${GREEN}Notes: if IP is used as the node name, data can NOT be migrated to other machine directly${NC}:" read localFqdn while true; do - if [ ! -z "$localFqdn" ]; then + if [ ! -z "$localFqdn" ]; then # Check if correct ip address is_correct_ipaddr $localFqdn retval=`echo $?` if [[ $retval != 0 ]]; then read -p "Please choose an IP from local IP list:" localFqdn else - # Write the local FQDN to configuration file - ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg + # Write the local FQDN to configuration file + ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg serverFqdn=$localFqdn break fi @@ -347,59 +393,59 @@ function local_fqdn_check() { echo echo -e -n "System hostname is: ${GREEN}$serverFqdn${NC}" echo - if [[ "$serverFqdn" == "" ]] || [[ "$serverFqdn" == "localhost" ]]; then - echo -e -n "${GREEN}It is strongly recommended to configure a hostname for this machine ${NC}" - echo - - while true - do - read -r -p "Set hostname now? [Y/n] " input - if [ ! -n "$input" ]; then - set_hostname - break - else - case $input in - [yY][eE][sS]|[yY]) - set_hostname - break - ;; - - [nN][oO]|[nN]) - set_ipAsFqdn - break - ;; - - *) - echo "Invalid input..." - ;; - esac - fi - done + if [[ "$serverFqdn" == "" ]] || [[ "$serverFqdn" == "localhost" ]]; then + echo -e -n "${GREEN}It is strongly recommended to configure a hostname for this machine ${NC}" + echo + + while true + do + read -r -p "Set hostname now? [Y/n] " input + if [ ! -n "$input" ]; then + set_hostname + break + else + case $input in + [yY][eE][sS]|[yY]) + set_hostname + break + ;; + + [nN][oO]|[nN]) + set_ipAsFqdn + break + ;; + + *) + echo "Invalid input..." + ;; + esac + fi + done fi } function install_config() { #${csudo} rm -f ${install_main_dir}/cfg/taos.cfg || : - + if [ ! -f ${cfg_install_dir}/taos.cfg ]; then ${csudo} mkdir -p ${cfg_install_dir} [ -f ${script_dir}/cfg/taos.cfg ] && ${csudo} cp ${script_dir}/cfg/taos.cfg ${cfg_install_dir} ${csudo} chmod 644 ${cfg_install_dir}/* - fi - + fi + ${csudo} cp -f ${script_dir}/cfg/taos.cfg ${install_main_dir}/cfg/taos.cfg.org ${csudo} ln -s ${cfg_install_dir}/taos.cfg ${install_main_dir}/cfg [ ! -z $1 ] && return 0 || : # only install client - + if ((${update_flag}==1)); then return 0 fi - + if [ "$interactiveFqdn" == "no" ]; then return 0 fi - + local_fqdn_check #FQDN_FORMAT="(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)" @@ -417,8 +463,8 @@ function install_config() { if [ ! -z "$firstEp" ]; then # check the format of the firstEp #if [[ $firstEp == $FQDN_PATTERN ]]; then - # Write the first FQDN to configuration file - ${csudo} sed -i -r "s/#*\s*(firstEp\s*).*/\1$firstEp/" ${cfg_install_dir}/taos.cfg + # Write the first FQDN to configuration file + ${csudo} sed -i -r "s/#*\s*(firstEp\s*).*/\1$firstEp/" ${cfg_install_dir}/taos.cfg break #else # read -p "Please enter the correct FQDN:port: " firstEp @@ -426,21 +472,21 @@ function install_config() { else break fi - done + done } function install_log() { ${csudo} rm -rf ${log_dir} || : ${csudo} mkdir -p ${log_dir} && ${csudo} chmod 777 ${log_dir} - + ${csudo} ln -s ${log_dir} ${install_main_dir}/log } function install_data() { ${csudo} mkdir -p ${data_dir} - - ${csudo} ln -s ${data_dir} ${install_main_dir}/data + + ${csudo} ln -s ${data_dir} ${install_main_dir}/data } function install_connector() { @@ -455,26 +501,26 @@ function install_examples() { function clean_service_on_sysvinit() { #restart_config_str="power:2345:respawn:${service_config_dir}/powerd start" - #${csudo} sed -i "\|${restart_config_str}|d" /etc/inittab || : - + #${csudo} sed -i "\|${restart_config_str}|d" /etc/inittab || : + if pidof powerd &> /dev/null; then ${csudo} service powerd stop || : fi - + if pidof tarbitrator &> /dev/null; then ${csudo} service tarbitratord stop || : fi if ((${initd_mod}==1)); then - if [ -e ${service_config_dir}/powerd ]; then + if [ -e ${service_config_dir}/powerd ]; then ${csudo} chkconfig --del powerd || : fi - if [ -e ${service_config_dir}/tarbitratord ]; then + if [ -e ${service_config_dir}/tarbitratord ]; then ${csudo} chkconfig --del tarbitratord || : fi elif ((${initd_mod}==2)); then - if [ -e ${service_config_dir}/powerd ]; then + if [ -e ${service_config_dir}/powerd ]; then ${csudo} insserv -r powerd || : fi if [ -e ${service_config_dir}/tarbitratord ]; then @@ -488,10 +534,10 @@ function clean_service_on_sysvinit() { ${csudo} update-rc.d -f tarbitratord remove || : fi fi - + ${csudo} rm -f ${service_config_dir}/powerd || : ${csudo} rm -f ${service_config_dir}/tarbitratord || : - + if $(which init &> /dev/null); then ${csudo} init q || : fi @@ -514,10 +560,10 @@ function install_service_on_sysvinit() { ${csudo} cp -f ${script_dir}/init.d/tarbitratord.rpm ${install_main_dir}/init.d/tarbitratord ${csudo} cp ${script_dir}/init.d/tarbitratord.rpm ${service_config_dir}/tarbitratord && ${csudo} chmod a+x ${service_config_dir}/tarbitratord fi - + #restart_config_str="power:2345:respawn:${service_config_dir}/powerd start" #${csudo} grep -q -F "$restart_config_str" /etc/inittab || ${csudo} bash -c "echo '${restart_config_str}' >> /etc/inittab" - + if ((${initd_mod}==1)); then ${csudo} chkconfig --add powerd || : ${csudo} chkconfig --level 2345 powerd on || : @@ -542,7 +588,7 @@ function clean_service_on_systemd() { fi ${csudo} systemctl disable powerd &> /dev/null || echo &> /dev/null ${csudo} rm -f ${powerd_service_config} - + tarbitratord_service_config="${service_config_dir}/tarbitratord.service" if systemctl is-active --quiet tarbitratord; then echo "tarbitrator is running, stopping it..." @@ -550,7 +596,7 @@ function clean_service_on_systemd() { fi ${csudo} systemctl disable tarbitratord &> /dev/null || echo &> /dev/null ${csudo} rm -f ${tarbitratord_service_config} - + if [ "$verMode" == "cluster" ]; then nginx_service_config="${service_config_dir}/nginxd.service" if systemctl is-active --quiet nginxd; then @@ -558,8 +604,8 @@ function clean_service_on_systemd() { ${csudo} systemctl stop nginxd &> /dev/null || echo &> /dev/null fi ${csudo} systemctl disable nginxd &> /dev/null || echo &> /dev/null - ${csudo} rm -f ${nginx_service_config} - fi + ${csudo} rm -f ${nginx_service_config} + fi } # power:2345:respawn:/etc/init.d/powerd start @@ -590,7 +636,7 @@ function install_service_on_systemd() { ${csudo} bash -c "echo '[Install]' >> ${powerd_service_config}" ${csudo} bash -c "echo 'WantedBy=multi-user.target' >> ${powerd_service_config}" ${csudo} systemctl enable powerd - + tarbitratord_service_config="${service_config_dir}/tarbitratord.service" ${csudo} bash -c "echo '[Unit]' >> ${tarbitratord_service_config}" ${csudo} bash -c "echo 'Description=TDengine arbitrator service' >> ${tarbitratord_service_config}" @@ -612,9 +658,9 @@ function install_service_on_systemd() { ${csudo} bash -c "echo >> ${tarbitratord_service_config}" ${csudo} bash -c "echo '[Install]' >> ${tarbitratord_service_config}" ${csudo} bash -c "echo 'WantedBy=multi-user.target' >> ${tarbitratord_service_config}" - #${csudo} systemctl enable tarbitratord - - if [ "$verMode" == "cluster" ]; then + #${csudo} systemctl enable tarbitratord + + if [ "$verMode" == "cluster" ]; then nginx_service_config="${service_config_dir}/nginxd.service" ${csudo} bash -c "echo '[Unit]' >> ${nginx_service_config}" ${csudo} bash -c "echo 'Description=Nginx For PowrDB Service' >> ${nginx_service_config}" @@ -643,7 +689,7 @@ function install_service_on_systemd() { ${csudo} systemctl enable nginxd fi ${csudo} systemctl start nginxd - fi + fi } function install_service() { @@ -725,8 +771,8 @@ function update_PowerDB() { kill_process powerd fi sleep 1 - fi - if [ "$verMode" == "cluster" ]; then + fi + if [ "$verMode" == "cluster" ]; then if pidof nginx &> /dev/null; then if ((${service_mod}==0)); then ${csudo} systemctl stop nginxd || : @@ -738,12 +784,13 @@ function update_PowerDB() { sleep 1 fi fi - + install_main_path install_log install_header install_lib + install_jemalloc if [ "$pagMode" != "lite" ]; then install_connector fi @@ -751,10 +798,10 @@ function update_PowerDB() { if [ -z $1 ]; then install_bin install_service - install_config - + install_config + openresty_work=false - if [ "$verMode" == "cluster" ]; then + if [ "$verMode" == "cluster" ]; then # Check if openresty is installed # Check if nginx is installed successfully if type curl &> /dev/null; then @@ -765,7 +812,7 @@ function update_PowerDB() { echo -e "\033[44;31;5mNginx for PowerDB does not work! Please try again!\033[0m" fi fi - fi + fi #echo #echo -e "\033[44;32;1mPowerDB is updated successfully!${NC}" @@ -784,7 +831,7 @@ function update_PowerDB() { else echo -e "${GREEN_DARK}To access PowerDB ${NC}: use ${GREEN_UNDERLINE}power -h $serverFqdn${NC} in shell${NC}" fi - + echo echo -e "\033[44;32;1mPowerDB is updated successfully!${NC}" else @@ -807,16 +854,17 @@ function install_PowerDB() { tar -zxf power.tar.gz echo -e "${GREEN}Start to install PowerDB...${NC}" - - install_main_path - + + install_main_path + if [ -z $1 ]; then install_data - fi - - install_log + fi + + install_log install_header install_lib + install_jemalloc if [ "$pagMode" != "lite" ]; then install_connector fi @@ -839,8 +887,8 @@ function install_PowerDB() { fi fi fi - - install_config + + install_config # Ask if to start the service #echo @@ -853,35 +901,35 @@ function install_PowerDB() { echo -e "${GREEN_DARK}To start PowerDB ${NC}: ${csudo} service powerd start${NC}" else echo -e "${GREEN_DARK}To start PowerDB ${NC}: powerd${NC}" - fi + fi #if [ ${openresty_work} = 'true' ]; then # echo -e "${GREEN_DARK}To access PowerDB ${NC}: use ${GREEN_UNDERLINE}power${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${nginx_port}${NC}" #else # echo -e "${GREEN_DARK}To access PowerDB ${NC}: use ${GREEN_UNDERLINE}power${NC} in shell${NC}" #fi - + if [ ! -z "$firstEp" ]; then - tmpFqdn=${firstEp%%:*} - substr=":" - if [[ $firstEp =~ $substr ]];then - tmpPort=${firstEp#*:} - else - tmpPort="" - fi - if [[ "$tmpPort" != "" ]];then - echo -e "${GREEN_DARK}To access PowerDB ${NC}: power -h $tmpFqdn -P $tmpPort${GREEN_DARK} to login into cluster, then${NC}" - else - echo -e "${GREEN_DARK}To access PowerDB ${NC}: power -h $tmpFqdn${GREEN_DARK} to login into cluster, then${NC}" - fi - echo -e "${GREEN_DARK}execute ${NC}: create dnode 'newDnodeFQDN:port'; ${GREEN_DARK}to add this new node${NC}" - echo + tmpFqdn=${firstEp%%:*} + substr=":" + if [[ $firstEp =~ $substr ]];then + tmpPort=${firstEp#*:} + else + tmpPort="" + fi + if [[ "$tmpPort" != "" ]];then + echo -e "${GREEN_DARK}To access PowerDB ${NC}: power -h $tmpFqdn -P $tmpPort${GREEN_DARK} to login into cluster, then${NC}" + else + echo -e "${GREEN_DARK}To access PowerDB ${NC}: power -h $tmpFqdn${GREEN_DARK} to login into cluster, then${NC}" + fi + echo -e "${GREEN_DARK}execute ${NC}: create dnode 'newDnodeFQDN:port'; ${GREEN_DARK}to add this new node${NC}" + echo elif [ ! -z "$serverFqdn" ]; then - echo -e "${GREEN_DARK}To access PowerDB ${NC}: power -h $serverFqdn${GREEN_DARK} to login into PowerDB server${NC}" - echo + echo -e "${GREEN_DARK}To access PowerDB ${NC}: power -h $serverFqdn${GREEN_DARK} to login into PowerDB server${NC}" + echo fi echo -e "\033[44;32;1mPowerDB is installed successfully!${NC}" - echo + echo else # Only install client install_bin install_config @@ -913,6 +961,6 @@ elif [ "$verType" == "client" ]; then else install_PowerDB client fi -else - echo "please input correct verType" +else + echo "please input correct verType" fi From 6da6c97053c72cbb4cd75fc46d8d8c339ab72420 Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Tue, 6 Jul 2021 13:26:59 +0800 Subject: [PATCH 21/26] [TD-5072] : describe detail about table name wildcard. --- documentation20/cn/12.taos-sql/docs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation20/cn/12.taos-sql/docs.md b/documentation20/cn/12.taos-sql/docs.md index e5570ed3be..34b9aba12b 100644 --- a/documentation20/cn/12.taos-sql/docs.md +++ b/documentation20/cn/12.taos-sql/docs.md @@ -218,7 +218,7 @@ TDengine 缺省的时间戳是毫秒精度，但通过在 CREATE DATABASE 时传 说明：可在like中使用通配符进行名称的匹配，这一通配符字符串最长不能超过24字节。 - 通配符匹配：1）’%’ (百分号)匹配0到任意个字符；2）’\_’下划线匹配一个字符。 + 通配符匹配：1）'%'（百分号）匹配0到任意个字符；2）'\_'下划线匹配单个任意字符。（如果希望匹配表名中带有的下划线，那么这里可以用反斜线进行转义，也就是说 '\\\_' 会被用于匹配表名中原始带有的下划线符号） - **显示一个数据表的创建语句** From 95b0cee27b433dd6ab7451faa5b4d3b88fb0da46 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 6 Jul 2021 14:07:52 +0800 Subject: [PATCH 22/26] [TD-5053]: taosdump support nanosecond (#6751) * [TD-5053]: taosdump support nanosecond huge code refactoring * fix arm32 compile issue. --- src/kit/taosdump/taosdump.c | 3930 ++++++++++++++++++----------------- 1 file changed, 2051 insertions(+), 1879 deletions(-) diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index ac5c58c591..c2b5e11e67 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -27,7 +27,9 @@ #include "tutil.h" #include -#define COMMAND_SIZE 65536 +#define MAX_FILE_NAME_LEN 256 // max file name length on linux is 255 +#define COMMAND_SIZE 65536 +#define MAX_RECORDS_PER_REQ 32766 //#define DEFAULT_DUMP_FILE "taosdump.sql" // for strncpy buffer overflow @@ -61,120 +63,120 @@ typedef struct { // -------------------------- SHOW DATABASE INTERFACE----------------------- enum _show_db_index { - TSDB_SHOW_DB_NAME_INDEX, - TSDB_SHOW_DB_CREATED_TIME_INDEX, - TSDB_SHOW_DB_NTABLES_INDEX, - TSDB_SHOW_DB_VGROUPS_INDEX, - TSDB_SHOW_DB_REPLICA_INDEX, - TSDB_SHOW_DB_QUORUM_INDEX, - TSDB_SHOW_DB_DAYS_INDEX, - TSDB_SHOW_DB_KEEP_INDEX, - TSDB_SHOW_DB_CACHE_INDEX, - TSDB_SHOW_DB_BLOCKS_INDEX, - TSDB_SHOW_DB_MINROWS_INDEX, - TSDB_SHOW_DB_MAXROWS_INDEX, - TSDB_SHOW_DB_WALLEVEL_INDEX, - TSDB_SHOW_DB_FSYNC_INDEX, - TSDB_SHOW_DB_COMP_INDEX, - TSDB_SHOW_DB_CACHELAST_INDEX, - TSDB_SHOW_DB_PRECISION_INDEX, - TSDB_SHOW_DB_UPDATE_INDEX, - TSDB_SHOW_DB_STATUS_INDEX, - TSDB_MAX_SHOW_DB + TSDB_SHOW_DB_NAME_INDEX, + TSDB_SHOW_DB_CREATED_TIME_INDEX, + TSDB_SHOW_DB_NTABLES_INDEX, + TSDB_SHOW_DB_VGROUPS_INDEX, + TSDB_SHOW_DB_REPLICA_INDEX, + TSDB_SHOW_DB_QUORUM_INDEX, + TSDB_SHOW_DB_DAYS_INDEX, + TSDB_SHOW_DB_KEEP_INDEX, + TSDB_SHOW_DB_CACHE_INDEX, + TSDB_SHOW_DB_BLOCKS_INDEX, + TSDB_SHOW_DB_MINROWS_INDEX, + TSDB_SHOW_DB_MAXROWS_INDEX, + TSDB_SHOW_DB_WALLEVEL_INDEX, + TSDB_SHOW_DB_FSYNC_INDEX, + TSDB_SHOW_DB_COMP_INDEX, + TSDB_SHOW_DB_CACHELAST_INDEX, + TSDB_SHOW_DB_PRECISION_INDEX, + TSDB_SHOW_DB_UPDATE_INDEX, + TSDB_SHOW_DB_STATUS_INDEX, + TSDB_MAX_SHOW_DB }; // -----------------------------------------SHOW TABLES CONFIGURE ------------------------------------- enum _show_tables_index { - TSDB_SHOW_TABLES_NAME_INDEX, - TSDB_SHOW_TABLES_CREATED_TIME_INDEX, - TSDB_SHOW_TABLES_COLUMNS_INDEX, - TSDB_SHOW_TABLES_METRIC_INDEX, - TSDB_SHOW_TABLES_UID_INDEX, - TSDB_SHOW_TABLES_TID_INDEX, - TSDB_SHOW_TABLES_VGID_INDEX, - TSDB_MAX_SHOW_TABLES + TSDB_SHOW_TABLES_NAME_INDEX, + TSDB_SHOW_TABLES_CREATED_TIME_INDEX, + TSDB_SHOW_TABLES_COLUMNS_INDEX, + TSDB_SHOW_TABLES_METRIC_INDEX, + TSDB_SHOW_TABLES_UID_INDEX, + TSDB_SHOW_TABLES_TID_INDEX, + TSDB_SHOW_TABLES_VGID_INDEX, + TSDB_MAX_SHOW_TABLES }; // ---------------------------------- DESCRIBE METRIC CONFIGURE ------------------------------ enum _describe_table_index { - TSDB_DESCRIBE_METRIC_FIELD_INDEX, - TSDB_DESCRIBE_METRIC_TYPE_INDEX, - TSDB_DESCRIBE_METRIC_LENGTH_INDEX, - TSDB_DESCRIBE_METRIC_NOTE_INDEX, - TSDB_MAX_DESCRIBE_METRIC + TSDB_DESCRIBE_METRIC_FIELD_INDEX, + TSDB_DESCRIBE_METRIC_TYPE_INDEX, + TSDB_DESCRIBE_METRIC_LENGTH_INDEX, + TSDB_DESCRIBE_METRIC_NOTE_INDEX, + TSDB_MAX_DESCRIBE_METRIC }; #define COL_NOTE_LEN 128 typedef struct { - char field[TSDB_COL_NAME_LEN + 1]; - char type[16]; - int length; - char note[COL_NOTE_LEN]; + char field[TSDB_COL_NAME_LEN + 1]; + char type[16]; + int length; + char note[COL_NOTE_LEN]; } SColDes; typedef struct { - char name[TSDB_TABLE_NAME_LEN]; - SColDes cols[]; + char name[TSDB_TABLE_NAME_LEN]; + SColDes cols[]; } STableDef; extern char version[]; typedef struct { - char name[TSDB_DB_NAME_LEN]; - char create_time[32]; - int32_t ntables; - int32_t vgroups; - int16_t replica; - int16_t quorum; - int16_t days; - char keeplist[32]; - //int16_t daysToKeep; - //int16_t daysToKeep1; - //int16_t daysToKeep2; - int32_t cache; //MB - int32_t blocks; - int32_t minrows; - int32_t maxrows; - int8_t wallevel; - int32_t fsync; - int8_t comp; - int8_t cachelast; - char precision[8]; // time resolution - int8_t update; - char status[16]; + char name[TSDB_DB_NAME_LEN]; + char create_time[32]; + int32_t ntables; + int32_t vgroups; + int16_t replica; + int16_t quorum; + int16_t days; + char keeplist[32]; + //int16_t daysToKeep; + //int16_t daysToKeep1; + //int16_t daysToKeep2; + int32_t cache; //MB + int32_t blocks; + int32_t minrows; + int32_t maxrows; + int8_t wallevel; + int32_t fsync; + int8_t comp; + int8_t cachelast; + char precision[8]; // time resolution + int8_t update; + char status[16]; } SDbInfo; typedef struct { - char name[TSDB_TABLE_NAME_LEN]; - char metric[TSDB_TABLE_NAME_LEN]; + char name[TSDB_TABLE_NAME_LEN]; + char metric[TSDB_TABLE_NAME_LEN]; } STableRecord; typedef struct { - bool isMetric; - STableRecord tableRecord; + bool isMetric; + STableRecord tableRecord; } STableRecordInfo; typedef struct { - pthread_t threadID; - int32_t threadIndex; - int32_t totalThreads; - char dbName[TSDB_DB_NAME_LEN]; - void *taosCon; - int64_t rowsOfDumpOut; - int64_t tablesOfDumpOut; + pthread_t threadID; + int32_t threadIndex; + int32_t totalThreads; + char dbName[TSDB_DB_NAME_LEN]; + void *taosCon; + int64_t rowsOfDumpOut; + int64_t tablesOfDumpOut; } SThreadParaObj; typedef struct { - int64_t totalRowsOfDumpOut; - int64_t totalChildTblsOfDumpOut; - int32_t totalSuperTblsOfDumpOut; - int32_t totalDatabasesOfDumpOut; + int64_t totalRowsOfDumpOut; + int64_t totalChildTblsOfDumpOut; + int32_t totalSuperTblsOfDumpOut; + int32_t totalDatabasesOfDumpOut; } resultStatistics; -static int64_t totalDumpOutRows = 0; +static int64_t g_totalDumpOutRows = 0; -SDbInfo **dbInfos = NULL; +SDbInfo **g_dbInfos = NULL; const char *argp_program_version = version; const char *argp_program_bug_address = ""; @@ -195,1283 +197,1377 @@ static char args_doc[] = "dbname [tbname ...]\n--databases dbname ...\n--all-dat /* The options we understand. */ static struct argp_option options[] = { - // connection option - {"host", 'h', "HOST", 0, "Server host dumping data from. Default is localhost.", 0}, - {"user", 'u', "USER", 0, "User name used to connect to server. Default is root.", 0}, - #ifdef _TD_POWER_ - {"password", 'p', "PASSWORD", 0, "User password to connect to server. Default is powerdb.", 0}, - #elif (_TD_TQ_ == true) - {"password", 'p', "PASSWORD", 0, "User password to connect to server. Default is tqueue.", 0}, - #else - {"password", 'p', "PASSWORD", 0, "User password to connect to server. Default is taosdata.", 0}, - #endif - {"port", 'P', "PORT", 0, "Port to connect", 0}, - {"cversion", 'v', "CVERION", 0, "client version", 0}, - {"mysqlFlag", 'q', "MYSQLFLAG", 0, "mysqlFlag, Default is 0", 0}, - // input/output file - {"outpath", 'o', "OUTPATH", 0, "Output file path.", 1}, - {"inpath", 'i', "INPATH", 0, "Input file path.", 1}, - {"resultFile", 'r', "RESULTFILE", 0, "DumpOut/In Result file path and name.", 1}, - #ifdef _TD_POWER_ - {"config", 'c', "CONFIG_DIR", 0, "Configure directory. Default is /etc/power/taos.cfg.", 1}, - #elif (_TD_TQ_ == true) - {"config", 'c', "CONFIG_DIR", 0, "Configure directory. Default is /etc/tq/taos.cfg.", 1}, - #else - {"config", 'c', "CONFIG_DIR", 0, "Configure directory. Default is /etc/taos/taos.cfg.", 1}, - #endif - {"encode", 'e', "ENCODE", 0, "Input file encoding.", 1}, - // dump unit options - {"all-databases", 'A', 0, 0, "Dump all databases.", 2}, - {"databases", 'D', 0, 0, "Dump assigned databases", 2}, - // dump format options - {"schemaonly", 's', 0, 0, "Only dump schema.", 3}, - {"without-property", 'N', 0, 0, "Dump schema without properties.", 3}, - {"start-time", 'S', "START_TIME", 0, "Start time to dump. Either Epoch or ISO8601/RFC3339 format is acceptable. Epoch precision millisecond. ISO8601 format example: 2017-10-01T18:00:00.000+0800 or 2017-10-0100:00:00.000+0800 or '2017-10-01 00:00:00.000+0800'", 3}, - {"end-time", 'E', "END_TIME", 0, "End time to dump. Either Epoch or ISO8601/RFC3339 format is acceptable. Epoch precision millisecond. ISO8601 format example: 2017-10-01T18:00:00.000+0800 or 2017-10-0100:00:00.000+0800 or '2017-10-01 00:00:00.000+0800'", 3}, - {"data-batch", 'B', "DATA_BATCH", 0, "Number of data point per insert statement. Default is 1.", 3}, - {"max-sql-len", 'L', "SQL_LEN", 0, "Max length of one sql. Default is 65480.", 3}, - {"table-batch", 't', "TABLE_BATCH", 0, "Number of table dumpout into one output file. Default is 1.", 3}, - {"thread_num", 'T', "THREAD_NUM", 0, "Number of thread for dump in file. Default is 5.", 3}, - {"allow-sys", 'a', 0, 0, "Allow to dump sys database", 3}, - {"debug", 'g', 0, 0, "Print debug info.", 1}, - {"verbose", 'v', 0, 0, "Print verbose debug info.", 1}, - {0}}; + // connection option + {"host", 'h', "HOST", 0, "Server host dumping data from. Default is localhost.", 0}, + {"user", 'u', "USER", 0, "User name used to connect to server. Default is root.", 0}, +#ifdef _TD_POWER_ + {"password", 'p', "PASSWORD", 0, "User password to connect to server. Default is powerdb.", 0}, +#else + {"password", 'p', "PASSWORD", 0, "User password to connect to server. Default is taosdata.", 0}, +#endif + {"port", 'P', "PORT", 0, "Port to connect", 0}, + {"cversion", 'v', "CVERION", 0, "client version", 0}, + {"mysqlFlag", 'q', "MYSQLFLAG", 0, "mysqlFlag, Default is 0", 0}, + // input/output file + {"outpath", 'o', "OUTPATH", 0, "Output file path.", 1}, + {"inpath", 'i', "INPATH", 0, "Input file path.", 1}, + {"resultFile", 'r', "RESULTFILE", 0, "DumpOut/In Result file path and name.", 1}, +#ifdef _TD_POWER_ + {"config", 'c', "CONFIG_DIR", 0, "Configure directory. Default is /etc/power/taos.cfg.", 1}, +#else + {"config", 'c', "CONFIG_DIR", 0, "Configure directory. Default is /etc/taos/taos.cfg.", 1}, +#endif + {"encode", 'e', "ENCODE", 0, "Input file encoding.", 1}, + // dump unit options + {"all-databases", 'A', 0, 0, "Dump all databases.", 2}, + {"databases", 'D', 0, 0, "Dump assigned databases", 2}, + {"allow-sys", 'a', 0, 0, "Allow to dump sys database", 2}, + // dump format options + {"schemaonly", 's', 0, 0, "Only dump schema.", 2}, + {"without-property", 'N', 0, 0, "Dump schema without properties.", 2}, + {"avro", 'V', 0, 0, "Dump apache avro format data file. By default, dump sql command sequence.", 2}, + {"start-time", 'S', "START_TIME", 0, "Start time to dump. Either Epoch or ISO8601/RFC3339 format is acceptable. Epoch precision millisecond. ISO8601 format example: 2017-10-01T18:00:00.000+0800 or 2017-10-0100:00:00.000+0800 or '2017-10-01 00:00:00.000+0800'", 3}, + {"end-time", 'E', "END_TIME", 0, "End time to dump. Either Epoch or ISO8601/RFC3339 format is acceptable. Epoch precision millisecond. ISO8601 format example: 2017-10-01T18:00:00.000+0800 or 2017-10-0100:00:00.000+0800 or '2017-10-01 00:00:00.000+0800'", 3}, + {"data-batch", 'B', "DATA_BATCH", 0, "Number of data point per insert statement. Max value is 32766. Default is 1.", 3}, + {"max-sql-len", 'L', "SQL_LEN", 0, "Max length of one sql. Default is 65480.", 3}, + {"table-batch", 't', "TABLE_BATCH", 0, "Number of table dumpout into one output file. Default is 1.", 3}, + {"thread_num", 'T', "THREAD_NUM", 0, "Number of thread for dump in file. Default is 5.", 3}, + {"debug", 'g', 0, 0, "Print debug info.", 4}, + {"verbose", 'b', 0, 0, "Print verbose debug info.", 5}, + {"performanceprint", 'm', 0, 0, "Print performance debug info.", 5}, + {0} +}; /* Used by main to communicate with parse_opt. */ typedef struct arguments { - // connection option - char *host; - char *user; - char *password; - uint16_t port; - char cversion[12]; - uint16_t mysqlFlag; - // output file - char outpath[TSDB_FILENAME_LEN+1]; - char inpath[TSDB_FILENAME_LEN+1]; - // result file - char *resultFile; - char *encode; - // dump unit option - bool all_databases; - bool databases; - // dump format option - bool schemaonly; - bool with_property; - int64_t start_time; - int64_t end_time; - int32_t data_batch; - int32_t max_sql_len; - int32_t table_batch; // num of table which will be dump into one output file. - bool allow_sys; - // other options - int32_t thread_num; - int abort; - char **arg_list; - int arg_list_len; - bool isDumpIn; - bool debug_print; - bool verbose_print; - bool performance_print; + // connection option + char *host; + char *user; + char *password; + uint16_t port; + char cversion[12]; + uint16_t mysqlFlag; + // output file + char outpath[MAX_FILE_NAME_LEN]; + char inpath[MAX_FILE_NAME_LEN]; + // result file + char *resultFile; + char *encode; + // dump unit option + bool all_databases; + bool databases; + // dump format option + bool schemaonly; + bool with_property; + bool avro; + int64_t start_time; + int64_t end_time; + int32_t data_batch; + int32_t max_sql_len; + int32_t table_batch; // num of table which will be dump into one output file. + bool allow_sys; + // other options + int32_t thread_num; + int abort; + char **arg_list; + int arg_list_len; + bool isDumpIn; + bool debug_print; + bool verbose_print; + bool performance_print; } SArguments; -/* Parse a single option. */ -static error_t parse_opt(int key, char *arg, struct argp_state *state) { - /* Get the input argument from argp_parse, which we - know is a pointer to our arguments structure. */ - struct arguments *arguments = state->input; - wordexp_t full_path; - - switch (key) { - // connection option - case 'a': - arguments->allow_sys = true; - break; - case 'h': - arguments->host = arg; - break; - case 'u': - arguments->user = arg; - break; - case 'p': - arguments->password = arg; - break; - case 'P': - arguments->port = atoi(arg); - break; - case 'q': - arguments->mysqlFlag = atoi(arg); - break; - case 'v': - if (wordexp(arg, &full_path, 0) != 0) { - fprintf(stderr, "Invalid client vesion %s\n", arg); - return -1; - } - tstrncpy(arguments->cversion, full_path.we_wordv[0], 11); - wordfree(&full_path); - break; - // output file path - case 'o': - if (wordexp(arg, &full_path, 0) != 0) { - fprintf(stderr, "Invalid path %s\n", arg); - return -1; - } - tstrncpy(arguments->outpath, full_path.we_wordv[0], TSDB_FILENAME_LEN); - wordfree(&full_path); - break; - case 'g': - arguments->debug_print = true; - break; - case 'i': - arguments->isDumpIn = true; - if (wordexp(arg, &full_path, 0) != 0) { - fprintf(stderr, "Invalid path %s\n", arg); - return -1; - } - tstrncpy(arguments->inpath, full_path.we_wordv[0], TSDB_FILENAME_LEN); - wordfree(&full_path); - break; - case 'r': - arguments->resultFile = arg; - break; - case 'c': - if (wordexp(arg, &full_path, 0) != 0) { - fprintf(stderr, "Invalid path %s\n", arg); - return -1; - } - tstrncpy(configDir, full_path.we_wordv[0], TSDB_FILENAME_LEN); - wordfree(&full_path); - break; - case 'e': - arguments->encode = arg; - break; - // dump unit option - case 'A': - arguments->all_databases = true; - break; - case 'D': - arguments->databases = true; - break; - // dump format option - case 's': - arguments->schemaonly = true; - break; - case 'N': - arguments->with_property = false; - break; - case 'S': - // parse time here. - arguments->start_time = atol(arg); - break; - case 'E': - arguments->end_time = atol(arg); - break; - case 'B': - arguments->data_batch = atoi(arg); - if (arguments->data_batch >= INT16_MAX) { - arguments->data_batch = INT16_MAX - 1; - } - break; - case 'L': - { - int32_t len = atoi(arg); - if (len > TSDB_MAX_ALLOWED_SQL_LEN) { - len = TSDB_MAX_ALLOWED_SQL_LEN; - } else if (len < TSDB_MAX_SQL_LEN) { - len = TSDB_MAX_SQL_LEN; - } - arguments->max_sql_len = len; - break; - } - case 't': - arguments->table_batch = atoi(arg); - break; - case 'T': - arguments->thread_num = atoi(arg); - break; - case OPT_ABORT: - arguments->abort = 1; - break; - case ARGP_KEY_ARG: - arguments->arg_list = &state->argv[state->next - 1]; - arguments->arg_list_len = state->argc - state->next + 1; - state->next = state->argc; - break; - - default: - return ARGP_ERR_UNKNOWN; - } - return 0; -} - /* Our argp parser. */ +static error_t parse_opt(int key, char *arg, struct argp_state *state); + static struct argp argp = {options, parse_opt, args_doc, doc}; static resultStatistics g_resultStatistics = {0}; static FILE *g_fpOfResult = NULL; static int g_numOfCores = 1; -static int taosDumpOut(struct arguments *arguments); -static int taosDumpIn(struct arguments *arguments); -static void taosDumpCreateDbClause(SDbInfo *dbInfo, bool isDumpProperty, FILE *fp); -static int taosDumpDb(SDbInfo *dbInfo, struct arguments *arguments, FILE *fp, TAOS *taosCon); -static int32_t taosDumpStable(char *table, FILE *fp, TAOS* taosCon, char* dbName); -static void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, FILE *fp, char* dbName); -static void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols, FILE *fp, char* dbName); -static int32_t taosDumpTable(char *table, char *metric, struct arguments *arguments, FILE *fp, TAOS* taosCon, char* dbName); -static int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* taosCon, char* dbName); +static int taosDumpOut(); +static int taosDumpIn(); +static void taosDumpCreateDbClause(SDbInfo *dbInfo, bool isDumpProperty, + FILE *fp); +static int taosDumpDb(SDbInfo *dbInfo, FILE *fp, TAOS *taosCon); +static int32_t taosDumpStable(char *table, FILE *fp, TAOS* taosCon, + char* dbName); +static void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, + FILE *fp, char* dbName); +static void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, + int numOfCols, FILE *fp, char* dbName); +static int32_t taosDumpTable(char *table, char *metric, + FILE *fp, TAOS* taosCon, char* dbName); +static int taosDumpTableData(FILE *fp, char *tbName, + TAOS* taosCon, char* dbName, + char *jsonAvroSchema); static int taosCheckParam(struct arguments *arguments); static void taosFreeDbInfos(); -static void taosStartDumpOutWorkThreads(void* taosCon, struct arguments* args, int32_t numOfThread, char *dbName); +static void taosStartDumpOutWorkThreads(int32_t numOfThread, char *dbName); struct arguments g_args = { - // connection option - NULL, - "root", - #ifdef _TD_POWER_ - "powerdb", - #elif (_TD_TQ_ == true) - "tqueue", - #else - "taosdata", - #endif - 0, - "", - 0, - // outpath and inpath - "", - "", - "./dump_result.txt", - NULL, - // dump unit option - false, - false, - // dump format option - false, // schemeonly - true, // with_property - 0, - INT64_MAX, - 1, - TSDB_MAX_SQL_LEN, - 1, - false, - // other options - 5, - 0, - NULL, - 0, - false, - false, // debug_print - false, // verbose_print - false // performance_print + // connection option + NULL, + "root", +#ifdef _TD_POWER_ + "powerdb", +#else + "taosdata", +#endif + 0, + "", + 0, + // outpath and inpath + "", + "", + "./dump_result.txt", + NULL, + // dump unit option + false, + false, + // dump format option + false, // schemeonly + true, // with_property + false, // avro format + 0, // start_time + INT64_MAX, // end_time + 1, // data_batch + TSDB_MAX_SQL_LEN, // max_sql_len + 1, // table_batch + false, // allow_sys + // other options + 5, // thread_num + 0, // abort + NULL, // arg_list + 0, // arg_list_len + false, // isDumpIn + false, // debug_print + false, // verbose_print + false // performance_print }; +/* Parse a single option. */ +static error_t parse_opt(int key, char *arg, struct argp_state *state) { + /* Get the input argument from argp_parse, which we + know is a pointer to our arguments structure. */ + wordexp_t full_path; + + switch (key) { + // connection option + case 'a': + g_args.allow_sys = true; + break; + case 'h': + g_args.host = arg; + break; + case 'u': + g_args.user = arg; + break; + case 'p': + g_args.password = arg; + break; + case 'P': + g_args.port = atoi(arg); + break; + case 'q': + g_args.mysqlFlag = atoi(arg); + break; + case 'v': + if (wordexp(arg, &full_path, 0) != 0) { + errorPrint("Invalid client vesion %s\n", arg); + return -1; + } + tstrncpy(g_args.cversion, full_path.we_wordv[0], 11); + wordfree(&full_path); + break; + // output file path + case 'o': + if (wordexp(arg, &full_path, 0) != 0) { + errorPrint("Invalid path %s\n", arg); + return -1; + } + tstrncpy(g_args.outpath, full_path.we_wordv[0], + MAX_FILE_NAME_LEN); + wordfree(&full_path); + break; + case 'g': + g_args.debug_print = true; + break; + case 'i': + g_args.isDumpIn = true; + if (wordexp(arg, &full_path, 0) != 0) { + errorPrint("Invalid path %s\n", arg); + return -1; + } + tstrncpy(g_args.inpath, full_path.we_wordv[0], + MAX_FILE_NAME_LEN); + wordfree(&full_path); + break; + case 'r': + g_args.resultFile = arg; + break; + case 'c': + if (wordexp(arg, &full_path, 0) != 0) { + errorPrint("Invalid path %s\n", arg); + return -1; + } + tstrncpy(configDir, full_path.we_wordv[0], MAX_FILE_NAME_LEN); + wordfree(&full_path); + break; + case 'e': + g_args.encode = arg; + break; + // dump unit option + case 'A': + g_args.all_databases = true; + break; + case 'D': + g_args.databases = true; + break; + // dump format option + case 's': + g_args.schemaonly = true; + break; + case 'N': + g_args.with_property = false; + break; + case 'V': + g_args.avro = true; + break; + case 'S': + // parse time here. + g_args.start_time = atol(arg); + break; + case 'E': + g_args.end_time = atol(arg); + break; + case 'B': + g_args.data_batch = atoi(arg); + if (g_args.data_batch > MAX_RECORDS_PER_REQ) { + g_args.data_batch = MAX_RECORDS_PER_REQ; + } + break; + case 'L': + { + int32_t len = atoi(arg); + if (len > TSDB_MAX_ALLOWED_SQL_LEN) { + len = TSDB_MAX_ALLOWED_SQL_LEN; + } else if (len < TSDB_MAX_SQL_LEN) { + len = TSDB_MAX_SQL_LEN; + } + g_args.max_sql_len = len; + break; + } + case 't': + g_args.table_batch = atoi(arg); + break; + case 'T': + g_args.thread_num = atoi(arg); + break; + case OPT_ABORT: + g_args.abort = 1; + break; + case ARGP_KEY_ARG: + g_args.arg_list = &state->argv[state->next - 1]; + g_args.arg_list_len = state->argc - state->next + 1; + state->next = state->argc; + break; + + default: + return ARGP_ERR_UNKNOWN; + } + return 0; +} + static int queryDbImpl(TAOS *taos, char *command) { - int i; - TAOS_RES *res = NULL; - int32_t code = -1; + int i; + TAOS_RES *res = NULL; + int32_t code = -1; - for (i = 0; i < 5; i++) { - if (NULL != res) { - taos_free_result(res); - res = NULL; + for (i = 0; i < 5; i++) { + if (NULL != res) { + taos_free_result(res); + res = NULL; + } + + res = taos_query(taos, command); + code = taos_errno(res); + if (0 == code) { + break; + } } - res = taos_query(taos, command); - code = taos_errno(res); - if (0 == code) { - break; + if (code != 0) { + errorPrint("Failed to run <%s>, reason: %s\n", command, taos_errstr(res)); + taos_free_result(res); + //taos_close(taos); + return -1; } - } - if (code != 0) { - fprintf(stderr, "Failed to run <%s>, reason: %s\n", command, taos_errstr(res)); taos_free_result(res); - //taos_close(taos); - return -1; - } - - taos_free_result(res); - return 0; + return 0; } static void parse_args(int argc, char *argv[], SArguments *arguments) { - for (int i = 1; i < argc; i++) { - if ((strcmp(argv[i], "-S") == 0) - || (strcmp(argv[i], "-E") == 0)) { - if (argv[i+1]) { - char *tmp = strdup(argv[++i]); + for (int i = 1; i < argc; i++) { + if ((strcmp(argv[i], "-S") == 0) + || (strcmp(argv[i], "-E") == 0)) { + if (argv[i+1]) { + char *tmp = strdup(argv[++i]); - if (tmp) { - int64_t tmpEpoch; - if (strchr(tmp, ':') && strchr(tmp, '-')) { - if (TSDB_CODE_SUCCESS != taosParseTime( - tmp, &tmpEpoch, strlen(tmp), TSDB_TIME_PRECISION_MILLI, 0)) { - fprintf(stderr, "Input end time error!\n"); - free(tmp); - return; + if (tmp) { + int64_t tmpEpoch; + if (strchr(tmp, ':') && strchr(tmp, '-')) { + if (TSDB_CODE_SUCCESS != taosParseTime( + tmp, &tmpEpoch, strlen(tmp), TSDB_TIME_PRECISION_MILLI, 0)) { + errorPrint("Input %s, end time error!\n", tmp); + free(tmp); + return; + } + } else { + tmpEpoch = atoll(tmp); + } + + sprintf(argv[i], "%"PRId64"", tmpEpoch); + debugPrint("%s() LN%d, tmp is: %s, argv[%d]: %s\n", + __func__, __LINE__, tmp, i, argv[i]); + + free(tmp); + } else { + errorPrint("%s() LN%d, strdup() cannot allocate memory\n", __func__, __LINE__); + exit(-1); + } + } else { + errorPrint("%s need a valid value following!\n", argv[i]); + exit(-1); } - } else { - tmpEpoch = atoll(tmp); - } - - sprintf(argv[i], "%"PRId64"", tmpEpoch); - debugPrint("%s() LN%d, tmp is: %s, argv[%d]: %s\n", - __func__, __LINE__, tmp, i, argv[i]); - - free(tmp); - } else { - errorPrint("%s() LN%d, strdup() cannot allocate memory\n", __func__, __LINE__); - exit(-1); + } else if (strcmp(argv[i], "-g") == 0) { + g_args.debug_print = true; } - } else { - errorPrint("%s need a valid value following!\n", argv[i]); - exit(-1); - } - } else if (strcmp(argv[i], "-g") == 0) { - arguments->debug_print = true; } - } } int main(int argc, char *argv[]) { - /* Parse our arguments; every option seen by parse_opt will be - reflected in arguments. */ - if (argc > 2) - parse_args(argc, argv, &g_args); + int ret = 0; + /* Parse our arguments; every option seen by parse_opt will be + reflected in arguments. */ + if (argc > 2) + parse_args(argc, argv, &g_args); - argp_parse(&argp, argc, argv, 0, 0, &g_args); + argp_parse(&argp, argc, argv, 0, 0, &g_args); - if (g_args.abort) { - #ifndef _ALPINE - error(10, 0, "ABORTED"); - #else - abort(); - #endif - } - - printf("====== arguments config ======\n"); - { - printf("host: %s\n", g_args.host); - printf("user: %s\n", g_args.user); - printf("password: %s\n", g_args.password); - printf("port: %u\n", g_args.port); - printf("cversion: %s\n", g_args.cversion); - printf("mysqlFlag: %d\n", g_args.mysqlFlag); - printf("outpath: %s\n", g_args.outpath); - printf("inpath: %s\n", g_args.inpath); - printf("resultFile: %s\n", g_args.resultFile); - printf("encode: %s\n", g_args.encode); - printf("all_databases: %d\n", g_args.all_databases); - printf("databases: %d\n", g_args.databases); - printf("schemaonly: %d\n", g_args.schemaonly); - printf("with_property: %d\n", g_args.with_property); - printf("start_time: %" PRId64 "\n", g_args.start_time); - printf("end_time: %" PRId64 "\n", g_args.end_time); - printf("data_batch: %d\n", g_args.data_batch); - printf("max_sql_len: %d\n", g_args.max_sql_len); - printf("table_batch: %d\n", g_args.table_batch); - printf("thread_num: %d\n", g_args.thread_num); - printf("allow_sys: %d\n", g_args.allow_sys); - printf("abort: %d\n", g_args.abort); - printf("isDumpIn: %d\n", g_args.isDumpIn); - printf("arg_list_len: %d\n", g_args.arg_list_len); - printf("debug_print: %d\n", g_args.debug_print); - - for (int32_t i = 0; i < g_args.arg_list_len; i++) { - printf("arg_list[%d]: %s\n", i, g_args.arg_list[i]); - } - } - printf("==============================\n"); - - if (g_args.cversion[0] != 0){ - tstrncpy(version, g_args.cversion, 11); - } - - if (taosCheckParam(&g_args) < 0) { - exit(EXIT_FAILURE); - } - - g_fpOfResult = fopen(g_args.resultFile, "a"); - if (NULL == g_fpOfResult) { - fprintf(stderr, "Failed to open %s for save result\n", g_args.resultFile); - return 1; - }; - - fprintf(g_fpOfResult, "#############################################################################\n"); - fprintf(g_fpOfResult, "============================== arguments config =============================\n"); - { - fprintf(g_fpOfResult, "host: %s\n", g_args.host); - fprintf(g_fpOfResult, "user: %s\n", g_args.user); - fprintf(g_fpOfResult, "password: %s\n", g_args.password); - fprintf(g_fpOfResult, "port: %u\n", g_args.port); - fprintf(g_fpOfResult, "cversion: %s\n", g_args.cversion); - fprintf(g_fpOfResult, "mysqlFlag: %d\n", g_args.mysqlFlag); - fprintf(g_fpOfResult, "outpath: %s\n", g_args.outpath); - fprintf(g_fpOfResult, "inpath: %s\n", g_args.inpath); - fprintf(g_fpOfResult, "resultFile: %s\n", g_args.resultFile); - fprintf(g_fpOfResult, "encode: %s\n", g_args.encode); - fprintf(g_fpOfResult, "all_databases: %s\n", g_args.all_databases?"true":"false"); - fprintf(g_fpOfResult, "databases: %d\n", g_args.databases); - fprintf(g_fpOfResult, "schemaonly: %s\n", g_args.schemaonly?"true":"false"); - fprintf(g_fpOfResult, "with_property: %s\n", g_args.with_property?"true":"false"); - fprintf(g_fpOfResult, "start_time: %" PRId64 "\n", g_args.start_time); - fprintf(g_fpOfResult, "end_time: %" PRId64 "\n", g_args.end_time); - fprintf(g_fpOfResult, "data_batch: %d\n", g_args.data_batch); - fprintf(g_fpOfResult, "max_sql_len: %d\n", g_args.max_sql_len); - fprintf(g_fpOfResult, "table_batch: %d\n", g_args.table_batch); - fprintf(g_fpOfResult, "thread_num: %d\n", g_args.thread_num); - fprintf(g_fpOfResult, "allow_sys: %d\n", g_args.allow_sys); - fprintf(g_fpOfResult, "abort: %d\n", g_args.abort); - fprintf(g_fpOfResult, "isDumpIn: %d\n", g_args.isDumpIn); - fprintf(g_fpOfResult, "arg_list_len: %d\n", g_args.arg_list_len); - - for (int32_t i = 0; i < g_args.arg_list_len; i++) { - fprintf(g_fpOfResult, "arg_list[%d]: %s\n", i, g_args.arg_list[i]); - } - } - - g_numOfCores = (int32_t)sysconf(_SC_NPROCESSORS_ONLN); - - time_t tTime = time(NULL); - struct tm tm = *localtime(&tTime); - - if (g_args.isDumpIn) { - fprintf(g_fpOfResult, "============================== DUMP IN ============================== \n"); - fprintf(g_fpOfResult, "# DumpIn start time: %d-%02d-%02d %02d:%02d:%02d\n", tm.tm_year + 1900, tm.tm_mon + 1, - tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); - if (taosDumpIn(&g_args) < 0) { - fprintf(g_fpOfResult, "\n"); - fclose(g_fpOfResult); - return -1; - } - } else { - fprintf(g_fpOfResult, "============================== DUMP OUT ============================== \n"); - fprintf(g_fpOfResult, "# DumpOut start time: %d-%02d-%02d %02d:%02d:%02d\n", tm.tm_year + 1900, tm.tm_mon + 1, - tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); - if (taosDumpOut(&g_args) < 0) { - fprintf(g_fpOfResult, "\n"); - fclose(g_fpOfResult); - return -1; + if (g_args.abort) { +#ifndef _ALPINE + error(10, 0, "ABORTED"); +#else + abort(); +#endif } - fprintf(g_fpOfResult, "\n============================== TOTAL STATISTICS ============================== \n"); - fprintf(g_fpOfResult, "# total database count: %d\n", g_resultStatistics.totalDatabasesOfDumpOut); - fprintf(g_fpOfResult, "# total super table count: %d\n", g_resultStatistics.totalSuperTblsOfDumpOut); - fprintf(g_fpOfResult, "# total child table count: %"PRId64"\n", g_resultStatistics.totalChildTblsOfDumpOut); - fprintf(g_fpOfResult, "# total row count: %"PRId64"\n", g_resultStatistics.totalRowsOfDumpOut); - } + printf("====== arguments config ======\n"); + { + printf("host: %s\n", g_args.host); + printf("user: %s\n", g_args.user); + printf("password: %s\n", g_args.password); + printf("port: %u\n", g_args.port); + printf("cversion: %s\n", g_args.cversion); + printf("mysqlFlag: %d\n", g_args.mysqlFlag); + printf("outpath: %s\n", g_args.outpath); + printf("inpath: %s\n", g_args.inpath); + printf("resultFile: %s\n", g_args.resultFile); + printf("encode: %s\n", g_args.encode); + printf("all_databases: %s\n", g_args.all_databases?"true":"false"); + printf("databases: %d\n", g_args.databases); + printf("schemaonly: %s\n", g_args.schemaonly?"true":"false"); + printf("with_property: %s\n", g_args.with_property?"true":"false"); + printf("avro format: %s\n", g_args.avro?"true":"false"); + printf("start_time: %" PRId64 "\n", g_args.start_time); + printf("end_time: %" PRId64 "\n", g_args.end_time); + printf("data_batch: %d\n", g_args.data_batch); + printf("max_sql_len: %d\n", g_args.max_sql_len); + printf("table_batch: %d\n", g_args.table_batch); + printf("thread_num: %d\n", g_args.thread_num); + printf("allow_sys: %d\n", g_args.allow_sys); + printf("abort: %d\n", g_args.abort); + printf("isDumpIn: %d\n", g_args.isDumpIn); + printf("arg_list_len: %d\n", g_args.arg_list_len); + printf("debug_print: %d\n", g_args.debug_print); - fprintf(g_fpOfResult, "\n"); - fclose(g_fpOfResult); + for (int32_t i = 0; i < g_args.arg_list_len; i++) { + printf("arg_list[%d]: %s\n", i, g_args.arg_list[i]); + } + } + printf("==============================\n"); - return 0; + if (g_args.cversion[0] != 0){ + tstrncpy(version, g_args.cversion, 11); + } + + if (taosCheckParam(&g_args) < 0) { + exit(EXIT_FAILURE); + } + + g_fpOfResult = fopen(g_args.resultFile, "a"); + if (NULL == g_fpOfResult) { + errorPrint("Failed to open %s for save result\n", g_args.resultFile); + exit(-1); + }; + + fprintf(g_fpOfResult, "#############################################################################\n"); + fprintf(g_fpOfResult, "============================== arguments config =============================\n"); + { + fprintf(g_fpOfResult, "host: %s\n", g_args.host); + fprintf(g_fpOfResult, "user: %s\n", g_args.user); + fprintf(g_fpOfResult, "password: %s\n", g_args.password); + fprintf(g_fpOfResult, "port: %u\n", g_args.port); + fprintf(g_fpOfResult, "cversion: %s\n", g_args.cversion); + fprintf(g_fpOfResult, "mysqlFlag: %d\n", g_args.mysqlFlag); + fprintf(g_fpOfResult, "outpath: %s\n", g_args.outpath); + fprintf(g_fpOfResult, "inpath: %s\n", g_args.inpath); + fprintf(g_fpOfResult, "resultFile: %s\n", g_args.resultFile); + fprintf(g_fpOfResult, "encode: %s\n", g_args.encode); + fprintf(g_fpOfResult, "all_databases: %s\n", g_args.all_databases?"true":"false"); + fprintf(g_fpOfResult, "databases: %d\n", g_args.databases); + fprintf(g_fpOfResult, "schemaonly: %s\n", g_args.schemaonly?"true":"false"); + fprintf(g_fpOfResult, "with_property: %s\n", g_args.with_property?"true":"false"); + fprintf(g_fpOfResult, "avro format: %s\n", g_args.avro?"true":"false"); + fprintf(g_fpOfResult, "start_time: %" PRId64 "\n", g_args.start_time); + fprintf(g_fpOfResult, "end_time: %" PRId64 "\n", g_args.end_time); + fprintf(g_fpOfResult, "data_batch: %d\n", g_args.data_batch); + fprintf(g_fpOfResult, "max_sql_len: %d\n", g_args.max_sql_len); + fprintf(g_fpOfResult, "table_batch: %d\n", g_args.table_batch); + fprintf(g_fpOfResult, "thread_num: %d\n", g_args.thread_num); + fprintf(g_fpOfResult, "allow_sys: %d\n", g_args.allow_sys); + fprintf(g_fpOfResult, "abort: %d\n", g_args.abort); + fprintf(g_fpOfResult, "isDumpIn: %d\n", g_args.isDumpIn); + fprintf(g_fpOfResult, "arg_list_len: %d\n", g_args.arg_list_len); + + for (int32_t i = 0; i < g_args.arg_list_len; i++) { + fprintf(g_fpOfResult, "arg_list[%d]: %s\n", i, g_args.arg_list[i]); + } + } + + g_numOfCores = (int32_t)sysconf(_SC_NPROCESSORS_ONLN); + + time_t tTime = time(NULL); + struct tm tm = *localtime(&tTime); + + if (g_args.isDumpIn) { + fprintf(g_fpOfResult, "============================== DUMP IN ============================== \n"); + fprintf(g_fpOfResult, "# DumpIn start time: %d-%02d-%02d %02d:%02d:%02d\n", + tm.tm_year + 1900, tm.tm_mon + 1, + tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); + if (taosDumpIn() < 0) { + ret = -1; + } + } else { + fprintf(g_fpOfResult, "============================== DUMP OUT ============================== \n"); + fprintf(g_fpOfResult, "# DumpOut start time: %d-%02d-%02d %02d:%02d:%02d\n", + tm.tm_year + 1900, tm.tm_mon + 1, + tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); + if (taosDumpOut() < 0) { + ret = -1; + } else { + fprintf(g_fpOfResult, "\n============================== TOTAL STATISTICS ============================== \n"); + fprintf(g_fpOfResult, "# total database count: %d\n", + g_resultStatistics.totalDatabasesOfDumpOut); + fprintf(g_fpOfResult, "# total super table count: %d\n", + g_resultStatistics.totalSuperTblsOfDumpOut); + fprintf(g_fpOfResult, "# total child table count: %"PRId64"\n", + g_resultStatistics.totalChildTblsOfDumpOut); + fprintf(g_fpOfResult, "# total row count: %"PRId64"\n", + g_resultStatistics.totalRowsOfDumpOut); + } + } + + fprintf(g_fpOfResult, "\n"); + fclose(g_fpOfResult); + + return ret; } static void taosFreeDbInfos() { - if (dbInfos == NULL) return; - for (int i = 0; i < 128; i++) tfree(dbInfos[i]); - tfree(dbInfos); + if (g_dbInfos == NULL) return; + for (int i = 0; i < 128; i++) tfree(g_dbInfos[i]); + tfree(g_dbInfos); } // check table is normal table or super table -static int taosGetTableRecordInfo(char *table, STableRecordInfo *pTableRecordInfo, TAOS *taosCon) { - TAOS_ROW row = NULL; - bool isSet = false; - TAOS_RES *result = NULL; +static int taosGetTableRecordInfo( + char *table, STableRecordInfo *pTableRecordInfo, TAOS *taosCon) { + TAOS_ROW row = NULL; + bool isSet = false; + TAOS_RES *result = NULL; - memset(pTableRecordInfo, 0, sizeof(STableRecordInfo)); + memset(pTableRecordInfo, 0, sizeof(STableRecordInfo)); - char* tempCommand = (char *)malloc(COMMAND_SIZE); - if (tempCommand == NULL) { - fprintf(stderr, "failed to allocate memory\n"); - return -1; - } + char* tempCommand = (char *)malloc(COMMAND_SIZE); + if (tempCommand == NULL) { + errorPrint("%s() LN%d, failed to allocate memory\n", + __func__, __LINE__); + return -1; + } - sprintf(tempCommand, "show tables like %s", table); + sprintf(tempCommand, "show tables like %s", table); - result = taos_query(taosCon, tempCommand); - int32_t code = taos_errno(result); + result = taos_query(taosCon, tempCommand); + int32_t code = taos_errno(result); + + if (code != 0) { + errorPrint("%s() LN%d, failed to run command %s\n", + __func__, __LINE__, tempCommand); + free(tempCommand); + taos_free_result(result); + return -1; + } + + TAOS_FIELD *fields = taos_fetch_fields(result); + + while ((row = taos_fetch_row(result)) != NULL) { + isSet = true; + pTableRecordInfo->isMetric = false; + strncpy(pTableRecordInfo->tableRecord.name, + (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], + fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes); + strncpy(pTableRecordInfo->tableRecord.metric, + (char *)row[TSDB_SHOW_TABLES_METRIC_INDEX], + fields[TSDB_SHOW_TABLES_METRIC_INDEX].bytes); + break; + } - if (code != 0) { - fprintf(stderr, "failed to run command %s\n", tempCommand); - free(tempCommand); taos_free_result(result); - return -1; - } + result = NULL; - TAOS_FIELD *fields = taos_fetch_fields(result); + if (isSet) { + free(tempCommand); + return 0; + } - while ((row = taos_fetch_row(result)) != NULL) { - isSet = true; - pTableRecordInfo->isMetric = false; - strncpy(pTableRecordInfo->tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], - fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes); - strncpy(pTableRecordInfo->tableRecord.metric, (char *)row[TSDB_SHOW_TABLES_METRIC_INDEX], - fields[TSDB_SHOW_TABLES_METRIC_INDEX].bytes); - break; - } + sprintf(tempCommand, "show stables like %s", table); - taos_free_result(result); - result = NULL; + result = taos_query(taosCon, tempCommand); + code = taos_errno(result); - if (isSet) { - free(tempCommand); - return 0; - } + if (code != 0) { + errorPrint("%s() LN%d, failed to run command %s\n", + __func__, __LINE__, tempCommand); + free(tempCommand); + taos_free_result(result); + return -1; + } - sprintf(tempCommand, "show stables like %s", table); + while ((row = taos_fetch_row(result)) != NULL) { + isSet = true; + pTableRecordInfo->isMetric = true; + tstrncpy(pTableRecordInfo->tableRecord.metric, table, + TSDB_TABLE_NAME_LEN); + break; + } - result = taos_query(taosCon, tempCommand); - code = taos_errno(result); - - if (code != 0) { - errorPrint("%s() LN%d, failed to run command %s\n", - __func__, __LINE__, tempCommand); - free(tempCommand); taos_free_result(result); - return -1; - } + result = NULL; - while ((row = taos_fetch_row(result)) != NULL) { - isSet = true; - pTableRecordInfo->isMetric = true; - tstrncpy(pTableRecordInfo->tableRecord.metric, table, TSDB_TABLE_NAME_LEN); - break; - } - - taos_free_result(result); - result = NULL; - - if (isSet) { + if (isSet) { + free(tempCommand); + return 0; + } + errorPrint("%s() LN%d, invalid table/metric %s\n", + __func__, __LINE__, table); free(tempCommand); - return 0; - } - errorPrint("%s() LN%d, invalid table/metric %s\n", - __func__, __LINE__, table); - free(tempCommand); - return -1; + return -1; } static int32_t taosSaveAllNormalTableToTempFile(TAOS *taosCon, char*meter, char* metric, int* fd) { - STableRecord tableRecord; + STableRecord tableRecord; - if (-1 == *fd) { - *fd = open(".tables.tmp.0", O_RDWR | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); - if (*fd == -1) { - fprintf(stderr, "failed to open temp file: .tables.tmp.0\n"); - return -1; + if (-1 == *fd) { + *fd = open(".tables.tmp.0", + O_RDWR | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); + if (*fd == -1) { + errorPrint("%s() LN%d, failed to open temp file: .tables.tmp.0\n", + __func__, __LINE__); + return -1; + } } - } - - memset(&tableRecord, 0, sizeof(STableRecord)); - tstrncpy(tableRecord.name, meter, TSDB_TABLE_NAME_LEN); - tstrncpy(tableRecord.metric, metric, TSDB_TABLE_NAME_LEN); - - taosWrite(*fd, &tableRecord, sizeof(STableRecord)); - return 0; -} - - -static int32_t taosSaveTableOfMetricToTempFile(TAOS *taosCon, char* metric, - struct arguments *arguments, int32_t* totalNumOfThread) { - TAOS_ROW row; - int fd = -1; - STableRecord tableRecord; - - char* tmpCommand = (char *)malloc(COMMAND_SIZE); - if (tmpCommand == NULL) { - fprintf(stderr, "failed to allocate memory\n"); - return -1; - } - - sprintf(tmpCommand, "select tbname from %s", metric); - - TAOS_RES *res = taos_query(taosCon, tmpCommand); - int32_t code = taos_errno(res); - if (code != 0) { - fprintf(stderr, "failed to run command %s\n", tmpCommand); - free(tmpCommand); - taos_free_result(res); - return -1; - } - free(tmpCommand); - - char tmpBuf[TSDB_FILENAME_LEN + 1]; - memset(tmpBuf, 0, TSDB_FILENAME_LEN); - sprintf(tmpBuf, ".select-tbname.tmp"); - fd = open(tmpBuf, O_RDWR | O_CREAT | O_TRUNC, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); - if (fd == -1) { - fprintf(stderr, "failed to open temp file: %s\n", tmpBuf); - taos_free_result(res); - return -1; - } - - TAOS_FIELD *fields = taos_fetch_fields(res); - - int32_t numOfTable = 0; - while ((row = taos_fetch_row(res)) != NULL) { memset(&tableRecord, 0, sizeof(STableRecord)); - tstrncpy(tableRecord.name, (char *)row[0], fields[0].bytes); + tstrncpy(tableRecord.name, meter, TSDB_TABLE_NAME_LEN); tstrncpy(tableRecord.metric, metric, TSDB_TABLE_NAME_LEN); - taosWrite(fd, &tableRecord, sizeof(STableRecord)); - numOfTable++; - } - taos_free_result(res); - lseek(fd, 0, SEEK_SET); - - int maxThreads = arguments->thread_num; - int tableOfPerFile ; - if (numOfTable <= arguments->thread_num) { - tableOfPerFile = 1; - maxThreads = numOfTable; - } else { - tableOfPerFile = numOfTable / arguments->thread_num; - if (0 != numOfTable % arguments->thread_num) { - tableOfPerFile += 1; - } - } - - char* tblBuf = (char*)calloc(1, tableOfPerFile * sizeof(STableRecord)); - if (NULL == tblBuf){ - fprintf(stderr, "failed to calloc %" PRIzu "\n", tableOfPerFile * sizeof(STableRecord)); - close(fd); - return -1; - } - - int32_t numOfThread = *totalNumOfThread; - int subFd = -1; - for (; numOfThread < maxThreads; numOfThread++) { - memset(tmpBuf, 0, TSDB_FILENAME_LEN); - sprintf(tmpBuf, ".tables.tmp.%d", numOfThread); - subFd = open(tmpBuf, O_RDWR | O_CREAT | O_TRUNC, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); - if (subFd == -1) { - fprintf(stderr, "failed to open temp file: %s\n", tmpBuf); - for (int32_t loopCnt = 0; loopCnt < numOfThread; loopCnt++) { - sprintf(tmpBuf, ".tables.tmp.%d", loopCnt); - (void)remove(tmpBuf); - } - sprintf(tmpBuf, ".select-tbname.tmp"); - (void)remove(tmpBuf); - free(tblBuf); - close(fd); - return -1; - } - - // read tableOfPerFile for fd, write to subFd - ssize_t readLen = read(fd, tblBuf, tableOfPerFile * sizeof(STableRecord)); - if (readLen <= 0) { - close(subFd); - break; - } - taosWrite(subFd, tblBuf, readLen); - close(subFd); - } - - sprintf(tmpBuf, ".select-tbname.tmp"); - (void)remove(tmpBuf); - - if (fd >= 0) { - close(fd); - fd = -1; - } - - *totalNumOfThread = numOfThread; - - free(tblBuf); - return 0; + taosWrite(*fd, &tableRecord, sizeof(STableRecord)); + return 0; } -static int taosDumpOut(struct arguments *arguments) { - TAOS *taos = NULL; - TAOS_RES *result = NULL; - char *command = NULL; +static int32_t taosSaveTableOfMetricToTempFile( + TAOS *taosCon, char* metric, + int32_t* totalNumOfThread) { + TAOS_ROW row; + int fd = -1; + STableRecord tableRecord; - TAOS_ROW row; - FILE *fp = NULL; - int32_t count = 0; - STableRecordInfo tableRecordInfo; - - char tmpBuf[TSDB_FILENAME_LEN+9] = {0}; - if (arguments->outpath[0] != 0) { - sprintf(tmpBuf, "%s/dbs.sql", arguments->outpath); - } else { - sprintf(tmpBuf, "dbs.sql"); - } - - fp = fopen(tmpBuf, "w"); - if (fp == NULL) { - fprintf(stderr, "failed to open file %s\n", tmpBuf); - return -1; - } - - dbInfos = (SDbInfo **)calloc(128, sizeof(SDbInfo *)); - if (dbInfos == NULL) { - fprintf(stderr, "failed to allocate memory\n"); - goto _exit_failure; - } - - command = (char *)malloc(COMMAND_SIZE); - if (command == NULL) { - fprintf(stderr, "failed to allocate memory\n"); - goto _exit_failure; - } - - /* Connect to server */ - taos = taos_connect(arguments->host, arguments->user, arguments->password, NULL, arguments->port); - if (taos == NULL) { - fprintf(stderr, "failed to connect to TDengine server\n"); - goto _exit_failure; - } - - /* --------------------------------- Main Code -------------------------------- */ - /* if (arguments->databases || arguments->all_databases) { // dump part of databases or all databases */ - /* */ - taosDumpCharset(fp); - - sprintf(command, "show databases"); - result = taos_query(taos, command); - int32_t code = taos_errno(result); - - if (code != 0) { - errorPrint("%s() LN%d, failed to run command: %s, reason: %s\n", - __func__, __LINE__, command, taos_errstr(result)); - goto _exit_failure; - } - - TAOS_FIELD *fields = taos_fetch_fields(result); - - while ((row = taos_fetch_row(result)) != NULL) { - // sys database name : 'log', but subsequent version changed to 'log' - if ((strncasecmp(row[TSDB_SHOW_DB_NAME_INDEX], "log", - fields[TSDB_SHOW_DB_NAME_INDEX].bytes) == 0) - && (!arguments->allow_sys)) { - continue; - } - - if (arguments->databases) { // input multi dbs - for (int i = 0; arguments->arg_list[i]; i++) { - if (strncasecmp(arguments->arg_list[i], (char *)row[TSDB_SHOW_DB_NAME_INDEX], - fields[TSDB_SHOW_DB_NAME_INDEX].bytes) == 0) - goto _dump_db_point; - } - continue; - } else if (!arguments->all_databases) { // only input one db - if (strncasecmp(arguments->arg_list[0], (char *)row[TSDB_SHOW_DB_NAME_INDEX], - fields[TSDB_SHOW_DB_NAME_INDEX].bytes) == 0) - goto _dump_db_point; - else - continue; + char* tmpCommand = (char *)malloc(COMMAND_SIZE); + if (tmpCommand == NULL) { + errorPrint("%s() LN%d, failed to allocate memory\n", __func__, __LINE__); + return -1; } + sprintf(tmpCommand, "select tbname from %s", metric); + + TAOS_RES *res = taos_query(taosCon, tmpCommand); + int32_t code = taos_errno(res); + if (code != 0) { + errorPrint("%s() LN%d, failed to run command %s\n", + __func__, __LINE__, tmpCommand); + free(tmpCommand); + taos_free_result(res); + return -1; + } + free(tmpCommand); + + char tmpBuf[MAX_FILE_NAME_LEN]; + memset(tmpBuf, 0, MAX_FILE_NAME_LEN); + sprintf(tmpBuf, ".select-tbname.tmp"); + fd = open(tmpBuf, O_RDWR | O_CREAT | O_TRUNC, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); + if (fd == -1) { + errorPrint("%s() LN%d, failed to open temp file: %s\n", + __func__, __LINE__, tmpBuf); + taos_free_result(res); + return -1; + } + + TAOS_FIELD *fields = taos_fetch_fields(res); + + int32_t numOfTable = 0; + while ((row = taos_fetch_row(res)) != NULL) { + + memset(&tableRecord, 0, sizeof(STableRecord)); + tstrncpy(tableRecord.name, (char *)row[0], fields[0].bytes); + tstrncpy(tableRecord.metric, metric, TSDB_TABLE_NAME_LEN); + + taosWrite(fd, &tableRecord, sizeof(STableRecord)); + numOfTable++; + } + taos_free_result(res); + lseek(fd, 0, SEEK_SET); + + int maxThreads = g_args.thread_num; + int tableOfPerFile ; + if (numOfTable <= g_args.thread_num) { + tableOfPerFile = 1; + maxThreads = numOfTable; + } else { + tableOfPerFile = numOfTable / g_args.thread_num; + if (0 != numOfTable % g_args.thread_num) { + tableOfPerFile += 1; + } + } + + char* tblBuf = (char*)calloc(1, tableOfPerFile * sizeof(STableRecord)); + if (NULL == tblBuf){ + errorPrint("%s() LN%d, failed to calloc %" PRIzu "\n", + __func__, __LINE__, tableOfPerFile * sizeof(STableRecord)); + close(fd); + return -1; + } + + int32_t numOfThread = *totalNumOfThread; + int subFd = -1; + for (; numOfThread < maxThreads; numOfThread++) { + memset(tmpBuf, 0, MAX_FILE_NAME_LEN); + sprintf(tmpBuf, ".tables.tmp.%d", numOfThread); + subFd = open(tmpBuf, O_RDWR | O_CREAT | O_TRUNC, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); + if (subFd == -1) { + errorPrint("%s() LN%d, failed to open temp file: %s\n", + __func__, __LINE__, tmpBuf); + for (int32_t loopCnt = 0; loopCnt < numOfThread; loopCnt++) { + sprintf(tmpBuf, ".tables.tmp.%d", loopCnt); + (void)remove(tmpBuf); + } + sprintf(tmpBuf, ".select-tbname.tmp"); + (void)remove(tmpBuf); + free(tblBuf); + close(fd); + return -1; + } + + // read tableOfPerFile for fd, write to subFd + ssize_t readLen = read(fd, tblBuf, tableOfPerFile * sizeof(STableRecord)); + if (readLen <= 0) { + close(subFd); + break; + } + taosWrite(subFd, tblBuf, readLen); + close(subFd); + } + + sprintf(tmpBuf, ".select-tbname.tmp"); + (void)remove(tmpBuf); + + if (fd >= 0) { + close(fd); + fd = -1; + } + + *totalNumOfThread = numOfThread; + + free(tblBuf); + return 0; +} + +static int taosDumpOut() { + TAOS *taos = NULL; + TAOS_RES *result = NULL; + char *command = NULL; + + TAOS_ROW row; + FILE *fp = NULL; + int32_t count = 0; + STableRecordInfo tableRecordInfo; + + char tmpBuf[4096] = {0}; + if (g_args.outpath[0] != 0) { + sprintf(tmpBuf, "%s/dbs.sql", g_args.outpath); + } else { + sprintf(tmpBuf, "dbs.sql"); + } + + fp = fopen(tmpBuf, "w"); + if (fp == NULL) { + errorPrint("%s() LN%d, failed to open file %s\n", + __func__, __LINE__, tmpBuf); + return -1; + } + + g_dbInfos = (SDbInfo **)calloc(128, sizeof(SDbInfo *)); + if (g_dbInfos == NULL) { + errorPrint("%s() LN%d, failed to allocate memory\n", + __func__, __LINE__); + goto _exit_failure; + } + + command = (char *)malloc(COMMAND_SIZE); + if (command == NULL) { + errorPrint("%s() LN%d, failed to allocate memory\n", __func__, __LINE__); + goto _exit_failure; + } + + /* Connect to server */ + taos = taos_connect(g_args.host, g_args.user, g_args.password, + NULL, g_args.port); + if (taos == NULL) { + errorPrint("Failed to connect to TDengine server %s\n", g_args.host); + goto _exit_failure; + } + + /* --------------------------------- Main Code -------------------------------- */ + /* if (g_args.databases || g_args.all_databases) { // dump part of databases or all databases */ + /* */ + taosDumpCharset(fp); + + sprintf(command, "show databases"); + result = taos_query(taos, command); + int32_t code = taos_errno(result); + + if (code != 0) { + errorPrint("%s() LN%d, failed to run command: %s, reason: %s\n", + __func__, __LINE__, command, taos_errstr(result)); + goto _exit_failure; + } + + TAOS_FIELD *fields = taos_fetch_fields(result); + + while ((row = taos_fetch_row(result)) != NULL) { + // sys database name : 'log', but subsequent version changed to 'log' + if ((strncasecmp(row[TSDB_SHOW_DB_NAME_INDEX], "log", + fields[TSDB_SHOW_DB_NAME_INDEX].bytes) == 0) + && (!g_args.allow_sys)) { + continue; + } + + if (g_args.databases) { // input multi dbs + for (int i = 0; g_args.arg_list[i]; i++) { + if (strncasecmp(g_args.arg_list[i], + (char *)row[TSDB_SHOW_DB_NAME_INDEX], + fields[TSDB_SHOW_DB_NAME_INDEX].bytes) == 0) + goto _dump_db_point; + } + continue; + } else if (!g_args.all_databases) { // only input one db + if (strncasecmp(g_args.arg_list[0], + (char *)row[TSDB_SHOW_DB_NAME_INDEX], + fields[TSDB_SHOW_DB_NAME_INDEX].bytes) == 0) + goto _dump_db_point; + else + continue; + } + _dump_db_point: - dbInfos[count] = (SDbInfo *)calloc(1, sizeof(SDbInfo)); - if (dbInfos[count] == NULL) { - errorPrint("%s() LN%d, failed to allocate %"PRIu64" memory\n", - __func__, __LINE__, (uint64_t)(sizeof(SDbInfo))); - goto _exit_failure; - } - - strncpy(dbInfos[count]->name, (char *)row[TSDB_SHOW_DB_NAME_INDEX], - fields[TSDB_SHOW_DB_NAME_INDEX].bytes); - if (arguments->with_property) { - dbInfos[count]->ntables = *((int32_t *)row[TSDB_SHOW_DB_NTABLES_INDEX]); - dbInfos[count]->vgroups = *((int32_t *)row[TSDB_SHOW_DB_VGROUPS_INDEX]); - dbInfos[count]->replica = *((int16_t *)row[TSDB_SHOW_DB_REPLICA_INDEX]); - dbInfos[count]->quorum = *((int16_t *)row[TSDB_SHOW_DB_QUORUM_INDEX]); - dbInfos[count]->days = *((int16_t *)row[TSDB_SHOW_DB_DAYS_INDEX]); - - strncpy(dbInfos[count]->keeplist, (char *)row[TSDB_SHOW_DB_KEEP_INDEX], - fields[TSDB_SHOW_DB_KEEP_INDEX].bytes); - //dbInfos[count]->daysToKeep = *((int16_t *)row[TSDB_SHOW_DB_KEEP_INDEX]); - //dbInfos[count]->daysToKeep1; - //dbInfos[count]->daysToKeep2; - dbInfos[count]->cache = *((int32_t *)row[TSDB_SHOW_DB_CACHE_INDEX]); - dbInfos[count]->blocks = *((int32_t *)row[TSDB_SHOW_DB_BLOCKS_INDEX]); - dbInfos[count]->minrows = *((int32_t *)row[TSDB_SHOW_DB_MINROWS_INDEX]); - dbInfos[count]->maxrows = *((int32_t *)row[TSDB_SHOW_DB_MAXROWS_INDEX]); - dbInfos[count]->wallevel = *((int8_t *)row[TSDB_SHOW_DB_WALLEVEL_INDEX]); - dbInfos[count]->fsync = *((int32_t *)row[TSDB_SHOW_DB_FSYNC_INDEX]); - dbInfos[count]->comp = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_COMP_INDEX])); - dbInfos[count]->cachelast = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_CACHELAST_INDEX])); - - strncpy(dbInfos[count]->precision, (char *)row[TSDB_SHOW_DB_PRECISION_INDEX], - fields[TSDB_SHOW_DB_PRECISION_INDEX].bytes); - //dbInfos[count]->precision = *((int8_t *)row[TSDB_SHOW_DB_PRECISION_INDEX]); - dbInfos[count]->update = *((int8_t *)row[TSDB_SHOW_DB_UPDATE_INDEX]); - } - count++; - - if (arguments->databases) { - if (count > arguments->arg_list_len) break; - - } else if (!arguments->all_databases) { - if (count >= 1) break; - } - } - - if (count == 0) { - fprintf(stderr, "No databases valid to dump\n"); - goto _exit_failure; - } - - if (arguments->databases || arguments->all_databases) { // case: taosdump --databases dbx dby ... OR taosdump --all-databases - for (int i = 0; i < count; i++) { - taosDumpDb(dbInfos[i], arguments, fp, taos); - } - } else { - if (arguments->arg_list_len == 1) { // case: taosdump - taosDumpDb(dbInfos[0], arguments, fp, taos); - } else { // case: taosdump tablex tabley ... - taosDumpCreateDbClause(dbInfos[0], arguments->with_property, fp); - fprintf(g_fpOfResult, "\n#### database: %s\n", dbInfos[0]->name); - g_resultStatistics.totalDatabasesOfDumpOut++; - - sprintf(command, "use %s", dbInfos[0]->name); - - result = taos_query(taos, command); - code = taos_errno(result); - if (code != 0) { - fprintf(stderr, "invalid database %s\n", dbInfos[0]->name); - goto _exit_failure; - } - - fprintf(fp, "USE %s;\n\n", dbInfos[0]->name); - - int32_t totalNumOfThread = 1; // 0: all normal talbe into .tables.tmp.0 - int normalTblFd = -1; - int32_t retCode; - int superTblCnt = 0 ; - for (int i = 1; arguments->arg_list[i]; i++) { - if (taosGetTableRecordInfo(arguments->arg_list[i], &tableRecordInfo, taos) < 0) { - fprintf(stderr, "input the invalide table %s\n", arguments->arg_list[i]); - continue; + g_dbInfos[count] = (SDbInfo *)calloc(1, sizeof(SDbInfo)); + if (g_dbInfos[count] == NULL) { + errorPrint("%s() LN%d, failed to allocate %"PRIu64" memory\n", + __func__, __LINE__, (uint64_t)sizeof(SDbInfo)); + goto _exit_failure; } - if (tableRecordInfo.isMetric) { // dump all table of this metric - int ret = taosDumpStable( - tableRecordInfo.tableRecord.metric, - fp, taos, dbInfos[0]->name); - if (0 == ret) { - superTblCnt++; - } - retCode = taosSaveTableOfMetricToTempFile( - taos, tableRecordInfo.tableRecord.metric, - arguments, &totalNumOfThread); - } else { - if (tableRecordInfo.tableRecord.metric[0] != '\0') { // dump this sub table and it's metric - int ret = taosDumpStable(tableRecordInfo.tableRecord.metric, - fp, taos, dbInfos[0]->name); - if (0 == ret) { - superTblCnt++; + strncpy(g_dbInfos[count]->name, (char *)row[TSDB_SHOW_DB_NAME_INDEX], + fields[TSDB_SHOW_DB_NAME_INDEX].bytes); + if (g_args.with_property) { + g_dbInfos[count]->ntables = *((int32_t *)row[TSDB_SHOW_DB_NTABLES_INDEX]); + g_dbInfos[count]->vgroups = *((int32_t *)row[TSDB_SHOW_DB_VGROUPS_INDEX]); + g_dbInfos[count]->replica = *((int16_t *)row[TSDB_SHOW_DB_REPLICA_INDEX]); + g_dbInfos[count]->quorum = *((int16_t *)row[TSDB_SHOW_DB_QUORUM_INDEX]); + g_dbInfos[count]->days = *((int16_t *)row[TSDB_SHOW_DB_DAYS_INDEX]); + + strncpy(g_dbInfos[count]->keeplist, (char *)row[TSDB_SHOW_DB_KEEP_INDEX], + fields[TSDB_SHOW_DB_KEEP_INDEX].bytes); + //g_dbInfos[count]->daysToKeep = *((int16_t *)row[TSDB_SHOW_DB_KEEP_INDEX]); + //g_dbInfos[count]->daysToKeep1; + //g_dbInfos[count]->daysToKeep2; + g_dbInfos[count]->cache = *((int32_t *)row[TSDB_SHOW_DB_CACHE_INDEX]); + g_dbInfos[count]->blocks = *((int32_t *)row[TSDB_SHOW_DB_BLOCKS_INDEX]); + g_dbInfos[count]->minrows = *((int32_t *)row[TSDB_SHOW_DB_MINROWS_INDEX]); + g_dbInfos[count]->maxrows = *((int32_t *)row[TSDB_SHOW_DB_MAXROWS_INDEX]); + g_dbInfos[count]->wallevel = *((int8_t *)row[TSDB_SHOW_DB_WALLEVEL_INDEX]); + g_dbInfos[count]->fsync = *((int32_t *)row[TSDB_SHOW_DB_FSYNC_INDEX]); + g_dbInfos[count]->comp = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_COMP_INDEX])); + g_dbInfos[count]->cachelast = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_CACHELAST_INDEX])); + + strncpy(g_dbInfos[count]->precision, (char *)row[TSDB_SHOW_DB_PRECISION_INDEX], + fields[TSDB_SHOW_DB_PRECISION_INDEX].bytes); + //g_dbInfos[count]->precision = *((int8_t *)row[TSDB_SHOW_DB_PRECISION_INDEX]); + g_dbInfos[count]->update = *((int8_t *)row[TSDB_SHOW_DB_UPDATE_INDEX]); + } + count++; + + if (g_args.databases) { + if (count > g_args.arg_list_len) break; + + } else if (!g_args.all_databases) { + if (count >= 1) break; + } + } + + if (count == 0) { + errorPrint("%d databases valid to dump\n", count); + goto _exit_failure; + } + + if (g_args.databases || g_args.all_databases) { // case: taosdump --databases dbx dby ... OR taosdump --all-databases + for (int i = 0; i < count; i++) { + taosDumpDb(g_dbInfos[i], fp, taos); + } + } else { + if (g_args.arg_list_len == 1) { // case: taosdump + taosDumpDb(g_dbInfos[0], fp, taos); + } else { // case: taosdump tablex tabley ... + taosDumpCreateDbClause(g_dbInfos[0], g_args.with_property, fp); + fprintf(g_fpOfResult, "\n#### database: %s\n", + g_dbInfos[0]->name); + g_resultStatistics.totalDatabasesOfDumpOut++; + + sprintf(command, "use %s", g_dbInfos[0]->name); + + result = taos_query(taos, command); + code = taos_errno(result); + if (code != 0) { + errorPrint("invalid database %s\n", g_dbInfos[0]->name); + goto _exit_failure; + } + + fprintf(fp, "USE %s;\n\n", g_dbInfos[0]->name); + + int32_t totalNumOfThread = 1; // 0: all normal talbe into .tables.tmp.0 + int normalTblFd = -1; + int32_t retCode; + int superTblCnt = 0 ; + for (int i = 1; g_args.arg_list[i]; i++) { + if (taosGetTableRecordInfo(g_args.arg_list[i], + &tableRecordInfo, taos) < 0) { + errorPrint("input the invalide table %s\n", + g_args.arg_list[i]); + continue; + } + + if (tableRecordInfo.isMetric) { // dump all table of this metric + int ret = taosDumpStable( + tableRecordInfo.tableRecord.metric, + fp, taos, g_dbInfos[0]->name); + if (0 == ret) { + superTblCnt++; + } + retCode = taosSaveTableOfMetricToTempFile( + taos, tableRecordInfo.tableRecord.metric, + &totalNumOfThread); + } else { + if (tableRecordInfo.tableRecord.metric[0] != '\0') { // dump this sub table and it's metric + int ret = taosDumpStable( + tableRecordInfo.tableRecord.metric, + fp, taos, g_dbInfos[0]->name); + if (0 == ret) { + superTblCnt++; + } + } + retCode = taosSaveAllNormalTableToTempFile( + taos, tableRecordInfo.tableRecord.name, + tableRecordInfo.tableRecord.metric, &normalTblFd); + } + + if (retCode < 0) { + if (-1 != normalTblFd){ + taosClose(normalTblFd); + } + goto _clean_tmp_file; + } + } + + // TODO: save dump super table into result_output.txt + fprintf(g_fpOfResult, "# super table counter: %d\n", + superTblCnt); + g_resultStatistics.totalSuperTblsOfDumpOut += superTblCnt; + + if (-1 != normalTblFd){ + taosClose(normalTblFd); + } + + // start multi threads to dumpout + taosStartDumpOutWorkThreads(totalNumOfThread, + g_dbInfos[0]->name); + + char tmpFileName[MAX_FILE_NAME_LEN]; +_clean_tmp_file: + for (int loopCnt = 0; loopCnt < totalNumOfThread; loopCnt++) { + sprintf(tmpFileName, ".tables.tmp.%d", loopCnt); + remove(tmpFileName); } - } - retCode = taosSaveAllNormalTableToTempFile( - taos, tableRecordInfo.tableRecord.name, - tableRecordInfo.tableRecord.metric, &normalTblFd); } - - if (retCode < 0) { - if (-1 != normalTblFd){ - taosClose(normalTblFd); - } - goto _clean_tmp_file; - } - } - - // TODO: save dump super table into result_output.txt - fprintf(g_fpOfResult, "# super table counter: %d\n", superTblCnt); - g_resultStatistics.totalSuperTblsOfDumpOut += superTblCnt; - - if (-1 != normalTblFd){ - taosClose(normalTblFd); - } - - // start multi threads to dumpout - taosStartDumpOutWorkThreads(taos, arguments, totalNumOfThread, dbInfos[0]->name); - - char tmpFileName[TSDB_FILENAME_LEN + 1]; - _clean_tmp_file: - for (int loopCnt = 0; loopCnt < totalNumOfThread; loopCnt++) { - sprintf(tmpFileName, ".tables.tmp.%d", loopCnt); - remove(tmpFileName); - } } - } - /* Close the handle and return */ - fclose(fp); - taos_close(taos); - taos_free_result(result); - tfree(command); - taosFreeDbInfos(); - fprintf(stderr, "dump out rows: %" PRId64 "\n", totalDumpOutRows); - return 0; + /* Close the handle and return */ + fclose(fp); + taos_close(taos); + taos_free_result(result); + tfree(command); + taosFreeDbInfos(); + fprintf(stderr, "dump out rows: %" PRId64 "\n", g_totalDumpOutRows); + return 0; _exit_failure: - fclose(fp); - taos_close(taos); - taos_free_result(result); - tfree(command); - taosFreeDbInfos(); - fprintf(stderr, "dump out rows: %" PRId64 "\n", totalDumpOutRows); - return -1; + fclose(fp); + taos_close(taos); + taos_free_result(result); + tfree(command); + taosFreeDbInfos(); + errorPrint("dump out rows: %" PRId64 "\n", g_totalDumpOutRows); + return -1; } static int taosGetTableDes( char* dbName, char *table, STableDef *tableDes, TAOS* taosCon, bool isSuperTable) { - TAOS_ROW row = NULL; - TAOS_RES* res = NULL; - int count = 0; + TAOS_ROW row = NULL; + TAOS_RES* res = NULL; + int count = 0; - char sqlstr[COMMAND_SIZE]; - sprintf(sqlstr, "describe %s.%s;", dbName, table); - - res = taos_query(taosCon, sqlstr); - int32_t code = taos_errno(res); - if (code != 0) { - errorPrint("%s() LN%d, failed to run command <%s>, reason:%s\n", - __func__, __LINE__, sqlstr, taos_errstr(res)); - taos_free_result(res); - return -1; - } - - TAOS_FIELD *fields = taos_fetch_fields(res); - - tstrncpy(tableDes->name, table, TSDB_TABLE_NAME_LEN); - while ((row = taos_fetch_row(res)) != NULL) { - strncpy(tableDes->cols[count].field, (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX], - fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes); - strncpy(tableDes->cols[count].type, (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], - min(15, fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes)); - tableDes->cols[count].length = *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]); - strncpy(tableDes->cols[count].note, (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], - fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes); - - count++; - } - - taos_free_result(res); - res = NULL; - - if (isSuperTable) { - return count; - } - - // if chidl-table have tag, using select tagName from table to get tagValue - for (int i = 0 ; i < count; i++) { - if (strcmp(tableDes->cols[i].note, "TAG") != 0) continue; - - - sprintf(sqlstr, "select %s from %s.%s", tableDes->cols[i].field, dbName, table); + char sqlstr[COMMAND_SIZE]; + sprintf(sqlstr, "describe %s.%s;", dbName, table); res = taos_query(taosCon, sqlstr); - code = taos_errno(res); + int32_t code = taos_errno(res); if (code != 0) { - errorPrint("%s() LN%d, failed to run command <%s>, reason:%s\n", - __func__, __LINE__, sqlstr, taos_errstr(res)); - taos_free_result(res); - return -1; + errorPrint("%s() LN%d, failed to run command <%s>, reason:%s\n", + __func__, __LINE__, sqlstr, taos_errstr(res)); + taos_free_result(res); + return -1; } - fields = taos_fetch_fields(res); + TAOS_FIELD *fields = taos_fetch_fields(res); - row = taos_fetch_row(res); - if (NULL == row) { - errorPrint("%s() LN%d, fetch failed to run command <%s>, reason:%s\n", - __func__, __LINE__, sqlstr, taos_errstr(res)); - taos_free_result(res); - return -1; - } + tstrncpy(tableDes->name, table, TSDB_TABLE_NAME_LEN); + while ((row = taos_fetch_row(res)) != NULL) { + strncpy(tableDes->cols[count].field, + (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX], + fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes); + strncpy(tableDes->cols[count].type, + (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], + min(15, fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes)); + tableDes->cols[count].length = + *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]); + strncpy(tableDes->cols[count].note, + (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], + fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes); - if (row[0] == NULL) { - sprintf(tableDes->cols[i].note, "%s", "NULL"); - taos_free_result(res); - res = NULL; - continue; - } - - int32_t* length = taos_fetch_lengths(res); - - //int32_t* length = taos_fetch_lengths(tmpResult); - switch (fields[0].type) { - case TSDB_DATA_TYPE_BOOL: - sprintf(tableDes->cols[i].note, "%d", ((((int32_t)(*((char *)row[0]))) == 1) ? 1 : 0)); - break; - case TSDB_DATA_TYPE_TINYINT: - sprintf(tableDes->cols[i].note, "%d", *((int8_t *)row[0])); - break; - case TSDB_DATA_TYPE_SMALLINT: - sprintf(tableDes->cols[i].note, "%d", *((int16_t *)row[0])); - break; - case TSDB_DATA_TYPE_INT: - sprintf(tableDes->cols[i].note, "%d", *((int32_t *)row[0])); - break; - case TSDB_DATA_TYPE_BIGINT: - sprintf(tableDes->cols[i].note, "%" PRId64 "", *((int64_t *)row[0])); - break; - case TSDB_DATA_TYPE_FLOAT: - sprintf(tableDes->cols[i].note, "%f", GET_FLOAT_VAL(row[0])); - break; - case TSDB_DATA_TYPE_DOUBLE: - sprintf(tableDes->cols[i].note, "%f", GET_DOUBLE_VAL(row[0])); - break; - case TSDB_DATA_TYPE_BINARY: { - memset(tableDes->cols[i].note, 0, sizeof(tableDes->cols[i].note)); - tableDes->cols[i].note[0] = '\''; - char tbuf[COL_NOTE_LEN]; - converStringToReadable((char *)row[0], length[0], tbuf, COL_NOTE_LEN); - char* pstr = stpcpy(&(tableDes->cols[i].note[1]), tbuf); - *(pstr++) = '\''; - break; - } - case TSDB_DATA_TYPE_NCHAR: { - memset(tableDes->cols[i].note, 0, sizeof(tableDes->cols[i].note)); - char tbuf[COL_NOTE_LEN-2]; // need reserve 2 bytes for ' ' - convertNCharToReadable((char *)row[0], length[0], tbuf, COL_NOTE_LEN); - sprintf(tableDes->cols[i].note, "\'%s\'", tbuf); - break; - } - case TSDB_DATA_TYPE_TIMESTAMP: - sprintf(tableDes->cols[i].note, "%" PRId64 "", *(int64_t *)row[0]); - #if 0 - if (!arguments->mysqlFlag) { - sprintf(tableDes->cols[i].note, "%" PRId64 "", *(int64_t *)row[0]); - } else { - char buf[64] = "\0"; - int64_t ts = *((int64_t *)row[0]); - time_t tt = (time_t)(ts / 1000); - struct tm *ptm = localtime(&tt); - strftime(buf, 64, "%y-%m-%d %H:%M:%S", ptm); - sprintf(tableDes->cols[i].note, "\'%s.%03d\'", buf, (int)(ts % 1000)); - } - #endif - break; - default: - break; + count++; } taos_free_result(res); res = NULL; - } - return count; + if (isSuperTable) { + return count; + } + + // if chidl-table have tag, using select tagName from table to get tagValue + for (int i = 0 ; i < count; i++) { + if (strcmp(tableDes->cols[i].note, "TAG") != 0) continue; + + + sprintf(sqlstr, "select %s from %s.%s", + tableDes->cols[i].field, dbName, table); + + res = taos_query(taosCon, sqlstr); + code = taos_errno(res); + if (code != 0) { + errorPrint("%s() LN%d, failed to run command <%s>, reason:%s\n", + __func__, __LINE__, sqlstr, taos_errstr(res)); + taos_free_result(res); + return -1; + } + + fields = taos_fetch_fields(res); + + row = taos_fetch_row(res); + if (NULL == row) { + errorPrint("%s() LN%d, fetch failed to run command <%s>, reason:%s\n", + __func__, __LINE__, sqlstr, taos_errstr(res)); + taos_free_result(res); + return -1; + } + + if (row[0] == NULL) { + sprintf(tableDes->cols[i].note, "%s", "NULL"); + taos_free_result(res); + res = NULL; + continue; + } + + int32_t* length = taos_fetch_lengths(res); + + //int32_t* length = taos_fetch_lengths(tmpResult); + switch (fields[0].type) { + case TSDB_DATA_TYPE_BOOL: + sprintf(tableDes->cols[i].note, "%d", + ((((int32_t)(*((char *)row[0]))) == 1) ? 1 : 0)); + break; + case TSDB_DATA_TYPE_TINYINT: + sprintf(tableDes->cols[i].note, "%d", *((int8_t *)row[0])); + break; + case TSDB_DATA_TYPE_SMALLINT: + sprintf(tableDes->cols[i].note, "%d", *((int16_t *)row[0])); + break; + case TSDB_DATA_TYPE_INT: + sprintf(tableDes->cols[i].note, "%d", *((int32_t *)row[0])); + break; + case TSDB_DATA_TYPE_BIGINT: + sprintf(tableDes->cols[i].note, "%" PRId64 "", *((int64_t *)row[0])); + break; + case TSDB_DATA_TYPE_FLOAT: + sprintf(tableDes->cols[i].note, "%f", GET_FLOAT_VAL(row[0])); + break; + case TSDB_DATA_TYPE_DOUBLE: + sprintf(tableDes->cols[i].note, "%f", GET_DOUBLE_VAL(row[0])); + break; + case TSDB_DATA_TYPE_BINARY: + { + memset(tableDes->cols[i].note, 0, sizeof(tableDes->cols[i].note)); + tableDes->cols[i].note[0] = '\''; + char tbuf[COL_NOTE_LEN]; + converStringToReadable((char *)row[0], length[0], tbuf, COL_NOTE_LEN); + char* pstr = stpcpy(&(tableDes->cols[i].note[1]), tbuf); + *(pstr++) = '\''; + break; + } + case TSDB_DATA_TYPE_NCHAR: + { + memset(tableDes->cols[i].note, 0, sizeof(tableDes->cols[i].note)); + char tbuf[COL_NOTE_LEN-2]; // need reserve 2 bytes for ' ' + convertNCharToReadable((char *)row[0], length[0], tbuf, COL_NOTE_LEN); + sprintf(tableDes->cols[i].note, "\'%s\'", tbuf); + break; + } + case TSDB_DATA_TYPE_TIMESTAMP: + sprintf(tableDes->cols[i].note, "%" PRId64 "", *(int64_t *)row[0]); +#if 0 + if (!g_args.mysqlFlag) { + sprintf(tableDes->cols[i].note, "%" PRId64 "", *(int64_t *)row[0]); + } else { + char buf[64] = "\0"; + int64_t ts = *((int64_t *)row[0]); + time_t tt = (time_t)(ts / 1000); + struct tm *ptm = localtime(&tt); + strftime(buf, 64, "%y-%m-%d %H:%M:%S", ptm); + sprintf(tableDes->cols[i].note, "\'%s.%03d\'", buf, (int)(ts % 1000)); + } +#endif + break; + default: + break; + } + + taos_free_result(res); + res = NULL; + } + + return count; +} + +static int convertSchemaToAvroSchema(STableDef *tableDes, char **avroSchema) +{ + errorPrint("%s() LN%d TODO: covert table schema to avro schema\n", + __func__, __LINE__); + return 0; } static int32_t taosDumpTable( - char *table, char *metric, struct arguments *arguments, + char *table, char *metric, FILE *fp, TAOS* taosCon, char* dbName) { - int count = 0; + int count = 0; - STableDef *tableDes = (STableDef *)calloc(1, sizeof(STableDef) - + sizeof(SColDes) * TSDB_MAX_COLUMNS); + STableDef *tableDes = (STableDef *)calloc(1, sizeof(STableDef) + + sizeof(SColDes) * TSDB_MAX_COLUMNS); - if (metric != NULL && metric[0] != '\0') { // dump table schema which is created by using super table - /* - count = taosGetTableDes(metric, tableDes, taosCon); + if (metric != NULL && metric[0] != '\0') { // dump table schema which is created by using super table + /* + count = taosGetTableDes(metric, tableDes, taosCon); - if (count < 0) { - free(tableDes); - return -1; + if (count < 0) { + free(tableDes); + return -1; + } + + taosDumpCreateTableClause(tableDes, count, fp); + + memset(tableDes, 0, sizeof(STableDef) + sizeof(SColDes) * TSDB_MAX_COLUMNS); + */ + + count = taosGetTableDes(dbName, table, tableDes, taosCon, false); + + if (count < 0) { + free(tableDes); + return -1; + } + + // create child-table using super-table + taosDumpCreateMTableClause(tableDes, metric, count, fp, dbName); + + } else { // dump table definition + count = taosGetTableDes(dbName, table, tableDes, taosCon, false); + + if (count < 0) { + free(tableDes); + return -1; + } + + // create normal-table or super-table + taosDumpCreateTableClause(tableDes, count, fp, dbName); } - taosDumpCreateTableClause(tableDes, count, fp); - - memset(tableDes, 0, sizeof(STableDef) + sizeof(SColDes) * TSDB_MAX_COLUMNS); - */ - - count = taosGetTableDes(dbName, table, tableDes, taosCon, false); - - if (count < 0) { - free(tableDes); - return -1; + char *jsonAvroSchema = NULL; + if (g_args.avro) { + convertSchemaToAvroSchema(tableDes, &jsonAvroSchema); } - // create child-table using super-table - taosDumpCreateMTableClause(tableDes, metric, count, fp, dbName); + free(tableDes); - } else { // dump table definition - count = taosGetTableDes(dbName, table, tableDes, taosCon, false); - - if (count < 0) { - free(tableDes); - return -1; + int32_t ret = 0; + if (!g_args.schemaonly) { + ret = taosDumpTableData(fp, table, taosCon, dbName, + jsonAvroSchema); } - // create normal-table or super-table - taosDumpCreateTableClause(tableDes, count, fp, dbName); - } - - free(tableDes); - - return taosDumpTableData(fp, table, arguments, taosCon, dbName); + return ret; } -static void taosDumpCreateDbClause(SDbInfo *dbInfo, bool isDumpProperty, FILE *fp) { - char sqlstr[TSDB_MAX_SQL_LEN] = {0}; +static void taosDumpCreateDbClause( + SDbInfo *dbInfo, bool isDumpProperty, FILE *fp) { + char sqlstr[TSDB_MAX_SQL_LEN] = {0}; - char *pstr = sqlstr; - pstr += sprintf(pstr, "CREATE DATABASE IF NOT EXISTS %s ", dbInfo->name); - if (isDumpProperty) { - pstr += sprintf(pstr, - "REPLICA %d QUORUM %d DAYS %d KEEP %s CACHE %d BLOCKS %d MINROWS %d MAXROWS %d FSYNC %d CACHELAST %d COMP %d PRECISION '%s' UPDATE %d", - dbInfo->replica, dbInfo->quorum, dbInfo->days, dbInfo->keeplist, dbInfo->cache, - dbInfo->blocks, dbInfo->minrows, dbInfo->maxrows, dbInfo->fsync, dbInfo->cachelast, - dbInfo->comp, dbInfo->precision, dbInfo->update); - } + char *pstr = sqlstr; + pstr += sprintf(pstr, "CREATE DATABASE IF NOT EXISTS %s ", dbInfo->name); + if (isDumpProperty) { + pstr += sprintf(pstr, + "REPLICA %d QUORUM %d DAYS %d KEEP %s CACHE %d BLOCKS %d MINROWS %d MAXROWS %d FSYNC %d CACHELAST %d COMP %d PRECISION '%s' UPDATE %d", + dbInfo->replica, dbInfo->quorum, dbInfo->days, + dbInfo->keeplist, + dbInfo->cache, + dbInfo->blocks, dbInfo->minrows, dbInfo->maxrows, + dbInfo->fsync, + dbInfo->cachelast, + dbInfo->comp, dbInfo->precision, dbInfo->update); + } - pstr += sprintf(pstr, ";"); - fprintf(fp, "%s\n\n", sqlstr); + pstr += sprintf(pstr, ";"); + fprintf(fp, "%s\n\n", sqlstr); } static void* taosDumpOutWorkThreadFp(void *arg) { - SThreadParaObj *pThread = (SThreadParaObj*)arg; - STableRecord tableRecord; - int fd; + SThreadParaObj *pThread = (SThreadParaObj*)arg; + STableRecord tableRecord; + int fd; - char tmpBuf[TSDB_FILENAME_LEN*4] = {0}; - sprintf(tmpBuf, ".tables.tmp.%d", pThread->threadIndex); - fd = open(tmpBuf, O_RDWR | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); - if (fd == -1) { - fprintf(stderr, "taosDumpTableFp() failed to open temp file: %s\n", tmpBuf); - return NULL; - } - - FILE *fp = NULL; - memset(tmpBuf, 0, TSDB_FILENAME_LEN + 128); - - if (g_args.outpath[0] != 0) { - sprintf(tmpBuf, "%s/%s.tables.%d.sql", g_args.outpath, pThread->dbName, pThread->threadIndex); - } else { - sprintf(tmpBuf, "%s.tables.%d.sql", pThread->dbName, pThread->threadIndex); - } - - fp = fopen(tmpBuf, "w"); - if (fp == NULL) { - fprintf(stderr, "failed to open file %s\n", tmpBuf); - close(fd); - return NULL; - } - - memset(tmpBuf, 0, TSDB_FILENAME_LEN); - sprintf(tmpBuf, "use %s", pThread->dbName); - - TAOS_RES* tmpResult = taos_query(pThread->taosCon, tmpBuf); - int32_t code = taos_errno(tmpResult); - if (code != 0) { - errorPrint("%s() LN%d, invalid database %s. reason: %s\n", - __func__, __LINE__, pThread->dbName, taos_errstr(tmpResult)); - taos_free_result(tmpResult); - fclose(fp); - close(fd); - return NULL; - } - - int fileNameIndex = 1; - int tablesInOneFile = 0; - int64_t lastRowsPrint = 5000000; - fprintf(fp, "USE %s;\n\n", pThread->dbName); - while (1) { - ssize_t readLen = read(fd, &tableRecord, sizeof(STableRecord)); - if (readLen <= 0) break; - - int ret = taosDumpTable( - tableRecord.name, tableRecord.metric, &g_args, - fp, pThread->taosCon, pThread->dbName); - if (ret >= 0) { - // TODO: sum table count and table rows by self - pThread->tablesOfDumpOut++; - pThread->rowsOfDumpOut += ret; - - if (pThread->rowsOfDumpOut >= lastRowsPrint) { - printf(" %"PRId64 " rows already be dumpout from database %s\n", - pThread->rowsOfDumpOut, pThread->dbName); - lastRowsPrint += 5000000; - } - - tablesInOneFile++; - if (tablesInOneFile >= g_args.table_batch) { - fclose(fp); - tablesInOneFile = 0; - - memset(tmpBuf, 0, TSDB_FILENAME_LEN + 128); - if (g_args.outpath[0] != 0) { - sprintf(tmpBuf, "%s/%s.tables.%d-%d.sql", - g_args.outpath, pThread->dbName, - pThread->threadIndex, fileNameIndex); - } else { - sprintf(tmpBuf, "%s.tables.%d-%d.sql", - pThread->dbName, pThread->threadIndex, fileNameIndex); - } - fileNameIndex++; - - fp = fopen(tmpBuf, "w"); - if (fp == NULL) { - fprintf(stderr, "failed to open file %s\n", tmpBuf); - close(fd); - taos_free_result(tmpResult); - return NULL; - } - } + char tmpBuf[4096] = {0}; + sprintf(tmpBuf, ".tables.tmp.%d", pThread->threadIndex); + fd = open(tmpBuf, O_RDWR | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); + if (fd == -1) { + errorPrint("%s() LN%d, failed to open temp file: %s\n", + __func__, __LINE__, tmpBuf); + return NULL; } - } - taos_free_result(tmpResult); - close(fd); - fclose(fp); + FILE *fp = NULL; + memset(tmpBuf, 0, 4096); - return NULL; + if (g_args.outpath[0] != 0) { + sprintf(tmpBuf, "%s/%s.tables.%d.sql", + g_args.outpath, pThread->dbName, pThread->threadIndex); + } else { + sprintf(tmpBuf, "%s.tables.%d.sql", + pThread->dbName, pThread->threadIndex); + } + + fp = fopen(tmpBuf, "w"); + if (fp == NULL) { + errorPrint("%s() LN%d, failed to open file %s\n", + __func__, __LINE__, tmpBuf); + close(fd); + return NULL; + } + + memset(tmpBuf, 0, 4096); + sprintf(tmpBuf, "use %s", pThread->dbName); + + TAOS_RES* tmpResult = taos_query(pThread->taosCon, tmpBuf); + int32_t code = taos_errno(tmpResult); + if (code != 0) { + errorPrint("%s() LN%d, invalid database %s. reason: %s\n", + __func__, __LINE__, pThread->dbName, taos_errstr(tmpResult)); + taos_free_result(tmpResult); + fclose(fp); + close(fd); + return NULL; + } + +#if 0 + int fileNameIndex = 1; + int tablesInOneFile = 0; +#endif + int64_t lastRowsPrint = 5000000; + fprintf(fp, "USE %s;\n\n", pThread->dbName); + while (1) { + ssize_t readLen = read(fd, &tableRecord, sizeof(STableRecord)); + if (readLen <= 0) break; + + int ret = taosDumpTable( + tableRecord.name, tableRecord.metric, + fp, pThread->taosCon, pThread->dbName); + if (ret >= 0) { + // TODO: sum table count and table rows by self + pThread->tablesOfDumpOut++; + pThread->rowsOfDumpOut += ret; + + if (pThread->rowsOfDumpOut >= lastRowsPrint) { + printf(" %"PRId64 " rows already be dumpout from database %s\n", + pThread->rowsOfDumpOut, pThread->dbName); + lastRowsPrint += 5000000; + } + +#if 0 + tablesInOneFile++; + if (tablesInOneFile >= g_args.table_batch) { + fclose(fp); + tablesInOneFile = 0; + + memset(tmpBuf, 0, 4096); + if (g_args.outpath[0] != 0) { + sprintf(tmpBuf, "%s/%s.tables.%d-%d.sql", + g_args.outpath, pThread->dbName, + pThread->threadIndex, fileNameIndex); + } else { + sprintf(tmpBuf, "%s.tables.%d-%d.sql", + pThread->dbName, pThread->threadIndex, fileNameIndex); + } + fileNameIndex++; + + fp = fopen(tmpBuf, "w"); + if (fp == NULL) { + errorPrint("%s() LN%d, failed to open file %s\n", + __func__, __LINE__, tmpBuf); + close(fd); + taos_free_result(tmpResult); + return NULL; + } + } +#endif + } + } + + taos_free_result(tmpResult); + close(fd); + fclose(fp); + + return NULL; } -static void taosStartDumpOutWorkThreads(void* taosCon, struct arguments* args, - int32_t numOfThread, char *dbName) +static void taosStartDumpOutWorkThreads(int32_t numOfThread, char *dbName) { - pthread_attr_t thattr; - SThreadParaObj *threadObj = - (SThreadParaObj *)calloc(numOfThread, sizeof(SThreadParaObj)); - for (int t = 0; t < numOfThread; ++t) { - SThreadParaObj *pThread = threadObj + t; - pThread->rowsOfDumpOut = 0; - pThread->tablesOfDumpOut = 0; - pThread->threadIndex = t; - pThread->totalThreads = numOfThread; - tstrncpy(pThread->dbName, dbName, TSDB_DB_NAME_LEN); - pThread->taosCon = taosCon; + pthread_attr_t thattr; + SThreadParaObj *threadObj = + (SThreadParaObj *)calloc(numOfThread, sizeof(SThreadParaObj)); - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); - - if (pthread_create(&(pThread->threadID), &thattr, taosDumpOutWorkThreadFp, - (void*)pThread) != 0) { - errorPrint("thread:%d failed to start\n", pThread->threadIndex); - exit(-1); + if (threadObj == NULL) { + errorPrint("%s() LN%d, memory allocation failed!\n", + __func__, __LINE__); + return; } - } - for (int32_t t = 0; t < numOfThread; ++t) { - pthread_join(threadObj[t].threadID, NULL); - } + for (int t = 0; t < numOfThread; ++t) { + SThreadParaObj *pThread = threadObj + t; + pThread->rowsOfDumpOut = 0; + pThread->tablesOfDumpOut = 0; + pThread->threadIndex = t; + pThread->totalThreads = numOfThread; + tstrncpy(pThread->dbName, dbName, TSDB_DB_NAME_LEN); + pThread->taosCon = taos_connect(g_args.host, g_args.user, g_args.password, + NULL, g_args.port); + if (pThread->taosCon == NULL) { + errorPrint("Failed to connect to TDengine server %s\n", g_args.host); + return; + } + pthread_attr_init(&thattr); + pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); - // TODO: sum all thread dump table count and rows of per table, then save into result_output.txt - int64_t totalRowsOfDumpOut = 0; - int64_t totalChildTblsOfDumpOut = 0; - for (int32_t t = 0; t < numOfThread; ++t) { - totalChildTblsOfDumpOut += threadObj[t].tablesOfDumpOut; - totalRowsOfDumpOut += threadObj[t].rowsOfDumpOut; - } + if (pthread_create(&(pThread->threadID), &thattr, + taosDumpOutWorkThreadFp, + (void*)pThread) != 0) { + errorPrint("%s() LN%d, thread:%d failed to start\n", + __func__, __LINE__, pThread->threadIndex); + exit(-1); + } + } - fprintf(g_fpOfResult, "# child table counter: %"PRId64"\n", totalChildTblsOfDumpOut); - fprintf(g_fpOfResult, "# row counter: %"PRId64"\n", totalRowsOfDumpOut); - g_resultStatistics.totalChildTblsOfDumpOut += totalChildTblsOfDumpOut; - g_resultStatistics.totalRowsOfDumpOut += totalRowsOfDumpOut; - free(threadObj); + for (int32_t t = 0; t < numOfThread; ++t) { + pthread_join(threadObj[t].threadID, NULL); + } + + // TODO: sum all thread dump table count and rows of per table, then save into result_output.txt + int64_t totalRowsOfDumpOut = 0; + int64_t totalChildTblsOfDumpOut = 0; + for (int32_t t = 0; t < numOfThread; ++t) { + totalChildTblsOfDumpOut += threadObj[t].tablesOfDumpOut; + totalRowsOfDumpOut += threadObj[t].rowsOfDumpOut; + } + + fprintf(g_fpOfResult, "# child table counter: %"PRId64"\n", + totalChildTblsOfDumpOut); + fprintf(g_fpOfResult, "# row counter: %"PRId64"\n", + totalRowsOfDumpOut); + g_resultStatistics.totalChildTblsOfDumpOut += totalChildTblsOfDumpOut; + g_resultStatistics.totalRowsOfDumpOut += totalRowsOfDumpOut; + free(threadObj); } static int32_t taosDumpStable(char *table, FILE *fp, @@ -1502,186 +1598,191 @@ static int32_t taosDumpStable(char *table, FILE *fp, static int32_t taosDumpCreateSuperTableClause(TAOS* taosCon, char* dbName, FILE *fp) { - TAOS_ROW row; - int fd = -1; - STableRecord tableRecord; - char sqlstr[TSDB_MAX_SQL_LEN] = {0}; + TAOS_ROW row; + int fd = -1; + STableRecord tableRecord; + char sqlstr[TSDB_MAX_SQL_LEN] = {0}; - sprintf(sqlstr, "show %s.stables", dbName); + sprintf(sqlstr, "show %s.stables", dbName); - TAOS_RES* res = taos_query(taosCon, sqlstr); - int32_t code = taos_errno(res); - if (code != 0) { - fprintf(stderr, "failed to run command <%s>, reason: %s\n", sqlstr, taos_errstr(res)); - taos_free_result(res); - exit(-1); - } - - TAOS_FIELD *fields = taos_fetch_fields(res); - - char tmpFileName[TSDB_FILENAME_LEN + 1]; - memset(tmpFileName, 0, TSDB_FILENAME_LEN); - sprintf(tmpFileName, ".stables.tmp"); - fd = open(tmpFileName, O_RDWR | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); - if (fd == -1) { - fprintf(stderr, "failed to open temp file: %s\n", tmpFileName); - taos_free_result(res); - (void)remove(".stables.tmp"); - exit(-1); - } - - while ((row = taos_fetch_row(res)) != NULL) { - memset(&tableRecord, 0, sizeof(STableRecord)); - strncpy(tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], - fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes); - taosWrite(fd, &tableRecord, sizeof(STableRecord)); - } - - taos_free_result(res); - (void)lseek(fd, 0, SEEK_SET); - - int superTblCnt = 0; - while (1) { - ssize_t readLen = read(fd, &tableRecord, sizeof(STableRecord)); - if (readLen <= 0) break; - - int ret = taosDumpStable(tableRecord.name, fp, taosCon, dbName); - if (0 == ret) { - superTblCnt++; + TAOS_RES* res = taos_query(taosCon, sqlstr); + int32_t code = taos_errno(res); + if (code != 0) { + errorPrint("%s() LN%d, failed to run command <%s>, reason: %s\n", + __func__, __LINE__, sqlstr, taos_errstr(res)); + taos_free_result(res); + exit(-1); } - } - // TODO: save dump super table into result_output.txt - fprintf(g_fpOfResult, "# super table counter: %d\n", superTblCnt); - g_resultStatistics.totalSuperTblsOfDumpOut += superTblCnt; + TAOS_FIELD *fields = taos_fetch_fields(res); - close(fd); - (void)remove(".stables.tmp"); + char tmpFileName[MAX_FILE_NAME_LEN]; + memset(tmpFileName, 0, MAX_FILE_NAME_LEN); + sprintf(tmpFileName, ".stables.tmp"); + fd = open(tmpFileName, O_RDWR | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); + if (fd == -1) { + errorPrint("%s() LN%d, failed to open temp file: %s\n", + __func__, __LINE__, tmpFileName); + taos_free_result(res); + (void)remove(".stables.tmp"); + exit(-1); + } - return 0; + while ((row = taos_fetch_row(res)) != NULL) { + memset(&tableRecord, 0, sizeof(STableRecord)); + strncpy(tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], + fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes); + taosWrite(fd, &tableRecord, sizeof(STableRecord)); + } + + taos_free_result(res); + (void)lseek(fd, 0, SEEK_SET); + + int superTblCnt = 0; + while (1) { + ssize_t readLen = read(fd, &tableRecord, sizeof(STableRecord)); + if (readLen <= 0) break; + + int ret = taosDumpStable(tableRecord.name, fp, taosCon, dbName); + if (0 == ret) { + superTblCnt++; + } + } + + // TODO: save dump super table into result_output.txt + fprintf(g_fpOfResult, "# super table counter: %d\n", superTblCnt); + g_resultStatistics.totalSuperTblsOfDumpOut += superTblCnt; + + close(fd); + (void)remove(".stables.tmp"); + + return 0; } -static int taosDumpDb(SDbInfo *dbInfo, struct arguments *arguments, FILE *fp, TAOS *taosCon) { - TAOS_ROW row; - int fd = -1; - STableRecord tableRecord; +static int taosDumpDb(SDbInfo *dbInfo, FILE *fp, TAOS *taosCon) { + TAOS_ROW row; + int fd = -1; + STableRecord tableRecord; - taosDumpCreateDbClause(dbInfo, arguments->with_property, fp); + taosDumpCreateDbClause(dbInfo, g_args.with_property, fp); - fprintf(g_fpOfResult, "\n#### database: %s\n", dbInfo->name); - g_resultStatistics.totalDatabasesOfDumpOut++; + fprintf(g_fpOfResult, "\n#### database: %s\n", + dbInfo->name); + g_resultStatistics.totalDatabasesOfDumpOut++; - char sqlstr[TSDB_MAX_SQL_LEN] = {0}; + char sqlstr[TSDB_MAX_SQL_LEN] = {0}; - fprintf(fp, "USE %s;\n\n", dbInfo->name); + fprintf(fp, "USE %s;\n\n", dbInfo->name); - (void)taosDumpCreateSuperTableClause(taosCon, dbInfo->name, fp); + (void)taosDumpCreateSuperTableClause(taosCon, dbInfo->name, fp); - sprintf(sqlstr, "show %s.tables", dbInfo->name); + sprintf(sqlstr, "show %s.tables", dbInfo->name); - TAOS_RES* res = taos_query(taosCon, sqlstr); - int code = taos_errno(res); - if (code != 0) { - fprintf(stderr, "failed to run command <%s>, reason:%s\n", sqlstr, taos_errstr(res)); - taos_free_result(res); - return -1; - } - - char tmpBuf[TSDB_FILENAME_LEN + 1]; - memset(tmpBuf, 0, TSDB_FILENAME_LEN); - sprintf(tmpBuf, ".show-tables.tmp"); - fd = open(tmpBuf, O_RDWR | O_CREAT | O_TRUNC, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); - if (fd == -1) { - fprintf(stderr, "failed to open temp file: %s\n", tmpBuf); - taos_free_result(res); - return -1; - } - - TAOS_FIELD *fields = taos_fetch_fields(res); - - int32_t numOfTable = 0; - while ((row = taos_fetch_row(res)) != NULL) { - memset(&tableRecord, 0, sizeof(STableRecord)); - tstrncpy(tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], - fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes); - tstrncpy(tableRecord.metric, (char *)row[TSDB_SHOW_TABLES_METRIC_INDEX], - min(TSDB_TABLE_NAME_LEN, fields[TSDB_SHOW_TABLES_METRIC_INDEX].bytes)); - - taosWrite(fd, &tableRecord, sizeof(STableRecord)); - - numOfTable++; - } - taos_free_result(res); - lseek(fd, 0, SEEK_SET); - - int maxThreads = g_args.thread_num; - int tableOfPerFile ; - if (numOfTable <= g_args.thread_num) { - tableOfPerFile = 1; - maxThreads = numOfTable; - } else { - tableOfPerFile = numOfTable / g_args.thread_num; - if (0 != numOfTable % g_args.thread_num) { - tableOfPerFile += 1; + TAOS_RES* res = taos_query(taosCon, sqlstr); + int code = taos_errno(res); + if (code != 0) { + errorPrint("%s() LN%d, failed to run command <%s>, reason:%s\n", + __func__, __LINE__, sqlstr, taos_errstr(res)); + taos_free_result(res); + return -1; } - } - char* tblBuf = (char*)calloc(1, tableOfPerFile * sizeof(STableRecord)); - if (NULL == tblBuf){ - fprintf(stderr, "failed to calloc %" PRIzu "\n", tableOfPerFile * sizeof(STableRecord)); - close(fd); - return -1; - } + char tmpBuf[MAX_FILE_NAME_LEN]; + memset(tmpBuf, 0, MAX_FILE_NAME_LEN); + sprintf(tmpBuf, ".show-tables.tmp"); + fd = open(tmpBuf, O_RDWR | O_CREAT | O_TRUNC, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); + if (fd == -1) { + errorPrint("%s() LN%d, failed to open temp file: %s\n", + __func__, __LINE__, tmpBuf); + taos_free_result(res); + return -1; + } - int32_t numOfThread = 0; - int subFd = -1; - for (numOfThread = 0; numOfThread < maxThreads; numOfThread++) { - memset(tmpBuf, 0, TSDB_FILENAME_LEN); - sprintf(tmpBuf, ".tables.tmp.%d", numOfThread); - subFd = open(tmpBuf, O_RDWR | O_CREAT | O_TRUNC, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); - if (subFd == -1) { - fprintf(stderr, "failed to open temp file: %s\n", tmpBuf); - for (int32_t loopCnt = 0; loopCnt < numOfThread; loopCnt++) { + TAOS_FIELD *fields = taos_fetch_fields(res); + + int32_t numOfTable = 0; + while ((row = taos_fetch_row(res)) != NULL) { + memset(&tableRecord, 0, sizeof(STableRecord)); + tstrncpy(tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], + fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes); + tstrncpy(tableRecord.metric, (char *)row[TSDB_SHOW_TABLES_METRIC_INDEX], + min(TSDB_TABLE_NAME_LEN, fields[TSDB_SHOW_TABLES_METRIC_INDEX].bytes)); + + taosWrite(fd, &tableRecord, sizeof(STableRecord)); + + numOfTable++; + } + taos_free_result(res); + lseek(fd, 0, SEEK_SET); + + int maxThreads = g_args.thread_num; + int tableOfPerFile ; + if (numOfTable <= g_args.thread_num) { + tableOfPerFile = 1; + maxThreads = numOfTable; + } else { + tableOfPerFile = numOfTable / g_args.thread_num; + if (0 != numOfTable % g_args.thread_num) { + tableOfPerFile += 1; + } + } + + char* tblBuf = (char*)calloc(1, tableOfPerFile * sizeof(STableRecord)); + if (NULL == tblBuf){ + errorPrint("failed to calloc %" PRIzu "\n", + tableOfPerFile * sizeof(STableRecord)); + close(fd); + return -1; + } + + int32_t numOfThread = 0; + int subFd = -1; + for (numOfThread = 0; numOfThread < maxThreads; numOfThread++) { + memset(tmpBuf, 0, MAX_FILE_NAME_LEN); + sprintf(tmpBuf, ".tables.tmp.%d", numOfThread); + subFd = open(tmpBuf, O_RDWR | O_CREAT | O_TRUNC, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); + if (subFd == -1) { + errorPrint("%s() LN%d, failed to open temp file: %s\n", + __func__, __LINE__, tmpBuf); + for (int32_t loopCnt = 0; loopCnt < numOfThread; loopCnt++) { + sprintf(tmpBuf, ".tables.tmp.%d", loopCnt); + (void)remove(tmpBuf); + } + sprintf(tmpBuf, ".show-tables.tmp"); + (void)remove(tmpBuf); + free(tblBuf); + close(fd); + return -1; + } + + // read tableOfPerFile for fd, write to subFd + ssize_t readLen = read(fd, tblBuf, tableOfPerFile * sizeof(STableRecord)); + if (readLen <= 0) { + close(subFd); + break; + } + taosWrite(subFd, tblBuf, readLen); + close(subFd); + } + + sprintf(tmpBuf, ".show-tables.tmp"); + (void)remove(tmpBuf); + + if (fd >= 0) { + close(fd); + fd = -1; + } + + // start multi threads to dumpout + taosStartDumpOutWorkThreads(numOfThread, dbInfo->name); + for (int loopCnt = 0; loopCnt < numOfThread; loopCnt++) { sprintf(tmpBuf, ".tables.tmp.%d", loopCnt); (void)remove(tmpBuf); - } - sprintf(tmpBuf, ".show-tables.tmp"); - (void)remove(tmpBuf); - free(tblBuf); - close(fd); - return -1; } - // read tableOfPerFile for fd, write to subFd - ssize_t readLen = read(fd, tblBuf, tableOfPerFile * sizeof(STableRecord)); - if (readLen <= 0) { - close(subFd); - break; - } - taosWrite(subFd, tblBuf, readLen); - close(subFd); - } - - sprintf(tmpBuf, ".show-tables.tmp"); - (void)remove(tmpBuf); - - if (fd >= 0) { - close(fd); - fd = -1; - } - - taos_free_result(res); - - // start multi threads to dumpout - taosStartDumpOutWorkThreads(taosCon, arguments, numOfThread, dbInfo->name); - for (int loopCnt = 0; loopCnt < numOfThread; loopCnt++) { - sprintf(tmpBuf, ".tables.tmp.%d", loopCnt); - (void)remove(tmpBuf); - } - - free(tblBuf); - return 0; + free(tblBuf); + return 0; } static void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, @@ -1736,259 +1837,285 @@ static void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, static void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols, FILE *fp, char* dbName) { - int counter = 0; - int count_temp = 0; + int counter = 0; + int count_temp = 0; - char* tmpBuf = (char *)malloc(COMMAND_SIZE); - if (tmpBuf == NULL) { - fprintf(stderr, "failed to allocate memory\n"); - return; - } - - char *pstr = NULL; - pstr = tmpBuf; - - pstr += sprintf(tmpBuf, "CREATE TABLE IF NOT EXISTS %s.%s USING %s.%s TAGS (", - dbName, tableDes->name, dbName, metric); - - for (; counter < numOfCols; counter++) { - if (tableDes->cols[counter].note[0] != '\0') break; - } - - assert(counter < numOfCols); - count_temp = counter; - - for (; counter < numOfCols; counter++) { - if (counter != count_temp) { - if (strcasecmp(tableDes->cols[counter].type, "binary") == 0 || - strcasecmp(tableDes->cols[counter].type, "nchar") == 0) { - //pstr += sprintf(pstr, ", \'%s\'", tableDes->cols[counter].note); - pstr += sprintf(pstr, ", %s", tableDes->cols[counter].note); - } else { - pstr += sprintf(pstr, ", %s", tableDes->cols[counter].note); - } - } else { - if (strcasecmp(tableDes->cols[counter].type, "binary") == 0 || - strcasecmp(tableDes->cols[counter].type, "nchar") == 0) { - //pstr += sprintf(pstr, "\'%s\'", tableDes->cols[counter].note); - pstr += sprintf(pstr, "%s", tableDes->cols[counter].note); - } else { - pstr += sprintf(pstr, "%s", tableDes->cols[counter].note); - } - /* pstr += sprintf(pstr, "%s", tableDes->cols[counter].note); */ + char* tmpBuf = (char *)malloc(COMMAND_SIZE); + if (tmpBuf == NULL) { + errorPrint("%s() LN%d, failed to allocate %d memory\n", + __func__, __LINE__, COMMAND_SIZE); + return; } - /* if (strcasecmp(tableDes->cols[counter].type, "binary") == 0 || strcasecmp(tableDes->cols[counter].type, "nchar") - * == 0) { */ - /* pstr += sprintf(pstr, "(%d)", tableDes->cols[counter].length); */ - /* } */ - } + char *pstr = NULL; + pstr = tmpBuf; - pstr += sprintf(pstr, ");"); + pstr += sprintf(tmpBuf, + "CREATE TABLE IF NOT EXISTS %s.%s USING %s.%s TAGS (", + dbName, tableDes->name, dbName, metric); - fprintf(fp, "%s\n", tmpBuf); - free(tmpBuf); + for (; counter < numOfCols; counter++) { + if (tableDes->cols[counter].note[0] != '\0') break; + } + + assert(counter < numOfCols); + count_temp = counter; + + for (; counter < numOfCols; counter++) { + if (counter != count_temp) { + if (strcasecmp(tableDes->cols[counter].type, "binary") == 0 || + strcasecmp(tableDes->cols[counter].type, "nchar") == 0) { + //pstr += sprintf(pstr, ", \'%s\'", tableDes->cols[counter].note); + pstr += sprintf(pstr, ", %s", tableDes->cols[counter].note); + } else { + pstr += sprintf(pstr, ", %s", tableDes->cols[counter].note); + } + } else { + if (strcasecmp(tableDes->cols[counter].type, "binary") == 0 || + strcasecmp(tableDes->cols[counter].type, "nchar") == 0) { + //pstr += sprintf(pstr, "\'%s\'", tableDes->cols[counter].note); + pstr += sprintf(pstr, "%s", tableDes->cols[counter].note); + } else { + pstr += sprintf(pstr, "%s", tableDes->cols[counter].note); + } + /* pstr += sprintf(pstr, "%s", tableDes->cols[counter].note); */ + } + + /* if (strcasecmp(tableDes->cols[counter].type, "binary") == 0 || strcasecmp(tableDes->cols[counter].type, "nchar") + * == 0) { */ + /* pstr += sprintf(pstr, "(%d)", tableDes->cols[counter].length); */ + /* } */ + } + + pstr += sprintf(pstr, ");"); + + fprintf(fp, "%s\n", tmpBuf); + free(tmpBuf); } -static int taosDumpTableData(FILE *fp, char *tbname, - struct arguments *arguments, TAOS* taosCon, char* dbName) { - int64_t lastRowsPrint = 5000000; - int64_t totalRows = 0; - int count = 0; - char *pstr = NULL; - TAOS_ROW row = NULL; - int numFields = 0; - - if (arguments->schemaonly) { +static int writeSchemaToAvro(char *jsonAvroSchema) +{ + errorPrint("%s() LN%d, TODO: implement write schema to avro", + __func__, __LINE__); return 0; - } +} - int32_t sql_buf_len = arguments->max_sql_len; - char* tmpBuffer = (char *)calloc(1, sql_buf_len + 128); - if (tmpBuffer == NULL) { - fprintf(stderr, "failed to allocate memory\n"); - return -1; - } +static int64_t writeResultToAvro(TAOS_RES *res) +{ + errorPrint("%s() LN%d, TODO: implementation need\n", __func__, __LINE__); + return 0; +} - pstr = tmpBuffer; +static int64_t writeResultToSql(TAOS_RES *res, FILE *fp, char *dbName, char *tbName) +{ + int64_t totalRows = 0; - char sqlstr[1024] = {0}; - sprintf(sqlstr, - "select * from %s.%s where _c0 >= %" PRId64 " and _c0 <= %" PRId64 " order by _c0 asc;", - dbName, tbname, arguments->start_time, arguments->end_time); + int32_t sql_buf_len = g_args.max_sql_len; + char* tmpBuffer = (char *)calloc(1, sql_buf_len + 128); + if (tmpBuffer == NULL) { + errorPrint("failed to allocate %d memory\n", sql_buf_len + 128); + return -1; + } - TAOS_RES* tmpResult = taos_query(taosCon, sqlstr); - int32_t code = taos_errno(tmpResult); - if (code != 0) { - fprintf(stderr, "failed to run command %s, reason: %s\n", sqlstr, taos_errstr(tmpResult)); - free(tmpBuffer); - taos_free_result(tmpResult); - return -1; - } + char *pstr = tmpBuffer; - numFields = taos_field_count(tmpResult); - assert(numFields > 0); - TAOS_FIELD *fields = taos_fetch_fields(tmpResult); + TAOS_ROW row = NULL; + int numFields = 0; + int rowFlag = 0; + int64_t lastRowsPrint = 5000000; + int count = 0; - int rowFlag = 0; - int32_t curr_sqlstr_len = 0; - int32_t total_sqlstr_len = 0; - count = 0; - while ((row = taos_fetch_row(tmpResult)) != NULL) { - pstr = tmpBuffer; - curr_sqlstr_len = 0; + numFields = taos_field_count(res); + assert(numFields > 0); + TAOS_FIELD *fields = taos_fetch_fields(res); - int32_t* length = taos_fetch_lengths(tmpResult); // act len + int32_t curr_sqlstr_len = 0; + int32_t total_sqlstr_len = 0; - if (count == 0) { - total_sqlstr_len = 0; - curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "INSERT INTO %s.%s VALUES (", dbName, tbname); - } else { - if (arguments->mysqlFlag) { - if (0 == rowFlag) { - curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "("); - rowFlag++; + while ((row = taos_fetch_row(res)) != NULL) { + curr_sqlstr_len = 0; + + int32_t* length = taos_fetch_lengths(res); // act len + + if (count == 0) { + total_sqlstr_len = 0; + curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, + "INSERT INTO %s.%s VALUES (", dbName, tbName); } else { - curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, ", ("); + if (g_args.mysqlFlag) { + if (0 == rowFlag) { + curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "("); + rowFlag++; + } else { + curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, ", ("); + } + } else { + curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "("); + } + } + + for (int col = 0; col < numFields; col++) { + if (col != 0) curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, ", "); + + if (row[col] == NULL) { + curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "NULL"); + continue; + } + + switch (fields[col].type) { + case TSDB_DATA_TYPE_BOOL: + curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "%d", + ((((int32_t)(*((char *)row[col]))) == 1) ? 1 : 0)); + break; + case TSDB_DATA_TYPE_TINYINT: + curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "%d", *((int8_t *)row[col])); + break; + case TSDB_DATA_TYPE_SMALLINT: + curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "%d", *((int16_t *)row[col])); + break; + case TSDB_DATA_TYPE_INT: + curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "%d", *((int32_t *)row[col])); + break; + case TSDB_DATA_TYPE_BIGINT: + curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "%" PRId64 "", + *((int64_t *)row[col])); + break; + case TSDB_DATA_TYPE_FLOAT: + curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "%f", GET_FLOAT_VAL(row[col])); + break; + case TSDB_DATA_TYPE_DOUBLE: + curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "%f", GET_DOUBLE_VAL(row[col])); + break; + case TSDB_DATA_TYPE_BINARY: + { + char tbuf[COMMAND_SIZE] = {0}; + //*(pstr++) = '\''; + converStringToReadable((char *)row[col], length[col], tbuf, COMMAND_SIZE); + //pstr = stpcpy(pstr, tbuf); + //*(pstr++) = '\''; + curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "\'%s\'", tbuf); + break; + } + case TSDB_DATA_TYPE_NCHAR: + { + char tbuf[COMMAND_SIZE] = {0}; + convertNCharToReadable((char *)row[col], length[col], tbuf, COMMAND_SIZE); + curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "\'%s\'", tbuf); + break; + } + case TSDB_DATA_TYPE_TIMESTAMP: + if (!g_args.mysqlFlag) { + curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "%" PRId64 "", + *(int64_t *)row[col]); + } else { + char buf[64] = "\0"; + int64_t ts = *((int64_t *)row[col]); + time_t tt = (time_t)(ts / 1000); + struct tm *ptm = localtime(&tt); + strftime(buf, 64, "%y-%m-%d %H:%M:%S", ptm); + curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "\'%s.%03d\'", + buf, (int)(ts % 1000)); + } + break; + default: + break; + } + } + + curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, ")"); + + totalRows++; + count++; + fprintf(fp, "%s", tmpBuffer); + + if (totalRows >= lastRowsPrint) { + printf(" %"PRId64 " rows already be dumpout from %s.%s\n", + totalRows, dbName, tbName); + lastRowsPrint += 5000000; + } + + total_sqlstr_len += curr_sqlstr_len; + + if ((count >= g_args.data_batch) + || (sql_buf_len - total_sqlstr_len < TSDB_MAX_BYTES_PER_ROW)) { + fprintf(fp, ";\n"); + count = 0; } - } else { - curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "("); - } } - for (int col = 0; col < numFields; col++) { - if (col != 0) curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, ", "); + debugPrint("total_sqlstr_len: %d\n", total_sqlstr_len); - if (row[col] == NULL) { - curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "NULL"); - continue; - } + fprintf(fp, "\n"); + atomic_add_fetch_64(&g_totalDumpOutRows, totalRows); + free(tmpBuffer); - switch (fields[col].type) { - case TSDB_DATA_TYPE_BOOL: - curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "%d", - ((((int32_t)(*((char *)row[col]))) == 1) ? 1 : 0)); - break; - case TSDB_DATA_TYPE_TINYINT: - curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "%d", *((int8_t *)row[col])); - break; - case TSDB_DATA_TYPE_SMALLINT: - curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "%d", *((int16_t *)row[col])); - break; - case TSDB_DATA_TYPE_INT: - curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "%d", *((int32_t *)row[col])); - break; - case TSDB_DATA_TYPE_BIGINT: - curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "%" PRId64 "", - *((int64_t *)row[col])); - break; - case TSDB_DATA_TYPE_FLOAT: - curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "%f", GET_FLOAT_VAL(row[col])); - break; - case TSDB_DATA_TYPE_DOUBLE: - curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "%f", GET_DOUBLE_VAL(row[col])); - break; - case TSDB_DATA_TYPE_BINARY: { - char tbuf[COMMAND_SIZE] = {0}; - //*(pstr++) = '\''; - converStringToReadable((char *)row[col], length[col], tbuf, COMMAND_SIZE); - //pstr = stpcpy(pstr, tbuf); - //*(pstr++) = '\''; - curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "\'%s\'", tbuf); - break; - } - case TSDB_DATA_TYPE_NCHAR: { - char tbuf[COMMAND_SIZE] = {0}; - convertNCharToReadable((char *)row[col], length[col], tbuf, COMMAND_SIZE); - curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "\'%s\'", tbuf); - break; - } - case TSDB_DATA_TYPE_TIMESTAMP: - if (!arguments->mysqlFlag) { - curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "%" PRId64 "", - *(int64_t *)row[col]); - } else { - char buf[64] = "\0"; - int64_t ts = *((int64_t *)row[col]); - time_t tt = (time_t)(ts / 1000); - struct tm *ptm = localtime(&tt); - strftime(buf, 64, "%y-%m-%d %H:%M:%S", ptm); - curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "\'%s.%03d\'", - buf, (int)(ts % 1000)); - } - break; - default: - break; - } + return 0; +} + +static int taosDumpTableData(FILE *fp, char *tbName, + TAOS* taosCon, char* dbName, + char *jsonAvroSchema) { + int64_t totalRows = 0; + + char sqlstr[1024] = {0}; + sprintf(sqlstr, + "select * from %s.%s where _c0 >= %" PRId64 " and _c0 <= %" PRId64 " order by _c0 asc;", + dbName, tbName, g_args.start_time, g_args.end_time); + + TAOS_RES* res = taos_query(taosCon, sqlstr); + int32_t code = taos_errno(res); + if (code != 0) { + errorPrint("failed to run command %s, reason: %s\n", + sqlstr, taos_errstr(res)); + taos_free_result(res); + return -1; } - curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, ") "); - - totalRows++; - count++; - fprintf(fp, "%s", tmpBuffer); - - if (totalRows >= lastRowsPrint) { - printf(" %"PRId64 " rows already be dumpout from %s.%s\n", - totalRows, dbName, tbname); - lastRowsPrint += 5000000; + if (g_args.avro) { + writeSchemaToAvro(jsonAvroSchema); + totalRows = writeResultToAvro(res); + } else { + totalRows = writeResultToSql(res, fp, dbName, tbName); } - total_sqlstr_len += curr_sqlstr_len; - - if ((count >= arguments->data_batch) - || (sql_buf_len - total_sqlstr_len < TSDB_MAX_BYTES_PER_ROW)) { - fprintf(fp, ";\n"); - count = 0; - } //else { - //fprintf(fp, "\\\n"); - //} - } - - printf("total_sqlstr_len: %d\n", total_sqlstr_len); - - fprintf(fp, "\n"); - atomic_add_fetch_64(&totalDumpOutRows, totalRows); - - taos_free_result(tmpResult); - free(tmpBuffer); - return totalRows; + taos_free_result(res); + return totalRows; } static int taosCheckParam(struct arguments *arguments) { - if (arguments->all_databases && arguments->databases) { - fprintf(stderr, "conflict option --all-databases and --databases\n"); - return -1; - } - - if (arguments->start_time > arguments->end_time) { - fprintf(stderr, "start time is larger than end time\n"); - return -1; - } - - if (arguments->arg_list_len == 0) { - if ((!arguments->all_databases) && (!arguments->isDumpIn)) { - fprintf(stderr, "taosdump requires parameters\n"); - return -1; + if (g_args.all_databases && g_args.databases) { + fprintf(stderr, "conflict option --all-databases and --databases\n"); + return -1; } - } -/* - if (arguments->isDumpIn && (strcmp(arguments->outpath, DEFAULT_DUMP_FILE) != 0)) { - fprintf(stderr, "duplicate parameter input and output file path\n"); - return -1; - } -*/ - if (!arguments->isDumpIn && arguments->encode != NULL) { - fprintf(stderr, "invalid option in dump out\n"); - return -1; - } - if (arguments->table_batch <= 0) { - fprintf(stderr, "invalid option in dump out\n"); - return -1; - } + if (g_args.start_time > g_args.end_time) { + fprintf(stderr, "start time is larger than end time\n"); + return -1; + } - return 0; + if (g_args.arg_list_len == 0) { + if ((!g_args.all_databases) && (!g_args.isDumpIn)) { + fprintf(stderr, "taosdump requires parameters\n"); + return -1; + } + } + /* + if (g_args.isDumpIn && (strcmp(g_args.outpath, DEFAULT_DUMP_FILE) != 0)) { + fprintf(stderr, "duplicate parameter input and output file path\n"); + return -1; + } + */ + if (!g_args.isDumpIn && g_args.encode != NULL) { + fprintf(stderr, "invalid option in dump out\n"); + return -1; + } + + if (g_args.table_batch <= 0) { + fprintf(stderr, "invalid option in dump out\n"); + return -1; + } + + return 0; } /* @@ -2071,374 +2198,419 @@ char *ascii_literal_list[] = { "\\xf7", "\\xf8", "\\xf9", "\\xfa", "\\xfb", "\\xfc", "\\xfd", "\\xfe", "\\xff"}; static int converStringToReadable(char *str, int size, char *buf, int bufsize) { - char *pstr = str; - char *pbuf = buf; - while (size > 0) { - if (*pstr == '\0') break; - pbuf = stpcpy(pbuf, ascii_literal_list[((uint8_t)(*pstr))]); - pstr++; - size--; - } - *pbuf = '\0'; - return 0; + char *pstr = str; + char *pbuf = buf; + while (size > 0) { + if (*pstr == '\0') break; + pbuf = stpcpy(pbuf, ascii_literal_list[((uint8_t)(*pstr))]); + pstr++; + size--; + } + *pbuf = '\0'; + return 0; } static int convertNCharToReadable(char *str, int size, char *buf, int bufsize) { - char *pstr = str; - char *pbuf = buf; - // TODO - wchar_t wc; - while (size > 0) { - if (*pstr == '\0') break; - int byte_width = mbtowc(&wc, pstr, MB_CUR_MAX); - if (byte_width < 0) { - errorPrint("%s() LN%d, mbtowc() return fail.\n", __func__, __LINE__); - exit(-1); + char *pstr = str; + char *pbuf = buf; + // TODO + wchar_t wc; + while (size > 0) { + if (*pstr == '\0') break; + int byte_width = mbtowc(&wc, pstr, MB_CUR_MAX); + if (byte_width < 0) { + errorPrint("%s() LN%d, mbtowc() return fail.\n", __func__, __LINE__); + exit(-1); + } + + if ((int)wc < 256) { + pbuf = stpcpy(pbuf, ascii_literal_list[(int)wc]); + } else { + memcpy(pbuf, pstr, byte_width); + pbuf += byte_width; + } + pstr += byte_width; } - if ((int)wc < 256) { - pbuf = stpcpy(pbuf, ascii_literal_list[(int)wc]); - } else { - memcpy(pbuf, pstr, byte_width); - pbuf += byte_width; - } - pstr += byte_width; - } + *pbuf = '\0'; - *pbuf = '\0'; - - return 0; + return 0; } static void taosDumpCharset(FILE *fp) { - char charsetline[256]; + char charsetline[256]; - (void)fseek(fp, 0, SEEK_SET); - sprintf(charsetline, "#!%s\n", tsCharset); - (void)fwrite(charsetline, strlen(charsetline), 1, fp); + (void)fseek(fp, 0, SEEK_SET); + sprintf(charsetline, "#!%s\n", tsCharset); + (void)fwrite(charsetline, strlen(charsetline), 1, fp); } static void taosLoadFileCharset(FILE *fp, char *fcharset) { - char * line = NULL; - size_t line_size = 0; + char * line = NULL; + size_t line_size = 0; - (void)fseek(fp, 0, SEEK_SET); - ssize_t size = getline(&line, &line_size, fp); - if (size <= 2) { - goto _exit_no_charset; - } + (void)fseek(fp, 0, SEEK_SET); + ssize_t size = getline(&line, &line_size, fp); + if (size <= 2) { + goto _exit_no_charset; + } - if (strncmp(line, "#!", 2) != 0) { - goto _exit_no_charset; - } - if (line[size - 1] == '\n') { - line[size - 1] = '\0'; - size--; - } - strcpy(fcharset, line + 2); + if (strncmp(line, "#!", 2) != 0) { + goto _exit_no_charset; + } + if (line[size - 1] == '\n') { + line[size - 1] = '\0'; + size--; + } + strcpy(fcharset, line + 2); - tfree(line); - return; + tfree(line); + return; _exit_no_charset: - (void)fseek(fp, 0, SEEK_SET); - *fcharset = '\0'; - tfree(line); - return; + (void)fseek(fp, 0, SEEK_SET); + *fcharset = '\0'; + tfree(line); + return; } // ======== dumpIn support multi threads functions ================================// -static char **tsDumpInSqlFiles = NULL; -static int32_t tsSqlFileNum = 0; -static char tsDbSqlFile[TSDB_FILENAME_LEN] = {0}; -static char tsfCharset[64] = {0}; -static int taosGetFilesNum(const char *directoryName, const char *prefix) +static char **g_tsDumpInSqlFiles = NULL; +static int32_t g_tsSqlFileNum = 0; +static char g_tsDbSqlFile[MAX_FILE_NAME_LEN] = {0}; +static char g_tsCharset[64] = {0}; + +static int taosGetFilesNum(const char *directoryName, + const char *prefix, const char *prefix2) { - char cmd[1024] = { 0 }; - sprintf(cmd, "ls %s/*.%s | wc -l ", directoryName, prefix); + char cmd[1024] = { 0 }; - FILE *fp = popen(cmd, "r"); - if (fp == NULL) { - errorPrint("failed to execute:%s, error:%s\n", cmd, strerror(errno)); - exit(-1); - } + if (prefix2) + sprintf(cmd, "ls %s/*.%s %s/*.%s | wc -l ", + directoryName, prefix, directoryName, prefix2); + else + sprintf(cmd, "ls %s/*.%s | wc -l ", directoryName, prefix); - int fileNum = 0; - if (fscanf(fp, "%d", &fileNum) != 1) { - errorPrint("failed to execute:%s, parse result error\n", cmd); - exit(-1); - } - - if (fileNum <= 0) { - errorPrint("directory:%s is empry\n", directoryName); - exit(-1); - } - - pclose(fp); - return fileNum; -} - -static void taosParseDirectory(const char *directoryName, const char *prefix, char **fileArray, int totalFiles) -{ - char cmd[1024] = { 0 }; - sprintf(cmd, "ls %s/*.%s | sort", directoryName, prefix); - - FILE *fp = popen(cmd, "r"); - if (fp == NULL) { - errorPrint("failed to execute:%s, error:%s\n", cmd, strerror(errno)); - exit(-1); - } - - int fileNum = 0; - while (fscanf(fp, "%128s", fileArray[fileNum++])) { - if (strcmp(fileArray[fileNum-1], tsDbSqlFile) == 0) { - fileNum--; + FILE *fp = popen(cmd, "r"); + if (fp == NULL) { + errorPrint("failed to execute:%s, error:%s\n", cmd, strerror(errno)); + exit(-1); + } + + int fileNum = 0; + if (fscanf(fp, "%d", &fileNum) != 1) { + errorPrint("failed to execute:%s, parse result error\n", cmd); + exit(-1); + } + + if (fileNum <= 0) { + errorPrint("directory:%s is empry\n", directoryName); + exit(-1); } - if (fileNum >= totalFiles) { - break; - } - } - if (fileNum != totalFiles) { - errorPrint("directory:%s changed while read\n", directoryName); pclose(fp); - exit(-1); - } - - pclose(fp); + return fileNum; } -static void taosCheckTablesSQLFile(const char *directoryName) +static void taosParseDirectory(const char *directoryName, + const char *prefix, const char *prefix2, + char **fileArray, int totalFiles) { - char cmd[1024] = { 0 }; - sprintf(cmd, "ls %s/dbs.sql", directoryName); + char cmd[1024] = { 0 }; - FILE *fp = popen(cmd, "r"); - if (fp == NULL) { - errorPrint("failed to execute:%s, error:%s\n", cmd, strerror(errno)); - exit(-1); - } + if (prefix2) { + sprintf(cmd, "ls %s/*.%s %s/*.%s | sort", + directoryName, prefix, directoryName, prefix2); + } else { + sprintf(cmd, "ls %s/*.%s | sort", directoryName, prefix); + } - while (fscanf(fp, "%128s", tsDbSqlFile)) { - break; - } + FILE *fp = popen(cmd, "r"); + if (fp == NULL) { + errorPrint("failed to execute:%s, error:%s\n", cmd, strerror(errno)); + exit(-1); + } - pclose(fp); + int fileNum = 0; + while (fscanf(fp, "%128s", fileArray[fileNum++])) { + if (strcmp(fileArray[fileNum-1], g_tsDbSqlFile) == 0) { + fileNum--; + } + if (fileNum >= totalFiles) { + break; + } + } + + if (fileNum != totalFiles) { + errorPrint("directory:%s changed while read\n", directoryName); + pclose(fp); + exit(-1); + } + + pclose(fp); } -static void taosMallocSQLFiles() +static void taosCheckDatabasesSQLFile(const char *directoryName) { - tsDumpInSqlFiles = (char**)calloc(tsSqlFileNum, sizeof(char*)); - for (int i = 0; i < tsSqlFileNum; i++) { - tsDumpInSqlFiles[i] = calloc(1, TSDB_FILENAME_LEN); - } + char cmd[1024] = { 0 }; + sprintf(cmd, "ls %s/dbs.sql", directoryName); + + FILE *fp = popen(cmd, "r"); + if (fp == NULL) { + errorPrint("failed to execute:%s, error:%s\n", cmd, strerror(errno)); + exit(-1); + } + + while (fscanf(fp, "%128s", g_tsDbSqlFile)) { + break; + } + + pclose(fp); } -static void taosFreeSQLFiles() +static void taosMallocDumpFiles() { - for (int i = 0; i < tsSqlFileNum; i++) { - tfree(tsDumpInSqlFiles[i]); - } - tfree(tsDumpInSqlFiles); + g_tsDumpInSqlFiles = (char**)calloc(g_tsSqlFileNum, sizeof(char*)); + for (int i = 0; i < g_tsSqlFileNum; i++) { + g_tsDumpInSqlFiles[i] = calloc(1, MAX_FILE_NAME_LEN); + } +} + +static void taosFreeDumpFiles() +{ + for (int i = 0; i < g_tsSqlFileNum; i++) { + tfree(g_tsDumpInSqlFiles[i]); + } + tfree(g_tsDumpInSqlFiles); } static void taosGetDirectoryFileList(char *inputDir) { - struct stat fileStat; - if (stat(inputDir, &fileStat) < 0) { - errorPrint("%s not exist\n", inputDir); - exit(-1); - } + struct stat fileStat; + if (stat(inputDir, &fileStat) < 0) { + errorPrint("%s not exist\n", inputDir); + exit(-1); + } - if (fileStat.st_mode & S_IFDIR) { - taosCheckTablesSQLFile(inputDir); - tsSqlFileNum = taosGetFilesNum(inputDir, "sql"); - int tsSqlFileNumOfTbls = tsSqlFileNum; - if (tsDbSqlFile[0] != 0) { - tsSqlFileNumOfTbls--; + if (fileStat.st_mode & S_IFDIR) { + taosCheckDatabasesSQLFile(inputDir); + if (g_args.avro) + g_tsSqlFileNum = taosGetFilesNum(inputDir, "sql", "avro"); + else + g_tsSqlFileNum += taosGetFilesNum(inputDir, "sql", NULL); + + int tsSqlFileNumOfTbls = g_tsSqlFileNum; + if (g_tsDbSqlFile[0] != 0) { + tsSqlFileNumOfTbls--; + } + taosMallocDumpFiles(); + if (0 != tsSqlFileNumOfTbls) { + if (g_args.avro) { + taosParseDirectory(inputDir, "sql", "avro", + g_tsDumpInSqlFiles, tsSqlFileNumOfTbls); + } else { + taosParseDirectory(inputDir, "sql", NULL, + g_tsDumpInSqlFiles, tsSqlFileNumOfTbls); + } + } + fprintf(stdout, "\nstart to dispose %d files in %s\n", + g_tsSqlFileNum, inputDir); + } else { + errorPrint("%s is not a directory\n", inputDir); + exit(-1); } - taosMallocSQLFiles(); - if (0 != tsSqlFileNumOfTbls) { - taosParseDirectory(inputDir, "sql", tsDumpInSqlFiles, tsSqlFileNumOfTbls); - } - fprintf(stdout, "\nstart to dispose %d files in %s\n", tsSqlFileNum, inputDir); - } - else { - errorPrint("%s is not a directory\n", inputDir); - exit(-1); - } } static FILE* taosOpenDumpInFile(char *fptr) { - wordexp_t full_path; + wordexp_t full_path; - if (wordexp(fptr, &full_path, 0) != 0) { - fprintf(stderr, "ERROR: illegal file name: %s\n", fptr); - return NULL; - } + if (wordexp(fptr, &full_path, 0) != 0) { + errorPrint("illegal file name: %s\n", fptr); + return NULL; + } - char *fname = full_path.we_wordv[0]; + char *fname = full_path.we_wordv[0]; - if ((fname) && (strlen(fname) > 0)) { - FILE *f = fopen(fname, "r"); - if (f == NULL) { - errorPrint("%s() LN%d, failed to open file %s\n", - __func__, __LINE__, fname); - } + FILE *f = NULL; + if ((fname) && (strlen(fname) > 0)) { + f = fopen(fname, "r"); + if (f == NULL) { + errorPrint("%s() LN%d, failed to open file %s\n", + __func__, __LINE__, fname); + } + } - wordfree(&full_path); - return f; - } - - return NULL; + wordfree(&full_path); + return f; } static int taosDumpInOneFile(TAOS* taos, FILE* fp, char* fcharset, char* encode, char* fileName) { - int read_len = 0; - char * cmd = NULL; - size_t cmd_len = 0; - char * line = NULL; - size_t line_len = 0; + int read_len = 0; + char * cmd = NULL; + size_t cmd_len = 0; + char * line = NULL; + size_t line_len = 0; - cmd = (char *)malloc(TSDB_MAX_ALLOWED_SQL_LEN); - if (cmd == NULL) { - fprintf(stderr, "failed to allocate memory\n"); - return -1; - } - - int lastRowsPrint = 5000000; - int lineNo = 0; - while ((read_len = getline(&line, &line_len, fp)) != -1) { - ++lineNo; - if (read_len >= TSDB_MAX_ALLOWED_SQL_LEN) continue; - line[--read_len] = '\0'; - - //if (read_len == 0 || isCommentLine(line)) { // line starts with # - if (read_len == 0 ) { - continue; + cmd = (char *)malloc(TSDB_MAX_ALLOWED_SQL_LEN); + if (cmd == NULL) { + errorPrint("%s() LN%d, failed to allocate memory\n", + __func__, __LINE__); + return -1; } - if (line[read_len - 1] == '\\') { - line[read_len - 1] = ' '; - memcpy(cmd + cmd_len, line, read_len); - cmd_len += read_len; - continue; + int lastRowsPrint = 5000000; + int lineNo = 0; + while ((read_len = getline(&line, &line_len, fp)) != -1) { + ++lineNo; + if (read_len >= TSDB_MAX_ALLOWED_SQL_LEN) continue; + line[--read_len] = '\0'; + + //if (read_len == 0 || isCommentLine(line)) { // line starts with # + if (read_len == 0 ) { + continue; + } + + if (line[read_len - 1] == '\\') { + line[read_len - 1] = ' '; + memcpy(cmd + cmd_len, line, read_len); + cmd_len += read_len; + continue; + } + + memcpy(cmd + cmd_len, line, read_len); + cmd[read_len + cmd_len]= '\0'; + if (queryDbImpl(taos, cmd)) { + errorPrint("%s() LN%d, error sql: linenu:%d, file:%s\n", + __func__, __LINE__, lineNo, fileName); + fprintf(g_fpOfResult, "error sql: linenu:%d, file:%s\n", lineNo, fileName); + } + + memset(cmd, 0, TSDB_MAX_ALLOWED_SQL_LEN); + cmd_len = 0; + + if (lineNo >= lastRowsPrint) { + printf(" %d lines already be executed from file %s\n", lineNo, fileName); + lastRowsPrint += 5000000; + } } - memcpy(cmd + cmd_len, line, read_len); - cmd[read_len + cmd_len]= '\0'; - if (queryDbImpl(taos, cmd)) { - fprintf(stderr, "error sql: linenu:%d, file:%s\n", lineNo, fileName); - fprintf(g_fpOfResult, "error sql: linenu:%d, file:%s\n", lineNo, fileName); - } - - memset(cmd, 0, TSDB_MAX_ALLOWED_SQL_LEN); - cmd_len = 0; - - if (lineNo >= lastRowsPrint) { - printf(" %d lines already be executed from file %s\n", lineNo, fileName); - lastRowsPrint += 5000000; - } - } - - tfree(cmd); - tfree(line); - fclose(fp); - return 0; + tfree(cmd); + tfree(line); + fclose(fp); + return 0; } static void* taosDumpInWorkThreadFp(void *arg) { - SThreadParaObj *pThread = (SThreadParaObj*)arg; - for (int32_t f = 0; f < tsSqlFileNum; ++f) { - if (f % pThread->totalThreads == pThread->threadIndex) { - char *SQLFileName = tsDumpInSqlFiles[f]; - FILE* fp = taosOpenDumpInFile(SQLFileName); - if (NULL == fp) { - continue; - } - fprintf(stderr, "Success Open input file: %s\n", SQLFileName); - taosDumpInOneFile(pThread->taosCon, fp, tsfCharset, g_args.encode, SQLFileName); + SThreadParaObj *pThread = (SThreadParaObj*)arg; + for (int32_t f = 0; f < g_tsSqlFileNum; ++f) { + if (f % pThread->totalThreads == pThread->threadIndex) { + char *SQLFileName = g_tsDumpInSqlFiles[f]; + FILE* fp = taosOpenDumpInFile(SQLFileName); + if (NULL == fp) { + continue; + } + fprintf(stderr, ", Success Open input file: %s\n", + SQLFileName); + taosDumpInOneFile(pThread->taosCon, fp, g_tsCharset, g_args.encode, SQLFileName); + } } - } - return NULL; + return NULL; } -static void taosStartDumpInWorkThreads(void* taosCon, struct arguments *args) +static void taosStartDumpInWorkThreads() { - pthread_attr_t thattr; - SThreadParaObj *pThread; - int32_t totalThreads = args->thread_num; + pthread_attr_t thattr; + SThreadParaObj *pThread; + int32_t totalThreads = g_args.thread_num; - if (totalThreads > tsSqlFileNum) { - totalThreads = tsSqlFileNum; - } - - SThreadParaObj *threadObj = (SThreadParaObj *)calloc(totalThreads, sizeof(SThreadParaObj)); - for (int32_t t = 0; t < totalThreads; ++t) { - pThread = threadObj + t; - pThread->threadIndex = t; - pThread->totalThreads = totalThreads; - pThread->taosCon = taosCon; - - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); - - if (pthread_create(&(pThread->threadID), &thattr, taosDumpInWorkThreadFp, (void*)pThread) != 0) { - fprintf(stderr, "ERROR: thread:%d failed to start\n", pThread->threadIndex); - exit(0); + if (totalThreads > g_tsSqlFileNum) { + totalThreads = g_tsSqlFileNum; } - } - for (int t = 0; t < totalThreads; ++t) { - pthread_join(threadObj[t].threadID, NULL); - } + SThreadParaObj *threadObj = (SThreadParaObj *)calloc( + totalThreads, sizeof(SThreadParaObj)); - for (int t = 0; t < totalThreads; ++t) { - taos_close(threadObj[t].taosCon); - } - free(threadObj); + if (NULL == threadObj) { + errorPrint("%s() LN%d, memory allocation failed\n", __func__, __LINE__); + } + + for (int32_t t = 0; t < totalThreads; ++t) { + pThread = threadObj + t; + pThread->threadIndex = t; + pThread->totalThreads = totalThreads; + pThread->taosCon = taos_connect(g_args.host, g_args.user, g_args.password, + NULL, g_args.port); + if (pThread->taosCon == NULL) { + errorPrint("Failed to connect to TDengine server %s\n", g_args.host); + return; + } + pthread_attr_init(&thattr); + pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + + if (pthread_create(&(pThread->threadID), &thattr, + taosDumpInWorkThreadFp, (void*)pThread) != 0) { + errorPrint("%s() LN%d, thread:%d failed to start\n", + __func__, __LINE__, pThread->threadIndex); + exit(0); + } + } + + for (int t = 0; t < totalThreads; ++t) { + pthread_join(threadObj[t].threadID, NULL); + } + + for (int t = 0; t < totalThreads; ++t) { + taos_close(threadObj[t].taosCon); + } + free(threadObj); } -static int taosDumpIn(struct arguments *arguments) { - assert(arguments->isDumpIn); +static int taosDumpIn() { + assert(g_args.isDumpIn); - TAOS *taos = NULL; - FILE *fp = NULL; + TAOS *taos = NULL; + FILE *fp = NULL; - taos = taos_connect(arguments->host, arguments->user, arguments->password, NULL, arguments->port); - if (taos == NULL) { - fprintf(stderr, "failed to connect to TDengine server\n"); - return -1; - } - - taosGetDirectoryFileList(arguments->inpath); - - int32_t tsSqlFileNumOfTbls = tsSqlFileNum; - if (tsDbSqlFile[0] != 0) { - tsSqlFileNumOfTbls--; - - fp = taosOpenDumpInFile(tsDbSqlFile); - if (NULL == fp) { - fprintf(stderr, "failed to open input file %s\n", tsDbSqlFile); - return -1; + taos = taos_connect( + g_args.host, g_args.user, g_args.password, + NULL, g_args.port); + if (taos == NULL) { + errorPrint("%s() LN%d, failed to connect to TDengine server\n", + __func__, __LINE__); + return -1; } - fprintf(stderr, "Success Open input file: %s\n", tsDbSqlFile); - taosLoadFileCharset(fp, tsfCharset); + taosGetDirectoryFileList(g_args.inpath); - taosDumpInOneFile(taos, fp, tsfCharset, arguments->encode, tsDbSqlFile); - } + int32_t tsSqlFileNumOfTbls = g_tsSqlFileNum; + if (g_tsDbSqlFile[0] != 0) { + tsSqlFileNumOfTbls--; - if (0 != tsSqlFileNumOfTbls) { - taosStartDumpInWorkThreads(taos, arguments); - } + fp = taosOpenDumpInFile(g_tsDbSqlFile); + if (NULL == fp) { + errorPrint("%s() LN%d, failed to open input file %s\n", + __func__, __LINE__, g_tsDbSqlFile); + return -1; + } + fprintf(stderr, "Success Open input file: %s\n", g_tsDbSqlFile); - taos_close(taos); - taosFreeSQLFiles(); - return 0; + taosLoadFileCharset(fp, g_tsCharset); + + taosDumpInOneFile(taos, fp, g_tsCharset, g_args.encode, + g_tsDbSqlFile); + } + + taos_close(taos); + + if (0 != tsSqlFileNumOfTbls) { + taosStartDumpInWorkThreads(); + } + + taosFreeDumpFiles(); + return 0; } From d53b734901ebb5d0fe1704d5209ad25cb2701e4b Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 6 Jul 2021 17:59:39 +0800 Subject: [PATCH 23/26] [TD-5067]: taosdemo stmt use sample data (#6759) --- src/kit/taosdemo/taosdemo.c | 158 +++++++++++++++++++++++++++--------- 1 file changed, 120 insertions(+), 38 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 6513f3e214..b162aa5299 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -3216,13 +3216,6 @@ static int readTagFromCsvFileToMem(SSuperTable * superTblInfo) { return 0; } -#if 0 -int readSampleFromJsonFileToMem(SSuperTable * superTblInfo) { - // TODO - return 0; -} -#endif - /* Read 10000 lines at most. If more than 10000 lines, continue to read after using */ @@ -5338,7 +5331,7 @@ static int64_t generateInterlaceDataWithoutStb( #if STMT_IFACE_ENABLED == 1 static int32_t prepareStmtBindArrayByType(TAOS_BIND *bind, - char *dataType, int32_t dataLen, char **ptr) + char *dataType, int32_t dataLen, char **ptr, char *value) { if (0 == strncasecmp(dataType, "BINARY", strlen("BINARY"))) { @@ -5348,12 +5341,18 @@ static int32_t prepareStmtBindArrayByType(TAOS_BIND *bind, return -1; } char *bind_binary = (char *)*ptr; - rand_string(bind_binary, dataLen); bind->buffer_type = TSDB_DATA_TYPE_BINARY; - bind->buffer_length = dataLen; - bind->buffer = bind_binary; + if (value) { + strncpy(bind_binary, value, strlen(value)); + bind->buffer_length = strlen(bind_binary); + } else { + rand_string(bind_binary, dataLen); + bind->buffer_length = dataLen; + } + bind->length = &bind->buffer_length; + bind->buffer = bind_binary; bind->is_null = NULL; *ptr += bind->buffer_length; @@ -5365,9 +5364,14 @@ static int32_t prepareStmtBindArrayByType(TAOS_BIND *bind, return -1; } char *bind_nchar = (char *)*ptr; - rand_string(bind_nchar, dataLen); bind->buffer_type = TSDB_DATA_TYPE_NCHAR; + if (value) { + strncpy(bind_nchar, value, strlen(value)); + } else { + rand_string(bind_nchar, dataLen); + } + bind->buffer_length = strlen(bind_nchar); bind->buffer = bind_nchar; bind->length = &bind->buffer_length; @@ -5378,7 +5382,11 @@ static int32_t prepareStmtBindArrayByType(TAOS_BIND *bind, "INT", strlen("INT"))) { int32_t *bind_int = (int32_t *)*ptr; - *bind_int = rand_int(); + if (value) { + *bind_int = atoi(value); + } else { + *bind_int = rand_int(); + } bind->buffer_type = TSDB_DATA_TYPE_INT; bind->buffer_length = sizeof(int32_t); bind->buffer = bind_int; @@ -5390,7 +5398,11 @@ static int32_t prepareStmtBindArrayByType(TAOS_BIND *bind, "BIGINT", strlen("BIGINT"))) { int64_t *bind_bigint = (int64_t *)*ptr; - *bind_bigint = rand_bigint(); + if (value) { + *bind_bigint = atoll(value); + } else { + *bind_bigint = rand_bigint(); + } bind->buffer_type = TSDB_DATA_TYPE_BIGINT; bind->buffer_length = sizeof(int64_t); bind->buffer = bind_bigint; @@ -5402,7 +5414,11 @@ static int32_t prepareStmtBindArrayByType(TAOS_BIND *bind, "FLOAT", strlen("FLOAT"))) { float *bind_float = (float *) *ptr; - *bind_float = rand_float(); + if (value) { + *bind_float = (float)atof(value); + } else { + *bind_float = rand_float(); + } bind->buffer_type = TSDB_DATA_TYPE_FLOAT; bind->buffer_length = sizeof(float); bind->buffer = bind_float; @@ -5414,7 +5430,11 @@ static int32_t prepareStmtBindArrayByType(TAOS_BIND *bind, "DOUBLE", strlen("DOUBLE"))) { double *bind_double = (double *)*ptr; - *bind_double = rand_double(); + if (value) { + *bind_double = atof(value); + } else { + *bind_double = rand_double(); + } bind->buffer_type = TSDB_DATA_TYPE_DOUBLE; bind->buffer_length = sizeof(double); bind->buffer = bind_double; @@ -5426,7 +5446,11 @@ static int32_t prepareStmtBindArrayByType(TAOS_BIND *bind, "SMALLINT", strlen("SMALLINT"))) { int16_t *bind_smallint = (int16_t *)*ptr; - *bind_smallint = rand_smallint(); + if (value) { + *bind_smallint = (int16_t)atoi(value); + } else { + *bind_smallint = rand_smallint(); + } bind->buffer_type = TSDB_DATA_TYPE_SMALLINT; bind->buffer_length = sizeof(int16_t); bind->buffer = bind_smallint; @@ -5438,7 +5462,11 @@ static int32_t prepareStmtBindArrayByType(TAOS_BIND *bind, "TINYINT", strlen("TINYINT"))) { int8_t *bind_tinyint = (int8_t *)*ptr; - *bind_tinyint = rand_tinyint(); + if (value) { + *bind_tinyint = (int8_t)atoi(value); + } else { + *bind_tinyint = rand_tinyint(); + } bind->buffer_type = TSDB_DATA_TYPE_TINYINT; bind->buffer_length = sizeof(int8_t); bind->buffer = bind_tinyint; @@ -5461,7 +5489,11 @@ static int32_t prepareStmtBindArrayByType(TAOS_BIND *bind, "TIMESTAMP", strlen("TIMESTAMP"))) { int64_t *bind_ts2 = (int64_t *) *ptr; - *bind_ts2 = rand_bigint(); + if (value) { + *bind_ts2 = atoll(value); + } else { + *bind_ts2 = rand_bigint(); + } bind->buffer_type = TSDB_DATA_TYPE_TIMESTAMP; bind->buffer_length = sizeof(int64_t); bind->buffer = bind_ts2; @@ -5527,12 +5559,13 @@ static int32_t prepareStmtWithoutStb( ptr += bind->buffer_length; for (int i = 0; i < g_args.num_of_CPR; i ++) { - bind = (TAOS_BIND *)((char *)bindArray + (sizeof(TAOS_BIND) * (i + 1))); + bind = (TAOS_BIND *)((char *)bindArray + + (sizeof(TAOS_BIND) * (i + 1))); if ( -1 == prepareStmtBindArrayByType( bind, data_type[i], g_args.len_of_binary, - &ptr)) { + &ptr, NULL)) { return -1; } } @@ -5551,12 +5584,14 @@ static int32_t prepareStmtWithoutStb( return k; } -static int32_t prepareStbStmt(SSuperTable *stbInfo, +static int32_t prepareStbStmt( + SSuperTable *stbInfo, TAOS_STMT *stmt, char *tableName, uint32_t batch, uint64_t insertRows, uint64_t recordFrom, - int64_t startTime, char *buffer) + int64_t startTime, + int64_t *pSamplePos) { int ret = taos_stmt_set_tbname(stmt, tableName); if (ret != 0) { @@ -5567,16 +5602,24 @@ static int32_t prepareStbStmt(SSuperTable *stbInfo, char *bindArray = malloc(sizeof(TAOS_BIND) * (stbInfo->columnCount + 1)); if (bindArray == NULL) { - errorPrint("Failed to allocate %d bind params\n", - (stbInfo->columnCount + 1)); + errorPrint("%s() LN%d, Failed to allocate %d bind params\n", + __func__, __LINE__, (stbInfo->columnCount + 1)); return -1; } - bool tsRand; + bool sourceRand; if (0 == strncasecmp(stbInfo->dataSource, "rand", strlen("rand"))) { - tsRand = true; + sourceRand = true; } else { - tsRand = false; + sourceRand = false; // from sample data file + } + + char *bindBuffer = malloc(g_args.len_of_binary); + if (bindBuffer == NULL) { + errorPrint("%s() LN%d, Failed to allocate %d bind buffer\n", + __func__, __LINE__, g_args.len_of_binary); + free(bindArray); + return -1; } uint32_t k; @@ -5592,7 +5635,7 @@ static int32_t prepareStbStmt(SSuperTable *stbInfo, bind_ts = (int64_t *)ptr; bind->buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - if (tsRand) { + if (sourceRand) { *bind_ts = startTime + getTSRandTail( stbInfo->timeStampStep, k, stbInfo->disorderRatio, @@ -5607,14 +5650,46 @@ static int32_t prepareStbStmt(SSuperTable *stbInfo, ptr += bind->buffer_length; + int cursor = 0; for (int i = 0; i < stbInfo->columnCount; i ++) { bind = (TAOS_BIND *)((char *)bindArray + (sizeof(TAOS_BIND) * (i + 1))); - if ( -1 == prepareStmtBindArrayByType( - bind, - stbInfo->columns[i].dataType, - stbInfo->columns[i].dataLen, - &ptr)) { - return -1; + + if (sourceRand) { + if ( -1 == prepareStmtBindArrayByType( + bind, + stbInfo->columns[i].dataType, + stbInfo->columns[i].dataLen, + &ptr, + NULL)) { + free(bindArray); + free(bindBuffer); + return -1; + } + } else { + char *restStr = stbInfo->sampleDataBuf + cursor; + int lengthOfRest = strlen(restStr); + + int index = 0; + for (index = 0; index < lengthOfRest; index ++) { + if (restStr[index] == ',') { + break; + } + } + + memset(bindBuffer, 0, g_args.len_of_binary); + strncpy(bindBuffer, restStr, index); + cursor += index + 1; // skip ',' too + + if ( -1 == prepareStmtBindArrayByType( + bind, + stbInfo->columns[i].dataType, + stbInfo->columns[i].dataLen, + &ptr, + bindBuffer)) { + free(bindArray); + free(bindBuffer); + return -1; + } } } taos_stmt_bind_param(stmt, (TAOS_BIND *)bindArray); @@ -5623,11 +5698,16 @@ static int32_t prepareStbStmt(SSuperTable *stbInfo, k++; recordFrom ++; + + if (!sourceRand) { + (*pSamplePos) ++; + } if (recordFrom >= insertRows) { break; } } + free(bindBuffer); free(bindArray); return k; } @@ -5820,13 +5900,14 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { if (superTblInfo) { if (superTblInfo->iface == STMT_IFACE) { #if STMT_IFACE_ENABLED == 1 - generated = prepareStbStmt(superTblInfo, + generated = prepareStbStmt( + superTblInfo, pThreadInfo->stmt, tableName, batchPerTbl, insertRows, i, startTime, - pThreadInfo->buffer); + &(pThreadInfo->samplePos)); #else generated = -1; #endif @@ -6051,7 +6132,8 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { pThreadInfo->stmt, tableName, g_args.num_of_RPR, - insertRows, i, start_time, pstr); + insertRows, i, start_time, + &(pThreadInfo->samplePos)); #else generated = -1; #endif From 7a80bf0417bcfbd33b0023063c757bd427d426cf Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 6 Jul 2021 17:59:51 +0800 Subject: [PATCH 24/26] Feature/sangshuduo/td 3973 use jemalloc (#6762) * [TD-3973]: add jemalloc as submodule. * add macro definitions in cmake. * [TD-3973]: use jemalloc. build works as following instructions: cmake .. -DJEMALLOC_ENABLED=true make * fix jemalloc at tag 5.2.1 * link jemalloc works. * make install works. * support jemalloc in release.sh. * release script works. * fix a typo. * [TD-3937]: support jemalloc add install funtion to all scripts. * adjust install_jemalloc() position for update check compatiblity. Co-authored-by: Shuduo Sang --- packaging/tools/install.sh | 2 +- packaging/tools/install_power.sh | 2 +- packaging/tools/install_tq.sh | 316 ++++++++++++++++++------------- 3 files changed, 184 insertions(+), 136 deletions(-) diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 3911b1bc72..7adeec487b 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -783,6 +783,7 @@ function update_TDengine() { echo "File taos.tar.gz does not exist" exit 1 fi + install_jemalloc tar -zxf taos.tar.gz # Check if version compatible @@ -822,7 +823,6 @@ function update_TDengine() { install_log install_header install_lib - install_jemalloc if [ "$pagMode" != "lite" ]; then install_connector fi diff --git a/packaging/tools/install_power.sh b/packaging/tools/install_power.sh index b1c4167e94..b21d609e14 100755 --- a/packaging/tools/install_power.sh +++ b/packaging/tools/install_power.sh @@ -752,6 +752,7 @@ function update_PowerDB() { echo "File power.tar.gz does not exist" exit 1 fi + install_jemalloc tar -zxf power.tar.gz # Check if version compatible @@ -790,7 +791,6 @@ function update_PowerDB() { install_log install_header install_lib - install_jemalloc if [ "$pagMode" != "lite" ]; then install_connector fi diff --git a/packaging/tools/install_tq.sh b/packaging/tools/install_tq.sh index 6579592fdf..33e781dd51 100755 --- a/packaging/tools/install_tq.sh +++ b/packaging/tools/install_tq.sh @@ -58,11 +58,11 @@ initd_mod=0 service_mod=2 if pidof systemd &> /dev/null; then service_mod=0 -elif $(which service &> /dev/null); then +elif $(which service &> /dev/null); then service_mod=1 - service_config_dir="/etc/init.d" + service_config_dir="/etc/init.d" if $(which chkconfig &> /dev/null); then - initd_mod=1 + initd_mod=1 elif $(which insserv &> /dev/null); then initd_mod=2 elif $(which update-rc.d &> /dev/null); then @@ -70,7 +70,7 @@ elif $(which service &> /dev/null); then else service_mod=2 fi -else +else service_mod=2 fi @@ -102,7 +102,7 @@ elif echo $osinfo | grep -qwi "fedora" ; then os_type=2 else echo " osinfo: ${osinfo}" - echo " This is an officially unverified linux system," + echo " This is an officially unverified linux system," echo " if there are any problems with the installation and operation, " echo " please feel free to contact taosdata.com for support." os_type=1 @@ -137,7 +137,7 @@ do echo "Usage: `basename $0` -v [server | client] -e [yes | no]" exit 0 ;; - ?) #unknow option + ?) #unknow option echo "unkonw argument" exit 1 ;; @@ -156,9 +156,9 @@ function kill_process() { function install_main_path() { #create install main dir and all sub dir ${csudo} rm -rf ${install_main_dir} || : - ${csudo} mkdir -p ${install_main_dir} + ${csudo} mkdir -p ${install_main_dir} ${csudo} mkdir -p ${install_main_dir}/cfg - ${csudo} mkdir -p ${install_main_dir}/bin + ${csudo} mkdir -p ${install_main_dir}/bin ${csudo} mkdir -p ${install_main_dir}/connector ${csudo} mkdir -p ${install_main_dir}/driver ${csudo} mkdir -p ${install_main_dir}/examples @@ -200,33 +200,79 @@ function install_lib() { ${csudo} rm -f ${lib_link_dir}/libtaos.* || : ${csudo} rm -f ${lib64_link_dir}/libtaos.* || : #${csudo} rm -rf ${v15_java_app_dir} || : - ${csudo} cp -rf ${script_dir}/driver/* ${install_main_dir}/driver && ${csudo} chmod 777 ${install_main_dir}/driver/* - + ${csudo} cp -rf ${script_dir}/driver/* ${install_main_dir}/driver && ${csudo} chmod 777 ${install_main_dir}/driver/* + ${csudo} ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1 ${csudo} ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so - + if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then ${csudo} ln -s ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : ${csudo} ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : fi - - #if [ "$verMode" == "cluster" ]; then + + #if [ "$verMode" == "cluster" ]; then # # Compatible with version 1.5 # ${csudo} mkdir -p ${v15_java_app_dir} # ${csudo} ln -s ${install_main_dir}/connector/taos-jdbcdriver-1.0.2-dist.jar ${v15_java_app_dir}/JDBCDriver-1.0.2-dist.jar # ${csudo} chmod 777 ${v15_java_app_dir} || : #fi - + ${csudo} ldconfig } function install_header() { ${csudo} rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taoserror.h || : - ${csudo} cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo} chmod 644 ${install_main_dir}/include/* + ${csudo} cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo} chmod 644 ${install_main_dir}/include/* ${csudo} ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h ${csudo} ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h } +function install_jemalloc() { + jemalloc_dir=${script_dir}/jemalloc + + if [ -d ${jemalloc_dir} ]; then + ${csudo} /usr/bin/install -c -d /usr/local/bin + + if [ -f ${jemalloc_dir}/bin/jemalloc-config ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc-config /usr/local/bin + fi + if [ -f ${jemalloc_dir}/bin/jemalloc.sh ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc.sh /usr/local/bin + fi + if [ -f ${jemalloc_dir}/bin/jeprof ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jeprof /usr/local/bin + fi + if [ -f ${jemalloc_dir}/include/jemalloc/jemalloc.h ]; then + ${csudo} /usr/bin/install -c -d /usr/local/include/jemalloc + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/include/jemalloc/jemalloc.h /usr/local/include/jemalloc + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc.so.2 ]; then + ${csudo} /usr/bin/install -c -d /usr/local/lib + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.so.2 /usr/local/lib + ${csudo} ln -sf libjemalloc.so.2 /usr/local/lib/libjemalloc.so + ${csudo} /usr/bin/install -c -d /usr/local/lib + if [ -f ${jemalloc_dir}/lib/libjemalloc.a ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.a /usr/local/lib + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc_pic.a /usr/local/lib + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then + ${csudo} /usr/bin/install -c -d /usr/local/lib/pkgconfig + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/lib/pkgconfig/jemalloc.pc /usr/local/lib/pkgconfig + fi + fi + if [ -f ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html ]; then + ${csudo} /usr/bin/install -c -d /usr/local/share/doc/jemalloc + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html /usr/local/share/doc/jemalloc + fi + if [ -f ${jemalloc_dir}/share/man/man3/jemalloc.3 ]; then + ${csudo} /usr/bin/install -c -d /usr/local/share/man/man3 + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/share/man/man3/jemalloc.3 /usr/local/share/man/man3 + fi + fi +} + function add_newHostname_to_hosts() { localIp="127.0.0.1" OLD_IFS="$IFS" @@ -239,13 +285,13 @@ function add_newHostname_to_hosts() { if [[ "$s" == "$localIp" ]]; then return fi - done + done ${csudo} echo "127.0.0.1 $1" >> /etc/hosts ||: } function set_hostname() { echo -e -n "${GREEN}Please enter one hostname(must not be 'localhost')${NC}:" - read newHostname + read newHostname while true; do if [[ ! -z "$newHostname" && "$newHostname" != "localhost" ]]; then break @@ -259,25 +305,25 @@ function set_hostname() { if [[ $retval != 0 ]]; then echo echo "set hostname fail!" - return + return fi #echo -e -n "$(hostnamectl status --static)" #echo -e -n "$(hostnamectl status --transient)" #echo -e -n "$(hostnamectl status --pretty)" - + #ubuntu/centos /etc/hostname if [[ -e /etc/hostname ]]; then ${csudo} echo $newHostname > /etc/hostname ||: fi - + #debian: #HOSTNAME=yourname if [[ -e /etc/sysconfig/network ]]; then ${csudo} sed -i -r "s/#*\s*(HOSTNAME=\s*).*/\1$newHostname/" /etc/sysconfig/network ||: fi ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$newHostname/" ${cfg_install_dir}/taos.cfg - serverFqdn=$newHostname - + serverFqdn=$newHostname + if [[ -e /etc/hosts ]]; then add_newHostname_to_hosts $newHostname fi @@ -295,7 +341,7 @@ function is_correct_ipaddr() { return 0 fi done - + return 1 } @@ -309,13 +355,13 @@ function set_ipAsFqdn() { echo echo -e -n "${GREEN}Unable to get local ip, use 127.0.0.1${NC}" localFqdn="127.0.0.1" - # Write the local FQDN to configuration file - ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg + # Write the local FQDN to configuration file + ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg serverFqdn=$localFqdn echo return - fi - + fi + echo -e -n "${GREEN}Please choose an IP from local IP list${NC}:" echo echo -e -n "${GREEN}$iplist${NC}" @@ -324,15 +370,15 @@ function set_ipAsFqdn() { echo -e -n "${GREEN}Notes: if IP is used as the node name, data can NOT be migrated to other machine directly${NC}:" read localFqdn while true; do - if [ ! -z "$localFqdn" ]; then + if [ ! -z "$localFqdn" ]; then # Check if correct ip address is_correct_ipaddr $localFqdn retval=`echo $?` if [[ $retval != 0 ]]; then read -p "Please choose an IP from local IP list:" localFqdn else - # Write the local FQDN to configuration file - ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg + # Write the local FQDN to configuration file + ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg serverFqdn=$localFqdn break fi @@ -343,63 +389,63 @@ function set_ipAsFqdn() { } function local_fqdn_check() { - #serverFqdn=$(hostname) - echo - echo -e -n "System hostname is: ${GREEN}$serverFqdn${NC}" - echo - if [[ "$serverFqdn" == "" ]] || [[ "$serverFqdn" == "localhost" ]]; then - echo -e -n "${GREEN}It is strongly recommended to configure a hostname for this machine ${NC}" + #serverFqdn=$(hostname) echo - - while true - do - read -r -p "Set hostname now? [Y/n] " input - if [ ! -n "$input" ]; then - set_hostname - break - else - case $input in - [yY][eE][sS]|[yY]) - set_hostname - break - ;; - - [nN][oO]|[nN]) - set_ipAsFqdn - break - ;; - - *) - echo "Invalid input..." - ;; - esac - fi - done - fi + echo -e -n "System hostname is: ${GREEN}$serverFqdn${NC}" + echo + if [[ "$serverFqdn" == "" ]] || [[ "$serverFqdn" == "localhost" ]]; then + echo -e -n "${GREEN}It is strongly recommended to configure a hostname for this machine ${NC}" + echo + + while true + do + read -r -p "Set hostname now? [Y/n] " input + if [ ! -n "$input" ]; then + set_hostname + break + else + case $input in + [yY][eE][sS]|[yY]) + set_hostname + break + ;; + + [nN][oO]|[nN]) + set_ipAsFqdn + break + ;; + + *) + echo "Invalid input..." + ;; + esac + fi + done + fi } function install_config() { #${csudo} rm -f ${install_main_dir}/cfg/taos.cfg || : - + if [ ! -f ${cfg_install_dir}/taos.cfg ]; then ${csudo} mkdir -p ${cfg_install_dir} [ -f ${script_dir}/cfg/taos.cfg ] && ${csudo} cp ${script_dir}/cfg/taos.cfg ${cfg_install_dir} ${csudo} chmod 644 ${cfg_install_dir}/* - fi - + fi + ${csudo} cp -f ${script_dir}/cfg/taos.cfg ${install_main_dir}/cfg/taos.cfg.org ${csudo} ln -s ${cfg_install_dir}/taos.cfg ${install_main_dir}/cfg [ ! -z $1 ] && return 0 || : # only install client - + if ((${update_flag}==1)); then return 0 fi - + if [ "$interactiveFqdn" == "no" ]; then return 0 fi - + local_fqdn_check #FQDN_FORMAT="(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)" @@ -417,8 +463,8 @@ function install_config() { if [ ! -z "$firstEp" ]; then # check the format of the firstEp #if [[ $firstEp == $FQDN_PATTERN ]]; then - # Write the first FQDN to configuration file - ${csudo} sed -i -r "s/#*\s*(firstEp\s*).*/\1$firstEp/" ${cfg_install_dir}/taos.cfg + # Write the first FQDN to configuration file + ${csudo} sed -i -r "s/#*\s*(firstEp\s*).*/\1$firstEp/" ${cfg_install_dir}/taos.cfg break #else # read -p "Please enter the correct FQDN:port: " firstEp @@ -426,21 +472,21 @@ function install_config() { else break fi - done + done } function install_log() { ${csudo} rm -rf ${log_dir} || : ${csudo} mkdir -p ${log_dir} && ${csudo} chmod 777 ${log_dir} - + ${csudo} ln -s ${log_dir} ${install_main_dir}/log } function install_data() { ${csudo} mkdir -p ${data_dir} - - ${csudo} ln -s ${data_dir} ${install_main_dir}/data + + ${csudo} ln -s ${data_dir} ${install_main_dir}/data } function install_connector() { @@ -455,12 +501,12 @@ function install_examples() { function clean_service_on_sysvinit() { #restart_config_str="tq:2345:respawn:${service_config_dir}/tqd start" - #${csudo} sed -i "\|${restart_config_str}|d" /etc/inittab || : - + #${csudo} sed -i "\|${restart_config_str}|d" /etc/inittab || : + if pidof tqd &> /dev/null; then ${csudo} service tqd stop || : fi - + if pidof tarbitrator &> /dev/null; then ${csudo} service tarbitratord stop || : fi @@ -470,7 +516,7 @@ function clean_service_on_sysvinit() { ${csudo} chkconfig --del tqd || : fi - if [ -e ${service_config_dir}/tarbitratord ]; then + if [ -e ${service_config_dir}/tarbitratord ]; then ${csudo} chkconfig --del tarbitratord || : fi elif ((${initd_mod}==2)); then @@ -488,10 +534,10 @@ function clean_service_on_sysvinit() { ${csudo} update-rc.d -f tarbitratord remove || : fi fi - + ${csudo} rm -f ${service_config_dir}/tqd || : ${csudo} rm -f ${service_config_dir}/tarbitratord || : - + if $(which init &> /dev/null); then ${csudo} init q || : fi @@ -514,10 +560,10 @@ function install_service_on_sysvinit() { ${csudo} cp -f ${script_dir}/init.d/tarbitratord.rpm ${install_main_dir}/init.d/tarbitratord ${csudo} cp ${script_dir}/init.d/tarbitratord.rpm ${service_config_dir}/tarbitratord && ${csudo} chmod a+x ${service_config_dir}/tarbitratord fi - + #restart_config_str="tq:2345:respawn:${service_config_dir}/tqd start" #${csudo} grep -q -F "$restart_config_str" /etc/inittab || ${csudo} bash -c "echo '${restart_config_str}' >> /etc/inittab" - + if ((${initd_mod}==1)); then ${csudo} chkconfig --add tqd || : ${csudo} chkconfig --level 2345 tqd on || : @@ -542,7 +588,7 @@ function clean_service_on_systemd() { fi ${csudo} systemctl disable tqd &> /dev/null || echo &> /dev/null ${csudo} rm -f ${tqd_service_config} - + tarbitratord_service_config="${service_config_dir}/tarbitratord.service" if systemctl is-active --quiet tarbitratord; then echo "tarbitrator is running, stopping it..." @@ -550,16 +596,16 @@ function clean_service_on_systemd() { fi ${csudo} systemctl disable tarbitratord &> /dev/null || echo &> /dev/null ${csudo} rm -f ${tarbitratord_service_config} - + if [ "$verMode" == "cluster" ]; then - nginx_service_config="${service_config_dir}/nginxd.service" - if systemctl is-active --quiet nginxd; then - echo "Nginx for TDengine is running, stopping it..." - ${csudo} systemctl stop nginxd &> /dev/null || echo &> /dev/null - fi - ${csudo} systemctl disable nginxd &> /dev/null || echo &> /dev/null - ${csudo} rm -f ${nginx_service_config} - fi + nginx_service_config="${service_config_dir}/nginxd.service" + if systemctl is-active --quiet nginxd; then + echo "Nginx for TDengine is running, stopping it..." + ${csudo} systemctl stop nginxd &> /dev/null || echo &> /dev/null + fi + ${csudo} systemctl disable nginxd &> /dev/null || echo &> /dev/null + ${csudo} rm -f ${nginx_service_config} + fi } # tq:2345:respawn:/etc/init.d/tqd start @@ -590,7 +636,7 @@ function install_service_on_systemd() { ${csudo} bash -c "echo '[Install]' >> ${tqd_service_config}" ${csudo} bash -c "echo 'WantedBy=multi-user.target' >> ${tqd_service_config}" ${csudo} systemctl enable tqd - + tarbitratord_service_config="${service_config_dir}/tarbitratord.service" ${csudo} bash -c "echo '[Unit]' >> ${tarbitratord_service_config}" ${csudo} bash -c "echo 'Description=TDengine arbitrator service' >> ${tarbitratord_service_config}" @@ -612,9 +658,9 @@ function install_service_on_systemd() { ${csudo} bash -c "echo >> ${tarbitratord_service_config}" ${csudo} bash -c "echo '[Install]' >> ${tarbitratord_service_config}" ${csudo} bash -c "echo 'WantedBy=multi-user.target' >> ${tarbitratord_service_config}" - #${csudo} systemctl enable tarbitratord - - if [ "$verMode" == "cluster" ]; then + #${csudo} systemctl enable tarbitratord + + if [ "$verMode" == "cluster" ]; then nginx_service_config="${service_config_dir}/nginxd.service" ${csudo} bash -c "echo '[Unit]' >> ${nginx_service_config}" ${csudo} bash -c "echo 'Description=Nginx For PowrDB Service' >> ${nginx_service_config}" @@ -643,7 +689,7 @@ function install_service_on_systemd() { ${csudo} systemctl enable nginxd fi ${csudo} systemctl start nginxd - fi + fi } function install_service() { @@ -706,6 +752,7 @@ function update_tq() { echo "File tq.tar.gz does not exist" exit 1 fi + install_jemalloc tar -zxf tq.tar.gz # Check if version compatible @@ -725,8 +772,8 @@ function update_tq() { kill_process tqd fi sleep 1 - fi - if [ "$verMode" == "cluster" ]; then + fi + if [ "$verMode" == "cluster" ]; then if pidof nginx &> /dev/null; then if ((${service_mod}==0)); then ${csudo} systemctl stop nginxd || : @@ -738,7 +785,7 @@ function update_tq() { sleep 1 fi fi - + install_main_path install_log @@ -751,10 +798,10 @@ function update_tq() { if [ -z $1 ]; then install_bin install_service - install_config - + install_config + openresty_work=false - if [ "$verMode" == "cluster" ]; then + if [ "$verMode" == "cluster" ]; then # Check if openresty is installed # Check if nginx is installed successfully if type curl &> /dev/null; then @@ -765,7 +812,7 @@ function update_tq() { echo -e "\033[44;31;5mNginx for TQ does not work! Please try again!\033[0m" fi fi - fi + fi #echo #echo -e "\033[44;32;1mTQ is updated successfully!${NC}" @@ -784,7 +831,7 @@ function update_tq() { else echo -e "${GREEN_DARK}To access TQ ${NC}: use ${GREEN_UNDERLINE}tq -h $serverFqdn${NC} in shell${NC}" fi - + echo echo -e "\033[44;32;1mTQ is updated successfully!${NC}" else @@ -807,16 +854,17 @@ function install_tq() { tar -zxf tq.tar.gz echo -e "${GREEN}Start to install TQ...${NC}" - - install_main_path - + + install_main_path + if [ -z $1 ]; then install_data - fi - - install_log + fi + + install_log install_header install_lib + install_jemalloc if [ "$pagMode" != "lite" ]; then install_connector fi @@ -839,8 +887,8 @@ function install_tq() { fi fi fi - - install_config + + install_config # Ask if to start the service #echo @@ -853,35 +901,35 @@ function install_tq() { echo -e "${GREEN_DARK}To start TQ ${NC}: ${csudo} service tqd start${NC}" else echo -e "${GREEN_DARK}To start TQ ${NC}: tqd${NC}" - fi + fi #if [ ${openresty_work} = 'true' ]; then # echo -e "${GREEN_DARK}To access TQ ${NC}: use ${GREEN_UNDERLINE}tq${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${nginx_port}${NC}" #else # echo -e "${GREEN_DARK}To access TQ ${NC}: use ${GREEN_UNDERLINE}tq${NC} in shell${NC}" #fi - + if [ ! -z "$firstEp" ]; then - tmpFqdn=${firstEp%%:*} - substr=":" - if [[ $firstEp =~ $substr ]];then - tmpPort=${firstEp#*:} - else - tmpPort="" - fi - if [[ "$tmpPort" != "" ]];then - echo -e "${GREEN_DARK}To access TQ ${NC}: tq -h $tmpFqdn -P $tmpPort${GREEN_DARK} to login into cluster, then${NC}" - else - echo -e "${GREEN_DARK}To access TQ ${NC}: tq -h $tmpFqdn${GREEN_DARK} to login into cluster, then${NC}" - fi - echo -e "${GREEN_DARK}execute ${NC}: create dnode 'newDnodeFQDN:port'; ${GREEN_DARK}to add this new node${NC}" - echo + tmpFqdn=${firstEp%%:*} + substr=":" + if [[ $firstEp =~ $substr ]];then + tmpPort=${firstEp#*:} + else + tmpPort="" + fi + if [[ "$tmpPort" != "" ]];then + echo -e "${GREEN_DARK}To access TQ ${NC}: tq -h $tmpFqdn -P $tmpPort${GREEN_DARK} to login into cluster, then${NC}" + else + echo -e "${GREEN_DARK}To access TQ ${NC}: tq -h $tmpFqdn${GREEN_DARK} to login into cluster, then${NC}" + fi + echo -e "${GREEN_DARK}execute ${NC}: create dnode 'newDnodeFQDN:port'; ${GREEN_DARK}to add this new node${NC}" + echo elif [ ! -z "$serverFqdn" ]; then - echo -e "${GREEN_DARK}To access TQ ${NC}: tq -h $serverFqdn${GREEN_DARK} to login into TQ server${NC}" - echo + echo -e "${GREEN_DARK}To access TQ ${NC}: tq -h $serverFqdn${GREEN_DARK} to login into TQ server${NC}" + echo fi echo -e "\033[44;32;1mTQ is installed successfully!${NC}" - echo + echo else # Only install client install_bin install_config @@ -913,6 +961,6 @@ elif [ "$verType" == "client" ]; then else install_tq client fi -else - echo "please input correct verType" +else + echo "please input correct verType" fi From 7d9efc95a28fd00d113dd9ab6b1ecd768845c0cb Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 6 Jul 2021 18:31:38 +0800 Subject: [PATCH 25/26] Feature/sangshuduo/td 3973 use jemalloc (#6766) * [TD-3973]: add jemalloc as submodule. * add macro definitions in cmake. * [TD-3973]: use jemalloc. build works as following instructions: cmake .. -DJEMALLOC_ENABLED=true make * fix jemalloc at tag 5.2.1 * link jemalloc works. * make install works. * support jemalloc in release.sh. * release script works. * fix a typo. * [TD-3937]: support jemalloc add install funtion to all scripts. * adjust install_jemalloc() position for update check compatiblity. * fix position bug. Co-authored-by: Shuduo Sang --- packaging/tools/install.sh | 1 + packaging/tools/install_power.sh | 1 + packaging/tools/install_tq.sh | 1 + 3 files changed, 3 insertions(+) diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 7adeec487b..226e09d846 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -785,6 +785,7 @@ function update_TDengine() { fi install_jemalloc tar -zxf taos.tar.gz + install_jemalloc # Check if version compatible if ! is_version_compatible; then diff --git a/packaging/tools/install_power.sh b/packaging/tools/install_power.sh index b21d609e14..e6941301c1 100755 --- a/packaging/tools/install_power.sh +++ b/packaging/tools/install_power.sh @@ -754,6 +754,7 @@ function update_PowerDB() { fi install_jemalloc tar -zxf power.tar.gz + install_jemalloc # Check if version compatible if ! is_version_compatible; then diff --git a/packaging/tools/install_tq.sh b/packaging/tools/install_tq.sh index 33e781dd51..e0726c0bb6 100755 --- a/packaging/tools/install_tq.sh +++ b/packaging/tools/install_tq.sh @@ -754,6 +754,7 @@ function update_tq() { fi install_jemalloc tar -zxf tq.tar.gz + install_jemalloc # Check if version compatible if ! is_version_compatible; then From 7c869c765a15891d22d462e9aa31f73c1e41ad5b Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 6 Jul 2021 19:32:21 +0800 Subject: [PATCH 26/26] Feature/sangshuduo/td 3973 use jemalloc (#6767) * [TD-3973]: add jemalloc as submodule. * add macro definitions in cmake. * [TD-3973]: use jemalloc. build works as following instructions: cmake .. -DJEMALLOC_ENABLED=true make * fix jemalloc at tag 5.2.1 * link jemalloc works. * make install works. * support jemalloc in release.sh. * release script works. * fix a typo. * [TD-3937]: support jemalloc add install funtion to all scripts. * adjust install_jemalloc() position for update check compatiblity. * fix position bug. * add ldconfig for jemalloc library cache refresh. Co-authored-by: Shuduo Sang --- packaging/tools/install.sh | 1 + packaging/tools/install_arbi.sh | 1 + packaging/tools/install_arbi_power.sh | 1 + packaging/tools/install_client.sh | 1 + packaging/tools/install_client_power.sh | 1 + packaging/tools/install_power.sh | 1 + packaging/tools/install_tq.sh | 1 + packaging/tools/make_install.sh | 2 + src/kit/taosdemo/taosdemo.c | 158 ++++++------------------ 9 files changed, 47 insertions(+), 120 deletions(-) diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 226e09d846..16a99d7a0c 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -270,6 +270,7 @@ function install_jemalloc() { ${csudo} /usr/bin/install -c -d /usr/local/share/man/man3 ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/share/man/man3/jemalloc.3 /usr/local/share/man/man3 fi + ${csudo} ldconfig fi } diff --git a/packaging/tools/install_arbi.sh b/packaging/tools/install_arbi.sh index cdfeef0543..74a7d8cd3d 100755 --- a/packaging/tools/install_arbi.sh +++ b/packaging/tools/install_arbi.sh @@ -165,6 +165,7 @@ function install_jemalloc() { ${csudo} /usr/bin/install -c -d /usr/local/share/man/man3 ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/share/man/man3/jemalloc.3 /usr/local/share/man/man3 fi + ${csudo} ldconfig fi } diff --git a/packaging/tools/install_arbi_power.sh b/packaging/tools/install_arbi_power.sh index 161b916b4c..864fbb9844 100755 --- a/packaging/tools/install_arbi_power.sh +++ b/packaging/tools/install_arbi_power.sh @@ -158,6 +158,7 @@ function install_jemalloc() { ${csudo} /usr/bin/install -c -d /usr/local/share/man/man3 ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/share/man/man3/jemalloc.3 /usr/local/share/man/man3 fi + ${csudo} ldconfig fi } diff --git a/packaging/tools/install_client.sh b/packaging/tools/install_client.sh index 2969a3cc98..9044f23672 100755 --- a/packaging/tools/install_client.sh +++ b/packaging/tools/install_client.sh @@ -182,6 +182,7 @@ function install_jemalloc() { ${csudo} /usr/bin/install -c -d /usr/local/share/man/man3 ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/share/man/man3/jemalloc.3 /usr/local/share/man/man3 fi + ${csudo} ldconfig fi } diff --git a/packaging/tools/install_client_power.sh b/packaging/tools/install_client_power.sh index d56527ecda..bb9be2493b 100755 --- a/packaging/tools/install_client_power.sh +++ b/packaging/tools/install_client_power.sh @@ -182,6 +182,7 @@ function install_jemalloc() { ${csudo} /usr/bin/install -c -d /usr/local/share/man/man3 ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/share/man/man3/jemalloc.3 /usr/local/share/man/man3 fi + ${csudo} ldconfig fi } diff --git a/packaging/tools/install_power.sh b/packaging/tools/install_power.sh index e6941301c1..bf4b1244ad 100755 --- a/packaging/tools/install_power.sh +++ b/packaging/tools/install_power.sh @@ -263,6 +263,7 @@ function install_jemalloc() { ${csudo} /usr/bin/install -c -d /usr/local/share/man/man3 ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/share/man/man3/jemalloc.3 /usr/local/share/man/man3 fi + ${csudo} ldconfig fi } diff --git a/packaging/tools/install_tq.sh b/packaging/tools/install_tq.sh index e0726c0bb6..7d4e374827 100755 --- a/packaging/tools/install_tq.sh +++ b/packaging/tools/install_tq.sh @@ -270,6 +270,7 @@ function install_jemalloc() { ${csudo} /usr/bin/install -c -d /usr/local/share/man/man3 ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/share/man/man3/jemalloc.3 /usr/local/share/man/man3 fi + ${csudo} ldconfig fi } diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh index 0c755d9f72..b1d56092c2 100755 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -176,6 +176,7 @@ function install_bin() { [ -x ${install_main_dir}/bin/remove_client.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove_client.sh ${bin_link_dir}/rmtaos || : fi } + function install_jemalloc() { if [ "$osType" != "Darwin" ]; then /usr/bin/install -c -d /usr/local/bin @@ -217,6 +218,7 @@ function install_jemalloc() { /usr/bin/install -c -d /usr/local/share/man/man3 /usr/bin/install -c -m 644 ${binary_dir}/build/share/man/man3/jemalloc.3 /usr/local/share/man/man3 fi + ${csudo} ldconfig fi } diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index b162aa5299..6513f3e214 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -3216,6 +3216,13 @@ static int readTagFromCsvFileToMem(SSuperTable * superTblInfo) { return 0; } +#if 0 +int readSampleFromJsonFileToMem(SSuperTable * superTblInfo) { + // TODO + return 0; +} +#endif + /* Read 10000 lines at most. If more than 10000 lines, continue to read after using */ @@ -5331,7 +5338,7 @@ static int64_t generateInterlaceDataWithoutStb( #if STMT_IFACE_ENABLED == 1 static int32_t prepareStmtBindArrayByType(TAOS_BIND *bind, - char *dataType, int32_t dataLen, char **ptr, char *value) + char *dataType, int32_t dataLen, char **ptr) { if (0 == strncasecmp(dataType, "BINARY", strlen("BINARY"))) { @@ -5341,18 +5348,12 @@ static int32_t prepareStmtBindArrayByType(TAOS_BIND *bind, return -1; } char *bind_binary = (char *)*ptr; + rand_string(bind_binary, dataLen); bind->buffer_type = TSDB_DATA_TYPE_BINARY; - if (value) { - strncpy(bind_binary, value, strlen(value)); - bind->buffer_length = strlen(bind_binary); - } else { - rand_string(bind_binary, dataLen); - bind->buffer_length = dataLen; - } - - bind->length = &bind->buffer_length; + bind->buffer_length = dataLen; bind->buffer = bind_binary; + bind->length = &bind->buffer_length; bind->is_null = NULL; *ptr += bind->buffer_length; @@ -5364,14 +5365,9 @@ static int32_t prepareStmtBindArrayByType(TAOS_BIND *bind, return -1; } char *bind_nchar = (char *)*ptr; + rand_string(bind_nchar, dataLen); bind->buffer_type = TSDB_DATA_TYPE_NCHAR; - if (value) { - strncpy(bind_nchar, value, strlen(value)); - } else { - rand_string(bind_nchar, dataLen); - } - bind->buffer_length = strlen(bind_nchar); bind->buffer = bind_nchar; bind->length = &bind->buffer_length; @@ -5382,11 +5378,7 @@ static int32_t prepareStmtBindArrayByType(TAOS_BIND *bind, "INT", strlen("INT"))) { int32_t *bind_int = (int32_t *)*ptr; - if (value) { - *bind_int = atoi(value); - } else { - *bind_int = rand_int(); - } + *bind_int = rand_int(); bind->buffer_type = TSDB_DATA_TYPE_INT; bind->buffer_length = sizeof(int32_t); bind->buffer = bind_int; @@ -5398,11 +5390,7 @@ static int32_t prepareStmtBindArrayByType(TAOS_BIND *bind, "BIGINT", strlen("BIGINT"))) { int64_t *bind_bigint = (int64_t *)*ptr; - if (value) { - *bind_bigint = atoll(value); - } else { - *bind_bigint = rand_bigint(); - } + *bind_bigint = rand_bigint(); bind->buffer_type = TSDB_DATA_TYPE_BIGINT; bind->buffer_length = sizeof(int64_t); bind->buffer = bind_bigint; @@ -5414,11 +5402,7 @@ static int32_t prepareStmtBindArrayByType(TAOS_BIND *bind, "FLOAT", strlen("FLOAT"))) { float *bind_float = (float *) *ptr; - if (value) { - *bind_float = (float)atof(value); - } else { - *bind_float = rand_float(); - } + *bind_float = rand_float(); bind->buffer_type = TSDB_DATA_TYPE_FLOAT; bind->buffer_length = sizeof(float); bind->buffer = bind_float; @@ -5430,11 +5414,7 @@ static int32_t prepareStmtBindArrayByType(TAOS_BIND *bind, "DOUBLE", strlen("DOUBLE"))) { double *bind_double = (double *)*ptr; - if (value) { - *bind_double = atof(value); - } else { - *bind_double = rand_double(); - } + *bind_double = rand_double(); bind->buffer_type = TSDB_DATA_TYPE_DOUBLE; bind->buffer_length = sizeof(double); bind->buffer = bind_double; @@ -5446,11 +5426,7 @@ static int32_t prepareStmtBindArrayByType(TAOS_BIND *bind, "SMALLINT", strlen("SMALLINT"))) { int16_t *bind_smallint = (int16_t *)*ptr; - if (value) { - *bind_smallint = (int16_t)atoi(value); - } else { - *bind_smallint = rand_smallint(); - } + *bind_smallint = rand_smallint(); bind->buffer_type = TSDB_DATA_TYPE_SMALLINT; bind->buffer_length = sizeof(int16_t); bind->buffer = bind_smallint; @@ -5462,11 +5438,7 @@ static int32_t prepareStmtBindArrayByType(TAOS_BIND *bind, "TINYINT", strlen("TINYINT"))) { int8_t *bind_tinyint = (int8_t *)*ptr; - if (value) { - *bind_tinyint = (int8_t)atoi(value); - } else { - *bind_tinyint = rand_tinyint(); - } + *bind_tinyint = rand_tinyint(); bind->buffer_type = TSDB_DATA_TYPE_TINYINT; bind->buffer_length = sizeof(int8_t); bind->buffer = bind_tinyint; @@ -5489,11 +5461,7 @@ static int32_t prepareStmtBindArrayByType(TAOS_BIND *bind, "TIMESTAMP", strlen("TIMESTAMP"))) { int64_t *bind_ts2 = (int64_t *) *ptr; - if (value) { - *bind_ts2 = atoll(value); - } else { - *bind_ts2 = rand_bigint(); - } + *bind_ts2 = rand_bigint(); bind->buffer_type = TSDB_DATA_TYPE_TIMESTAMP; bind->buffer_length = sizeof(int64_t); bind->buffer = bind_ts2; @@ -5559,13 +5527,12 @@ static int32_t prepareStmtWithoutStb( ptr += bind->buffer_length; for (int i = 0; i < g_args.num_of_CPR; i ++) { - bind = (TAOS_BIND *)((char *)bindArray - + (sizeof(TAOS_BIND) * (i + 1))); + bind = (TAOS_BIND *)((char *)bindArray + (sizeof(TAOS_BIND) * (i + 1))); if ( -1 == prepareStmtBindArrayByType( bind, data_type[i], g_args.len_of_binary, - &ptr, NULL)) { + &ptr)) { return -1; } } @@ -5584,14 +5551,12 @@ static int32_t prepareStmtWithoutStb( return k; } -static int32_t prepareStbStmt( - SSuperTable *stbInfo, +static int32_t prepareStbStmt(SSuperTable *stbInfo, TAOS_STMT *stmt, char *tableName, uint32_t batch, uint64_t insertRows, uint64_t recordFrom, - int64_t startTime, - int64_t *pSamplePos) + int64_t startTime, char *buffer) { int ret = taos_stmt_set_tbname(stmt, tableName); if (ret != 0) { @@ -5602,24 +5567,16 @@ static int32_t prepareStbStmt( char *bindArray = malloc(sizeof(TAOS_BIND) * (stbInfo->columnCount + 1)); if (bindArray == NULL) { - errorPrint("%s() LN%d, Failed to allocate %d bind params\n", - __func__, __LINE__, (stbInfo->columnCount + 1)); + errorPrint("Failed to allocate %d bind params\n", + (stbInfo->columnCount + 1)); return -1; } - bool sourceRand; + bool tsRand; if (0 == strncasecmp(stbInfo->dataSource, "rand", strlen("rand"))) { - sourceRand = true; + tsRand = true; } else { - sourceRand = false; // from sample data file - } - - char *bindBuffer = malloc(g_args.len_of_binary); - if (bindBuffer == NULL) { - errorPrint("%s() LN%d, Failed to allocate %d bind buffer\n", - __func__, __LINE__, g_args.len_of_binary); - free(bindArray); - return -1; + tsRand = false; } uint32_t k; @@ -5635,7 +5592,7 @@ static int32_t prepareStbStmt( bind_ts = (int64_t *)ptr; bind->buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - if (sourceRand) { + if (tsRand) { *bind_ts = startTime + getTSRandTail( stbInfo->timeStampStep, k, stbInfo->disorderRatio, @@ -5650,46 +5607,14 @@ static int32_t prepareStbStmt( ptr += bind->buffer_length; - int cursor = 0; for (int i = 0; i < stbInfo->columnCount; i ++) { bind = (TAOS_BIND *)((char *)bindArray + (sizeof(TAOS_BIND) * (i + 1))); - - if (sourceRand) { - if ( -1 == prepareStmtBindArrayByType( - bind, - stbInfo->columns[i].dataType, - stbInfo->columns[i].dataLen, - &ptr, - NULL)) { - free(bindArray); - free(bindBuffer); - return -1; - } - } else { - char *restStr = stbInfo->sampleDataBuf + cursor; - int lengthOfRest = strlen(restStr); - - int index = 0; - for (index = 0; index < lengthOfRest; index ++) { - if (restStr[index] == ',') { - break; - } - } - - memset(bindBuffer, 0, g_args.len_of_binary); - strncpy(bindBuffer, restStr, index); - cursor += index + 1; // skip ',' too - - if ( -1 == prepareStmtBindArrayByType( - bind, - stbInfo->columns[i].dataType, - stbInfo->columns[i].dataLen, - &ptr, - bindBuffer)) { - free(bindArray); - free(bindBuffer); - return -1; - } + if ( -1 == prepareStmtBindArrayByType( + bind, + stbInfo->columns[i].dataType, + stbInfo->columns[i].dataLen, + &ptr)) { + return -1; } } taos_stmt_bind_param(stmt, (TAOS_BIND *)bindArray); @@ -5698,16 +5623,11 @@ static int32_t prepareStbStmt( k++; recordFrom ++; - - if (!sourceRand) { - (*pSamplePos) ++; - } if (recordFrom >= insertRows) { break; } } - free(bindBuffer); free(bindArray); return k; } @@ -5900,14 +5820,13 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { if (superTblInfo) { if (superTblInfo->iface == STMT_IFACE) { #if STMT_IFACE_ENABLED == 1 - generated = prepareStbStmt( - superTblInfo, + generated = prepareStbStmt(superTblInfo, pThreadInfo->stmt, tableName, batchPerTbl, insertRows, i, startTime, - &(pThreadInfo->samplePos)); + pThreadInfo->buffer); #else generated = -1; #endif @@ -6132,8 +6051,7 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { pThreadInfo->stmt, tableName, g_args.num_of_RPR, - insertRows, i, start_time, - &(pThreadInfo->samplePos)); + insertRows, i, start_time, pstr); #else generated = -1; #endif