From 2e3e34f330812217bf6a585b2b4c7c60470773f3 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 1 Mar 2021 11:15:35 +0800 Subject: [PATCH 01/66] [TD-3096]: fix query dead lock --- src/tsdb/inc/tsdbMemTable.h | 3 +- src/tsdb/src/tsdbMemTable.c | 85 +++++++++++++++++++++++++++++++------ src/tsdb/src/tsdbRead.c | 2 +- 3 files changed, 74 insertions(+), 16 deletions(-) diff --git a/src/tsdb/inc/tsdbMemTable.h b/src/tsdb/inc/tsdbMemTable.h index 3b3f1dd1f6..988483ec78 100644 --- a/src/tsdb/inc/tsdbMemTable.h +++ b/src/tsdb/inc/tsdbMemTable.h @@ -37,6 +37,7 @@ typedef struct { TSKEY keyLast; int64_t numOfRows; SSkipList* pData; + T_REF_DECLARE(); } STableData; typedef struct { @@ -76,7 +77,7 @@ typedef struct { int tsdbRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable); int tsdbUnRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable); -int tsdbTakeMemSnapshot(STsdbRepo* pRepo, SMemTable** pMem, SMemTable** pIMem); +int tsdbTakeMemSnapshot(STsdbRepo* pRepo, SMemTable** pMem, SMemTable** pIMem, SArray* pATable); void tsdbUnTakeMemSnapShot(STsdbRepo* pRepo, SMemTable* pMem, SMemTable* pIMem); void* tsdbAllocBytes(STsdbRepo* pRepo, int bytes); int tsdbAsyncCommit(STsdbRepo* pRepo); diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 73a1270799..6818f2ed14 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -124,17 +124,66 @@ int tsdbUnRefMemTable(STsdbRepo *pRepo, SMemTable *pMemTable) { return 0; } -int tsdbTakeMemSnapshot(STsdbRepo *pRepo, SMemTable **pMem, SMemTable **pIMem) { +int tsdbTakeMemSnapshot(STsdbRepo *pRepo, SMemTable **pMem, SMemTable **pIMem, SArray *pATable) { + SMemTable *tmem; + + // Get snap object if (tsdbLockRepo(pRepo) < 0) return -1; - *pMem = pRepo->mem; + tmem = pRepo->mem; *pIMem = pRepo->imem; - tsdbRefMemTable(pRepo, *pMem); + tsdbRefMemTable(pRepo, tmem); tsdbRefMemTable(pRepo, *pIMem); if (tsdbUnlockRepo(pRepo) < 0) return -1; - if (*pMem != NULL) taosRLockLatch(&((*pMem)->latch)); + // Copy mem objects and ref needed STableData + if (tmem) { + taosRLockLatch(&(tmem->latch)); + + *pMem = (SMemTable *)calloc(1, sizeof(**pMem)); + if (*pMem == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + taosRUnLockLatch(&(tmem->latch)); + tsdbUnRefMemTable(pRepo, tmem); + tsdbUnRefMemTable(pRepo, *pIMem); + *pMem = NULL; + *pIMem = NULL; + return -1; + } + + (*pMem)->tData = (STableData **)calloc(tmem->maxTables, sizeof(STableData *)); + if ((*pMem)->tData == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + taosRUnLockLatch(&(tmem->latch)); + free(*pMem); + tsdbUnRefMemTable(pRepo, tmem); + tsdbUnRefMemTable(pRepo, *pIMem); + *pMem = NULL; + *pIMem = NULL; + return -1; + } + + (*pMem)->keyFirst = tmem->keyFirst; + (*pMem)->keyLast = tmem->keyLast; + (*pMem)->numOfRows = tmem->numOfRows; + (*pMem)->maxTables = tmem->maxTables; + + for (size_t i = 0; i < taosArrayGetSize(pATable); i++) { + STable * pTable = *(STable **)taosArrayGet(pATable, i); + int32_t tid = TABLE_TID(pTable); + STableData *pTableData = (tid < tmem->maxTables) ? tmem->tData[tid] : NULL; + + if ((pTableData == NULL) || (TABLE_UID(pTable) != pTableData->uid)) continue; + + (*pMem)->tData[tid] = tmem->tData[tid]; + T_REF_INC(tmem->tData[tid]); + } + + taosRUnLockLatch(&(tmem->latch)); + } + + tsdbUnRefMemTable(pRepo, tmem); tsdbDebug("vgId:%d take memory snapshot, pMem %p pIMem %p", REPO_ID(pRepo), *pMem, *pIMem); return 0; @@ -144,8 +193,14 @@ void tsdbUnTakeMemSnapShot(STsdbRepo *pRepo, SMemTable *pMem, SMemTable *pIMem) tsdbDebug("vgId:%d untake memory snapshot, pMem %p pIMem %p", REPO_ID(pRepo), pMem, pIMem); if (pMem != NULL) { - taosRUnLockLatch(&(pMem->latch)); - tsdbUnRefMemTable(pRepo, pMem); + for (size_t i = 0; i < pMem->maxTables; i++) { + STableData *pTableData = pMem->tData[i]; + if (pTableData) { + tsdbFreeTableData(pTableData); + } + } + free(pMem->tData); + free(pMem); } if (pIMem != NULL) { @@ -436,7 +491,7 @@ static STableData *tsdbNewTableData(STsdbCfg *pCfg, STable *pTable) { STableData *pTableData = (STableData *)calloc(1, sizeof(*pTableData)); if (pTableData == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; + return NULL; } pTableData->uid = TABLE_UID(pTable); @@ -449,20 +504,22 @@ static STableData *tsdbNewTableData(STsdbCfg *pCfg, STable *pTable) { tkeyComparFn, pCfg->update ? SL_UPDATE_DUP_KEY : SL_DISCARD_DUP_KEY, tsdbGetTsTupleKey); if (pTableData->pData == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; + free(pTableData); + return NULL; } - return pTableData; + T_REF_INC(pTableData); -_err: - tsdbFreeTableData(pTableData); - return NULL; + return pTableData; } static void tsdbFreeTableData(STableData *pTableData) { if (pTableData) { - tSkipListDestroy(pTableData->pData); - free(pTableData); + int32_t ref = T_REF_DEC(pTableData); + if (ref == 0) { + tSkipListDestroy(pTableData->pData); + free(pTableData); + } } } diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 7162f74d3e..b6a4109e68 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -192,7 +192,7 @@ static void tsdbMayTakeMemSnapshot(STsdbQueryHandle* pQueryHandle) { SMemRef* pMemRef = pQueryHandle->pMemRef; if (pQueryHandle->pMemRef->ref++ == 0) { - tsdbTakeMemSnapshot(pQueryHandle->pTsdb, (SMemTable**)&(pMemRef->mem), (SMemTable**)&(pMemRef->imem)); + tsdbTakeMemSnapshot(pQueryHandle->pTsdb, (SMemTable**)&(pMemRef->mem), (SMemTable**)&(pMemRef->imem), NULL); } } From 2290b3e49243b93aaa117fabe641e7b90682b471 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Mon, 1 Mar 2021 16:46:34 +0800 Subject: [PATCH 02/66] fix bug --- src/tsdb/src/tsdbRead.c | 38 ++++++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index b6a4109e68..08f0e9e19e 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -187,7 +187,7 @@ static SArray* getDefaultLoadColumns(STsdbQueryHandle* pQueryHandle, bool loadTS return pLocalIdList; } -static void tsdbMayTakeMemSnapshot(STsdbQueryHandle* pQueryHandle) { +static void tsdbMayTakeMemSnapshot(STsdbQueryHandle* pQueryHandle, SArray* psTable) { assert(pQueryHandle != NULL && pQueryHandle->pMemRef != NULL); SMemRef* pMemRef = pQueryHandle->pMemRef; @@ -242,7 +242,7 @@ int64_t tsdbGetNumOfRowsInMemTable(TsdbQueryHandleT* pHandle) { return rows; } -static SArray* createCheckInfoFromTableGroup(STsdbQueryHandle* pQueryHandle, STableGroupInfo* pGroupList, STsdbMeta* pMeta) { +static SArray* createCheckInfoFromTableGroup(STsdbQueryHandle* pQueryHandle, STableGroupInfo* pGroupList, STsdbMeta* pMeta, SArray** psTable) { size_t sizeOfGroup = taosArrayGetSize(pGroupList->pGroupList); assert(sizeOfGroup >= 1 && pMeta != NULL); @@ -252,6 +252,12 @@ static SArray* createCheckInfoFromTableGroup(STsdbQueryHandle* pQueryHandle, STa return NULL; } + SArray* pTable = taosArrayInit(4, sizeof(STable*)); + if (pTable == NULL) { + taosArrayDestroy(pTableCheckInfo); + return NULL; + } + // todo apply the lastkey of table check to avoid to load header file for (int32_t i = 0; i < sizeOfGroup; ++i) { SArray* group = *(SArray**) taosArrayGet(pGroupList->pGroupList, i); @@ -277,31 +283,40 @@ static SArray* createCheckInfoFromTableGroup(STsdbQueryHandle* pQueryHandle, STa assert(info.lastKey <= pQueryHandle->window.skey); } + taosArrayPush(pTable, &pKeyInfo->pTable); + taosArrayPush(pTableCheckInfo, &info); tsdbDebug("%p check table uid:%"PRId64", tid:%d from lastKey:%"PRId64" %p", pQueryHandle, info.tableId.uid, info.tableId.tid, info.lastKey, pQueryHandle->qinfo); } } + *psTable = pTable; + taosArraySort(pTableCheckInfo, tsdbCheckInfoCompar); return pTableCheckInfo; } -static SArray* createCheckInfoFromCheckInfo(SArray* pTableCheckInfo, TSKEY skey) { +static SArray* createCheckInfoFromCheckInfo(SArray* pTableCheckInfo, TSKEY skey, SArray** psTable) { size_t si = taosArrayGetSize(pTableCheckInfo); SArray* pNew = taosArrayInit(si, sizeof(STableCheckInfo)); if (pNew == NULL) { return NULL; } + SArray* pTable = taosArrayInit(si, sizeof(STable*)); + for (int32_t j = 0; j < si; ++j) { STableCheckInfo* pCheckInfo = (STableCheckInfo*) taosArrayGet(pTableCheckInfo, j); STableCheckInfo info = { .lastKey = skey, .pTableObj = pCheckInfo->pTableObj}; info.tableId = pCheckInfo->tableId; taosArrayPush(pNew, &info); + taosArrayPush(pTable, &pCheckInfo->pTableObj); } + *psTable = pTable; + // it is ordered already, no need to sort again. taosArraySort(pNew, tsdbCheckInfoCompar); return pNew; @@ -332,7 +347,7 @@ static STsdbQueryHandle* tsdbQueryTablesImpl(STsdbRepo* tsdb, STsdbQueryCond* pC goto out_of_memory; } - tsdbMayTakeMemSnapshot(pQueryHandle); + //tsdbMayTakeMemSnapshot(pQueryHandle); assert(pCond != NULL && pCond->numOfCols > 0 && pMemRef != NULL); if (ASCENDING_TRAVERSE(pCond->order)) { @@ -393,14 +408,18 @@ TsdbQueryHandleT* tsdbQueryTables(STsdbRepo* tsdb, STsdbQueryCond* pCond, STable STsdbMeta* pMeta = tsdbGetMeta(tsdb); assert(pMeta != NULL); + SArray* psTable = NULL; + // todo apply the lastkey of table check to avoid to load header file - pQueryHandle->pTableCheckInfo = createCheckInfoFromTableGroup(pQueryHandle, groupList, pMeta); + pQueryHandle->pTableCheckInfo = createCheckInfoFromTableGroup(pQueryHandle, groupList, pMeta, &psTable); if (pQueryHandle->pTableCheckInfo == NULL) { tsdbCleanupQueryHandle(pQueryHandle); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return NULL; } + tsdbMayTakeMemSnapshot(pQueryHandle, psTable); + tsdbDebug("%p total numOfTable:%" PRIzu " in query, %p", pQueryHandle, taosArrayGetSize(pQueryHandle->pTableCheckInfo), pQueryHandle->qinfo); return (TsdbQueryHandleT) pQueryHandle; } @@ -2337,12 +2356,18 @@ static int32_t doGetExternalRow(STsdbQueryHandle* pQueryHandle, int16_t type, SM pSecQueryHandle = tsdbQueryTablesImpl(pQueryHandle->pTsdb, &cond, pQueryHandle->qinfo, pMemRef); tfree(cond.colList); - pSecQueryHandle->pTableCheckInfo = createCheckInfoFromCheckInfo(pQueryHandle->pTableCheckInfo, pSecQueryHandle->window.skey); + + SArray* psTable = NULL; + + pSecQueryHandle->pTableCheckInfo = createCheckInfoFromCheckInfo(pQueryHandle->pTableCheckInfo, pSecQueryHandle->window.skey, &psTable); if (pSecQueryHandle->pTableCheckInfo == NULL) { terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; goto out_of_memory; } + + tsdbMayTakeMemSnapshot(pSecQueryHandle, psTable); + if (!tsdbNextDataBlock((void*)pSecQueryHandle)) { // no result in current query, free the corresponding result rows structure if (type == TSDB_PREV_ROW) { @@ -3360,3 +3385,4 @@ void getTableListfromSkipList(tExprNode *pExpr, SSkipList *pSkipList, SArray *re //apply the hierarchical filter expression to every node in skiplist to find the qualified nodes applyFilterToSkipListNode(pSkipList, pExpr, result, param); } + \ No newline at end of file From 2b462aafdcbd8e27f4a157e8ddd6d2ab2b28d798 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Mon, 1 Mar 2021 18:15:42 +0800 Subject: [PATCH 03/66] fix bug --- src/tsdb/src/tsdbRead.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 08f0e9e19e..486ff49f09 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -192,8 +192,10 @@ static void tsdbMayTakeMemSnapshot(STsdbQueryHandle* pQueryHandle, SArray* psTab SMemRef* pMemRef = pQueryHandle->pMemRef; if (pQueryHandle->pMemRef->ref++ == 0) { - tsdbTakeMemSnapshot(pQueryHandle->pTsdb, (SMemTable**)&(pMemRef->mem), (SMemTable**)&(pMemRef->imem), NULL); + tsdbTakeMemSnapshot(pQueryHandle->pTsdb, (SMemTable**)&(pMemRef->mem), (SMemTable**)&(pMemRef->imem), psTable); } + + taosArrayDestroy(psTable); } static void tsdbMayUnTakeMemSnapshot(STsdbQueryHandle* pQueryHandle) { @@ -3385,4 +3387,3 @@ void getTableListfromSkipList(tExprNode *pExpr, SSkipList *pSkipList, SArray *re //apply the hierarchical filter expression to every node in skiplist to find the qualified nodes applyFilterToSkipListNode(pSkipList, pExpr, result, param); } - \ No newline at end of file From e9a4f1c6201a9178c298b7a6649884103b4957b1 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 2 Mar 2021 22:47:03 +0800 Subject: [PATCH 04/66] [TD-3124]: stop & free TAOS_RES *result post http queue shutdown --- src/plugins/http/src/httpQueue.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/plugins/http/src/httpQueue.c b/src/plugins/http/src/httpQueue.c index aebba97fb8..7f7ce40460 100644 --- a/src/plugins/http/src/httpQueue.c +++ b/src/plugins/http/src/httpQueue.c @@ -59,7 +59,9 @@ void httpDispatchToResultQueue(void *param, TAOS_RES *result, int32_t code, int3 pMsg->fp = fp; taosWriteQitem(tsHttpQueue, TAOS_QTYPE_RPC, pMsg); } else { - (*fp)(param, result, code, rows); + taos_stop_query(result); + taos_free_result(result); + //(*fp)(param, result, code, rows); } } From 1488c1e7fc859ea3760b1cab82279f8d122da140 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Wed, 3 Mar 2021 09:54:53 +0800 Subject: [PATCH 05/66] return sqlobj --- src/client/inc/tsclient.h | 2 ++ src/client/src/tscAsync.c | 13 ++++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 8f76f812ac..0bfbf3f946 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -442,6 +442,8 @@ void tscCloseTscObj(void *pObj); TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int), void *param, TAOS **taos); TAOS_RES* taos_query_h(TAOS* taos, const char *sqlstr, int64_t* res); +TAOS_RES * taos_query_ra(TAOS *taos, const char *sqlstr, __async_cb_func_t fp, void *param); + void waitForQueryRsp(void *param, TAOS_RES *tres, int code); void doAsyncQuery(STscObj *pObj, SSqlObj *pSql, __async_cb_func_t fp, void *param, const char *sqlstr, size_t sqlLen); diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index 7d5b76cc2b..5cba897b30 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -74,12 +74,16 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, __async_cb_func_t fp, void* para // TODO return the correct error code to client in tscQueueAsyncError void taos_query_a(TAOS *taos, const char *sqlstr, __async_cb_func_t fp, void *param) { + taos_query_ra(taos, sqlstr, fp, param); +} + +TAOS_RES * taos_query_ra(TAOS *taos, const char *sqlstr, __async_cb_func_t fp, void *param) { STscObj *pObj = (STscObj *)taos; if (pObj == NULL || pObj->signature != pObj) { tscError("bug!!! pObj:%p", pObj); terrno = TSDB_CODE_TSC_DISCONNECTED; tscQueueAsyncError(fp, param, TSDB_CODE_TSC_DISCONNECTED); - return; + return NULL; } int32_t sqlLen = (int32_t)strlen(sqlstr); @@ -87,7 +91,7 @@ void taos_query_a(TAOS *taos, const char *sqlstr, __async_cb_func_t fp, void *pa tscError("sql string exceeds max length:%d", tsMaxSQLStringLen); terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT; tscQueueAsyncError(fp, param, terrno); - return; + return NULL; } nPrintTsc("%s", sqlstr); @@ -96,12 +100,15 @@ void taos_query_a(TAOS *taos, const char *sqlstr, __async_cb_func_t fp, void *pa if (pSql == NULL) { tscError("failed to malloc sqlObj"); tscQueueAsyncError(fp, param, TSDB_CODE_TSC_OUT_OF_MEMORY); - return; + return NULL; } doAsyncQuery(pObj, pSql, fp, param, sqlstr, sqlLen); + + return pSql; } + static void tscAsyncFetchRowsProxy(void *param, TAOS_RES *tres, int numOfRows) { if (tres == NULL) { return; From b82a1c71be377a07c2458288a3e92a18f3ec918f Mon Sep 17 00:00:00 2001 From: zyyang Date: Wed, 3 Mar 2021 10:02:07 +0800 Subject: [PATCH 06/66] add count and select tbname SQL test in springbootdemo --- .../controller/WeatherController.java | 10 ++++++++ .../springbootdemo/dao/WeatherMapper.java | 6 ++++- .../springbootdemo/dao/WeatherMapper.xml | 25 ++++++++++++------- .../service/WeatherService.java | 8 ++++++ .../src/main/resources/application.properties | 13 ++++++---- 5 files changed, 47 insertions(+), 15 deletions(-) diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/controller/WeatherController.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/controller/WeatherController.java index 4a4109dcf3..1ce3de4948 100644 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/controller/WeatherController.java +++ b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/controller/WeatherController.java @@ -59,4 +59,14 @@ public class WeatherController { return weatherService.save(weatherList); } + @GetMapping("/count") + public int count() { + return weatherService.count(); + } + + @GetMapping("/subTables") + public List getSubTables() { + return weatherService.getSubTables(); + } + } diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.java index cae1a1aec0..8687e66772 100644 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.java +++ b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.java @@ -11,9 +11,13 @@ public interface WeatherMapper { int batchInsert(List weatherList); - List select(@Param("limit") Long limit, @Param("offset")Long offset); + List select(@Param("limit") Long limit, @Param("offset") Long offset); void createDB(); void createTable(); + + int count(); + + List getSubTables(); } diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.xml b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.xml index a9bcda0b00..ffdabf9a35 100644 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.xml +++ b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.xml @@ -4,16 +4,16 @@ - - - + + + - + create database if not exists test; - + create table if not exists test.weather(ts timestamp, temperature int, humidity float); @@ -23,7 +23,7 @@ - + insert into test.weather (ts, temperature, humidity) values (now, #{temperature,jdbcType=INTEGER}, #{humidity,jdbcType=FLOAT}) - + insert into test.weather (ts, temperature, humidity) values - + (now + #{index}a, #{weather.temperature}, #{weather.humidity}) + + + \ No newline at end of file diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/service/WeatherService.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/service/WeatherService.java index 31ce8f1dd9..2f3bc78991 100644 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/service/WeatherService.java +++ b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/service/WeatherService.java @@ -35,4 +35,12 @@ public class WeatherService { return weatherMapper.batchInsert(weatherList); } + public int count() { + return weatherMapper.count(); + } + + public List getSubTables() { + return weatherMapper.getSubTables(); + } + } diff --git a/tests/examples/JDBC/springbootdemo/src/main/resources/application.properties b/tests/examples/JDBC/springbootdemo/src/main/resources/application.properties index 4fb68758c4..594a1fa4d0 100644 --- a/tests/examples/JDBC/springbootdemo/src/main/resources/application.properties +++ b/tests/examples/JDBC/springbootdemo/src/main/resources/application.properties @@ -1,12 +1,15 @@ # datasource config - JDBC-JNI -spring.datasource.driver-class-name=com.taosdata.jdbc.TSDBDriver -spring.datasource.url=jdbc:TAOS://127.0.0.1:6030/test?timezone=UTC-8&charset=UTF-8&locale=en_US.UTF-8 +#spring.datasource.driver-class-name=com.taosdata.jdbc.TSDBDriver +#spring.datasource.url=jdbc:TAOS://127.0.0.1:6030/test?timezone=UTC-8&charset=UTF-8&locale=en_US.UTF-8 +#spring.datasource.username=root +#spring.datasource.password=taosdata + +# datasource config - JDBC-RESTful +spring.datasource.driver-class-name=com.taosdata.jdbc.rs.RestfulDriver +spring.datasource.url=jdbc:TAOS-RS://master:6041/test?timezone=UTC-8&charset=UTF-8&locale=en_US.UTF-8 spring.datasource.username=root spring.datasource.password=taosdata -# datasource config - JDBC-RESTful -#spring.datasource.driver-class-name=com.taosdata.jdbc.rs.RestfulDriver -#spring.datasource.url=jdbc:TAOS-RS://master:6041/test?user=root&password=taosdata spring.datasource.druid.initial-size=5 spring.datasource.druid.min-idle=5 From 04f06f9d4dd3a6aa3d6c371b22ebf8a2a6132239 Mon Sep 17 00:00:00 2001 From: zyyang Date: Wed, 3 Mar 2021 10:09:07 +0800 Subject: [PATCH 07/66] change driver --- tests/examples/JDBC/springbootdemo/pom.xml | 2 +- .../src/main/resources/application.properties | 15 +++++++-------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/tests/examples/JDBC/springbootdemo/pom.xml b/tests/examples/JDBC/springbootdemo/pom.xml index 52fb8caa90..4b7272323d 100644 --- a/tests/examples/JDBC/springbootdemo/pom.xml +++ b/tests/examples/JDBC/springbootdemo/pom.xml @@ -63,7 +63,7 @@ com.taosdata.jdbc taos-jdbcdriver - 2.0.18 + 2.0.20 diff --git a/tests/examples/JDBC/springbootdemo/src/main/resources/application.properties b/tests/examples/JDBC/springbootdemo/src/main/resources/application.properties index 594a1fa4d0..e957e00112 100644 --- a/tests/examples/JDBC/springbootdemo/src/main/resources/application.properties +++ b/tests/examples/JDBC/springbootdemo/src/main/resources/application.properties @@ -1,15 +1,14 @@ # datasource config - JDBC-JNI -#spring.datasource.driver-class-name=com.taosdata.jdbc.TSDBDriver -#spring.datasource.url=jdbc:TAOS://127.0.0.1:6030/test?timezone=UTC-8&charset=UTF-8&locale=en_US.UTF-8 -#spring.datasource.username=root -#spring.datasource.password=taosdata - -# datasource config - JDBC-RESTful -spring.datasource.driver-class-name=com.taosdata.jdbc.rs.RestfulDriver -spring.datasource.url=jdbc:TAOS-RS://master:6041/test?timezone=UTC-8&charset=UTF-8&locale=en_US.UTF-8 +spring.datasource.driver-class-name=com.taosdata.jdbc.TSDBDriver +spring.datasource.url=jdbc:TAOS://127.0.0.1:6030/test?timezone=UTC-8&charset=UTF-8&locale=en_US.UTF-8 spring.datasource.username=root spring.datasource.password=taosdata +# datasource config - JDBC-RESTful +#spring.datasource.driver-class-name=com.taosdata.jdbc.rs.RestfulDriver +#spring.datasource.url=jdbc:TAOS-RS://localhost:6041/test?timezone=UTC-8&charset=UTF-8&locale=en_US.UTF-8 +#spring.datasource.username=root +#spring.datasource.password=taosdata spring.datasource.druid.initial-size=5 spring.datasource.druid.min-idle=5 From 2a683c5b9ce5d1421d465a262eef43ecaecfe6cb Mon Sep 17 00:00:00 2001 From: zyyang Date: Wed, 3 Mar 2021 11:00:43 +0800 Subject: [PATCH 08/66] change --- .../controller/WeatherController.java | 4 +- .../springbootdemo/dao/WeatherMapper.java | 10 +++-- .../springbootdemo/dao/WeatherMapper.xml | 17 ++++---- .../springbootdemo/domain/Weather.java | 39 +++++++++++++++---- .../service/WeatherService.java | 15 +++++-- 5 files changed, 59 insertions(+), 26 deletions(-) diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/controller/WeatherController.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/controller/WeatherController.java index 1ce3de4948..a2470713b8 100644 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/controller/WeatherController.java +++ b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/controller/WeatherController.java @@ -20,7 +20,7 @@ public class WeatherController { * @return */ @GetMapping("/init") - public boolean init() { + public int init() { return weatherService.init(); } @@ -44,7 +44,7 @@ public class WeatherController { * @return */ @PostMapping("/{temperature}/{humidity}") - public int saveWeather(@PathVariable int temperature, @PathVariable float humidity) { + public int saveWeather(@PathVariable float temperature, @PathVariable int humidity) { return weatherService.save(temperature, humidity); } diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.java index 8687e66772..ddbc0e997d 100644 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.java +++ b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.java @@ -7,15 +7,17 @@ import java.util.List; public interface WeatherMapper { - int insert(Weather weather); + void createDB(); - int batchInsert(List weatherList); + void createSuperTable(); + + void createTable(); List select(@Param("limit") Long limit, @Param("offset") Long offset); - void createDB(); + int insert(Weather weather); - void createTable(); + int batchInsert(List weatherList); int count(); diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.xml b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.xml index ffdabf9a35..f9a9002d76 100644 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.xml +++ b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.xml @@ -13,19 +13,16 @@ create database if not exists test; - - create table if not exists test.weather(ts timestamp, temperature int, humidity float); + + create table if not exists test.weather(ts timestamp, temporary float, humidity int) tags(location nchar(64), groupId int) - - ts, temperature, humidity - + + create table test.t#{groupId} using test.weather tags(#{location}, #{groupId}) + \ No newline at end of file diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/domain/Weather.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/domain/Weather.java index 60565448ad..c57eb26ca9 100644 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/domain/Weather.java +++ b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/domain/Weather.java @@ -6,12 +6,21 @@ import java.sql.Timestamp; public class Weather { - @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss.SSS",timezone = "GMT+8") + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss.SSS", timezone = "GMT+8") private Timestamp ts; + private float temperature; + private int humidity; + private String location; + private int groupId; - private int temperature; + public Weather() { + } - private float humidity; + public Weather(Timestamp ts, float temperature, int humidity) { + this.ts = ts; + this.temperature = temperature; + this.humidity = humidity; + } public Timestamp getTs() { return ts; @@ -21,19 +30,35 @@ public class Weather { this.ts = ts; } - public int getTemperature() { + public float getTemperature() { return temperature; } - public void setTemperature(int temperature) { + public void setTemperature(float temperature) { this.temperature = temperature; } - public float getHumidity() { + public int getHumidity() { return humidity; } - public void setHumidity(float humidity) { + public void setHumidity(int humidity) { this.humidity = humidity; } + + public String getLocation() { + return location; + } + + public void setLocation(String location) { + this.location = location; + } + + public int getGroupId() { + return groupId; + } + + public void setGroupId(int groupId) { + this.groupId = groupId; + } } diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/service/WeatherService.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/service/WeatherService.java index 2f3bc78991..08a1321ff1 100644 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/service/WeatherService.java +++ b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/service/WeatherService.java @@ -5,25 +5,34 @@ import com.taosdata.example.springbootdemo.domain.Weather; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import java.sql.Timestamp; import java.util.List; +import java.util.Random; @Service public class WeatherService { @Autowired private WeatherMapper weatherMapper; + private Random random = new Random(System.currentTimeMillis()); - public boolean init() { + public int init() { weatherMapper.createDB(); + weatherMapper.createSuperTable(); weatherMapper.createTable(); - return true; + long ts = System.currentTimeMillis(); + int count = 0; + for (int i = 0; i < 10; i++) { + count += weatherMapper.insert(new Weather(new Timestamp(ts + (1000 * i)), 30 * random.nextFloat(), random.nextInt(100))); + } + return count; } public List query(Long limit, Long offset) { return weatherMapper.select(limit, offset); } - public int save(int temperature, float humidity) { + public int save(float temperature, int humidity) { Weather weather = new Weather(); weather.setTemperature(temperature); weather.setHumidity(humidity); From 0b5729fbf7364ce46601206a52e2ad5c5b2be115 Mon Sep 17 00:00:00 2001 From: zyyang Date: Wed, 3 Mar 2021 11:16:30 +0800 Subject: [PATCH 09/66] change --- .../controller/WeatherController.java | 11 ----------- .../springbootdemo/dao/WeatherMapper.java | 4 +--- .../example/springbootdemo/dao/WeatherMapper.xml | 11 ++--------- .../springbootdemo/service/WeatherService.java | 12 ++++++------ .../src/main/resources/application.properties | 16 ++++++++-------- 5 files changed, 17 insertions(+), 37 deletions(-) diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/controller/WeatherController.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/controller/WeatherController.java index a2470713b8..e595e26f3e 100644 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/controller/WeatherController.java +++ b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/controller/WeatherController.java @@ -48,17 +48,6 @@ public class WeatherController { return weatherService.save(temperature, humidity); } - /** - * upload multi weather info - * - * @param weatherList - * @return - */ - @PostMapping("/batch") - public int batchSaveWeather(@RequestBody List weatherList) { - return weatherService.save(weatherList); - } - @GetMapping("/count") public int count() { return weatherService.count(); diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.java index ddbc0e997d..7e35e19e61 100644 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.java +++ b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.java @@ -11,14 +11,12 @@ public interface WeatherMapper { void createSuperTable(); - void createTable(); + void createTable(Weather weather); List select(@Param("limit") Long limit, @Param("offset") Long offset); int insert(Weather weather); - int batchInsert(List weatherList); - int count(); List getSubTables(); diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.xml b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.xml index f9a9002d76..49ae45fad9 100644 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.xml +++ b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.xml @@ -10,7 +10,7 @@ - create database if not exists test; + create database if not exists test @@ -32,14 +32,7 @@ - insert into test.weather (ts, temperature, humidity) values (now, #{temperature,jdbcType=INTEGER}, #{humidity,jdbcType=FLOAT}) - - - - insert into test.weather (ts, temperature, humidity) values - - (now + #{index}a, #{weather.temperature}, #{weather.humidity}) - + insert into test.t#{groupId} (ts, temperature, humidity) values (#{ts}, #{temperature}, #{humidity}) @@ -32,7 +36,7 @@ - insert into test.t#{groupId} (ts, temperature, humidity) values (#{ts}, #{temperature}, #{humidity}) + insert into test.t#{groupId} (ts, temperature, humidity) values (#{ts}, ${temperature}, ${humidity}) + + \ No newline at end of file diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/service/WeatherService.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/service/WeatherService.java index a5ba27658d..dfd7ce23fd 100644 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/service/WeatherService.java +++ b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/service/WeatherService.java @@ -7,6 +7,7 @@ import org.springframework.stereotype.Service; import java.sql.Timestamp; import java.util.List; +import java.util.Map; import java.util.Random; @Service @@ -18,12 +19,14 @@ public class WeatherService { private String[] locations = {"北京", "上海", "广州", "深圳", "天津"}; public int init() { + weatherMapper.dropDB(); weatherMapper.createDB(); weatherMapper.createSuperTable(); long ts = System.currentTimeMillis(); + long thirtySec = 1000 * 30; int count = 0; - for (int i = 0; i < 10; i++) { - Weather weather = new Weather(new Timestamp(ts + (1000 * i)), 30 * random.nextFloat(), random.nextInt(100)); + for (int i = 0; i < 20; i++) { + Weather weather = new Weather(new Timestamp(ts + (thirtySec * i)), 30 * random.nextFloat(), random.nextInt(100)); weather.setLocation(locations[random.nextInt(locations.length)]); weather.setGroupId(i % locations.length); weatherMapper.createTable(weather); @@ -52,4 +55,7 @@ public class WeatherService { return weatherMapper.getSubTables(); } + public Map avg() { + return weatherMapper.avg(); + } } diff --git a/tests/examples/JDBC/springbootdemo/src/main/resources/application.properties b/tests/examples/JDBC/springbootdemo/src/main/resources/application.properties index 4d7e64d105..6b1689bb3e 100644 --- a/tests/examples/JDBC/springbootdemo/src/main/resources/application.properties +++ b/tests/examples/JDBC/springbootdemo/src/main/resources/application.properties @@ -1,15 +1,15 @@ # datasource config - JDBC-JNI -#spring.datasource.driver-class-name=com.taosdata.jdbc.TSDBDriver -#spring.datasource.url=jdbc:TAOS://127.0.0.1:6030/test?timezone=UTC-8&charset=UTF-8&locale=en_US.UTF-8 -#spring.datasource.username=root -#spring.datasource.password=taosdata - -# datasource config - JDBC-RESTful -spring.datasource.driver-class-name=com.taosdata.jdbc.rs.RestfulDriver -spring.datasource.url=jdbc:TAOS-RS://master:6041/test?timezone=UTC-8&charset=UTF-8&locale=en_US.UTF-8 +spring.datasource.driver-class-name=com.taosdata.jdbc.TSDBDriver +spring.datasource.url=jdbc:TAOS://127.0.0.1:6030/test?timezone=UTC-8&charset=UTF-8&locale=en_US.UTF-8 spring.datasource.username=root spring.datasource.password=taosdata +# datasource config - JDBC-RESTful +#spring.datasource.driver-class-name=com.taosdata.jdbc.rs.RestfulDriver +#spring.datasource.url=jdbc:TAOS-RS://master:6041/test?timezone=UTC-8&charset=UTF-8&locale=en_US.UTF-8 +#spring.datasource.username=root +#spring.datasource.password=taosdata + spring.datasource.druid.initial-size=5 spring.datasource.druid.min-idle=5 spring.datasource.druid.max-active=5 From bc95793da07fe1d687dcc7887f579ab294c1aad2 Mon Sep 17 00:00:00 2001 From: zyyang Date: Wed, 3 Mar 2021 14:27:58 +0800 Subject: [PATCH 12/66] change --- .../com/taosdata/example/springbootdemo/dao/WeatherMapper.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.xml b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.xml index e8647826df..bdf83a77f6 100644 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.xml +++ b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.xml @@ -39,7 +39,7 @@ insert into test.t#{groupId} (ts, temperature, humidity) values (#{ts}, ${temperature}, ${humidity}) - select tbname from test.weather From a8bb365a6930a38060ad4d84be008f930067faa8 Mon Sep 17 00:00:00 2001 From: zyyang Date: Wed, 3 Mar 2021 14:28:47 +0800 Subject: [PATCH 13/66] change --- .../com/taosdata/example/springbootdemo/dao/WeatherMapper.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.xml b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.xml index bdf83a77f6..b4a36d0e19 100644 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.xml +++ b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.xml @@ -43,7 +43,7 @@ select tbname from test.weather - select count(*) from test.weather From 11e042a13838c48d84610360515c4a605a47b632 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 3 Mar 2021 14:34:53 +0800 Subject: [PATCH 14/66] [TD-3042]: fix unsigned to signed conversion with serverPort max(serverPort) changed from 65535 to 65056 --- src/common/src/tglobal.c | 4 ++-- src/dnode/src/dnodeCheck.c | 6 +++--- src/util/inc/tconfig.h | 1 + src/util/src/tconfig.c | 26 ++++++++++++++++++++++++++ 4 files changed, 32 insertions(+), 5 deletions(-) diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 80bf48364c..4ed4e0473b 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -430,10 +430,10 @@ static void doInitGlobalConfig(void) { // port cfg.option = "serverPort"; cfg.ptr = &tsServerPort; - cfg.valType = TAOS_CFG_VTYPE_INT16; + cfg.valType = TAOS_CFG_VTYPE_UINT16; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; cfg.minValue = 1; - cfg.maxValue = 65535; + cfg.maxValue = 65056; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); diff --git a/src/dnode/src/dnodeCheck.c b/src/dnode/src/dnodeCheck.c index 94d2360950..87baff3067 100644 --- a/src/dnode/src/dnodeCheck.c +++ b/src/dnode/src/dnodeCheck.c @@ -29,7 +29,7 @@ typedef struct { static SCheckItem tsCheckItem[TSDB_CHECK_ITEM_MAX] = {{0}}; int64_t tsMinFreeMemSizeForStart = 0; -static int32_t bindTcpPort(int16_t port) { +static int32_t bindTcpPort(uint16_t port) { SOCKET serverSocket; struct sockaddr_in server_addr; @@ -85,9 +85,9 @@ static int32_t bindUdpPort(int16_t port) { static int32_t dnodeCheckNetwork() { int32_t ret; - int16_t startPort = tsServerPort; + uint16_t startPort = tsServerPort; - for (int16_t port = startPort; port < startPort + 12; port++) { + for (uint16_t port = startPort; port < startPort + 12; port++) { ret = bindTcpPort(port); if (0 != ret) { dError("failed to tcp bind port %d, quit", port); diff --git a/src/util/inc/tconfig.h b/src/util/inc/tconfig.h index 9923409885..fdb2595fd8 100644 --- a/src/util/inc/tconfig.h +++ b/src/util/inc/tconfig.h @@ -44,6 +44,7 @@ enum { TAOS_CFG_VTYPE_INT8, TAOS_CFG_VTYPE_INT16, TAOS_CFG_VTYPE_INT32, + TAOS_CFG_VTYPE_UINT16, TAOS_CFG_VTYPE_FLOAT, TAOS_CFG_VTYPE_STRING, TAOS_CFG_VTYPE_IPSTR, diff --git a/src/util/src/tconfig.c b/src/util/src/tconfig.c index 7a92750f8f..c4bd577602 100644 --- a/src/util/src/tconfig.c +++ b/src/util/src/tconfig.c @@ -95,6 +95,23 @@ static void taosReadInt16Config(SGlobalCfg *cfg, char *input_value) { } } +static void taosReadUInt16Config(SGlobalCfg *cfg, char *input_value) { + int32_t value = atoi(input_value); + uint16_t *option = (uint16_t *)cfg->ptr; + if (value < cfg->minValue || value > cfg->maxValue) { + uError("config option:%s, input value:%s, out of range[%f, %f], use default value:%d", + cfg->option, input_value, cfg->minValue, cfg->maxValue, *option); + } else { + if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { + *option = (uint16_t)value; + cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE; + } else { + uWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, input_value, + tsCfgStatusStr[cfg->cfgStatus], *option); + } + } +} + static void taosReadInt8Config(SGlobalCfg *cfg, char *input_value) { int32_t value = atoi(input_value); int8_t *option = (int8_t *)cfg->ptr; @@ -239,6 +256,9 @@ static void taosReadConfigOption(const char *option, char *value, char *value2, case TAOS_CFG_VTYPE_INT32: taosReadInt32Config(cfg, value); break; + case TAOS_CFG_VTYPE_UINT16: + taosReadUInt16Config(cfg, value); + break; case TAOS_CFG_VTYPE_FLOAT: taosReadFloatConfig(cfg, value); break; @@ -422,6 +442,9 @@ void taosPrintGlobalCfg() { case TAOS_CFG_VTYPE_INT32: uInfo(" %s:%s%d%s", cfg->option, blank, *((int32_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); break; + case TAOS_CFG_VTYPE_UINT16: + uInfo(" %s:%s%d%s", cfg->option, blank, *((uint16_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); + break; case TAOS_CFG_VTYPE_FLOAT: uInfo(" %s:%s%f%s", cfg->option, blank, *((float *)cfg->ptr), tsGlobalUnit[cfg->unitType]); break; @@ -459,6 +482,9 @@ static void taosDumpCfg(SGlobalCfg *cfg) { case TAOS_CFG_VTYPE_INT32: printf(" %s:%s%d%s\n", cfg->option, blank, *((int32_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); break; + case TAOS_CFG_VTYPE_UINT16: + printf(" %s:%s%d%s\n", cfg->option, blank, *((uint16_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); + break; case TAOS_CFG_VTYPE_FLOAT: printf(" %s:%s%f%s\n", cfg->option, blank, *((float *)cfg->ptr), tsGlobalUnit[cfg->unitType]); break; From c9c6bd42de28275f9b3e05d240b74bf20f08adca Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Wed, 3 Mar 2021 15:19:49 +0800 Subject: [PATCH 15/66] : move DataX link into document index page. --- documentation20/cn/00.index/docs.md | 1 + documentation20/cn/09.connections/docs.md | 8 -------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/documentation20/cn/00.index/docs.md b/documentation20/cn/00.index/docs.md index 7e5f98d909..5194ec9ae5 100644 --- a/documentation20/cn/00.index/docs.md +++ b/documentation20/cn/00.index/docs.md @@ -120,6 +120,7 @@ TDengine是一个高效的存储、查询、分析时序大数据的平台，专 * [TDengine性能对比测试工具](https://www.taosdata.com/blog/2020/01/18/1166.html) * [IDEA数据库管理工具可视化使用TDengine](https://www.taosdata.com/blog/2020/08/27/1767.html) * [基于eletron开发的跨平台TDengine图形化管理工具](https://github.com/skye0207/TDengineGUI) +* [DataX，支持TDengine的离线数据采集/同步工具](https://github.com/alibaba/DataX) ## TDengine与其他数据库的对比测试 diff --git a/documentation20/cn/09.connections/docs.md b/documentation20/cn/09.connections/docs.md index 9a1d7f9d17..79380f3bbd 100644 --- a/documentation20/cn/09.connections/docs.md +++ b/documentation20/cn/09.connections/docs.md @@ -155,11 +155,3 @@ TDengine客户端暂不支持如下函数： - dbExistsTable(conn, "test")：是否存在表test - dbListTables(conn)：显示连接中的所有表 - -## DataX - -[DataX](https://github.com/alibaba/DataX) 是阿里巴巴集团开源的一款通用离线数据采集/同步工具，能够简单、高效地接入 TDengine 进行数据写入和读取。 - -* 数据读取集成的方法请参见 [TSDBReader 插件文档](https://github.com/alibaba/DataX/blob/master/tsdbreader/doc/tsdbreader.md) -* 数据写入集成的方法请参见 [TSDBWriter 插件文档](https://github.com/alibaba/DataX/blob/master/tsdbwriter/doc/tsdbhttpwriter.md) - From fb1773f04444eea1397855edb5ba0db0d072c509 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 3 Mar 2021 16:11:18 +0800 Subject: [PATCH 16/66] [TD-3158]: add head_type fileds with RESTful API --- src/plugins/http/inc/httpRestJson.h | 4 +++- src/plugins/http/src/httpRestJson.c | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/plugins/http/inc/httpRestJson.h b/src/plugins/http/inc/httpRestJson.h index 112e845f36..b3b1b33e4e 100644 --- a/src/plugins/http/inc/httpRestJson.h +++ b/src/plugins/http/inc/httpRestJson.h @@ -34,6 +34,8 @@ #define REST_JSON_DATA_LEN 4 #define REST_JSON_HEAD "head" #define REST_JSON_HEAD_LEN 4 +#define REST_JSON_HEAD_TYPE "head_type" +#define REST_JSON_HEAD_TYPE_LEN 9 #define REST_JSON_ROWS "rows" #define REST_JSON_ROWS_LEN 4 #define REST_JSON_AFFECT_ROWS "affected_rows" @@ -51,4 +53,4 @@ bool restBuildSqlLocalTimeStringJson(HttpContext *pContext, HttpSqlCmd *cmd, TAO bool restBuildSqlUtcTimeStringJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows); void restStopSqlJson(HttpContext *pContext, HttpSqlCmd *cmd); -#endif \ No newline at end of file +#endif diff --git a/src/plugins/http/src/httpRestJson.c b/src/plugins/http/src/httpRestJson.c index a620625d25..1172c4eef6 100644 --- a/src/plugins/http/src/httpRestJson.c +++ b/src/plugins/http/src/httpRestJson.c @@ -75,6 +75,26 @@ void restStartSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result) // head array end httpJsonToken(jsonBuf, JsonArrEnd); + // head_type begin + httpJsonItemToken(jsonBuf); + httpJsonPairHead(jsonBuf, REST_JSON_HEAD_TYPE, REST_JSON_HEAD_TYPE_LEN); + // head_type array begin + httpJsonItemToken(jsonBuf); + httpJsonToken(jsonBuf, JsonArrStt); + + if (num_fields == 0) { + httpJsonItemToken(jsonBuf); + httpJsonInt(jsonBuf, TSDB_DATA_TYPE_INT); + } else { + for (int32_t i = 0; i < num_fields; ++i) { + httpJsonItemToken(jsonBuf); + httpJsonInt(jsonBuf, fields[i].type); + } + } + + // head_type array end + httpJsonToken(jsonBuf, JsonArrEnd); + // data begin httpJsonItemToken(jsonBuf); httpJsonPairHead(jsonBuf, REST_JSON_DATA, REST_JSON_DATA_LEN); From e891ecc8655cdcef45c471e73913da9e6f5ed8f3 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 3 Mar 2021 16:41:53 +0800 Subject: [PATCH 17/66] [TD-3147] : change insert rate to insert interval. --- src/kit/taosdemo/taosdemo.c | 48 ++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 9549c72429..6a5d0d6e90 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -266,7 +266,7 @@ typedef struct SSuperTable_S { char childTblPrefix[MAX_TB_NAME_SIZE]; char dataSource[MAX_TB_NAME_SIZE]; // rand_gen or sample char insertMode[MAX_TB_NAME_SIZE]; // taosc, restful - int insertRate; // 0: unlimit > 0 rows/s + int insertInterval; // interval time between insert twice int multiThreadWriteOneTbl; // 0: no, 1: yes int numberOfTblInOneSql; // 0/1: one table, > 1: number of tbl @@ -1015,7 +1015,7 @@ static int printfInsertMeta() { printf(" childTblPrefix: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].childTblPrefix); printf(" dataSource: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].dataSource); printf(" insertMode: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].insertMode); - printf(" insertRate: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].insertRate); + printf(" insertInterval: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].insertInterval); printf(" insertRows: \033[33m%"PRId64"\033[0m\n", g_Dbs.db[i].superTbls[j].insertRows); if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) { @@ -1153,7 +1153,7 @@ static void printfInsertMetaToFile(FILE* fp) { fprintf(fp, " childTblPrefix: %s\n", g_Dbs.db[i].superTbls[j].childTblPrefix); fprintf(fp, " dataSource: %s\n", g_Dbs.db[i].superTbls[j].dataSource); fprintf(fp, " insertMode: %s\n", g_Dbs.db[i].superTbls[j].insertMode); - fprintf(fp, " insertRate: %d\n", g_Dbs.db[i].superTbls[j].insertRate); + fprintf(fp, " insertInterval: %d\n", g_Dbs.db[i].superTbls[j].insertInterval); fprintf(fp, " insertRows: %"PRId64"\n", g_Dbs.db[i].superTbls[j].insertRows); if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) { @@ -2997,11 +2997,11 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { goto PARSE_OVER; } - cJSON* insertRate = cJSON_GetObjectItem(stbInfo, "insert_rate"); - if (insertRate && insertRate->type == cJSON_Number) { - g_Dbs.db[i].superTbls[j].insertRate = insertRate->valueint; - } else if (!insertRate) { - g_Dbs.db[i].superTbls[j].insertRate = 0; + cJSON* insertInterval = cJSON_GetObjectItem(stbInfo, "insert_interval"); + if (insertInterval && insertInterval->type == cJSON_Number) { + g_Dbs.db[i].superTbls[j].insertInterval = insertInterval->valueint; + } else if (!insertInterval) { + g_Dbs.db[i].superTbls[j].insertInterval = 0; } else { printf("failed to read json, insert_rate not found"); goto PARSE_OVER; @@ -3558,12 +3558,11 @@ void syncWriteForNumberOfTblInOneSql(threadInfo *winfo, FILE *fp, char* sampleDa int64_t st = 0; int64_t et = 0; for (int i = 0; i < superTblInfo->insertRows;) { - if (superTblInfo->insertRate && (et - st) < 1000) { - taosMsleep(1000 - (et - st)); // ms - //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, winfo->start_table_id, winfo->end_table_id); + if (superTblInfo->insertInterval && (superTblInfo->insertInterval > (et - st))) { + taosMsleep(superTblInfo->insertInterval - (et - st)); // ms } - if (superTblInfo->insertRate) { + if (superTblInfo->insertInterval) { st = taosGetTimestampMs(); } @@ -3709,7 +3708,7 @@ void syncWriteForNumberOfTblInOneSql(threadInfo *winfo, FILE *fp, char* sampleDa } } - if (superTblInfo->insertRate) { + if (superTblInfo->insertInterval) { et = taosGetTimestampMs(); } //printf("========loop %d childTables duration:%"PRId64 "========inserted rows:%d\n", winfo->end_table_id - winfo->start_table_id, et - st, i); @@ -3817,12 +3816,11 @@ void *syncWrite(void *sarg) { int64_t st = 0; int64_t et = 0; for (int i = 0; i < superTblInfo->insertRows;) { - if (superTblInfo->insertRate && (et - st) < 1000) { - taosMsleep(1000 - (et - st)); // ms - //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, winfo->start_table_id, winfo->end_table_id); + if (superTblInfo->insertInterval && (superTblInfo->insertInterval > (et - st) )) { + taosMsleep(superTblInfo->insertInterval - (et - st)); // ms } - if (superTblInfo->insertRate) { + if (superTblInfo->insertInterval) { st = taosGetTimestampMs(); } @@ -3950,7 +3948,7 @@ void *syncWrite(void *sarg) { } } - if (superTblInfo->insertRate) { + if (superTblInfo->insertInterval) { et = taosGetTimestampMs(); } //printf("========loop %d childTables duration:%"PRId64 "========inserted rows:%d\n", winfo->end_table_id - winfo->start_table_id, et - st, i); @@ -3971,7 +3969,7 @@ void *syncWrite(void *sarg) { void callBack(void *param, TAOS_RES *res, int code) { threadInfo* winfo = (threadInfo*)param; - if (winfo->superTblInfo->insertRate) { + if (winfo->superTblInfo->insertInterval) { winfo->et = taosGetTimestampMs(); if (winfo->et - winfo->st < 1000) { taosMsleep(1000 - (winfo->et - winfo->st)); // ms @@ -4013,7 +4011,7 @@ void callBack(void *param, TAOS_RES *res, int code) { } } - if (winfo->superTblInfo->insertRate) { + if (winfo->superTblInfo->insertInterval) { winfo->st = taosGetTimestampMs(); } taos_query_a(winfo->taos, buffer, callBack, winfo); @@ -4033,9 +4031,9 @@ void *asyncWrite(void *sarg) { // winfo->nrecords_per_request = (winfo->superTblInfo->maxSqlLen - 1280) / winfo->superTblInfo->lenOfOneRow; //} - if (0 != winfo->superTblInfo->insertRate) { - if (winfo->nrecords_per_request >= winfo->superTblInfo->insertRate) { - winfo->nrecords_per_request = winfo->superTblInfo->insertRate; + if (0 != winfo->superTblInfo->insertInterval) { + if (winfo->nrecords_per_request >= winfo->superTblInfo->insertInterval) { + winfo->nrecords_per_request = winfo->superTblInfo->insertInterval; } } @@ -4055,7 +4053,7 @@ void *asyncWrite(void *sarg) { winfo->et = 0; winfo->lastTs = winfo->start_time; - if (winfo->superTblInfo->insertRate) { + if (winfo->superTblInfo->insertInterval) { winfo->st = taosGetTimestampMs(); } taos_query_a(winfo->taos, "show databases", callBack, winfo); @@ -4951,7 +4949,7 @@ void setParaFromArg(){ g_Dbs.db[0].superTbls[0].autoCreateTable = PRE_CREATE_SUBTBL; g_Dbs.db[0].superTbls[0].superTblExists = TBL_NO_EXISTS; g_Dbs.db[0].superTbls[0].childTblExists = TBL_NO_EXISTS; - g_Dbs.db[0].superTbls[0].insertRate = 0; + g_Dbs.db[0].superTbls[0].insertInterval = 0; g_Dbs.db[0].superTbls[0].disorderRange = g_args.disorderRange; g_Dbs.db[0].superTbls[0].disorderRatio = g_args.disorderRatio; tstrncpy(g_Dbs.db[0].superTbls[0].childTblPrefix, g_args.tb_prefix, MAX_TB_NAME_SIZE); From f73383da2792386fa3046cd74b6a126db2d396bf Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Wed, 3 Mar 2021 17:06:37 +0800 Subject: [PATCH 18/66] [TD-3154]: update test case --- tests/pytest/query/last_row_cache.py | 30 ++++++++++++++++++---------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/tests/pytest/query/last_row_cache.py b/tests/pytest/query/last_row_cache.py index d9e09dae7a..a0e8147709 100644 --- a/tests/pytest/query/last_row_cache.py +++ b/tests/pytest/query/last_row_cache.py @@ -25,18 +25,23 @@ class TDTestCase: self.tables = 10 self.rows = 20 + self.columns = 50 self.perfix = 't' self.ts = 1601481600000 def insertData(self): - print("==============step1") - tdSql.execute("create table st (ts timestamp, c1 int) tags(t1 int)") + print("==============step1") + sql = "create table st(ts timestamp, " + for i in range(self.columns - 1): + sql += "c%d int, " % (i + 1) + sql += "c50 int) tags(t1 int)" + tdSql.execute(sql) for i in range(self.tables): tdSql.execute("create table %s%d using st tags(%d)" % (self.perfix, i, i)) for j in range(self.rows): tc = self.ts + j * 60000 - tdSql.execute("insert into %s%d values(%d, %d)" %(self.perfix, i, tc, j)) + tdSql.execute("insert into %s%d(ts, c1) values(%d, %d)" %(self.perfix, i, tc, j)) def executeQueries(self): print("==============step2") @@ -66,29 +71,29 @@ class TDTestCase: tdSql.checkData(0, 0, 19) tc = self.ts + 1 * 3600000 - tdSql.execute("insert into %s%d values(%d, %d)" %(self.perfix, 1, tc, 10)) + tdSql.execute("insert into %s%d(ts, c1) values(%d, %d)" %(self.perfix, 1, tc, 10)) tc = self.ts + 3 * 3600000 - tdSql.execute("insert into %s%d values(%d, null)" %(self.perfix, 1, tc)) + tdSql.execute("insert into %s%d(ts, c1) values(%d, null)" %(self.perfix, 1, tc)) tc = self.ts + 5 * 3600000 - tdSql.execute("insert into %s%d values(%d, %d)" %(self.perfix, 1, tc, -1)) + tdSql.execute("insert into %s%d(ts, c1) values(%d, %d)" %(self.perfix, 1, tc, -1)) tc = self.ts + 7 * 3600000 - tdSql.execute("insert into %s%d values(%d, null)" %(self.perfix, 1, tc)) + tdSql.execute("insert into %s%d(ts, c1) values(%d, null)" %(self.perfix, 1, tc)) def insertData2(self): tc = self.ts + 1 * 3600000 - tdSql.execute("insert into %s%d values(%d, %d)" %(self.perfix, 1, tc, 10)) + tdSql.execute("insert into %s%d(ts, c1) values(%d, %d)" %(self.perfix, 1, tc, 10)) tc = self.ts + 3 * 3600000 - tdSql.execute("insert into %s%d values(%d, null)" %(self.perfix, 1, tc)) + tdSql.execute("insert into %s%d(ts, c1) values(%d, null)" %(self.perfix, 1, tc)) tc = self.ts + 5 * 3600000 - tdSql.execute("insert into %s%d values(%d, %d)" %(self.perfix, 1, tc, -1)) + tdSql.execute("insert into %s%d(ts, c1) values(%d, %d)" %(self.perfix, 1, tc, -1)) tc = self.ts + 7 * 3600000 - tdSql.execute("insert into %s%d values(%d, null)" %(self.perfix, 1, tc)) + tdSql.execute("insert into %s%d(ts, c1) values(%d, null)" %(self.perfix, 1, tc)) def executeQueries2(self): # For stable @@ -164,6 +169,9 @@ class TDTestCase: self.executeQueries() self.insertData2() self.executeQueries2() + tdDnodes.stop(1) + tdDnodes.start(1) + self.executeQueries2() tdSql.execute("alter database test2 cachelast 0") self.executeQueries2() From 8236cbae8b892fe7de35821bc9976b02908494d1 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 3 Mar 2021 16:41:04 +0800 Subject: [PATCH 19/66] http/head_type: fix test cases --- tests/script/general/http/restful.sim | 8 ++++---- tests/script/general/http/restful_full.sim | 22 +++++++++++----------- tests/script/general/http/telegraf.sim | 4 ++-- tests/script/unique/http/opentsdb.sim | 14 +++++++------- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/tests/script/general/http/restful.sim b/tests/script/general/http/restful.sim index fdde975238..0e9b9de132 100644 --- a/tests/script/general/http/restful.sim +++ b/tests/script/general/http/restful.sim @@ -39,14 +39,14 @@ print =============== step3 - query data system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'select * from d1.table_rest' 127.0.0.1:7111/rest/sql print curl 127.0.0.1:7111/rest/sql -----> $system_content -if $system_content != @{"status":"succ","head":["ts","i"],"data":[["2017-12-25 21:28:41.022",1],["2017-12-25 21:28:42.022",2],["2017-12-25 21:28:43.022",3],["2017-12-25 21:28:44.022",4],["2017-12-25 21:28:45.022",5],["2017-12-25 21:28:46.022",6],["2017-12-25 21:28:47.022",7],["2017-12-25 21:28:48.022",8],["2017-12-25 21:28:49.022",9],["2017-12-25 21:28:50.022",10]],"rows":10}@ then +if $system_content != @{"status":"succ","head":["ts","i"],"head_type":[9,4],"data":[["2017-12-25 21:28:41.022",1],["2017-12-25 21:28:42.022",2],["2017-12-25 21:28:43.022",3],["2017-12-25 21:28:44.022",4],["2017-12-25 21:28:45.022",5],["2017-12-25 21:28:46.022",6],["2017-12-25 21:28:47.022",7],["2017-12-25 21:28:48.022",8],["2017-12-25 21:28:49.022",9],["2017-12-25 21:28:50.022",10]],"rows":10}@ then return -1 endi print =============== step4 - insert data system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d "insert into d1.table_rest values('2017-12-25 21:28:51.022', 11)" 127.0.0.1:7111/rest/sql print curl 127.0.0.1:7111/rest/sql -----> $system_content -if $system_content != @{"status":"succ","head":["affected_rows"],"data":[[1]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["affected_rows"],"head_type":[4],"data":[[1]],"rows":1}@ then return -1 endi @@ -54,7 +54,7 @@ print =============== step5 - query data system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'select * from d1.table_rest' 127.0.0.1:7111/rest/sql print curl 127.0.0.1:7111/rest/sql -----> $system_content -if $system_content != @{"status":"succ","head":["ts","i"],"data":[["2017-12-25 21:28:41.022",1],["2017-12-25 21:28:42.022",2],["2017-12-25 21:28:43.022",3],["2017-12-25 21:28:44.022",4],["2017-12-25 21:28:45.022",5],["2017-12-25 21:28:46.022",6],["2017-12-25 21:28:47.022",7],["2017-12-25 21:28:48.022",8],["2017-12-25 21:28:49.022",9],["2017-12-25 21:28:50.022",10],["2017-12-25 21:28:51.022",11]],"rows":11}@ then +if $system_content != @{"status":"succ","head":["ts","i"],"head_type":[9,4],"data":[["2017-12-25 21:28:41.022",1],["2017-12-25 21:28:42.022",2],["2017-12-25 21:28:43.022",3],["2017-12-25 21:28:44.022",4],["2017-12-25 21:28:45.022",5],["2017-12-25 21:28:46.022",6],["2017-12-25 21:28:47.022",7],["2017-12-25 21:28:48.022",8],["2017-12-25 21:28:49.022",9],["2017-12-25 21:28:50.022",10],["2017-12-25 21:28:51.022",11]],"rows":11}@ then return -1 endi @@ -79,4 +79,4 @@ if $system_content != @{"status":"error","code":3,"desc":"Authentication failure return -1 endi -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/http/restful_full.sim b/tests/script/general/http/restful_full.sim index 05f6a9ac9c..64128d02dc 100644 --- a/tests/script/general/http/restful_full.sim +++ b/tests/script/general/http/restful_full.sim @@ -88,13 +88,13 @@ print =============== step2 - no db #11 system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'show databases' 127.0.0.1:7111/rest/sql print 11-> $system_content -if $system_content != @{"status":"succ","head":["name","created_time","ntables","vgroups","replica","quorum","days","keep0,keep1,keep(D)","cache(MB)","blocks","minrows","maxrows","wallevel","fsync","comp","cachelast","precision","update","status"],"data":[],"rows":0}@ then +if $system_content != @{"status":"succ","head":["name","created_time","ntables","vgroups","replica","quorum","days","keep0,keep1,keep(D)","cache(MB)","blocks","minrows","maxrows","wallevel","fsync","comp","cachelast","precision","update","status"],"head_type":[8,9,4,4,3,3,3,8,4,4,4,4,2,4,2,2,8,2,8],"data":[],"rows":0}@ then return -1 endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'create database d1' 127.0.0.1:7111/rest/sql print 12-> $system_content -if $system_content != @{"status":"succ","head":["affected_rows"],"data":[[0]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["affected_rows"],"head_type":[4],"data":[[0]],"rows":1}@ then return -1 endi @@ -160,26 +160,26 @@ endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' create table d1.t1 (ts timestamp, speed int)' 127.0.0.1:7111/rest/sql print 22-> $system_content -if $system_content != @{"status":"succ","head":["affected_rows"],"data":[[0]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["affected_rows"],"head_type":[4],"data":[[0]],"rows":1}@ then return -1 endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' select * from d1.t1 ' 127.0.0.1:7111/rest/sql print 23-> $system_content -if $system_content != @{"status":"succ","head":["ts","speed"],"data":[],"rows":0}@ then +if $system_content != @{"status":"succ","head":["ts","speed"],"head_type":[9,4],"data":[],"rows":0}@ then return -1 endi #24 system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d "insert into d1.t1 values('2017-12-25 21:28:41.022', 1)" 127.0.0.1:7111/rest/sql print 24-> $system_content -if $system_content != @{"status":"succ","head":["affected_rows"],"data":[[1]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["affected_rows"],"head_type":[4],"data":[[1]],"rows":1}@ then return -1 endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' select * from d1.t1 ' 127.0.0.1:7111/rest/sql print 25-> $system_content -if $system_content != @{"status":"succ","head":["ts","speed"],"data":[["2017-12-25 21:28:41.022",1]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["ts","speed"],"head_type":[9,4],"data":[["2017-12-25 21:28:41.022",1]],"rows":1}@ then return -1 endi @@ -208,32 +208,32 @@ system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl #27 system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' select * from d1.t1 ' 127.0.0.1:7111/rest/sql print 27-> $system_content -if $system_content != @{"status":"succ","head":["ts","speed"],"data":[["2017-12-25 21:28:41.022",1],["2017-12-25 21:28:42.022",2],["2017-12-25 21:28:43.022",3],["2017-12-25 21:28:44.022",4],["2017-12-25 21:28:45.022",5],["2017-12-25 21:28:46.022",6],["2017-12-25 21:28:47.022",7],["2017-12-25 21:28:48.022",8],["2017-12-25 21:28:49.022",9],["2017-12-25 21:28:50.022",10],["2017-12-25 21:28:51.022",11]],"rows":11}@ then +if $system_content != @{"status":"succ","head":["ts","speed"],"head_type":[9,4],"data":[["2017-12-25 21:28:41.022",1],["2017-12-25 21:28:42.022",2],["2017-12-25 21:28:43.022",3],["2017-12-25 21:28:44.022",4],["2017-12-25 21:28:45.022",5],["2017-12-25 21:28:46.022",6],["2017-12-25 21:28:47.022",7],["2017-12-25 21:28:48.022",8],["2017-12-25 21:28:49.022",9],["2017-12-25 21:28:50.022",10],["2017-12-25 21:28:51.022",11]],"rows":11}@ then return -1 endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'create database d2' 127.0.0.1:7111/rest/sql print 28-> $system_content -if $system_content != @{"status":"succ","head":["affected_rows"],"data":[[0]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["affected_rows"],"head_type":[4],"data":[[0]],"rows":1}@ then return -1 endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' create table d2.t1 (ts timestamp, speed int)' 127.0.0.1:7111/rest/sql print 29-> $system_content -if $system_content != @{"status":"succ","head":["affected_rows"],"data":[[0]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["affected_rows"],"head_type":[4],"data":[[0]],"rows":1}@ then return -1 endi #30 system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d "insert into d2.t1 values('2017-12-25 21:28:41.022', 1)" 127.0.0.1:7111/rest/sql print 30-> $system_content -if $system_content != @{"status":"succ","head":["affected_rows"],"data":[[1]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["affected_rows"],"head_type":[4],"data":[[1]],"rows":1}@ then return -1 endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' select * from d2.t1 ' 127.0.0.1:7111/rest/sql print 31-> $system_content -if $system_content != @{"status":"succ","head":["ts","speed"],"data":[["2017-12-25 21:28:41.022",1]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["ts","speed"],"head_type":[9,4],"data":[["2017-12-25 21:28:41.022",1]],"rows":1}@ then return -1 endi diff --git a/tests/script/general/http/telegraf.sim b/tests/script/general/http/telegraf.sim index 6825e5c479..03116e9f3b 100644 --- a/tests/script/general/http/telegraf.sim +++ b/tests/script/general/http/telegraf.sim @@ -285,8 +285,8 @@ system_content curl -u root:taosdata -d 'select count(*) from db.win_cpu' 127.0 print $system_content -if $system_content != @{"status":"succ","head":["count(*)"],"data":[[3]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["count(*)"],"head_type":[5],"data":[[3]],"rows":1}@ then return -1 endi -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/unique/http/opentsdb.sim b/tests/script/unique/http/opentsdb.sim index 3d8e5a8c44..aad9b87cbe 100644 --- a/tests/script/unique/http/opentsdb.sim +++ b/tests/script/unique/http/opentsdb.sim @@ -169,7 +169,7 @@ endi system_content curl -u root:taosdata -d 'select * from db.sys_cpu_d_bbb_lga_1_web01' 127.0.0.1:7111/rest/sql/ print $system_content -if $system_content != @{"status":"succ","head":["ts","value"],"data":[["2012-09-05 20:00:00.000",18.000000000]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["ts","value"],"head_type":[9,7],"data":[["2012-09-05 20:00:00.000",18.000000000]],"rows":1}@ then return -1 endi @@ -186,7 +186,7 @@ system_content curl -u root:taosdata -d 'select * from db.sys_cpu_d_bbb_lga_1_w print $system_content -if $system_content != @{"status":"succ","head":["ts","value"],"data":[["2012-09-05 20:00:00.000",18.000000000],["2012-09-05 20:00:05.000",18.000000000]],"rows":2}@ then +if $system_content != @{"status":"succ","head":["ts","value"],"head_type":[9,7],"data":[["2012-09-05 20:00:00.000",18.000000000],["2012-09-05 20:00:05.000",18.000000000]],"rows":2}@ then return -1 endi @@ -194,7 +194,7 @@ system_content curl -u root:taosdata -d 'select count(*) from db.sys_cpu_d_bbb' print $system_content -if $system_content != @{"status":"succ","head":["count(*)"],"data":[[3]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["count(*)"],"head_type":[5],"data":[[3]],"rows":1}@ then return -1 endi @@ -211,7 +211,7 @@ system_content curl -u root:taosdata -d 'select * from db.sys_mem_d_bbb_lga_1_w print $system_content -if $system_content != @{"status":"succ","head":["ts","value"],"data":[["2012-09-05 20:00:00.000",8.000000000],["2012-09-05 20:00:05.000",9.000000000]],"rows":2}@ then +if $system_content != @{"status":"succ","head":["ts","value"],"head_type":[9,7],"data":[["2012-09-05 20:00:00.000",8.000000000],["2012-09-05 20:00:05.000",9.000000000]],"rows":2}@ then return -1 endi @@ -219,7 +219,7 @@ system_content curl -u root:taosdata -d 'select count(*) from db.sys_mem_d_bbb' print $system_content -if $system_content != @{"status":"succ","head":["count(*)"],"data":[[2]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["count(*)"],"head_type":[5],"data":[[2]],"rows":1}@ then return -1 endi @@ -233,7 +233,7 @@ system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": 134 system_content curl -u root:taosdata -d 'select count(*) from db.sys_cpu_d_bbb' 127.0.0.1:7111/rest/sql/ print $system_content -if $system_content != @{"status":"succ","head":["count(*)"],"data":[[7]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["count(*)"],"head_type":[5],"data":[[7]],"rows":1}@ then return -1 endi @@ -244,4 +244,4 @@ system sh/exec.sh -n dnode4 -s stop -x SIGINT system sh/exec.sh -n dnode5 -s stop -x SIGINT system sh/exec.sh -n dnode6 -s stop -x SIGINT system sh/exec.sh -n dnode7 -s stop -x SIGINT -system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode8 -s stop -x SIGINT From 33c34970413c24b423cba368e1d2ee5204c294dc Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 3 Mar 2021 19:00:54 +0800 Subject: [PATCH 20/66] fix windows compile error --- src/tsdb/inc/tsdbMemTable.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tsdb/inc/tsdbMemTable.h b/src/tsdb/inc/tsdbMemTable.h index 988483ec78..bd64ed4a52 100644 --- a/src/tsdb/inc/tsdbMemTable.h +++ b/src/tsdb/inc/tsdbMemTable.h @@ -37,7 +37,7 @@ typedef struct { TSKEY keyLast; int64_t numOfRows; SSkipList* pData; - T_REF_DECLARE(); + T_REF_DECLARE() } STableData; typedef struct { From 7ee81346e272ab20a3bb66a751b86755149a4c09 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 3 Mar 2021 22:01:43 +0800 Subject: [PATCH 21/66] [TD-3043]: allow monitor db table to be dropped & altered --- src/mnode/src/mnodeTable.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 4fcedd82e3..b1ae118b4b 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -916,11 +916,13 @@ static int32_t mnodeProcessDropTableMsg(SMnodeMsg *pMsg) { return TSDB_CODE_MND_DB_IN_DROPPING; } +#if 0 if (mnodeCheckIsMonitorDB(pMsg->pDb->name, tsMonitorDbName)) { mError("msg:%p, app:%p table:%s, failed to drop table, in monitor database", pMsg, pMsg->rpcMsg.ahandle, pDrop->name); return TSDB_CODE_MND_MONITOR_DB_FORBIDDEN; } +#endif if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(pDrop->name); if (pMsg->pTable == NULL) { @@ -3020,10 +3022,12 @@ static int32_t mnodeProcessAlterTableMsg(SMnodeMsg *pMsg) { return TSDB_CODE_MND_DB_IN_DROPPING; } +#if 0 if (mnodeCheckIsMonitorDB(pMsg->pDb->name, tsMonitorDbName)) { mError("msg:%p, app:%p table:%s, failed to alter table, its log db", pMsg, pMsg->rpcMsg.ahandle, pAlter->tableFname); return TSDB_CODE_MND_MONITOR_DB_FORBIDDEN; } +#endif if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(pAlter->tableFname); if (pMsg->pTable == NULL) { From 60d8315431adc4763dbd3d6d2fca73b054553146 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 3 Mar 2021 22:26:31 +0800 Subject: [PATCH 22/66] monitor: update test cases to allow drop & alter monit tables --- tests/script/general/user/monitor.sim | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/script/general/user/monitor.sim b/tests/script/general/user/monitor.sim index fe12df9baa..016848c06d 100644 --- a/tests/script/general/user/monitor.sim +++ b/tests/script/general/user/monitor.sim @@ -15,18 +15,18 @@ print ========== step2 # return -1 #step21: sql drop table log.dn -x step22 - return -1 +# return -1 step22: sql drop user log -x step23 - return -1 +# return -1 step23: print ========== step3 sleep 2000 -sql select * from log.dn -if $rows == 0 then - return -1 -endi +#sql select * from log.dn +#if $rows == 0 then +# return -1 +#endi -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode1 -s stop -x SIGINT From 76a6b2ff1fb27c514e734ba6e3747aac6701f897 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 3 Mar 2021 23:17:57 +0800 Subject: [PATCH 23/66] [TD-3147] : support insert internal instead of insert rate. --- src/kit/taosdemo/insert.json | 5 +- src/kit/taosdemo/taosdemo.c | 532 +++++++++++++++++++++++++---------- 2 files changed, 388 insertions(+), 149 deletions(-) diff --git a/src/kit/taosdemo/insert.json b/src/kit/taosdemo/insert.json index 56a64b7b85..ebc0cfd607 100644 --- a/src/kit/taosdemo/insert.json +++ b/src/kit/taosdemo/insert.json @@ -29,13 +29,14 @@ }, "super_tables": [{ "name": "stb", - "child_table_exists":"no", + "child_table_exists":"no", "childtable_count": 1, "childtable_prefix": "stb_", "auto_create_table": "no", "data_source": "rand", "insert_mode": "taosc", - "insert_rate": 0, + "insert_interval": 0, + "num_of_records_per_req": 100, "insert_rows": 100000, "multi_thread_write_one_tbl": "no", "number_of_tbl_in_one_sql": 1, diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 6a5d0d6e90..a9a5ec0898 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -266,7 +266,8 @@ typedef struct SSuperTable_S { char childTblPrefix[MAX_TB_NAME_SIZE]; char dataSource[MAX_TB_NAME_SIZE]; // rand_gen or sample char insertMode[MAX_TB_NAME_SIZE]; // taosc, restful - int insertInterval; // interval time between insert twice + uint32_t insertInterval; // interval time between insert twice + uint32_t numRecPerReq; int multiThreadWriteOneTbl; // 0: no, 1: yes int numberOfTblInOneSql; // 0/1: one table, > 1: number of tbl @@ -431,7 +432,7 @@ typedef struct SThreadInfo_S { int start_table_id; int end_table_id; int data_of_rate; - int64_t start_time; + uint64_t start_time; char* cols; bool use_metric; SSuperTable* superTblInfo; @@ -439,10 +440,9 @@ typedef struct SThreadInfo_S { // for async insert tsem_t lock_sem; int64_t counter; - int64_t st; - int64_t et; + uint64_t st; + uint64_t et; int64_t lastTs; - int nrecords_per_request; // statistics int64_t totalRowsInserted; @@ -458,6 +458,7 @@ typedef struct SThreadInfo_S { } threadInfo; +#if 0 #ifdef LINUX /* The options we understand. */ static struct argp_option options[] = { @@ -645,23 +646,213 @@ void parse_args(int argc, char *argv[], SArguments *arguments) { } #else +#endif +#endif void printHelp() { char indent[10] = " "; printf("%s%s\n", indent, "-f"); - printf("%s%s%s\n", indent, indent, "The meta file to the execution procedure. Default is './meta.json'."); + printf("%s%s%s\n", indent, indent, + "The meta file to the execution procedure. Default is './meta.json'."); +#ifdef _TD_POWER_ printf("%s%s\n", indent, "-c"); - printf("%s%s%s\n", indent, indent, "config_directory, Configuration directory. Default is '/etc/taos/'."); + printf("%s%s%s\n", indent, indent, + "Configuration directory. Default is '/etc/power/'."); + printf("%s%s\n", indent, "-P"); + printf("%s%s%s\n", indent, indent, + "The password to use when connecting to the server. Default is 'powerdb'."); +#else + printf("%s%s\n", indent, "-c"); + printf("%s%s%s\n", indent, indent, + "Configuration directory. Default is '/etc/taos/'."); + printf("%s%s\n", indent, "-P"); + printf("%s%s%s\n", indent, indent, + "The password to use when connecting to the server. Default is 'taosdata'."); +#endif + printf("%s%s\n", indent, "-h"); + printf("%s%s%s\n", indent, indent, + "The host to connect to TDengine. Default is localhost."); + printf("%s%s\n", indent, "-p"); + printf("%s%s%s\n", indent, indent, + "The TCP/IP port number to use for the connection. Default is 0."); + printf("%s%s\n", indent, "-u"); + printf("%s%s%s\n", indent, indent, + "The TDengine user name to use when connecting to the server. Default is 'root'."); + printf("%s%s\n", indent, "-d"); + printf("%s%s%s\n", indent, indent, + "Destination database. Default is 'test'."); + printf("%s%s\n", indent, "-a"); + printf("%s%s%s\n", indent, indent, + "Set the replica parameters of the database, Default 1, min: 1, max: 3."); + printf("%s%s\n", indent, "-m"); + printf("%s%s%s\n", indent, indent, + "Table prefix name. Default is 't'."); + printf("%s%s\n", indent, "-s"); + printf("%s%s%s\n", indent, indent, + "The select sql file."); + printf("%s%s\n", indent, "-M"); + printf("%s%s%s\n", indent, indent, + "Use metric flag."); + printf("%s%s\n", indent, "-o"); + printf("%s%s%s\n", indent, indent, + "Direct output to the named file. Default is './output.txt'."); + printf("%s%s\n", indent, "-q"); + printf("%s%s%s\n", indent, indent, + "Query mode--0: SYNC, 1: ASYNC. Default is SYNC."); + printf("%s%s\n", indent, "-b"); + printf("%s%s%s\n", indent, indent, + "The data_type of columns, default: TINYINT,SMALLINT,INT,BIGINT,FLOAT,DOUBLE,BINARY,NCHAR,BOOL,TIMESTAMP."); + printf("%s%s\n", indent, "-w"); + printf("%s%s%s\n", indent, indent, + "The length of data_type 'BINARY' or 'NCHAR'. Default is 16"); + printf("%s%s\n", indent, "-l"); + printf("%s%s%s\n", indent, indent, + "The number of columns per record. Default is 10."); + printf("%s%s\n", indent, "-T"); + printf("%s%s%s\n", indent, indent, + "The number of threads. Default is 10."); + printf("%s%s\n", indent, "-r"); + printf("%s%s%s\n", indent, indent, + "The number of records per request. Default is 100."); + printf("%s%s\n", indent, "-t"); + printf("%s%s%s\n", indent, indent, + "The number of tables. Default is 10000."); + printf("%s%s\n", indent, "-n"); + printf("%s%s%s\n", indent, indent, + "The number of records per table. Default is 10000."); + printf("%s%s\n", indent, "-x"); + printf("%s%s%s\n", indent, indent, + "Not insert only flag."); + printf("%s%s\n", indent, "-y"); + printf("%s%s%s\n", indent, indent, + "Default input yes for prompt."); + printf("%s%s\n", indent, "-O"); + printf("%s%s%s\n", indent, indent, + "Insert mode--0: In order, > 0: disorder ratio. Default is in order."); + printf("%s%s\n", indent, "-R"); + printf("%s%s%s\n", indent, indent, + "Out of order data's range, ms, default is 1000."); +/* printf("%s%s\n", indent, "-D"); + printf("%s%s%s\n", indent, indent, + "if elete database if exists. 0: no, 1: yes, default is 1"); + */ } void parse_args(int argc, char *argv[], SArguments *arguments) { + char **sptr; + wordexp_t full_path; + for (int i = 1; i < argc; i++) { if (strcmp(argv[i], "-f") == 0) { arguments->metaFile = argv[++i]; + } else if (strcmp(argv[i], "-c") == 0) { + char *configPath = argv[++i]; + if (wordexp(configPath, &full_path, 0) != 0) { + fprintf(stderr, "Invalid path %s\n", configPath); + return; + } + taos_options(TSDB_OPTION_CONFIGDIR, full_path.we_wordv[0]); + wordfree(&full_path); + } else if (strcmp(argv[i], "-h") == 0) { + arguments->host = argv[++i]; + } else if (strcmp(argv[i], "-p") == 0) { + arguments->port = atoi(argv[++i]); + } else if (strcmp(argv[i], "-u") == 0) { + arguments->user = argv[++i]; + } else if (strcmp(argv[i], "-P") == 0) { + arguments->password = argv[++i]; + } else if (strcmp(argv[i], "-o") == 0) { + arguments->output_file = argv[++i]; + } else if (strcmp(argv[i], "-s") == 0) { + arguments->sqlFile = argv[++i]; + } else if (strcmp(argv[i], "-q") == 0) { + arguments->mode = atoi(argv[++i]); + } else if (strcmp(argv[i], "-T") == 0) { + arguments->num_of_threads = atoi(argv[++i]); + } else if (strcmp(argv[i], "-r") == 0) { + arguments->num_of_RPR = atoi(argv[++i]); + } else if (strcmp(argv[i], "-t") == 0) { + arguments->num_of_tables = atoi(argv[++i]); + } else if (strcmp(argv[i], "-n") == 0) { + arguments->num_of_DPT = atoi(argv[++i]); + } else if (strcmp(argv[i], "-d") == 0) { + arguments->database = argv[++i]; + } else if (strcmp(argv[i], "-l") == 0) { + arguments->num_of_CPR = atoi(argv[++i]); + } else if (strcmp(argv[i], "-b") == 0) { + sptr = arguments->datatype; + ++i; + if (strstr(argv[i], ",") == NULL) { + if (strcasecmp(argv[i], "INT") != 0 && strcasecmp(argv[i], "FLOAT") != 0 && + strcasecmp(argv[i], "TINYINT") != 0 && strcasecmp(argv[i], "BOOL") != 0 && + strcasecmp(argv[i], "SMALLINT") != 0 && + strcasecmp(argv[i], "BIGINT") != 0 && strcasecmp(argv[i], "DOUBLE") != 0 && + strcasecmp(argv[i], "BINARY") && strcasecmp(argv[i], "NCHAR")) { + fprintf(stderr, "Invalid data_type!\n"); + printHelp(); + exit(EXIT_FAILURE); + } + sptr[0] = argv[i]; + } else { + int index = 0; + char *dupstr = strdup(argv[i]); + char *running = dupstr; + char *token = strsep(&running, ","); + while (token != NULL) { + if (strcasecmp(token, "INT") != 0 && + strcasecmp(token, "FLOAT") != 0 && + strcasecmp(token, "TINYINT") != 0 && + strcasecmp(token, "BOOL") != 0 && + strcasecmp(token, "SMALLINT") != 0 && + strcasecmp(token, "BIGINT") != 0 && + strcasecmp(token, "DOUBLE") != 0 && strcasecmp(token, "BINARY") && strcasecmp(token, "NCHAR")) { + fprintf(stderr, "Invalid data_type!\n"); + printHelp(); + exit(EXIT_FAILURE); + } + sptr[index++] = token; + token = strsep(&running, ","); + if (index >= MAX_NUM_DATATYPE) break; + } + } + } else if (strcmp(argv[i], "-w") == 0) { + arguments->len_of_binary = atoi(argv[++i]); + } else if (strcmp(argv[i], "-m") == 0) { + arguments->tb_prefix = argv[++i]; + } else if (strcmp(argv[i], "-M") == 0) { + arguments->use_metric = true; + } else if (strcmp(argv[i], "-x") == 0) { + arguments->insert_only = true; + } else if (strcmp(argv[i], "-y") == 0) { + arguments->answer_yes = true; } else if (strcmp(argv[i], "-c") == 0) { strcpy(configDir, argv[++i]); + } else if (strcmp(argv[i], "-O") == 0) { + arguments->disorderRatio = atoi(argv[++i]); + if (arguments->disorderRatio > 1 || arguments->disorderRatio < 0) { + arguments->disorderRatio = 0; + } else if (arguments->disorderRatio == 1) { + arguments->disorderRange = 10; + } + } else if (strcmp(argv[i], "-R") == 0) { + arguments->disorderRange = atoi(argv[++i]); + if (arguments->disorderRange == 1 + && (arguments->disorderRange > 50 + || arguments->disorderRange <= 0)) { + arguments->disorderRange = 10; + } + } else if (strcmp(argv[i], "-a") == 0) { + arguments->replica = atoi(argv[++i]); + if (arguments->replica > 3 || arguments->replica < 1) { + arguments->replica = 1; + } + } else if (strcmp(argv[i], "-D") == 0) { + arguments->method_of_delete = atoi(argv[++i]); + if (arguments->method_of_delete < 0 || arguments->method_of_delete > 3) { + arguments->method_of_delete = 0; + } } else if (strcmp(argv[i], "--help") == 0) { printHelp(); - exit(EXIT_FAILURE); + exit(0); } else { fprintf(stderr, "wrong options\n"); printHelp(); @@ -669,7 +860,7 @@ void parse_args(int argc, char *argv[], SArguments *arguments) { } } } -#endif +//#endif static bool getInfoFromJsonFile(char* file); //static int generateOneRowDataForStb(SSuperTable* stbInfo); @@ -1016,6 +1207,7 @@ static int printfInsertMeta() { printf(" dataSource: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].dataSource); printf(" insertMode: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].insertMode); printf(" insertInterval: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].insertInterval); + printf(" numRecPerReq: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].numRecPerReq); printf(" insertRows: \033[33m%"PRId64"\033[0m\n", g_Dbs.db[i].superTbls[j].insertRows); if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) { @@ -1154,6 +1346,7 @@ static void printfInsertMetaToFile(FILE* fp) { fprintf(fp, " dataSource: %s\n", g_Dbs.db[i].superTbls[j].dataSource); fprintf(fp, " insertMode: %s\n", g_Dbs.db[i].superTbls[j].insertMode); fprintf(fp, " insertInterval: %d\n", g_Dbs.db[i].superTbls[j].insertInterval); + fprintf(fp, " numRecPerReq: %d\n", g_Dbs.db[i].superTbls[j].numRecPerReq); fprintf(fp, " insertRows: %"PRId64"\n", g_Dbs.db[i].superTbls[j].insertRows); if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) { @@ -3003,10 +3196,21 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!insertInterval) { g_Dbs.db[i].superTbls[j].insertInterval = 0; } else { - printf("failed to read json, insert_rate not found"); + printf("failed to read json, insert_interval not found"); goto PARSE_OVER; } + cJSON* numRecPerReq = cJSON_GetObjectItem(stbInfo, "num_of_records_per_req"); + if (numRecPerReq && numRecPerReq->type == cJSON_Number) { + g_Dbs.db[i].superTbls[j].numRecPerReq = numRecPerReq->valueint; + } else if (!numRecPerReq) { + g_Dbs.db[i].superTbls[j].numRecPerReq = 0; + } else { + printf("failed to read json, num_of_records_per_req not found"); + goto PARSE_OVER; + } + + cJSON* insertRows = cJSON_GetObjectItem(stbInfo, "insert_rows"); if (insertRows && insertRows->type == cJSON_Number) { g_Dbs.db[i].superTbls[j].insertRows = insertRows->valueint; @@ -3570,10 +3774,8 @@ void syncWriteForNumberOfTblInOneSql(threadInfo *winfo, FILE *fp, char* sampleDa for (int tID = winfo->start_table_id; tID <= winfo->end_table_id; ) { int inserted = i; - int k = 0; - int batchRowsSql = 0; - while (1) - { + for (int k = 0; k < winfo->superTblInfo->numRecPerReq;) + { int len = 0; memset(buffer, 0, superTblInfo->maxSqlLen); char *pstr = buffer; @@ -3582,6 +3784,7 @@ void syncWriteForNumberOfTblInOneSql(threadInfo *winfo, FILE *fp, char* sampleDa if (end_tbl_id > winfo->end_table_id) { end_tbl_id = winfo->end_table_id+1; } + for (tbl_id = tID; tbl_id < end_tbl_id; tbl_id++) { sampleUsePos = samplePos; if (AUTO_CREATE_SUBTBL == superTblInfo->autoCreateTable) { @@ -3589,47 +3792,96 @@ void syncWriteForNumberOfTblInOneSql(threadInfo *winfo, FILE *fp, char* sampleDa if (0 == superTblInfo->tagSource) { tagsValBuf = generateTagVaulesForStb(superTblInfo); } else { - tagsValBuf = getTagValueFromTagSample(superTblInfo, tbl_id % superTblInfo->tagSampleCount); + tagsValBuf = getTagValueFromTagSample( + superTblInfo, tbl_id % superTblInfo->tagSampleCount); } if (NULL == tagsValBuf) { goto free_and_statistics; } if (0 == len) { - len += snprintf(pstr + len, superTblInfo->maxSqlLen - len, "insert into %s.%s%d using %s.%s tags %s values ", winfo->db_name, superTblInfo->childTblPrefix, tbl_id, winfo->db_name, superTblInfo->sTblName, tagsValBuf); + len += snprintf(pstr + len, + superTblInfo->maxSqlLen - len, + "insert into %s.%s%d using %s.%s tags %s values ", + winfo->db_name, + superTblInfo->childTblPrefix, + tbl_id, + winfo->db_name, + superTblInfo->sTblName, + tagsValBuf); } else { - len += snprintf(pstr + len, superTblInfo->maxSqlLen - len, " %s.%s%d using %s.%s tags %s values ", winfo->db_name, superTblInfo->childTblPrefix, tbl_id, winfo->db_name, superTblInfo->sTblName, tagsValBuf); + len += snprintf(pstr + len, + superTblInfo->maxSqlLen - len, + " %s.%s%d using %s.%s tags %s values ", + winfo->db_name, + superTblInfo->childTblPrefix, + tbl_id, + winfo->db_name, + superTblInfo->sTblName, + tagsValBuf); } tmfree(tagsValBuf); } else if (TBL_ALREADY_EXISTS == superTblInfo->childTblExists) { if (0 == len) { - len += snprintf(pstr + len, superTblInfo->maxSqlLen - len, "insert into %s.%s values ", winfo->db_name, superTblInfo->childTblName + tbl_id * TSDB_TABLE_NAME_LEN); + len += snprintf(pstr + len, + superTblInfo->maxSqlLen - len, + "insert into %s.%s values ", + winfo->db_name, + superTblInfo->childTblName + tbl_id * TSDB_TABLE_NAME_LEN); } else { - len += snprintf(pstr + len, superTblInfo->maxSqlLen - len, " %s.%s values ", winfo->db_name, superTblInfo->childTblName + tbl_id * TSDB_TABLE_NAME_LEN); + len += snprintf(pstr + len, + superTblInfo->maxSqlLen - len, + " %s.%s values ", + winfo->db_name, + superTblInfo->childTblName + tbl_id * TSDB_TABLE_NAME_LEN); } } else { // pre-create child table if (0 == len) { - len += snprintf(pstr + len, superTblInfo->maxSqlLen - len, "insert into %s.%s%d values ", winfo->db_name, superTblInfo->childTblPrefix, tbl_id); + len += snprintf(pstr + len, + superTblInfo->maxSqlLen - len, + "insert into %s.%s%d values ", + winfo->db_name, + superTblInfo->childTblPrefix, + tbl_id); } else { - len += snprintf(pstr + len, superTblInfo->maxSqlLen - len, " %s.%s%d values ", winfo->db_name, superTblInfo->childTblPrefix, tbl_id); + len += snprintf(pstr + len, + superTblInfo->maxSqlLen - len, + " %s.%s%d values ", + winfo->db_name, + superTblInfo->childTblPrefix, + tbl_id); } } - + tmp_time = time_counter; for (k = 0; k < superTblInfo->rowsPerTbl;) { int retLen = 0; - if (0 == strncasecmp(superTblInfo->dataSource, "sample", 6)) { - retLen = getRowDataFromSample(pstr + len, superTblInfo->maxSqlLen - len, tmp_time += superTblInfo->timeStampStep, superTblInfo, &sampleUsePos, fp, sampleDataBuf); + if (0 == strncasecmp(superTblInfo->dataSource, "sample", strlen("sample"))) { + retLen = getRowDataFromSample(pstr + len, + superTblInfo->maxSqlLen - len, + tmp_time += superTblInfo->timeStampStep, + superTblInfo, + &sampleUsePos, + fp, + sampleDataBuf); if (retLen < 0) { goto free_and_statistics; } - } else if (0 == strncasecmp(superTblInfo->dataSource, "rand", 8)) { + } else if (0 == strncasecmp( + superTblInfo->dataSource, "rand", strlen("rand"))) { int rand_num = rand_tinyint() % 100; - if (0 != superTblInfo->disorderRatio && rand_num < superTblInfo->disorderRatio) { + if (0 != superTblInfo->disorderRatio + && rand_num < superTblInfo->disorderRatio) { int64_t d = tmp_time - rand() % superTblInfo->disorderRange; - retLen = generateRowData(pstr + len, superTblInfo->maxSqlLen - len, d, superTblInfo); + retLen = generateRowData(pstr + len, + superTblInfo->maxSqlLen - len, + d, + superTblInfo); } else { - retLen = generateRowData(pstr + len, superTblInfo->maxSqlLen - len, tmp_time += superTblInfo->timeStampStep, superTblInfo); + retLen = generateRowData(pstr + len, + superTblInfo->maxSqlLen - len, + tmp_time += superTblInfo->timeStampStep, + superTblInfo); } if (retLen < 0) { goto free_and_statistics; @@ -3639,11 +3891,12 @@ void syncWriteForNumberOfTblInOneSql(threadInfo *winfo, FILE *fp, char* sampleDa //inserted++; k++; totalRowsInserted++; - batchRowsSql++; - if (inserted >= superTblInfo->insertRows || (superTblInfo->maxSqlLen - len) < (superTblInfo->lenOfOneRow + 128) || batchRowsSql >= INT16_MAX - 1) { + if (inserted >= superTblInfo->insertRows || + (superTblInfo->maxSqlLen - len) < (superTblInfo->lenOfOneRow + 128)) { tID = tbl_id + 1; - printf("config rowsPerTbl and numberOfTblInOneSql not match with max_sql_lenth, please reconfig![lenOfOneRow:%d]\n", superTblInfo->lenOfOneRow); + printf("config rowsPerTbl and numberOfTblInOneSql not match with max_sql_lenth, please reconfig![lenOfOneRow:%d]\n", + superTblInfo->lenOfOneRow); goto send_to_server; } } @@ -3654,15 +3907,17 @@ void syncWriteForNumberOfTblInOneSql(threadInfo *winfo, FILE *fp, char* sampleDa inserted += superTblInfo->rowsPerTbl; send_to_server: - batchRowsSql = 0; - if (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5)) { + if (0 == strncasecmp(superTblInfo->insertMode, + "taosc", + strlen("taosc"))) { //printf("multi table===== sql: %s \n\n", buffer); //int64_t t1 = taosGetTimestampMs(); int64_t startTs; int64_t endTs; startTs = taosGetTimestampUs(); - int affectedRows = queryDbExec(winfo->taos, buffer, INSERT_TYPE); + int affectedRows = queryDbExec( + winfo->taos, buffer, INSERT_TYPE); if (0 > affectedRows) { goto free_and_statistics; } else { @@ -3678,29 +3933,31 @@ void syncWriteForNumberOfTblInOneSql(threadInfo *winfo, FILE *fp, char* sampleDa int64_t currentPrintTime = taosGetTimestampMs(); if (currentPrintTime - lastPrintTime > 30*1000) { - printf("thread[%d] has currently inserted rows: %"PRId64 ", affected rows: %"PRId64 "\n", winfo->threadID, totalRowsInserted, totalAffectedRows); + printf("thread[%d] has currently inserted rows: %"PRId64 ", affected rows: %"PRId64 "\n", + winfo->threadID, + totalRowsInserted, + totalAffectedRows); lastPrintTime = currentPrintTime; } //int64_t t2 = taosGetTimestampMs(); - //printf("taosc insert sql return, Spent %.4f seconds \n", (double)(t2 - t1)/1000.0); + //printf("taosc insert sql return, Spent %.4f seconds \n", (double)(t2 - t1)/1000.0); } else { //int64_t t1 = taosGetTimestampMs(); int retCode = postProceSql(g_Dbs.host, g_Dbs.port, buffer); //int64_t t2 = taosGetTimestampMs(); //printf("http insert sql return, Spent %ld ms \n", t2 - t1); - + if (0 != retCode) { printf("========restful return fail, threadID[%d]\n", winfo->threadID); goto free_and_statistics; } } - //printf("========tID:%d, k:%d, loop_cnt:%d\n", tID, k, loop_cnt); break; } if (tID > winfo->end_table_id) { - if (0 == strncasecmp(superTblInfo->dataSource, "sample", 6)) { + if (0 == strncasecmp(superTblInfo->dataSource, "sample", strlen("sample"))) { samplePos = sampleUsePos; } i = inserted; @@ -3731,9 +3988,9 @@ void syncWriteForNumberOfTblInOneSql(threadInfo *winfo, FILE *fp, char* sampleDa 2 taosinsertdata , 1 thread: 10 tables * 20000 rows/s */ void *syncWrite(void *sarg) { - int64_t totalRowsInserted = 0; - int64_t totalAffectedRows = 0; - int64_t lastPrintTime = taosGetTimestampMs(); + uint64_t totalRowsInserted = 0; + uint64_t totalAffectedRows = 0; + uint64_t lastPrintTime = taosGetTimestampMs(); threadInfo *winfo = (threadInfo *)sarg; SSuperTable* superTblInfo = winfo->superTblInfo; @@ -3743,20 +4000,27 @@ void *syncWrite(void *sarg) { int samplePos = 0; // each thread read sample data from csv file - if (0 == strncasecmp(superTblInfo->dataSource, "sample", 6)) { - sampleDataBuf = calloc(superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE, 1); + if (0 == strncasecmp(superTblInfo->dataSource, + "sample", + strlen("sample"))) { + sampleDataBuf = calloc( + superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE, 1); if (sampleDataBuf == NULL) { - printf("Failed to calloc %d Bytes, reason:%s\n", superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE, strerror(errno)); + printf("Failed to calloc %d Bytes, reason:%s\n", + superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE, + strerror(errno)); return NULL; } - + fp = fopen(superTblInfo->sampleFile, "r"); if (fp == NULL) { - printf("Failed to open sample file: %s, reason:%s\n", superTblInfo->sampleFile, strerror(errno)); + printf("Failed to open sample file: %s, reason:%s\n", + superTblInfo->sampleFile, strerror(errno)); tmfree(sampleDataBuf); return NULL; } - int ret = readSampleFromCsvFileToMem(fp, superTblInfo, sampleDataBuf); + int ret = readSampleFromCsvFileToMem(fp, + superTblInfo, sampleDataBuf); if (0 != ret) { tmfree(sampleDataBuf); tmfclose(fp); @@ -3771,62 +4035,33 @@ void *syncWrite(void *sarg) { return NULL; } - //printf("========threadID[%d], table rang: %d - %d \n", winfo->threadID, winfo->start_table_id, winfo->end_table_id); - char* buffer = calloc(superTblInfo->maxSqlLen, 1); - - int nrecords_per_request = 0; - if (AUTO_CREATE_SUBTBL == superTblInfo->autoCreateTable) { - nrecords_per_request = (superTblInfo->maxSqlLen - 1280 - superTblInfo->lenOfTagOfOneRow) / superTblInfo->lenOfOneRow; - } else { - nrecords_per_request = (superTblInfo->maxSqlLen - 1280) / superTblInfo->lenOfOneRow; - } - - int nrecords_no_last_req = nrecords_per_request; - int nrecords_last_req = 0; - int loop_cnt = 0; - if (0 != superTblInfo->insertRate) { - if (nrecords_no_last_req >= superTblInfo->insertRate) { - nrecords_no_last_req = superTblInfo->insertRate; - } else { - nrecords_last_req = superTblInfo->insertRate % nrecords_per_request; - loop_cnt = (superTblInfo->insertRate / nrecords_per_request) + (superTblInfo->insertRate % nrecords_per_request ? 1 : 0) ; - } - } - - if (nrecords_no_last_req <= 0) { - nrecords_no_last_req = 1; + if (NULL == buffer) { + printf("Failed to calloc %d Bytes, reason:%s\n", + superTblInfo->maxSqlLen, + strerror(errno)); + tmfree(sampleDataBuf); + tmfclose(fp); + return NULL; } - if (nrecords_no_last_req >= INT16_MAX) { - nrecords_no_last_req = INT16_MAX - 1; - } + uint64_t time_counter = winfo->start_time; + uint64_t st = 0; + uint64_t et = 0; - if (nrecords_last_req >= INT16_MAX) { - nrecords_last_req = INT16_MAX - 1; - } - - int nrecords_cur_req = nrecords_no_last_req; - int loop_cnt_orig = loop_cnt; - - //printf("========nrecords_per_request:%d, nrecords_no_last_req:%d, nrecords_last_req:%d, loop_cnt:%d\n", nrecords_per_request, nrecords_no_last_req, nrecords_last_req, loop_cnt); - - int64_t time_counter = winfo->start_time; - - int64_t st = 0; - int64_t et = 0; for (int i = 0; i < superTblInfo->insertRows;) { - if (superTblInfo->insertInterval && (superTblInfo->insertInterval > (et - st) )) { + if (i > 0 && superTblInfo->insertInterval + && (superTblInfo->insertInterval > (et - st) )) { taosMsleep(superTblInfo->insertInterval - (et - st)); // ms } if (superTblInfo->insertInterval) { st = taosGetTimestampMs(); } - - for (int tID = winfo->start_table_id; tID <= winfo->end_table_id; tID++) { - int inserted = i; - int64_t tmp_time = time_counter; + + for (uint32_t tID = winfo->start_table_id; tID <= winfo->end_table_id; tID++) { + uint64_t inserted = i; + uint64_t tmp_time = time_counter; int sampleUsePos = samplePos; int k = 0; @@ -3841,24 +4076,50 @@ void *syncWrite(void *sarg) { if (0 == superTblInfo->tagSource) { tagsValBuf = generateTagVaulesForStb(superTblInfo); } else { - tagsValBuf = getTagValueFromTagSample(superTblInfo, tID % superTblInfo->tagSampleCount); + tagsValBuf = getTagValueFromTagSample( + superTblInfo, + tID % superTblInfo->tagSampleCount); } if (NULL == tagsValBuf) { goto free_and_statistics_2; } - len += snprintf(pstr + len, superTblInfo->maxSqlLen - len, "insert into %s.%s%d using %s.%s tags %s values", winfo->db_name, superTblInfo->childTblPrefix, tID, winfo->db_name, superTblInfo->sTblName, tagsValBuf); + len += snprintf(pstr + len, + superTblInfo->maxSqlLen - len, + "insert into %s.%s%d using %s.%s tags %s values", + winfo->db_name, + superTblInfo->childTblPrefix, + tID, + winfo->db_name, + superTblInfo->sTblName, + tagsValBuf); tmfree(tagsValBuf); } else if (TBL_ALREADY_EXISTS == superTblInfo->childTblExists) { - len += snprintf(pstr + len, superTblInfo->maxSqlLen - len, "insert into %s.%s values", winfo->db_name, superTblInfo->childTblName + tID * TSDB_TABLE_NAME_LEN); + len += snprintf(pstr + len, + superTblInfo->maxSqlLen - len, + "insert into %s.%s values", + winfo->db_name, + superTblInfo->childTblName + tID * TSDB_TABLE_NAME_LEN); } else { - len += snprintf(pstr + len, superTblInfo->maxSqlLen - len, "insert into %s.%s%d values", winfo->db_name, superTblInfo->childTblPrefix, tID); + len += snprintf(pstr + len, + superTblInfo->maxSqlLen - len, + "insert into %s.%s%d values", + winfo->db_name, + superTblInfo->childTblPrefix, + tID); } - for (k = 0; k < nrecords_cur_req;) { + for (k = 0; k < superTblInfo->numRecPerReq;) { int retLen = 0; if (0 == strncasecmp(superTblInfo->dataSource, "sample", 6)) { - retLen = getRowDataFromSample(pstr + len, superTblInfo->maxSqlLen - len, tmp_time += superTblInfo->timeStampStep, superTblInfo, &sampleUsePos, fp, sampleDataBuf); + retLen = getRowDataFromSample( + pstr + len, + superTblInfo->maxSqlLen - len, + tmp_time += superTblInfo->timeStampStep, + superTblInfo, + &sampleUsePos, + fp, + sampleDataBuf); if (retLen < 0) { goto free_and_statistics_2; } @@ -3866,10 +4127,17 @@ void *syncWrite(void *sarg) { int rand_num = rand_tinyint() % 100; if (0 != superTblInfo->disorderRatio && rand_num < superTblInfo->disorderRatio) { int64_t d = tmp_time - rand() % superTblInfo->disorderRange; - retLen = generateRowData(pstr + len, superTblInfo->maxSqlLen - len, d, superTblInfo); + retLen = generateRowData( + pstr + len, + superTblInfo->maxSqlLen - len, d, + superTblInfo); //printf("disorder rows, rand_num:%d, last ts:%"PRId64" current ts:%"PRId64"\n", rand_num, tmp_time, d); } else { - retLen = generateRowData(pstr + len, superTblInfo->maxSqlLen - len, tmp_time += superTblInfo->timeStampStep, superTblInfo); + retLen = generateRowData( + pstr + len, + superTblInfo->maxSqlLen - len, + tmp_time += superTblInfo->timeStampStep, + superTblInfo); } if (retLen < 0) { goto free_and_statistics_2; @@ -3880,7 +4148,9 @@ void *syncWrite(void *sarg) { k++; totalRowsInserted++; - if (inserted >= superTblInfo->insertRows || (superTblInfo->maxSqlLen - len) < (superTblInfo->lenOfOneRow + 128)) break; + if (inserted >= superTblInfo->insertRows + || (superTblInfo->maxSqlLen - len) < (superTblInfo->lenOfOneRow + 128)) + break; } if (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5)) { @@ -3906,7 +4176,10 @@ void *syncWrite(void *sarg) { int64_t currentPrintTime = taosGetTimestampMs(); if (currentPrintTime - lastPrintTime > 30*1000) { - printf("thread[%d] has currently inserted rows: %"PRId64 ", affected rows: %"PRId64 "\n", winfo->threadID, totalRowsInserted, totalAffectedRows); + printf("thread[%d] has currently inserted rows: %"PRId64 ", affected rows: %"PRId64 "\n", + winfo->threadID, + totalRowsInserted, + totalAffectedRows); lastPrintTime = currentPrintTime; } //int64_t t2 = taosGetTimestampMs(); @@ -3922,25 +4195,11 @@ void *syncWrite(void *sarg) { goto free_and_statistics_2; } } - - //printf("========tID:%d, k:%d, loop_cnt:%d\n", tID, k, loop_cnt); - - if (loop_cnt) { - loop_cnt--; - if ((1 == loop_cnt) && (0 != nrecords_last_req)) { - nrecords_cur_req = nrecords_last_req; - } else if (0 == loop_cnt){ - nrecords_cur_req = nrecords_no_last_req; - loop_cnt = loop_cnt_orig; - break; - } - } else { - break; - } } if (tID == winfo->end_table_id) { - if (0 == strncasecmp(superTblInfo->dataSource, "sample", 6)) { + if (0 == strncasecmp( + superTblInfo->dataSource, "sample", 6)) { samplePos = sampleUsePos; } i = inserted; @@ -3954,7 +4213,7 @@ void *syncWrite(void *sarg) { //printf("========loop %d childTables duration:%"PRId64 "========inserted rows:%d\n", winfo->end_table_id - winfo->start_table_id, et - st, i); } - free_and_statistics_2: +free_and_statistics_2: tmfree(buffer); tmfree(sampleDataBuf); tmfclose(fp); @@ -3962,7 +4221,10 @@ void *syncWrite(void *sarg) { winfo->totalRowsInserted = totalRowsInserted; winfo->totalAffectedRows = totalAffectedRows; - printf("====thread[%d] completed total inserted rows: %"PRId64 ", total affected rows: %"PRId64 "====\n", winfo->threadID, totalRowsInserted, totalAffectedRows); + printf("====thread[%d] completed total inserted rows: %"PRId64 ", total affected rows: %"PRId64 "====\n", + winfo->threadID, + totalRowsInserted, + totalAffectedRows); return NULL; } @@ -3992,7 +4254,7 @@ void callBack(void *param, TAOS_RES *res, int code) { return; } - for (int i = 0; i < winfo->nrecords_per_request; i++) { + for (int i = 0; i < winfo->superTblInfo->numRecPerReq; i++) { int rand_num = rand() % 100; if (0 != winfo->superTblInfo->disorderRatio && rand_num < winfo->superTblInfo->disorderRatio) { @@ -4024,31 +4286,6 @@ void callBack(void *param, TAOS_RES *res, int code) { void *asyncWrite(void *sarg) { threadInfo *winfo = (threadInfo *)sarg; - winfo->nrecords_per_request = 0; - //if (AUTO_CREATE_SUBTBL == winfo->superTblInfo->autoCreateTable) { - winfo->nrecords_per_request = (winfo->superTblInfo->maxSqlLen - 1280 - winfo->superTblInfo->lenOfTagOfOneRow) / winfo->superTblInfo->lenOfOneRow; - //} else { - // winfo->nrecords_per_request = (winfo->superTblInfo->maxSqlLen - 1280) / winfo->superTblInfo->lenOfOneRow; - //} - - if (0 != winfo->superTblInfo->insertInterval) { - if (winfo->nrecords_per_request >= winfo->superTblInfo->insertInterval) { - winfo->nrecords_per_request = winfo->superTblInfo->insertInterval; - } - } - - if (winfo->nrecords_per_request <= 0) { - winfo->nrecords_per_request = 1; - } - - if (winfo->nrecords_per_request >= INT16_MAX) { - winfo->nrecords_per_request = INT16_MAX - 1; - } - - if (winfo->nrecords_per_request >= INT16_MAX) { - winfo->nrecords_per_request = INT16_MAX - 1; - } - winfo->st = 0; winfo->et = 0; winfo->lastTs = winfo->start_time; @@ -4950,6 +5187,7 @@ void setParaFromArg(){ g_Dbs.db[0].superTbls[0].superTblExists = TBL_NO_EXISTS; g_Dbs.db[0].superTbls[0].childTblExists = TBL_NO_EXISTS; g_Dbs.db[0].superTbls[0].insertInterval = 0; + g_Dbs.db[0].superTbls[0].numRecPerReq = 0; g_Dbs.db[0].superTbls[0].disorderRange = g_args.disorderRange; g_Dbs.db[0].superTbls[0].disorderRatio = g_args.disorderRatio; tstrncpy(g_Dbs.db[0].superTbls[0].childTblPrefix, g_args.tb_prefix, MAX_TB_NAME_SIZE); From 42ac4ff8fdc36a6a31382abab02ee6b24b818f57 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Thu, 4 Mar 2021 09:42:14 +0800 Subject: [PATCH 24/66] [TD-3127]add unsigned type --- tests/pytest/concurrent_inquiry.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/tests/pytest/concurrent_inquiry.py b/tests/pytest/concurrent_inquiry.py index e832c9a74e..524ba31918 100644 --- a/tests/pytest/concurrent_inquiry.py +++ b/tests/pytest/concurrent_inquiry.py @@ -349,18 +349,27 @@ class ConcurrentInquiry: cl.execute("create database if not exists %s;" %self.dbname) cl.execute("use %s" % self.dbname) for k in range(stableNum): - sql="create table %s (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool,c8 binary(20),c9 nchar(20)) \ - tags(t1 int, t2 float, t3 bigint, t4 smallint, t5 tinyint, t6 double, t7 bool,t8 binary(20),t9 nchar(20))" % (self.stb_prefix+str(k)) + sql="create table %s (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool,c8 binary(20),c9 nchar(20),c11 int unsigned,c12 smallint unsigned,c13 tinyint unsigned,c14 bigint unsigned) \ + tags(t1 int, t2 float, t3 bigint, t4 smallint, t5 tinyint, t6 double, t7 bool,t8 binary(20),t9 nchar(20), t11 int unsigned , t12 smallint unsigned , t13 tinyint unsigned , t14 bigint unsigned)" % (self.stb_prefix+str(k)) cl.execute(sql) for j in range(subtableNum): - sql = "create table %s using %s tags(%d,%d,%d,%d,%d,%d,%d,'%s','%s')" % \ - (self.subtb_prefix+str(k)+'_'+str(j),self.stb_prefix+str(k),j,j/2.0,j%41,j%51,j%53,j*1.0,j%2,'taos'+str(j),'涛思'+str(j)) + if j % 100 == 0: + sql = "create table %s using %s tags(NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL)" % \ + (self.subtb_prefix+str(k)+'_'+str(j),self.stb_prefix+str(k)) + else: + sql = "create table %s using %s tags(%d,%d,%d,%d,%d,%d,%d,'%s','%s',%d,%d,%d,%d)" % \ + (self.subtb_prefix+str(k)+'_'+str(j),self.stb_prefix+str(k),j,j/2.0,j%41,j%51,j%53,j*1.0,j%2,'taos'+str(j),'涛思'+str(j), j%43, j%23 , j%17 , j%3167) print(sql) cl.execute(sql) for i in range(insertRows): - ret = cl.execute( - "insert into %s values (%d , %d,%d,%d,%d,%d,%d,%d,'%s','%s')" % - (self.subtb_prefix+str(k)+'_'+str(j),t0+i,i%100,i/2.0,i%41,i%51,i%53,i*1.0,i%2,'taos'+str(i),'涛思'+str(i))) + if i % 100 == 0 : + ret = cl.execute( + "insert into %s values (%d , NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL)" % + (self.subtb_prefix+str(k)+'_'+str(j), t0+i)) + else: + ret = cl.execute( + "insert into %s values (%d , %d,%d,%d,%d,%d,%d,%d,'%s','%s',%d,%d,%d,%d)" % + (self.subtb_prefix+str(k)+'_'+str(j), t0+i, i%100, i/2.0, i%41, i%51, i%53, i*1.0, i%2,'taos'+str(i),'涛思'+str(i), i%43, i%23 , i%17 , i%3167)) cl.close() conn.close() From 767c74a69484d282eee3d4722837fce5de6cbfec Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Thu, 4 Mar 2021 10:09:06 +0800 Subject: [PATCH 25/66] fix bug --- src/client/src/tscUtil.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index cfa73b969d..ebf6e5127c 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -2714,7 +2714,11 @@ STableMeta* createSuperTableMeta(STableMetaMsg* pChild) { uint32_t tscGetTableMetaSize(STableMeta* pTableMeta) { assert(pTableMeta != NULL); - int32_t totalCols = pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; + int32_t totalCols = 0; + if (pTableMeta->tableInfo.numOfColumns >= 0 && pTableMeta->tableInfo.numOfTags >= 0) { + totalCols = pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; + } + return sizeof(STableMeta) + totalCols * sizeof(SSchema); } From 15418181ad1043888d590b55fcd670460a2bd07a Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 4 Mar 2021 11:02:49 +0800 Subject: [PATCH 26/66] [TD-3147] : support insert interval instead of insert rate. cleanup --- src/kit/taosdemo/taosdemo.c | 629 +++++++++++++----------------------- 1 file changed, 227 insertions(+), 402 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index a9a5ec0898..28a41b7925 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -239,6 +239,7 @@ typedef struct SArguments_S { int len_of_binary; int num_of_CPR; int num_of_threads; + int insert_interval; int num_of_RPR; int num_of_tables; int num_of_DPT; @@ -458,409 +459,199 @@ typedef struct SThreadInfo_S { } threadInfo; -#if 0 -#ifdef LINUX - /* The options we understand. */ - static struct argp_option options[] = { - {0, 'f', "meta file", 0, "The meta data to the execution procedure, if use -f, all others options invalid. Default is NULL.", 0}, - #ifdef _TD_POWER_ - {0, 'c', "config_directory", 0, "Configuration directory. Default is '/etc/power/'.", 1}, - {0, 'P', "password", 0, "The password to use when connecting to the server. Default is 'powerdb'.", 2}, - #else - {0, 'c', "config_directory", 0, "Configuration directory. Default is '/etc/taos/'.", 1}, - {0, 'P', "password", 0, "The password to use when connecting to the server. Default is 'taosdata'.", 2}, - #endif - {0, 'h', "host", 0, "The host to connect to TDengine. Default is localhost.", 2}, - {0, 'p', "port", 0, "The TCP/IP port number to use for the connection. Default is 0.", 2}, - {0, 'u', "user", 0, "The TDengine user name to use when connecting to the server. Default is 'root'.", 2}, - {0, 'd', "database", 0, "Destination database. Default is 'test'.", 3}, - {0, 'a', "replica", 0, "Set the replica parameters of the database, Default 1, min: 1, max: 3.", 4}, - {0, 'm', "table_prefix", 0, "Table prefix name. Default is 't'.", 4}, - {0, 's', "sql file", 0, "The select sql file.", 6}, - {0, 'M', 0, 0, "Use metric flag.", 4}, - {0, 'o', "outputfile", 0, "Direct output to the named file. Default is './output.txt'.", 6}, - {0, 'q', "query_mode", 0, "Query mode--0: SYNC, 1: ASYNC. Default is SYNC.", 4}, - {0, 'b', "type_of_cols", 0, "The data_type of columns, default: TINYINT,SMALLINT,INT,BIGINT,FLOAT,DOUBLE,BINARY,NCHAR,BOOL,TIMESTAMP.", 4}, - {0, 'w', "length_of_chartype", 0, "The length of data_type 'BINARY' or 'NCHAR'. Default is 16", 4}, - {0, 'l', "num_of_cols_per_record", 0, "The number of columns per record. Default is 10.", 4}, - {0, 'T', "num_of_threads", 0, "The number of threads. Default is 10.", 4}, - // {0, 'r', "num_of_records_per_req", 0, "The number of records per request. Default is 100.", 4}, - {0, 't', "num_of_tables", 0, "The number of tables. Default is 10000.", 4}, - {0, 'n', "num_of_records_per_table", 0, "The number of records per table. Default is 10000.", 4}, - {0, 'x', 0, 0, "Not insert only flag.", 4}, - {0, 'y', 0, 0, "Default input yes for prompt.", 4}, - {0, 'O', "disorderRatio", 0, "Insert mode--0: In order, > 0: disorder ratio. Default is in order.", 4}, - {0, 'R', "disorderRang", 0, "Out of order data's range, ms, default is 1000.", 4}, - //{0, 'D', "delete database", 0, "if elete database if exists. 0: no, 1: yes, default is 1", 5}, - {0}}; +void printHelp() { + char indent[10] = " "; + printf("%s%s%s%s\n", indent, "-f", indent, + "The meta file to the execution procedure. Default is './meta.json'."); + printf("%s%s%s%s\n", indent, "-u", indent, + "The TDengine user name to use when connecting to the server. Default is 'root'."); +#ifdef _TD_POWER_ + printf("%s%s%s%s\n", indent, "-P", indent, + "The password to use when connecting to the server. Default is 'powerdb'."); + printf("%s%s%s%s\n", indent, "-c", indent, + "Configuration directory. Default is '/etc/power/'."); +#else + printf("%s%s%s%s\n", indent, "-P", indent, + "The password to use when connecting to the server. Default is 'taosdata'."); + printf("%s%s%s%s\n", indent, "-c", indent, + "Configuration directory. Default is '/etc/taos/'."); +#endif + printf("%s%s%s%s\n", indent, "-h", indent, + "The host to connect to TDengine. Default is localhost."); + printf("%s%s%s%s\n", indent, "-p", indent, + "The TCP/IP port number to use for the connection. Default is 0."); + printf("%s%s%s%s\n", indent, "-d", indent, + "Destination database. Default is 'test'."); + printf("%s%s%s%s\n", indent, "-a", indent, + "Set the replica parameters of the database, Default 1, min: 1, max: 3."); + printf("%s%s%s%s\n", indent, "-m", indent, + "Table prefix name. Default is 't'."); + printf("%s%s%s%s\n", indent, "-s", indent, "The select sql file."); + printf("%s%s%s%s\n", indent, "-M", indent, "Use metric flag."); + printf("%s%s%s%s\n", indent, "-o", indent, + "Direct output to the named file. Default is './output.txt'."); + printf("%s%s%s%s\n", indent, "-q", indent, + "Query mode--0: SYNC, 1: ASYNC. Default is SYNC."); + printf("%s%s%s%s\n", indent, "-b", indent, + "The data_type of columns, default: TINYINT,SMALLINT,INT,BIGINT,FLOAT,DOUBLE,BINARY,NCHAR,BOOL,TIMESTAMP."); + printf("%s%s%s%s\n", indent, "-w", indent, + "The length of data_type 'BINARY' or 'NCHAR'. Default is 16"); + printf("%s%s%s%s\n", indent, "-l", indent, + "The number of columns per record. Default is 10."); + printf("%s%s%s%s\n", indent, "-T", indent, + "The number of threads. Default is 10."); + printf("%s%s%s%s\n", indent, "-i", indent, + "The sleep time (ms) between insertion. Default is 0."); + printf("%s%s%s%s\n", indent, "-r", indent, + "The number of records per request. Default is 100."); + printf("%s%s%s%s\n", indent, "-t", indent, + "The number of tables. Default is 10000."); + printf("%s%s%s%s\n", indent, "-n", indent, + "The number of records per table. Default is 10000."); + printf("%s%s%s%s\n", indent, "-x", indent, "Not insert only flag."); + printf("%s%s%s%s\n", indent, "-y", indent, "Default input yes for prompt."); + printf("%s%s%s%s\n", indent, "-O", indent, + "Insert mode--0: In order, > 0: disorder ratio. Default is in order."); + printf("%s%s%s%s\n", indent, "-R", indent, + "Out of order data's range, ms, default is 1000."); +/* printf("%s%s%s%s\n", indent, "-D", indent, + "if elete database if exists. 0: no, 1: yes, default is 1"); + */ +} -/* Parse a single option. */ -static error_t parse_opt(int key, char *arg, struct argp_state *state) { - // Get the input argument from argp_parse, which we know is a pointer to our arguments structure. - SArguments *arguments = state->input; - wordexp_t full_path; +void parse_args(int argc, char *argv[], SArguments *arguments) { char **sptr; - switch (key) { - case 'f': - arguments->metaFile = arg; - break; - case 'h': - arguments->host = arg; - break; - case 'p': - arguments->port = atoi(arg); - break; - case 'u': - arguments->user = arg; - break; - case 'P': - arguments->password = arg; - break; - case 'o': - arguments->output_file = arg; - break; - case 's': - arguments->sqlFile = arg; - break; - case 'q': - arguments->mode = atoi(arg); - break; - case 'T': - arguments->num_of_threads = atoi(arg); - break; - //case 'r': - // arguments->num_of_RPR = atoi(arg); - // break; - case 't': - arguments->num_of_tables = atoi(arg); - break; - case 'n': - arguments->num_of_DPT = atoi(arg); - break; - case 'd': - arguments->database = arg; - break; - case 'l': - arguments->num_of_CPR = atoi(arg); - break; - case 'b': + wordexp_t full_path; + + for (int i = 1; i < argc; i++) { + if (strcmp(argv[i], "-f") == 0) { + arguments->metaFile = argv[++i]; + } else if (strcmp(argv[i], "-c") == 0) { + char *configPath = argv[++i]; + if (wordexp(configPath, &full_path, 0) != 0) { + fprintf(stderr, "Invalid path %s\n", configPath); + return; + } + taos_options(TSDB_OPTION_CONFIGDIR, full_path.we_wordv[0]); + wordfree(&full_path); + } else if (strcmp(argv[i], "-h") == 0) { + arguments->host = argv[++i]; + } else if (strcmp(argv[i], "-p") == 0) { + arguments->port = atoi(argv[++i]); + } else if (strcmp(argv[i], "-u") == 0) { + arguments->user = argv[++i]; + } else if (strcmp(argv[i], "-P") == 0) { + arguments->password = argv[++i]; + } else if (strcmp(argv[i], "-o") == 0) { + arguments->output_file = argv[++i]; + } else if (strcmp(argv[i], "-s") == 0) { + arguments->sqlFile = argv[++i]; + } else if (strcmp(argv[i], "-q") == 0) { + arguments->mode = atoi(argv[++i]); + } else if (strcmp(argv[i], "-T") == 0) { + arguments->num_of_threads = atoi(argv[++i]); + } else if (strcmp(argv[i], "-i") == 0) { + arguments->insert_interval = atoi(argv[++i]); + } else if (strcmp(argv[i], "-r") == 0) { + arguments->num_of_RPR = atoi(argv[++i]); + } else if (strcmp(argv[i], "-t") == 0) { + arguments->num_of_tables = atoi(argv[++i]); + } else if (strcmp(argv[i], "-n") == 0) { + arguments->num_of_DPT = atoi(argv[++i]); + } else if (strcmp(argv[i], "-d") == 0) { + arguments->database = argv[++i]; + } else if (strcmp(argv[i], "-l") == 0) { + arguments->num_of_CPR = atoi(argv[++i]); + } else if (strcmp(argv[i], "-b") == 0) { sptr = arguments->datatype; - if (strstr(arg, ",") == NULL) { - if (strcasecmp(arg, "INT") != 0 && strcasecmp(arg, "FLOAT") != 0 && - strcasecmp(arg, "TINYINT") != 0 && strcasecmp(arg, "BOOL") != 0 && - strcasecmp(arg, "SMALLINT") != 0 && strcasecmp(arg, "TIMESTAMP") != 0 && - strcasecmp(arg, "BIGINT") != 0 && strcasecmp(arg, "DOUBLE") != 0 && - strcasecmp(arg, "BINARY") != 0 && strcasecmp(arg, "NCHAR") != 0) { - argp_error(state, "Invalid data_type!"); + ++i; + if (strstr(argv[i], ",") == NULL) { + if (strcasecmp(argv[i], "INT") + && strcasecmp(argv[i], "FLOAT") + && strcasecmp(argv[i], "TINYINT") + && strcasecmp(argv[i], "BOOL") + && strcasecmp(argv[i], "SMALLINT") + && strcasecmp(argv[i], "BIGINT") + && strcasecmp(argv[i], "DOUBLE") + && strcasecmp(argv[i], "BINARY") + && strcasecmp(argv[i], "NCHAR")) { + fprintf(stderr, "Invalid data_type!\n"); + printHelp(); + exit(EXIT_FAILURE); } - sptr[0] = arg; + sptr[0] = argv[i]; } else { int index = 0; - char *dupstr = strdup(arg); + char *dupstr = strdup(argv[i]); char *running = dupstr; char *token = strsep(&running, ","); while (token != NULL) { - if (strcasecmp(token, "INT") != 0 && strcasecmp(token, "FLOAT") != 0 && - strcasecmp(token, "TINYINT") != 0 && strcasecmp(token, "BOOL") != 0 && - strcasecmp(token, "SMALLINT") != 0 && strcasecmp(token, "TIMESTAMP") != 0 && - strcasecmp(token, "BIGINT") != 0 && strcasecmp(token, "DOUBLE") != 0 && - strcasecmp(token, "BINARY") != 0 && strcasecmp(token, "NCHAR") != 0) { - argp_error(state, "Invalid data_type!"); + if (strcasecmp(token, "INT") + && strcasecmp(token, "FLOAT") + && strcasecmp(token, "TINYINT") + && strcasecmp(token, "BOOL") + && strcasecmp(token, "SMALLINT") + && strcasecmp(token, "BIGINT") + && strcasecmp(token, "DOUBLE") + && strcasecmp(token, "BINARY") + && strcasecmp(token, "NCHAR")) { + fprintf(stderr, "Invalid data_type!\n"); + printHelp(); + exit(EXIT_FAILURE); } sptr[index++] = token; token = strsep(&running, ","); if (index >= MAX_NUM_DATATYPE) break; } } - break; - case 'w': - arguments->len_of_binary = atoi(arg); - break; - case 'm': - arguments->tb_prefix = arg; - break; - case 'M': + } else if (strcmp(argv[i], "-w") == 0) { + arguments->len_of_binary = atoi(argv[++i]); + } else if (strcmp(argv[i], "-m") == 0) { + arguments->tb_prefix = argv[++i]; + } else if (strcmp(argv[i], "-M") == 0) { arguments->use_metric = true; - break; - case 'x': + } else if (strcmp(argv[i], "-x") == 0) { arguments->insert_only = true; - break; - - case 'y': + } else if (strcmp(argv[i], "-y") == 0) { arguments->answer_yes = true; - break; - case 'c': - if (wordexp(arg, &full_path, 0) != 0) { - fprintf(stderr, "Invalid path %s\n", arg); - return -1; + } else if (strcmp(argv[i], "-c") == 0) { + strcpy(configDir, argv[++i]); + } else if (strcmp(argv[i], "-O") == 0) { + arguments->disorderRatio = atoi(argv[++i]); + if (arguments->disorderRatio > 1 + || arguments->disorderRatio < 0) { + arguments->disorderRatio = 0; + } else if (arguments->disorderRatio == 1) { + arguments->disorderRange = 10; } - taos_options(TSDB_OPTION_CONFIGDIR, full_path.we_wordv[0]); - wordfree(&full_path); - break; - case 'O': - arguments->disorderRatio = atoi(arg); - if (arguments->disorderRatio < 0 || arguments->disorderRatio > 100) - { - argp_error(state, "Invalid disorder ratio, should 1 ~ 100!"); + } else if (strcmp(argv[i], "-R") == 0) { + arguments->disorderRange = atoi(argv[++i]); + if (arguments->disorderRange == 1 + && (arguments->disorderRange > 50 + || arguments->disorderRange <= 0)) { + arguments->disorderRange = 10; } - break; - case 'R': - arguments->disorderRange = atoi(arg); - break; - case 'a': - arguments->replica = atoi(arg); - if (arguments->replica > 3 || arguments->replica < 1) - { - arguments->replica = 1; + } else if (strcmp(argv[i], "-a") == 0) { + arguments->replica = atoi(argv[++i]); + if (arguments->replica > 3 || arguments->replica < 1) { + arguments->replica = 1; } - break; - //case 'D': - // arguments->method_of_delete = atoi(arg); - // break; - case OPT_ABORT: - arguments->abort = 1; - break; - case ARGP_KEY_ARG: - /*arguments->arg_list = &state->argv[state->next-1]; - state->next = state->argc;*/ - argp_usage(state); - break; - - default: - return ARGP_ERR_UNKNOWN; - } - return 0; -} - -static struct argp argp = {options, parse_opt, 0, 0}; - -void parse_args(int argc, char *argv[], SArguments *arguments) { - argp_parse(&argp, argc, argv, 0, 0, arguments); - if (arguments->abort) { - #ifndef _ALPINE - error(10, 0, "ABORTED"); - #else - abort(); - #endif - } -} - -#else -#endif -#endif - void printHelp() { - char indent[10] = " "; - printf("%s%s\n", indent, "-f"); - printf("%s%s%s\n", indent, indent, - "The meta file to the execution procedure. Default is './meta.json'."); -#ifdef _TD_POWER_ - printf("%s%s\n", indent, "-c"); - printf("%s%s%s\n", indent, indent, - "Configuration directory. Default is '/etc/power/'."); - printf("%s%s\n", indent, "-P"); - printf("%s%s%s\n", indent, indent, - "The password to use when connecting to the server. Default is 'powerdb'."); -#else - printf("%s%s\n", indent, "-c"); - printf("%s%s%s\n", indent, indent, - "Configuration directory. Default is '/etc/taos/'."); - printf("%s%s\n", indent, "-P"); - printf("%s%s%s\n", indent, indent, - "The password to use when connecting to the server. Default is 'taosdata'."); -#endif - printf("%s%s\n", indent, "-h"); - printf("%s%s%s\n", indent, indent, - "The host to connect to TDengine. Default is localhost."); - printf("%s%s\n", indent, "-p"); - printf("%s%s%s\n", indent, indent, - "The TCP/IP port number to use for the connection. Default is 0."); - printf("%s%s\n", indent, "-u"); - printf("%s%s%s\n", indent, indent, - "The TDengine user name to use when connecting to the server. Default is 'root'."); - printf("%s%s\n", indent, "-d"); - printf("%s%s%s\n", indent, indent, - "Destination database. Default is 'test'."); - printf("%s%s\n", indent, "-a"); - printf("%s%s%s\n", indent, indent, - "Set the replica parameters of the database, Default 1, min: 1, max: 3."); - printf("%s%s\n", indent, "-m"); - printf("%s%s%s\n", indent, indent, - "Table prefix name. Default is 't'."); - printf("%s%s\n", indent, "-s"); - printf("%s%s%s\n", indent, indent, - "The select sql file."); - printf("%s%s\n", indent, "-M"); - printf("%s%s%s\n", indent, indent, - "Use metric flag."); - printf("%s%s\n", indent, "-o"); - printf("%s%s%s\n", indent, indent, - "Direct output to the named file. Default is './output.txt'."); - printf("%s%s\n", indent, "-q"); - printf("%s%s%s\n", indent, indent, - "Query mode--0: SYNC, 1: ASYNC. Default is SYNC."); - printf("%s%s\n", indent, "-b"); - printf("%s%s%s\n", indent, indent, - "The data_type of columns, default: TINYINT,SMALLINT,INT,BIGINT,FLOAT,DOUBLE,BINARY,NCHAR,BOOL,TIMESTAMP."); - printf("%s%s\n", indent, "-w"); - printf("%s%s%s\n", indent, indent, - "The length of data_type 'BINARY' or 'NCHAR'. Default is 16"); - printf("%s%s\n", indent, "-l"); - printf("%s%s%s\n", indent, indent, - "The number of columns per record. Default is 10."); - printf("%s%s\n", indent, "-T"); - printf("%s%s%s\n", indent, indent, - "The number of threads. Default is 10."); - printf("%s%s\n", indent, "-r"); - printf("%s%s%s\n", indent, indent, - "The number of records per request. Default is 100."); - printf("%s%s\n", indent, "-t"); - printf("%s%s%s\n", indent, indent, - "The number of tables. Default is 10000."); - printf("%s%s\n", indent, "-n"); - printf("%s%s%s\n", indent, indent, - "The number of records per table. Default is 10000."); - printf("%s%s\n", indent, "-x"); - printf("%s%s%s\n", indent, indent, - "Not insert only flag."); - printf("%s%s\n", indent, "-y"); - printf("%s%s%s\n", indent, indent, - "Default input yes for prompt."); - printf("%s%s\n", indent, "-O"); - printf("%s%s%s\n", indent, indent, - "Insert mode--0: In order, > 0: disorder ratio. Default is in order."); - printf("%s%s\n", indent, "-R"); - printf("%s%s%s\n", indent, indent, - "Out of order data's range, ms, default is 1000."); -/* printf("%s%s\n", indent, "-D"); - printf("%s%s%s\n", indent, indent, - "if elete database if exists. 0: no, 1: yes, default is 1"); - */ - } - - void parse_args(int argc, char *argv[], SArguments *arguments) { - char **sptr; - wordexp_t full_path; - - for (int i = 1; i < argc; i++) { - if (strcmp(argv[i], "-f") == 0) { - arguments->metaFile = argv[++i]; - } else if (strcmp(argv[i], "-c") == 0) { - char *configPath = argv[++i]; - if (wordexp(configPath, &full_path, 0) != 0) { - fprintf(stderr, "Invalid path %s\n", configPath); - return; - } - taos_options(TSDB_OPTION_CONFIGDIR, full_path.we_wordv[0]); - wordfree(&full_path); - } else if (strcmp(argv[i], "-h") == 0) { - arguments->host = argv[++i]; - } else if (strcmp(argv[i], "-p") == 0) { - arguments->port = atoi(argv[++i]); - } else if (strcmp(argv[i], "-u") == 0) { - arguments->user = argv[++i]; - } else if (strcmp(argv[i], "-P") == 0) { - arguments->password = argv[++i]; - } else if (strcmp(argv[i], "-o") == 0) { - arguments->output_file = argv[++i]; - } else if (strcmp(argv[i], "-s") == 0) { - arguments->sqlFile = argv[++i]; - } else if (strcmp(argv[i], "-q") == 0) { - arguments->mode = atoi(argv[++i]); - } else if (strcmp(argv[i], "-T") == 0) { - arguments->num_of_threads = atoi(argv[++i]); - } else if (strcmp(argv[i], "-r") == 0) { - arguments->num_of_RPR = atoi(argv[++i]); - } else if (strcmp(argv[i], "-t") == 0) { - arguments->num_of_tables = atoi(argv[++i]); - } else if (strcmp(argv[i], "-n") == 0) { - arguments->num_of_DPT = atoi(argv[++i]); - } else if (strcmp(argv[i], "-d") == 0) { - arguments->database = argv[++i]; - } else if (strcmp(argv[i], "-l") == 0) { - arguments->num_of_CPR = atoi(argv[++i]); - } else if (strcmp(argv[i], "-b") == 0) { - sptr = arguments->datatype; - ++i; - if (strstr(argv[i], ",") == NULL) { - if (strcasecmp(argv[i], "INT") != 0 && strcasecmp(argv[i], "FLOAT") != 0 && - strcasecmp(argv[i], "TINYINT") != 0 && strcasecmp(argv[i], "BOOL") != 0 && - strcasecmp(argv[i], "SMALLINT") != 0 && - strcasecmp(argv[i], "BIGINT") != 0 && strcasecmp(argv[i], "DOUBLE") != 0 && - strcasecmp(argv[i], "BINARY") && strcasecmp(argv[i], "NCHAR")) { - fprintf(stderr, "Invalid data_type!\n"); - printHelp(); - exit(EXIT_FAILURE); - } - sptr[0] = argv[i]; - } else { - int index = 0; - char *dupstr = strdup(argv[i]); - char *running = dupstr; - char *token = strsep(&running, ","); - while (token != NULL) { - if (strcasecmp(token, "INT") != 0 && - strcasecmp(token, "FLOAT") != 0 && - strcasecmp(token, "TINYINT") != 0 && - strcasecmp(token, "BOOL") != 0 && - strcasecmp(token, "SMALLINT") != 0 && - strcasecmp(token, "BIGINT") != 0 && - strcasecmp(token, "DOUBLE") != 0 && strcasecmp(token, "BINARY") && strcasecmp(token, "NCHAR")) { - fprintf(stderr, "Invalid data_type!\n"); - printHelp(); - exit(EXIT_FAILURE); - } - sptr[index++] = token; - token = strsep(&running, ","); - if (index >= MAX_NUM_DATATYPE) break; - } - } - } else if (strcmp(argv[i], "-w") == 0) { - arguments->len_of_binary = atoi(argv[++i]); - } else if (strcmp(argv[i], "-m") == 0) { - arguments->tb_prefix = argv[++i]; - } else if (strcmp(argv[i], "-M") == 0) { - arguments->use_metric = true; - } else if (strcmp(argv[i], "-x") == 0) { - arguments->insert_only = true; - } else if (strcmp(argv[i], "-y") == 0) { - arguments->answer_yes = true; - } else if (strcmp(argv[i], "-c") == 0) { - strcpy(configDir, argv[++i]); - } else if (strcmp(argv[i], "-O") == 0) { - arguments->disorderRatio = atoi(argv[++i]); - if (arguments->disorderRatio > 1 || arguments->disorderRatio < 0) { - arguments->disorderRatio = 0; - } else if (arguments->disorderRatio == 1) { - arguments->disorderRange = 10; - } - } else if (strcmp(argv[i], "-R") == 0) { - arguments->disorderRange = atoi(argv[++i]); - if (arguments->disorderRange == 1 - && (arguments->disorderRange > 50 - || arguments->disorderRange <= 0)) { - arguments->disorderRange = 10; - } - } else if (strcmp(argv[i], "-a") == 0) { - arguments->replica = atoi(argv[++i]); - if (arguments->replica > 3 || arguments->replica < 1) { - arguments->replica = 1; - } - } else if (strcmp(argv[i], "-D") == 0) { - arguments->method_of_delete = atoi(argv[++i]); - if (arguments->method_of_delete < 0 || arguments->method_of_delete > 3) { - arguments->method_of_delete = 0; - } - } else if (strcmp(argv[i], "--help") == 0) { - printHelp(); - exit(0); - } else { - fprintf(stderr, "wrong options\n"); - printHelp(); - exit(EXIT_FAILURE); + } else if (strcmp(argv[i], "-D") == 0) { + arguments->method_of_delete = atoi(argv[++i]); + if (arguments->method_of_delete < 0 + || arguments->method_of_delete > 3) { + arguments->method_of_delete = 0; } + } else if (strcmp(argv[i], "--help") == 0) { + printHelp(); + exit(0); + } else { + fprintf(stderr, "wrong options\n"); + printHelp(); + exit(EXIT_FAILURE); } } -//#endif +} static bool getInfoFromJsonFile(char* file); //static int generateOneRowDataForStb(SSuperTable* stbInfo); @@ -876,7 +667,8 @@ int32_t randint[MAX_PREPARED_RAND]; int64_t randbigint[MAX_PREPARED_RAND]; float randfloat[MAX_PREPARED_RAND]; double randdouble[MAX_PREPARED_RAND]; -char *aggreFunc[] = {"*", "count(*)", "avg(col0)", "sum(col0)", "max(col0)", "min(col0)", "first(col0)", "last(col0)"}; +char *aggreFunc[] = {"*", "count(*)", "avg(col0)", "sum(col0)", + "max(col0)", "min(col0)", "first(col0)", "last(col0)"}; SArguments g_args = {NULL, "127.0.0.1", // host @@ -911,6 +703,7 @@ SArguments g_args = {NULL, 16, // len_of_binary 10, // num_of_CPR 10, // num_of_connections/thread + 0, // insert_interval 100, // num_of_RPR 10000, // num_of_tables 10000, // num_of_DPT @@ -4952,7 +4745,9 @@ void *subSubscribeProcess(void *sarg) { if (res) { char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; if (g_queryInfo.subQueryInfo.result[i][0] != 0) { - sprintf(tmpFile, "%s-%d", g_queryInfo.subQueryInfo.result[i], winfo->threadID); + sprintf(tmpFile, "%s-%d", + g_queryInfo.subQueryInfo.result[i], + winfo->threadID); } getResult(res, tmpFile); } @@ -4961,7 +4756,8 @@ void *subSubscribeProcess(void *sarg) { taos_free_result(res); for (int i = 0; i < g_queryInfo.subQueryInfo.sqlCount; i++) { - taos_unsubscribe(g_queryInfo.subQueryInfo.tsub[i], g_queryInfo.subQueryInfo.subscribeKeepProgress); + taos_unsubscribe(g_queryInfo.subQueryInfo.tsub[i], + g_queryInfo.subQueryInfo.subscribeKeepProgress); } return NULL; } @@ -4989,9 +4785,13 @@ void *superSubscribeProcess(void *sarg) { sprintf(topic, "taosdemo-subscribe-%d", i); char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; if (g_queryInfo.subQueryInfo.result[i][0] != 0) { - sprintf(tmpFile, "%s-%d", g_queryInfo.superQueryInfo.result[i], winfo->threadID); + sprintf(tmpFile, "%s-%d", + g_queryInfo.superQueryInfo.result[i], winfo->threadID); } - g_queryInfo.superQueryInfo.tsub[i] = subscribeImpl(winfo->taos, g_queryInfo.superQueryInfo.sql[i], topic, tmpFile); + g_queryInfo.superQueryInfo.tsub[i] = + subscribeImpl(winfo->taos, + g_queryInfo.superQueryInfo.sql[i], + topic, tmpFile); if (NULL == g_queryInfo.superQueryInfo.tsub[i]) { return NULL; } @@ -5012,7 +4812,8 @@ void *superSubscribeProcess(void *sarg) { if (res) { char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; if (g_queryInfo.superQueryInfo.result[i][0] != 0) { - sprintf(tmpFile, "%s-%d", g_queryInfo.superQueryInfo.result[i], winfo->threadID); + sprintf(tmpFile, "%s-%d", + g_queryInfo.superQueryInfo.result[i], winfo->threadID); } getResult(res, tmpFile); } @@ -5021,7 +4822,8 @@ void *superSubscribeProcess(void *sarg) { taos_free_result(res); for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { - taos_unsubscribe(g_queryInfo.superQueryInfo.tsub[i], g_queryInfo.superQueryInfo.subscribeKeepProgress); + taos_unsubscribe(g_queryInfo.superQueryInfo.tsub[i], + g_queryInfo.superQueryInfo.subscribeKeepProgress); } return NULL; } @@ -5042,14 +4844,19 @@ int subscribeTestProcess() { } if (0 != g_queryInfo.subQueryInfo.sqlCount) { - (void)getAllChildNameOfSuperTable(taos, g_queryInfo.dbName, g_queryInfo.subQueryInfo.sTblName, &g_queryInfo.subQueryInfo.childTblName, &g_queryInfo.subQueryInfo.childTblCount); + (void)getAllChildNameOfSuperTable(taos, + g_queryInfo.dbName, + g_queryInfo.subQueryInfo.sTblName, + &g_queryInfo.subQueryInfo.childTblName, + &g_queryInfo.subQueryInfo.childTblCount); } pthread_t *pids = NULL; threadInfo *infos = NULL; //==== create sub threads for query from super table - if (g_queryInfo.superQueryInfo.sqlCount > 0 && g_queryInfo.superQueryInfo.concurrent > 0) { + if (g_queryInfo.superQueryInfo.sqlCount > 0 + && g_queryInfo.superQueryInfo.concurrent > 0) { pids = malloc(g_queryInfo.superQueryInfo.concurrent * sizeof(pthread_t)); infos = malloc(g_queryInfo.superQueryInfo.concurrent * sizeof(threadInfo)); if ((NULL == pids) || (NULL == infos)) { @@ -5069,9 +4876,12 @@ int subscribeTestProcess() { //==== create sub threads for query from sub table pthread_t *pidsOfSub = NULL; threadInfo *infosOfSub = NULL; - if ((g_queryInfo.subQueryInfo.sqlCount > 0) && (g_queryInfo.subQueryInfo.threadCnt > 0)) { - pidsOfSub = malloc(g_queryInfo.subQueryInfo.threadCnt * sizeof(pthread_t)); - infosOfSub = malloc(g_queryInfo.subQueryInfo.threadCnt * sizeof(threadInfo)); + if ((g_queryInfo.subQueryInfo.sqlCount > 0) + && (g_queryInfo.subQueryInfo.threadCnt > 0)) { + pidsOfSub = malloc(g_queryInfo.subQueryInfo.threadCnt * + sizeof(pthread_t)); + infosOfSub = malloc(g_queryInfo.subQueryInfo.threadCnt * + sizeof(threadInfo)); if ((NULL == pidsOfSub) || (NULL == infosOfSub)) { printf("malloc failed for create threads\n"); taos_close(taos); @@ -5170,7 +4980,6 @@ void setParaFromArg(){ g_Dbs.db[0].dbCfg.replica = g_args.replica; tstrncpy(g_Dbs.db[0].dbCfg.precision, "ms", MAX_DB_NAME_SIZE); - tstrncpy(g_Dbs.resultFile, g_args.output_file, MAX_FILE_NAME_LEN); g_Dbs.use_metric = g_args.use_metric; @@ -5190,10 +4999,12 @@ void setParaFromArg(){ g_Dbs.db[0].superTbls[0].numRecPerReq = 0; g_Dbs.db[0].superTbls[0].disorderRange = g_args.disorderRange; g_Dbs.db[0].superTbls[0].disorderRatio = g_args.disorderRatio; - tstrncpy(g_Dbs.db[0].superTbls[0].childTblPrefix, g_args.tb_prefix, MAX_TB_NAME_SIZE); + tstrncpy(g_Dbs.db[0].superTbls[0].childTblPrefix, + g_args.tb_prefix, MAX_TB_NAME_SIZE); tstrncpy(g_Dbs.db[0].superTbls[0].dataSource, "rand", MAX_TB_NAME_SIZE); tstrncpy(g_Dbs.db[0].superTbls[0].insertMode, "taosc", MAX_TB_NAME_SIZE); - tstrncpy(g_Dbs.db[0].superTbls[0].startTimestamp, "2017-07-14 10:40:00.000", MAX_TB_NAME_SIZE); + tstrncpy(g_Dbs.db[0].superTbls[0].startTimestamp, + "2017-07-14 10:40:00.000", MAX_TB_NAME_SIZE); g_Dbs.db[0].superTbls[0].timeStampStep = 10; // g_args.num_of_RPR; @@ -5207,7 +5018,9 @@ void setParaFromArg(){ memset(dataString, 0, STRING_LEN); - if (strcasecmp(data_type[0], "BINARY") == 0 || strcasecmp(data_type[0], "BOOL") == 0 || strcasecmp(data_type[0], "NCHAR") == 0 ) { + if (strcasecmp(data_type[0], "BINARY") == 0 + || strcasecmp(data_type[0], "BOOL") == 0 + || strcasecmp(data_type[0], "NCHAR") == 0 ) { g_Dbs.do_aggreFunc = false; } @@ -5217,7 +5030,8 @@ void setParaFromArg(){ break; } - tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, data_type[i], MAX_TB_NAME_SIZE); + tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, + data_type[i], MAX_TB_NAME_SIZE); g_Dbs.db[0].superTbls[0].columns[i].dataLen = g_args.len_of_binary; g_Dbs.db[0].superTbls[0].columnCount++; } @@ -5339,23 +5153,28 @@ int main(int argc, char *argv[]) { if (g_Dbs.cfgDir[0]) taos_options(TSDB_OPTION_CONFIGDIR, g_Dbs.cfgDir); (void)insertTestProcess(); } else if (QUERY_MODE == g_jsonType) { - if (g_queryInfo.cfgDir[0]) taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir); + if (g_queryInfo.cfgDir[0]) + taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir); (void)queryTestProcess(); } else if (SUBSCRIBE_MODE == g_jsonType) { - if (g_queryInfo.cfgDir[0]) taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir); + if (g_queryInfo.cfgDir[0]) + taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir); (void)subscribeTestProcess(); } else { ; } } else { - memset(&g_Dbs, 0, sizeof(SDbs)); g_jsonType = INSERT_MODE; setParaFromArg(); if (NULL != g_args.sqlFile) { TAOS* qtaos = taos_connect( - g_Dbs.host, g_Dbs.user, g_Dbs.password, g_Dbs.db[0].dbName, g_Dbs.port); + g_Dbs.host, + g_Dbs.user, + g_Dbs.password, + g_Dbs.db[0].dbName, + g_Dbs.port); querySqlFile(qtaos, g_args.sqlFile); taos_close(qtaos); return 0; @@ -5376,8 +5195,14 @@ int main(int argc, char *argv[]) { //rInfo->do_aggreFunc = g_Dbs.do_aggreFunc; //rInfo->nrecords_per_table = g_Dbs.db[0].superTbls[0].insertRows; rInfo->superTblInfo = &g_Dbs.db[0].superTbls[0]; - rInfo->taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, g_Dbs.db[0].dbName, g_Dbs.port); - strcpy(rInfo->tb_prefix, g_Dbs.db[0].superTbls[0].childTblPrefix); + rInfo->taos = taos_connect( + g_Dbs.host, + g_Dbs.user, + g_Dbs.password, + g_Dbs.db[0].dbName, + g_Dbs.port); + strcpy(rInfo->tb_prefix, + g_Dbs.db[0].superTbls[0].childTblPrefix); strcpy(rInfo->fp, g_Dbs.resultFile); if (!g_Dbs.use_metric) { From 9d875dba456aba2ee2b11c2598da2886ee7a98f9 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 4 Mar 2021 10:57:33 +0800 Subject: [PATCH 27/66] [TD-3158]: replace head_type with column_meta in http response --- src/plugins/http/inc/httpRestJson.h | 4 ++-- src/plugins/http/src/httpRestJson.c | 26 ++++++++++++++++++++++---- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/src/plugins/http/inc/httpRestJson.h b/src/plugins/http/inc/httpRestJson.h index b3b1b33e4e..5f19983826 100644 --- a/src/plugins/http/inc/httpRestJson.h +++ b/src/plugins/http/inc/httpRestJson.h @@ -34,8 +34,8 @@ #define REST_JSON_DATA_LEN 4 #define REST_JSON_HEAD "head" #define REST_JSON_HEAD_LEN 4 -#define REST_JSON_HEAD_TYPE "head_type" -#define REST_JSON_HEAD_TYPE_LEN 9 +#define REST_JSON_HEAD_INFO "column_meta" +#define REST_JSON_HEAD_INFO_LEN 11 #define REST_JSON_ROWS "rows" #define REST_JSON_ROWS_LEN 4 #define REST_JSON_AFFECT_ROWS "affected_rows" diff --git a/src/plugins/http/src/httpRestJson.c b/src/plugins/http/src/httpRestJson.c index 1172c4eef6..61a5a361c4 100644 --- a/src/plugins/http/src/httpRestJson.c +++ b/src/plugins/http/src/httpRestJson.c @@ -75,24 +75,42 @@ void restStartSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result) // head array end httpJsonToken(jsonBuf, JsonArrEnd); - // head_type begin + // column_meta begin httpJsonItemToken(jsonBuf); - httpJsonPairHead(jsonBuf, REST_JSON_HEAD_TYPE, REST_JSON_HEAD_TYPE_LEN); - // head_type array begin + httpJsonPairHead(jsonBuf, REST_JSON_HEAD_INFO, REST_JSON_HEAD_INFO_LEN); + // column_meta array begin httpJsonItemToken(jsonBuf); httpJsonToken(jsonBuf, JsonArrStt); if (num_fields == 0) { + httpJsonItemToken(jsonBuf); + httpJsonToken(jsonBuf, JsonArrStt); + + httpJsonItemToken(jsonBuf); + httpJsonString(jsonBuf, REST_JSON_AFFECT_ROWS, REST_JSON_AFFECT_ROWS_LEN); httpJsonItemToken(jsonBuf); httpJsonInt(jsonBuf, TSDB_DATA_TYPE_INT); + httpJsonItemToken(jsonBuf); + httpJsonInt(jsonBuf, 4); + + httpJsonToken(jsonBuf, JsonArrEnd); } else { for (int32_t i = 0; i < num_fields; ++i) { + httpJsonItemToken(jsonBuf); + httpJsonToken(jsonBuf, JsonArrStt); + + httpJsonItemToken(jsonBuf); + httpJsonString(jsonBuf, fields[i].name, (int32_t)strlen(fields[i].name)); httpJsonItemToken(jsonBuf); httpJsonInt(jsonBuf, fields[i].type); + httpJsonItemToken(jsonBuf); + httpJsonInt(jsonBuf, fields[i].bytes); + + httpJsonToken(jsonBuf, JsonArrEnd); } } - // head_type array end + // column_meta array end httpJsonToken(jsonBuf, JsonArrEnd); // data begin From 8359b0be240332e2c1359060b7b4ef6e4dce3877 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Thu, 4 Mar 2021 11:46:29 +0800 Subject: [PATCH 28/66] sort table --- src/tsdb/src/tsdbRead.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 486ff49f09..e0648c33a0 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -285,17 +285,24 @@ static SArray* createCheckInfoFromTableGroup(STsdbQueryHandle* pQueryHandle, STa assert(info.lastKey <= pQueryHandle->window.skey); } - taosArrayPush(pTable, &pKeyInfo->pTable); - taosArrayPush(pTableCheckInfo, &info); tsdbDebug("%p check table uid:%"PRId64", tid:%d from lastKey:%"PRId64" %p", pQueryHandle, info.tableId.uid, info.tableId.tid, info.lastKey, pQueryHandle->qinfo); } } - *psTable = pTable; - taosArraySort(pTableCheckInfo, tsdbCheckInfoCompar); + + size_t gsize = taosArrayGetSize(pTableCheckInfo); + + for (int32_t i = 0; i < gsize; ++i) { + STableCheckInfo* pInfo = (STableCheckInfo*) taosArrayGet(pTableCheckInfo, i); + + taosArrayPush(pTable, &pInfo->pTableObj); + } + + *psTable = pTable; + return pTableCheckInfo; } From c34c8164e56fcc264fa279a92951b01e78931737 Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Thu, 4 Mar 2021 13:36:17 +0800 Subject: [PATCH 29/66] [TD-2639] : reserve directory when remove server data by hand. --- documentation20/cn/10.cluster/docs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation20/cn/10.cluster/docs.md b/documentation20/cn/10.cluster/docs.md index 15ac449c1a..6d7d68fe1b 100644 --- a/documentation20/cn/10.cluster/docs.md +++ b/documentation20/cn/10.cluster/docs.md @@ -13,7 +13,7 @@ TDengine的集群管理极其简单，除添加和删除节点需要人工干预 **第零步**：规划集群所有物理节点的FQDN，将规划好的FQDN分别添加到每个物理节点的/etc/hostname；修改每个物理节点的/etc/hosts，将所有集群物理节点的IP与FQDN的对应添加好。【如部署了DNS，请联系网络管理员在DNS上做好相关配置】 **第一步**：如果搭建集群的物理节点中，存有之前的测试数据、装过1.X的版本，或者装过其他版本的TDengine，请先将其删除，并清空所有数据，具体步骤请参考博客[《TDengine多种安装包的安装和卸载》](https://www.taosdata.com/blog/2019/08/09/566.html ) -**注意1：**因为FQDN的信息会写进文件，如果之前没有配置或者更改FQDN，且启动了TDengine。请一定在确保数据无用或者备份的前提下，清理一下之前的数据（rm -rf /var/lib/taos/）； +**注意1：**因为FQDN的信息会写进文件，如果之前没有配置或者更改FQDN，且启动了TDengine。请一定在确保数据无用或者备份的前提下，清理一下之前的数据（`rm -rf /var/lib/taos/*`）； **注意2：**客户端也需要配置，确保它可以正确解析每个节点的FQDN配置，不管是通过DNS服务，还是 Host 文件。 **第二步**：建议关闭所有物理节点的防火墙，至少保证端口：6030 - 6042的TCP和UDP端口都是开放的。**强烈建议**先关闭防火墙，集群搭建完毕之后，再来配置端口； From b786f914cb6c81d9e2f95cc5a085ab4954b350fd Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 4 Mar 2021 14:24:41 +0800 Subject: [PATCH 30/66] tsim: enlarge variable size from 512 to 1024 --- tests/tsim/src/simExe.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tsim/src/simExe.c b/tests/tsim/src/simExe.c index 83ca46599c..dbda5f1bbd 100644 --- a/tests/tsim/src/simExe.c +++ b/tests/tsim/src/simExe.c @@ -146,7 +146,7 @@ char *simGetVariable(SScript *script, char *varName, int32_t varLen) { int32_t simExecuteExpression(SScript *script, char *exp) { char * op1, *op2, *var1, *var2, *var3, *rest; int32_t op1Len, op2Len, var1Len, var2Len, var3Len, val0, val1; - char t0[512], t1[512], t2[512], t3[1024]; + char t0[1024], t1[1024], t2[1024], t3[2048]; int32_t result; rest = paGetToken(exp, &var1, &var1Len); From 4d52e8cb4e406a85b76a6eabb977d452f59d398e Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 4 Mar 2021 14:26:25 +0800 Subject: [PATCH 31/66] [TD-3158]: fix http column_meta test cases --- tests/script/general/http/restful.sim | 6 +++--- tests/script/general/http/restful_full.sim | 22 +++++++++++----------- tests/script/general/http/telegraf.sim | 2 +- tests/script/unique/http/opentsdb.sim | 12 ++++++------ 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/tests/script/general/http/restful.sim b/tests/script/general/http/restful.sim index 0e9b9de132..6ebf5644e7 100644 --- a/tests/script/general/http/restful.sim +++ b/tests/script/general/http/restful.sim @@ -39,14 +39,14 @@ print =============== step3 - query data system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'select * from d1.table_rest' 127.0.0.1:7111/rest/sql print curl 127.0.0.1:7111/rest/sql -----> $system_content -if $system_content != @{"status":"succ","head":["ts","i"],"head_type":[9,4],"data":[["2017-12-25 21:28:41.022",1],["2017-12-25 21:28:42.022",2],["2017-12-25 21:28:43.022",3],["2017-12-25 21:28:44.022",4],["2017-12-25 21:28:45.022",5],["2017-12-25 21:28:46.022",6],["2017-12-25 21:28:47.022",7],["2017-12-25 21:28:48.022",8],["2017-12-25 21:28:49.022",9],["2017-12-25 21:28:50.022",10]],"rows":10}@ then +if $system_content != @{"status":"succ","head":["ts","i"],"column_meta":[["ts",9,8],["i",4,4]],"data":[["2017-12-25 21:28:41.022",1],["2017-12-25 21:28:42.022",2],["2017-12-25 21:28:43.022",3],["2017-12-25 21:28:44.022",4],["2017-12-25 21:28:45.022",5],["2017-12-25 21:28:46.022",6],["2017-12-25 21:28:47.022",7],["2017-12-25 21:28:48.022",8],["2017-12-25 21:28:49.022",9],["2017-12-25 21:28:50.022",10]],"rows":10}@ then return -1 endi print =============== step4 - insert data system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d "insert into d1.table_rest values('2017-12-25 21:28:51.022', 11)" 127.0.0.1:7111/rest/sql print curl 127.0.0.1:7111/rest/sql -----> $system_content -if $system_content != @{"status":"succ","head":["affected_rows"],"head_type":[4],"data":[[1]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["affected_rows"],"column_meta":[["affected_rows",4,4]],"data":[[1]],"rows":1}@ then return -1 endi @@ -54,7 +54,7 @@ print =============== step5 - query data system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'select * from d1.table_rest' 127.0.0.1:7111/rest/sql print curl 127.0.0.1:7111/rest/sql -----> $system_content -if $system_content != @{"status":"succ","head":["ts","i"],"head_type":[9,4],"data":[["2017-12-25 21:28:41.022",1],["2017-12-25 21:28:42.022",2],["2017-12-25 21:28:43.022",3],["2017-12-25 21:28:44.022",4],["2017-12-25 21:28:45.022",5],["2017-12-25 21:28:46.022",6],["2017-12-25 21:28:47.022",7],["2017-12-25 21:28:48.022",8],["2017-12-25 21:28:49.022",9],["2017-12-25 21:28:50.022",10],["2017-12-25 21:28:51.022",11]],"rows":11}@ then +if $system_content != @{"status":"succ","head":["ts","i"],"column_meta":[["ts",9,8],["i",4,4]],"data":[["2017-12-25 21:28:41.022",1],["2017-12-25 21:28:42.022",2],["2017-12-25 21:28:43.022",3],["2017-12-25 21:28:44.022",4],["2017-12-25 21:28:45.022",5],["2017-12-25 21:28:46.022",6],["2017-12-25 21:28:47.022",7],["2017-12-25 21:28:48.022",8],["2017-12-25 21:28:49.022",9],["2017-12-25 21:28:50.022",10],["2017-12-25 21:28:51.022",11]],"rows":11}@ then return -1 endi diff --git a/tests/script/general/http/restful_full.sim b/tests/script/general/http/restful_full.sim index 64128d02dc..8094a943f5 100644 --- a/tests/script/general/http/restful_full.sim +++ b/tests/script/general/http/restful_full.sim @@ -88,13 +88,13 @@ print =============== step2 - no db #11 system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'show databases' 127.0.0.1:7111/rest/sql print 11-> $system_content -if $system_content != @{"status":"succ","head":["name","created_time","ntables","vgroups","replica","quorum","days","keep0,keep1,keep(D)","cache(MB)","blocks","minrows","maxrows","wallevel","fsync","comp","cachelast","precision","update","status"],"head_type":[8,9,4,4,3,3,3,8,4,4,4,4,2,4,2,2,8,2,8],"data":[],"rows":0}@ then +if $system_content != @{"status":"succ","head":["name","created_time","ntables","vgroups","replica","quorum","days","keep0,keep1,keep(D)","cache(MB)","blocks","minrows","maxrows","wallevel","fsync","comp","cachelast","precision","update","status"],"column_meta":[["name",8,32],["created_time",9,8],["ntables",4,4],["vgroups",4,4],["replica",3,2],["quorum",3,2],["days",3,2],["keep0,keep1,keep(D)",8,24],["cache(MB)",4,4],["blocks",4,4],["minrows",4,4],["maxrows",4,4],["wallevel",2,1],["fsync",4,4],["comp",2,1],["cachelast",2,1],["precision",8,3],["update",2,1],["status",8,10]],"data":[],"rows":0}@ then return -1 endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'create database d1' 127.0.0.1:7111/rest/sql print 12-> $system_content -if $system_content != @{"status":"succ","head":["affected_rows"],"head_type":[4],"data":[[0]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["affected_rows"],"column_meta":[["affected_rows",4,4]],"data":[[0]],"rows":1}@ then return -1 endi @@ -160,26 +160,26 @@ endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' create table d1.t1 (ts timestamp, speed int)' 127.0.0.1:7111/rest/sql print 22-> $system_content -if $system_content != @{"status":"succ","head":["affected_rows"],"head_type":[4],"data":[[0]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["affected_rows"],"column_meta":[["affected_rows",4,4]],"data":[[0]],"rows":1}@ then return -1 endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' select * from d1.t1 ' 127.0.0.1:7111/rest/sql print 23-> $system_content -if $system_content != @{"status":"succ","head":["ts","speed"],"head_type":[9,4],"data":[],"rows":0}@ then +if $system_content != @{"status":"succ","head":["ts","speed"],"column_meta":[["ts",9,8],["speed",4,4]],"data":[],"rows":0}@ then return -1 endi #24 system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d "insert into d1.t1 values('2017-12-25 21:28:41.022', 1)" 127.0.0.1:7111/rest/sql print 24-> $system_content -if $system_content != @{"status":"succ","head":["affected_rows"],"head_type":[4],"data":[[1]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["affected_rows"],"column_meta":[["affected_rows",4,4]],"data":[[1]],"rows":1}@ then return -1 endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' select * from d1.t1 ' 127.0.0.1:7111/rest/sql print 25-> $system_content -if $system_content != @{"status":"succ","head":["ts","speed"],"head_type":[9,4],"data":[["2017-12-25 21:28:41.022",1]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["ts","speed"],"column_meta":[["ts",9,8],["speed",4,4]],"data":[["2017-12-25 21:28:41.022",1]],"rows":1}@ then return -1 endi @@ -208,32 +208,32 @@ system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl #27 system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' select * from d1.t1 ' 127.0.0.1:7111/rest/sql print 27-> $system_content -if $system_content != @{"status":"succ","head":["ts","speed"],"head_type":[9,4],"data":[["2017-12-25 21:28:41.022",1],["2017-12-25 21:28:42.022",2],["2017-12-25 21:28:43.022",3],["2017-12-25 21:28:44.022",4],["2017-12-25 21:28:45.022",5],["2017-12-25 21:28:46.022",6],["2017-12-25 21:28:47.022",7],["2017-12-25 21:28:48.022",8],["2017-12-25 21:28:49.022",9],["2017-12-25 21:28:50.022",10],["2017-12-25 21:28:51.022",11]],"rows":11}@ then +if $system_content != @{"status":"succ","head":["ts","speed"],"column_meta":[["ts",9,8],["speed",4,4]],"data":[["2017-12-25 21:28:41.022",1],["2017-12-25 21:28:42.022",2],["2017-12-25 21:28:43.022",3],["2017-12-25 21:28:44.022",4],["2017-12-25 21:28:45.022",5],["2017-12-25 21:28:46.022",6],["2017-12-25 21:28:47.022",7],["2017-12-25 21:28:48.022",8],["2017-12-25 21:28:49.022",9],["2017-12-25 21:28:50.022",10],["2017-12-25 21:28:51.022",11]],"rows":11}@ then return -1 endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'create database d2' 127.0.0.1:7111/rest/sql print 28-> $system_content -if $system_content != @{"status":"succ","head":["affected_rows"],"head_type":[4],"data":[[0]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["affected_rows"],"column_meta":[["affected_rows",4,4]],"data":[[0]],"rows":1}@ then return -1 endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' create table d2.t1 (ts timestamp, speed int)' 127.0.0.1:7111/rest/sql print 29-> $system_content -if $system_content != @{"status":"succ","head":["affected_rows"],"head_type":[4],"data":[[0]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["affected_rows"],"column_meta":[["affected_rows",4,4]],"data":[[0]],"rows":1}@ then return -1 endi #30 system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d "insert into d2.t1 values('2017-12-25 21:28:41.022', 1)" 127.0.0.1:7111/rest/sql print 30-> $system_content -if $system_content != @{"status":"succ","head":["affected_rows"],"head_type":[4],"data":[[1]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["affected_rows"],"column_meta":[["affected_rows",4,4]],"data":[[1]],"rows":1}@ then return -1 endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' select * from d2.t1 ' 127.0.0.1:7111/rest/sql print 31-> $system_content -if $system_content != @{"status":"succ","head":["ts","speed"],"head_type":[9,4],"data":[["2017-12-25 21:28:41.022",1]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["ts","speed"],"column_meta":[["ts",9,8],["speed",4,4]],"data":[["2017-12-25 21:28:41.022",1]],"rows":1}@ then return -1 endi diff --git a/tests/script/general/http/telegraf.sim b/tests/script/general/http/telegraf.sim index 03116e9f3b..f342697186 100644 --- a/tests/script/general/http/telegraf.sim +++ b/tests/script/general/http/telegraf.sim @@ -285,7 +285,7 @@ system_content curl -u root:taosdata -d 'select count(*) from db.win_cpu' 127.0 print $system_content -if $system_content != @{"status":"succ","head":["count(*)"],"head_type":[5],"data":[[3]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["count(*)"],"column_meta":[["count(*)",5,8]],"data":[[3]],"rows":1}@ then return -1 endi diff --git a/tests/script/unique/http/opentsdb.sim b/tests/script/unique/http/opentsdb.sim index aad9b87cbe..7d1e6b03d4 100644 --- a/tests/script/unique/http/opentsdb.sim +++ b/tests/script/unique/http/opentsdb.sim @@ -169,7 +169,7 @@ endi system_content curl -u root:taosdata -d 'select * from db.sys_cpu_d_bbb_lga_1_web01' 127.0.0.1:7111/rest/sql/ print $system_content -if $system_content != @{"status":"succ","head":["ts","value"],"head_type":[9,7],"data":[["2012-09-05 20:00:00.000",18.000000000]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["ts","value"],"column_meta":[["ts",9,8],["value",7,8]],"data":[["2012-09-05 20:00:00.000",18.000000000]],"rows":1}@ then return -1 endi @@ -186,7 +186,7 @@ system_content curl -u root:taosdata -d 'select * from db.sys_cpu_d_bbb_lga_1_w print $system_content -if $system_content != @{"status":"succ","head":["ts","value"],"head_type":[9,7],"data":[["2012-09-05 20:00:00.000",18.000000000],["2012-09-05 20:00:05.000",18.000000000]],"rows":2}@ then +if $system_content != @{"status":"succ","head":["ts","value"],"column_meta":[["ts",9,8],["value",7,8]],"data":[["2012-09-05 20:00:00.000",18.000000000],["2012-09-05 20:00:05.000",18.000000000]],"rows":2}@ then return -1 endi @@ -194,7 +194,7 @@ system_content curl -u root:taosdata -d 'select count(*) from db.sys_cpu_d_bbb' print $system_content -if $system_content != @{"status":"succ","head":["count(*)"],"head_type":[5],"data":[[3]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["count(*)"],"column_meta":[["count(*)",5,8]],"data":[[3]],"rows":1}@ then return -1 endi @@ -211,7 +211,7 @@ system_content curl -u root:taosdata -d 'select * from db.sys_mem_d_bbb_lga_1_w print $system_content -if $system_content != @{"status":"succ","head":["ts","value"],"head_type":[9,7],"data":[["2012-09-05 20:00:00.000",8.000000000],["2012-09-05 20:00:05.000",9.000000000]],"rows":2}@ then +if $system_content != @{"status":"succ","head":["ts","value"],"column_meta":[["ts",9,8],["value",7,8]],"data":[["2012-09-05 20:00:00.000",8.000000000],["2012-09-05 20:00:05.000",9.000000000]],"rows":2}@ then return -1 endi @@ -219,7 +219,7 @@ system_content curl -u root:taosdata -d 'select count(*) from db.sys_mem_d_bbb' print $system_content -if $system_content != @{"status":"succ","head":["count(*)"],"head_type":[5],"data":[[2]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["count(*)"],"column_meta":[["count(*)",5,8]],"data":[[2]],"rows":1}@ then return -1 endi @@ -233,7 +233,7 @@ system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": 134 system_content curl -u root:taosdata -d 'select count(*) from db.sys_cpu_d_bbb' 127.0.0.1:7111/rest/sql/ print $system_content -if $system_content != @{"status":"succ","head":["count(*)"],"head_type":[5],"data":[[7]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["count(*)"],"column_meta":[["count(*)",5,8]],"data":[[7]],"rows":1}@ then return -1 endi From fd3eefa04cc1ed833c0f9299ae7d5ea73443b731 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Thu, 4 Mar 2021 14:40:54 +0800 Subject: [PATCH 32/66] [TD-3167]: fix cannot use chinese charset in docker images --- packaging/docker/Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packaging/docker/Dockerfile b/packaging/docker/Dockerfile index e13cad4120..230741d036 100644 --- a/packaging/docker/Dockerfile +++ b/packaging/docker/Dockerfile @@ -13,9 +13,8 @@ WORKDIR /root/${dirName}/ RUN /bin/bash install.sh -e no ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib" -ENV LANG=en_US.UTF-8 -ENV LANGUAGE=en_US:en -ENV LC_ALL=en_US.UTF-8 +ENV LANG=C.UTF-8 +ENV LC_ALL=C.UTF-8 EXPOSE 6030 6031 6032 6033 6034 6035 6036 6037 6038 6039 6040 6041 6042 CMD ["taosd"] VOLUME [ "/var/lib/taos", "/var/log/taos","/etc/taos/" ] From 93a20c16bc526c48f86680e329523cc404ddc386 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Thu, 4 Mar 2021 15:13:31 +0800 Subject: [PATCH 33/66] fix bug --- src/client/inc/tscUtil.h | 2 +- src/client/src/tscSubquery.c | 2 +- src/client/src/tscUtil.c | 17 ++++++++++++----- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index e78e259eb2..502d044d75 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -270,7 +270,7 @@ void tscPrintSelectClause(SSqlObj* pSql, int32_t subClauseIndex); bool hasMoreVnodesToTry(SSqlObj *pSql); bool hasMoreClauseToTry(SSqlObj* pSql); -void tscFreeQueryInfo(SSqlCmd* pCmd); +void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeMeta); void tscTryQueryNextVnode(SSqlObj *pSql, __async_cb_func_t fp); void tscAsyncQuerySingleRowForNextVnode(void *param, TAOS_RES *tres, int numOfRows); diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 812027aa65..78e9c68290 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -2802,7 +2802,7 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows) numOfFailed += 1; // clean up tableMeta in cache - tscFreeQueryInfo(&pSql->cmd); + tscFreeQueryInfo(&pSql->cmd, false); SQueryInfo* pQueryInfo = tscGetQueryInfoDetailSafely(&pSql->cmd, 0); STableMetaInfo* pMasterTableMetaInfo = tscGetTableMetaInfoFromCmd(&pParentObj->cmd, pSql->cmd.clauseIndex, 0); tscAddTableMetaInfo(pQueryInfo, &pMasterTableMetaInfo->name, NULL, NULL, NULL, NULL); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index cfa73b969d..859cfe4782 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -30,7 +30,7 @@ #include "ttokendef.h" static void freeQueryInfoImpl(SQueryInfo* pQueryInfo); -static void clearAllTableMetaInfo(SQueryInfo* pQueryInfo); +static void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, bool removeMeta); static void tscStrToLower(char *str, int32_t n) { if (str == NULL || n <= 0) { return;} @@ -367,7 +367,7 @@ static void tscDestroyResPointerInfo(SSqlRes* pRes) { pRes->data = NULL; // pRes->data points to the buffer of pRsp, no need to free } -void tscFreeQueryInfo(SSqlCmd* pCmd) { +void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeMeta) { if (pCmd == NULL || pCmd->numOfClause == 0) { return; } @@ -376,7 +376,7 @@ void tscFreeQueryInfo(SSqlCmd* pCmd) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, i); freeQueryInfoImpl(pQueryInfo); - clearAllTableMetaInfo(pQueryInfo); + clearAllTableMetaInfo(pQueryInfo, removeMeta); tfree(pQueryInfo); } @@ -404,7 +404,7 @@ void tscResetSqlCmd(SSqlCmd* pCmd, bool removeMeta) { pCmd->pTableBlockHashList = tscDestroyBlockHashTable(pCmd->pTableBlockHashList, removeMeta); pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); - tscFreeQueryInfo(pCmd); + tscFreeQueryInfo(pCmd, removeMeta); } void tscFreeSqlResult(SSqlObj* pSql) { @@ -1847,10 +1847,17 @@ SArray* tscVgroupTableInfoDup(SArray* pVgroupTables) { return pa; } -void clearAllTableMetaInfo(SQueryInfo* pQueryInfo) { +void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, bool removeMeta) { for(int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); + if (removeMeta) { + char name[TSDB_TABLE_FNAME_LEN] = {0}; + tNameExtractFullName(&pTableMetaInfo->name, name); + + taosHashRemove(tscTableMetaInfo, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); + } + tscFreeVgroupTableInfo(pTableMetaInfo->pVgroupTables); tscClearTableMetaInfo(pTableMetaInfo); free(pTableMetaInfo); From a5b22e6510700b8953d1808bef4bff6c38741b20 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 4 Mar 2021 16:05:29 +0800 Subject: [PATCH 34/66] [TD-3110]: TSDB_MIN_WAL_LEVEL <- 0 to allow db wal no log --- src/inc/taosdef.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 9f3c31f225..f4712198ee 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -286,7 +286,7 @@ do { \ #define TSDB_MAX_COMP_LEVEL 2 #define TSDB_DEFAULT_COMP_LEVEL 2 -#define TSDB_MIN_WAL_LEVEL 1 +#define TSDB_MIN_WAL_LEVEL 0 #define TSDB_MAX_WAL_LEVEL 2 #define TSDB_DEFAULT_WAL_LEVEL 1 From 6808d72b1e02aa8174c7abeed1cf79d7eb10c832 Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Thu, 4 Mar 2021 16:07:23 +0800 Subject: [PATCH 35/66] [TD-2992] : add "between and" support in where clause. --- documentation20/cn/12.taos-sql/docs.md | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/documentation20/cn/12.taos-sql/docs.md b/documentation20/cn/12.taos-sql/docs.md index ff6adad045..aba8faa677 100644 --- a/documentation20/cn/12.taos-sql/docs.md +++ b/documentation20/cn/12.taos-sql/docs.md @@ -627,19 +627,21 @@ Query OK, 1 row(s) in set (0.001091s) ### 支持的条件过滤操作 -| Operation | Note | Applicable Data Types | -| --------- | ----------------------------- | ------------------------------------- | -| > | larger than | **`timestamp`** and all numeric types | -| < | smaller than | **`timestamp`** and all numeric types | -| >= | larger than or equal to | **`timestamp`** and all numeric types | -| <= | smaller than or equal to | **`timestamp`** and all numeric types | -| = | equal to | all types | -| <> | not equal to | all types | -| % | match with any char sequences | **`binary`** **`nchar`** | -| _ | match with a single char | **`binary`** **`nchar`** | +| Operation | Note | Applicable Data Types | +| ----------- | ----------------------------- | ------------------------------------- | +| > | larger than | **`timestamp`** and all numeric types | +| < | smaller than | **`timestamp`** and all numeric types | +| >= | larger than or equal to | **`timestamp`** and all numeric types | +| <= | smaller than or equal to | **`timestamp`** and all numeric types | +| = | equal to | all types | +| <> | not equal to | all types | +| between and | within a certain range | **`timestamp`** and all numeric types | +| % | match with any char sequences | **`binary`** **`nchar`** | +| _ | match with a single char | **`binary`** **`nchar`** | 1. 同时进行多个字段的范围过滤，需要使用关键词 AND 来连接不同的查询条件，暂不支持 OR 连接的不同列之间的查询过滤条件。 -2. 针对单一字段的过滤，如果是时间过滤条件，则一条语句中只支持设定一个；但针对其他的（普通）列或标签列，则可以使用``` OR``` 关键字进行组合条件的查询过滤。例如：((value > 20 and value < 30) OR (value < 12)) 。 +2. 针对单一字段的过滤，如果是时间过滤条件，则一条语句中只支持设定一个；但针对其他的（普通）列或标签列，则可以使用 `OR` 关键字进行组合条件的查询过滤。例如：((value > 20 AND value < 30) OR (value < 12)) 。 +3. 从 2.0.17 版本开始，条件过滤开始支持 BETWEEN AND 语法，例如 `WHERE col2 BETWEEN 1.5 AND 3.25` 表示查询条件为“1.5 ≤ col2 ≤ 3.25”。 ### SQL 示例 From e437f5c823b0202ac07711cd9be414305b8042db Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 4 Mar 2021 16:17:59 +0800 Subject: [PATCH 36/66] [TD-3147] : support insert interval. json works. --- src/kit/taosdemo/insert.json | 4 +- src/kit/taosdemo/taosdemo.c | 503 ++++++++++++++++++++--------------- 2 files changed, 295 insertions(+), 212 deletions(-) diff --git a/src/kit/taosdemo/insert.json b/src/kit/taosdemo/insert.json index ebc0cfd607..33fd587509 100644 --- a/src/kit/taosdemo/insert.json +++ b/src/kit/taosdemo/insert.json @@ -9,6 +9,8 @@ "thread_count_create_tbl": 1, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", + "insert_interval": 0, + "num_of_records_per_req": 100, "databases": [{ "dbinfo": { "name": "db", @@ -35,8 +37,6 @@ "auto_create_table": "no", "data_source": "rand", "insert_mode": "taosc", - "insert_interval": 0, - "num_of_records_per_req": 100, "insert_rows": 100000, "multi_thread_write_one_tbl": "no", "number_of_tbl_in_one_sql": 1, diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 28a41b7925..449768bff5 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -61,56 +61,6 @@ #define REQ_EXTRA_BUF_LEN 1024 #define RESP_BUF_LEN 4096 -#ifdef WINDOWS -#include -// Some old MinGW/CYGWIN distributions don't define this: -#ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING -#define ENABLE_VIRTUAL_TERMINAL_PROCESSING 0x0004 -#endif - -static HANDLE g_stdoutHandle; -static DWORD g_consoleMode; - -void setupForAnsiEscape(void) { - DWORD mode = 0; - g_stdoutHandle = GetStdHandle(STD_OUTPUT_HANDLE); - - if(g_stdoutHandle == INVALID_HANDLE_VALUE) { - exit(GetLastError()); - } - - if(!GetConsoleMode(g_stdoutHandle, &mode)) { - exit(GetLastError()); - } - - g_consoleMode = mode; - - // Enable ANSI escape codes - mode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING; - - if(!SetConsoleMode(g_stdoutHandle, mode)) { - exit(GetLastError()); - } -} - -void resetAfterAnsiEscape(void) { - // Reset colors - printf("\x1b[0m"); - - // Reset console mode - if(!SetConsoleMode(g_stdoutHandle, g_consoleMode)) { - exit(GetLastError()); - } -} -#else -void setupForAnsiEscape(void) {} - -void resetAfterAnsiEscape(void) { - // Reset colors - printf("\x1b[0m"); -} -#endif - extern char configDir[]; #define INSERT_JSON_NAME "insert.json" @@ -163,7 +113,7 @@ enum MODE { ASYNC, MODE_BUT }; - + enum QUERY_TYPE { NO_INSERT_TYPE, INSERT_TYPE, @@ -233,6 +183,7 @@ typedef struct SArguments_S { bool use_metric; bool insert_only; bool answer_yes; + bool debug_print; char * output_file; int mode; char * datatype[MAX_NUM_DATATYPE + 1]; @@ -267,8 +218,6 @@ typedef struct SSuperTable_S { char childTblPrefix[MAX_TB_NAME_SIZE]; char dataSource[MAX_TB_NAME_SIZE]; // rand_gen or sample char insertMode[MAX_TB_NAME_SIZE]; // taosc, restful - uint32_t insertInterval; // interval time between insert twice - uint32_t numRecPerReq; int multiThreadWriteOneTbl; // 0: no, 1: yes int numberOfTblInOneSql; // 0/1: one table, > 1: number of tbl @@ -374,6 +323,9 @@ typedef struct SDbs_S { int dbCount; SDataBase db[MAX_DB_COUNT]; + int insert_interval; + int num_of_RPR; + // statistics int64_t totalRowsInserted; int64_t totalAffectedRows; @@ -458,6 +410,125 @@ typedef struct SThreadInfo_S { } threadInfo; +#ifdef WINDOWS +#include +// Some old MinGW/CYGWIN distributions don't define this: +#ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING +#define ENABLE_VIRTUAL_TERMINAL_PROCESSING 0x0004 +#endif + +static HANDLE g_stdoutHandle; +static DWORD g_consoleMode; + +void setupForAnsiEscape(void) { + DWORD mode = 0; + g_stdoutHandle = GetStdHandle(STD_OUTPUT_HANDLE); + + if(g_stdoutHandle == INVALID_HANDLE_VALUE) { + exit(GetLastError()); + } + + if(!GetConsoleMode(g_stdoutHandle, &mode)) { + exit(GetLastError()); + } + + g_consoleMode = mode; + + // Enable ANSI escape codes + mode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING; + + if(!SetConsoleMode(g_stdoutHandle, mode)) { + exit(GetLastError()); + } +} + +void resetAfterAnsiEscape(void) { + // Reset colors + printf("\x1b[0m"); + + // Reset console mode + if(!SetConsoleMode(g_stdoutHandle, g_consoleMode)) { + exit(GetLastError()); + } +} +#else +void setupForAnsiEscape(void) {} + +void resetAfterAnsiEscape(void) { + // Reset colors + printf("\x1b[0m"); +} +#endif + +static int createDatabases(); +static void createChildTables(); +static int queryDbExec(TAOS *taos, char *command, int type); + +/* ************ Global variables ************ */ + +int32_t randint[MAX_PREPARED_RAND]; +int64_t randbigint[MAX_PREPARED_RAND]; +float randfloat[MAX_PREPARED_RAND]; +double randdouble[MAX_PREPARED_RAND]; +char *aggreFunc[] = {"*", "count(*)", "avg(col0)", "sum(col0)", + "max(col0)", "min(col0)", "first(col0)", "last(col0)"}; + +SArguments g_args = {NULL, + "127.0.0.1", // host + 6030, // port + "root", // user + #ifdef _TD_POWER_ + "powerdb", // password + #else + "taosdata", // password + #endif + "test", // database + 1, // replica + "t", // tb_prefix + NULL, // sqlFile + false, // use_metric + false, // insert_only + false, // debug_print + false, // answer_yes; + "./output.txt", // output_file + 0, // mode : sync or async + { + "TINYINT", // datatype + "SMALLINT", + "INT", + "BIGINT", + "FLOAT", + "DOUBLE", + "BINARY", + "NCHAR", + "BOOL", + "TIMESTAMP" + }, + 16, // len_of_binary + 10, // num_of_CPR + 10, // num_of_connections/thread + 0, // insert_interval + 100, // num_of_RPR + 10000, // num_of_tables + 10000, // num_of_DPT + 0, // abort + 0, // disorderRatio + 1000, // disorderRange + 1, // method_of_delete + NULL // arg_list +}; + + +static int g_jsonType = 0; + +static SDbs g_Dbs; +static int g_totalChildTables = 0; +static SQueryMetaInfo g_queryInfo; +static FILE * g_fpOfInsertResult = NULL; + +#define debugPrint(fmt, ...) \ + do { if (g_args.debug_print) fprintf(stderr, fmt, __VA_ARGS__); } while(0) +/////////////////////////////////////////////////// void printHelp() { char indent[10] = " "; @@ -514,6 +585,8 @@ void printHelp() { "Insert mode--0: In order, > 0: disorder ratio. Default is in order."); printf("%s%s%s%s\n", indent, "-R", indent, "Out of order data's range, ms, default is 1000."); + printf("%s%s%s%s\n", indent, "-g", indent, + "Print debug info."); /* printf("%s%s%s%s\n", indent, "-D", indent, "if elete database if exists. 0: no, 1: yes, default is 1"); */ @@ -614,6 +687,8 @@ void parse_args(int argc, char *argv[], SArguments *arguments) { arguments->insert_only = true; } else if (strcmp(argv[i], "-y") == 0) { arguments->answer_yes = true; + } else if (strcmp(argv[i], "-g") == 0) { + arguments->debug_print = true; } else if (strcmp(argv[i], "-c") == 0) { strcpy(configDir, argv[++i]); } else if (strcmp(argv[i], "-O") == 0) { @@ -651,77 +726,42 @@ void parse_args(int argc, char *argv[], SArguments *arguments) { exit(EXIT_FAILURE); } } + + if (arguments->debug_print) { + printf("###################################################################\n"); + printf("# Server IP: %s:%hu\n", + arguments->host == NULL ? "localhost" : arguments->host, + arguments->port ); + printf("# User: %s\n", arguments->user); + printf("# Password: %s\n", arguments->password); + printf("# Use metric: %s\n", arguments->use_metric ? "true" : "false"); + printf("# Insertion interval: %d\n", arguments->insert_interval); + printf("# Number of Columns per record: %d\n", arguments->num_of_RPR); + printf("# Number of Threads: %d\n", arguments->num_of_threads); + printf("# Number of Tables: %d\n", arguments->num_of_tables); + printf("# Number of Data per Table: %d\n", arguments->num_of_DPT); + printf("# Database name: %s\n", arguments->database); + printf("# Table prefix: %s\n", arguments->tb_prefix); + if (arguments->disorderRatio) { + printf("# Data order: %d\n", arguments->disorderRatio); + printf("# Data out of order rate: %d\n", arguments->disorderRange); + + } + printf("# Delete method: %d\n", arguments->method_of_delete); + printf("# Answer yes when prompt: %d\n", arguments->answer_yes); + printf("# Print debug info: %d\n", arguments->debug_print); + printf("###################################################################\n"); + if (!arguments->answer_yes) { + printf("Press enter key to continue\n\n"); + (void) getchar(); + } + } } static bool getInfoFromJsonFile(char* file); //static int generateOneRowDataForStb(SSuperTable* stbInfo); //static int getDataIntoMemForStb(SSuperTable* stbInfo); static void init_rand_data(); -static int createDatabases(); -static void createChildTables(); -static int queryDbExec(TAOS *taos, char *command, int type); - -/* ************ Global variables ************ */ - -int32_t randint[MAX_PREPARED_RAND]; -int64_t randbigint[MAX_PREPARED_RAND]; -float randfloat[MAX_PREPARED_RAND]; -double randdouble[MAX_PREPARED_RAND]; -char *aggreFunc[] = {"*", "count(*)", "avg(col0)", "sum(col0)", - "max(col0)", "min(col0)", "first(col0)", "last(col0)"}; - -SArguments g_args = {NULL, - "127.0.0.1", // host - 6030, // port - "root", // user - #ifdef _TD_POWER_ - "powerdb", // password - #else - "taosdata", // password - #endif - "test", // database - 1, // replica - "t", // tb_prefix - NULL, // sqlFile - false, // use_metric - false, // insert_only - false, // answer_yes; - "./output.txt", // output_file - 0, // mode : sync or async - { - "TINYINT", // datatype - "SMALLINT", - "INT", - "BIGINT", - "FLOAT", - "DOUBLE", - "BINARY", - "NCHAR", - "BOOL", - "TIMESTAMP" - }, - 16, // len_of_binary - 10, // num_of_CPR - 10, // num_of_connections/thread - 0, // insert_interval - 100, // num_of_RPR - 10000, // num_of_tables - 10000, // num_of_DPT - 0, // abort - 0, // disorderRatio - 1000, // disorderRange - 1, // method_of_delete - NULL // arg_list -}; - - -static int g_jsonType = 0; -static SDbs g_Dbs; -static int g_totalChildTables = 0; -static SQueryMetaInfo g_queryInfo; -static FILE * g_fpOfInsertResult = NULL; - - void tmfclose(FILE *fp) { if (NULL != fp) { fclose(fp); @@ -917,6 +957,8 @@ static int printfInsertMeta() { printf("resultFile: \033[33m%s\033[0m\n", g_Dbs.resultFile); printf("thread num of insert data: \033[33m%d\033[0m\n", g_Dbs.threadCount); printf("thread num of create table: \033[33m%d\033[0m\n", g_Dbs.threadCountByCreateTbl); + printf("insert interval: \033[33m%d\033[0m\n", g_Dbs.insert_interval); + printf("number of records per req: \033[33m%d\033[0m\n", g_Dbs.num_of_RPR); printf("database count: \033[33m%d\033[0m\n", g_Dbs.dbCount); for (int i = 0; i < g_Dbs.dbCount; i++) { @@ -999,8 +1041,6 @@ static int printfInsertMeta() { printf(" childTblPrefix: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].childTblPrefix); printf(" dataSource: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].dataSource); printf(" insertMode: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].insertMode); - printf(" insertInterval: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].insertInterval); - printf(" numRecPerReq: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].numRecPerReq); printf(" insertRows: \033[33m%"PRId64"\033[0m\n", g_Dbs.db[i].superTbls[j].insertRows); if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) { @@ -1138,8 +1178,6 @@ static void printfInsertMetaToFile(FILE* fp) { fprintf(fp, " childTblPrefix: %s\n", g_Dbs.db[i].superTbls[j].childTblPrefix); fprintf(fp, " dataSource: %s\n", g_Dbs.db[i].superTbls[j].dataSource); fprintf(fp, " insertMode: %s\n", g_Dbs.db[i].superTbls[j].insertMode); - fprintf(fp, " insertInterval: %d\n", g_Dbs.db[i].superTbls[j].insertInterval); - fprintf(fp, " numRecPerReq: %d\n", g_Dbs.db[i].superTbls[j].numRecPerReq); fprintf(fp, " insertRows: %"PRId64"\n", g_Dbs.db[i].superTbls[j].insertRows); if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) { @@ -2530,6 +2568,26 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { goto PARSE_OVER; } + cJSON* insertInterval = cJSON_GetObjectItem(root, "insert_interval"); + if (insertInterval && insertInterval->type == cJSON_Number) { + g_Dbs.insert_interval = insertInterval->valueint; + } else if (!insertInterval) { + g_Dbs.insert_interval = 0; + } else { + printf("failed to read json, insert_interval not found"); + goto PARSE_OVER; + } + + cJSON* numRecPerReq = cJSON_GetObjectItem(root, "num_of_records_per_req"); + if (numRecPerReq && numRecPerReq->type == cJSON_Number) { + g_Dbs.num_of_RPR = numRecPerReq->valueint; + } else if (!numRecPerReq) { + g_Dbs.num_of_RPR = 0; + } else { + printf("failed to read json, num_of_records_per_req not found"); + goto PARSE_OVER; + } + cJSON *answerPrompt = cJSON_GetObjectItem(root, "confirm_parameter_prompt"); // yes, no, if (answerPrompt && answerPrompt->type == cJSON_String && answerPrompt->valuestring != NULL) { if (0 == strncasecmp(answerPrompt->valuestring, "yes", 3)) { @@ -2983,26 +3041,6 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { goto PARSE_OVER; } - cJSON* insertInterval = cJSON_GetObjectItem(stbInfo, "insert_interval"); - if (insertInterval && insertInterval->type == cJSON_Number) { - g_Dbs.db[i].superTbls[j].insertInterval = insertInterval->valueint; - } else if (!insertInterval) { - g_Dbs.db[i].superTbls[j].insertInterval = 0; - } else { - printf("failed to read json, insert_interval not found"); - goto PARSE_OVER; - } - - cJSON* numRecPerReq = cJSON_GetObjectItem(stbInfo, "num_of_records_per_req"); - if (numRecPerReq && numRecPerReq->type == cJSON_Number) { - g_Dbs.db[i].superTbls[j].numRecPerReq = numRecPerReq->valueint; - } else if (!numRecPerReq) { - g_Dbs.db[i].superTbls[j].numRecPerReq = 0; - } else { - printf("failed to read json, num_of_records_per_req not found"); - goto PARSE_OVER; - } - cJSON* insertRows = cJSON_GetObjectItem(stbInfo, "insert_rows"); if (insertRows && insertRows->type == cJSON_Number) { @@ -3414,7 +3452,7 @@ static bool getInfoFromJsonFile(char* file) { } else { printf("input json file type error! please input correct file type: insert or query or subscribe\n"); goto PARSE_OVER; - } + } PARSE_OVER: free(content); @@ -3423,7 +3461,6 @@ PARSE_OVER: return ret; } - void prePareSampleData() { for (int i = 0; i < g_Dbs.dbCount; i++) { for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { @@ -3526,7 +3563,8 @@ int generateRowData(char* dataBuf, int maxLen, int64_t timestamp, SSuperTable* return dataLen; } -void syncWriteForNumberOfTblInOneSql(threadInfo *winfo, FILE *fp, char* sampleDataBuf) { +void syncWriteForNumberOfTblInOneSql( + threadInfo *winfo, FILE *fp, char* sampleDataBuf) { SSuperTable* superTblInfo = winfo->superTblInfo; int samplePos = 0; @@ -3555,11 +3593,11 @@ void syncWriteForNumberOfTblInOneSql(threadInfo *winfo, FILE *fp, char* sampleDa int64_t st = 0; int64_t et = 0; for (int i = 0; i < superTblInfo->insertRows;) { - if (superTblInfo->insertInterval && (superTblInfo->insertInterval > (et - st))) { - taosMsleep(superTblInfo->insertInterval - (et - st)); // ms + if (g_Dbs.insert_interval && (g_Dbs.insert_interval > (et - st))) { + taosMsleep(g_Dbs.insert_interval - (et - st)); // ms } - if (superTblInfo->insertInterval) { + if (g_Dbs.insert_interval) { st = taosGetTimestampMs(); } @@ -3567,8 +3605,7 @@ void syncWriteForNumberOfTblInOneSql(threadInfo *winfo, FILE *fp, char* sampleDa for (int tID = winfo->start_table_id; tID <= winfo->end_table_id; ) { int inserted = i; - for (int k = 0; k < winfo->superTblInfo->numRecPerReq;) - { + for (int k = 0; k < g_Dbs.num_of_RPR;) { int len = 0; memset(buffer, 0, superTblInfo->maxSqlLen); char *pstr = buffer; @@ -3649,7 +3686,8 @@ void syncWriteForNumberOfTblInOneSql(threadInfo *winfo, FILE *fp, char* sampleDa tmp_time = time_counter; for (k = 0; k < superTblInfo->rowsPerTbl;) { int retLen = 0; - if (0 == strncasecmp(superTblInfo->dataSource, "sample", strlen("sample"))) { + if (0 == strncasecmp(superTblInfo->dataSource, + "sample", strlen("sample"))) { retLen = getRowDataFromSample(pstr + len, superTblInfo->maxSqlLen - len, tmp_time += superTblInfo->timeStampStep, @@ -3684,7 +3722,7 @@ void syncWriteForNumberOfTblInOneSql(threadInfo *winfo, FILE *fp, char* sampleDa //inserted++; k++; totalRowsInserted++; - + if (inserted >= superTblInfo->insertRows || (superTblInfo->maxSqlLen - len) < (superTblInfo->lenOfOneRow + 128)) { tID = tbl_id + 1; @@ -3693,13 +3731,12 @@ void syncWriteForNumberOfTblInOneSql(threadInfo *winfo, FILE *fp, char* sampleDa goto send_to_server; } } - } tID = tbl_id; inserted += superTblInfo->rowsPerTbl; - send_to_server: +send_to_server: if (0 == strncasecmp(superTblInfo->insertMode, "taosc", strlen("taosc"))) { @@ -3758,7 +3795,7 @@ void syncWriteForNumberOfTblInOneSql(threadInfo *winfo, FILE *fp, char* sampleDa } } - if (superTblInfo->insertInterval) { + if (g_Dbs.insert_interval) { et = taosGetTimestampMs(); } //printf("========loop %d childTables duration:%"PRId64 "========inserted rows:%d\n", winfo->end_table_id - winfo->start_table_id, et - st, i); @@ -3843,12 +3880,12 @@ void *syncWrite(void *sarg) { uint64_t et = 0; for (int i = 0; i < superTblInfo->insertRows;) { - if (i > 0 && superTblInfo->insertInterval - && (superTblInfo->insertInterval > (et - st) )) { - taosMsleep(superTblInfo->insertInterval - (et - st)); // ms + if (i > 0 && g_Dbs.insert_interval + && (g_Dbs.insert_interval > (et - st) )) { + taosMsleep(g_Dbs.insert_interval - (et - st)); // ms } - if (superTblInfo->insertInterval) { + if (g_Dbs.insert_interval) { st = taosGetTimestampMs(); } @@ -3901,10 +3938,10 @@ void *syncWrite(void *sarg) { superTblInfo->childTblPrefix, tID); } - - for (k = 0; k < superTblInfo->numRecPerReq;) { + + for (k = 0; k < g_Dbs.num_of_RPR;) { int retLen = 0; - if (0 == strncasecmp(superTblInfo->dataSource, "sample", 6)) { + if (0 == strncasecmp(superTblInfo->dataSource, "sample", strlen("sample"))) { retLen = getRowDataFromSample( pstr + len, superTblInfo->maxSqlLen - len, @@ -3916,9 +3953,10 @@ void *syncWrite(void *sarg) { if (retLen < 0) { goto free_and_statistics_2; } - } else if (0 == strncasecmp(superTblInfo->dataSource, "rand", 8)) { + } else if (0 == strncasecmp(superTblInfo->dataSource, "rand", strlen("rand"))) { int rand_num = rand_tinyint() % 100; - if (0 != superTblInfo->disorderRatio && rand_num < superTblInfo->disorderRatio) { + if (0 != superTblInfo->disorderRatio + && rand_num < superTblInfo->disorderRatio) { int64_t d = tmp_time - rand() % superTblInfo->disorderRange; retLen = generateRowData( pstr + len, @@ -3946,7 +3984,7 @@ void *syncWrite(void *sarg) { break; } - if (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5)) { + if (0 == strncasecmp(superTblInfo->insertMode, "taosc", strlen("taosc"))) { //printf("===== sql: %s \n\n", buffer); //int64_t t1 = taosGetTimestampMs(); int64_t startTs; @@ -3975,7 +4013,7 @@ void *syncWrite(void *sarg) { totalAffectedRows); lastPrintTime = currentPrintTime; } - //int64_t t2 = taosGetTimestampMs(); + //int64_t t2 = taosGetTimestampMs(); //printf("taosc insert sql return, Spent %.4f seconds \n", (double)(t2 - t1)/1000.0); } else { //int64_t t1 = taosGetTimestampMs(); @@ -3992,15 +4030,15 @@ void *syncWrite(void *sarg) { if (tID == winfo->end_table_id) { if (0 == strncasecmp( - superTblInfo->dataSource, "sample", 6)) { + superTblInfo->dataSource, "sample", strlen("sample"))) { samplePos = sampleUsePos; } i = inserted; time_counter = tmp_time; } - } - - if (superTblInfo->insertInterval) { + } + + if (g_Dbs.insert_interval) { et = taosGetTimestampMs(); } //printf("========loop %d childTables duration:%"PRId64 "========inserted rows:%d\n", winfo->end_table_id - winfo->start_table_id, et - st, i); @@ -4024,7 +4062,7 @@ free_and_statistics_2: void callBack(void *param, TAOS_RES *res, int code) { threadInfo* winfo = (threadInfo*)param; - if (winfo->superTblInfo->insertInterval) { + if (g_Dbs.insert_interval) { winfo->et = taosGetTimestampMs(); if (winfo->et - winfo->st < 1000) { taosMsleep(1000 - (winfo->et - winfo->st)); // ms @@ -4047,7 +4085,7 @@ void callBack(void *param, TAOS_RES *res, int code) { return; } - for (int i = 0; i < winfo->superTblInfo->numRecPerReq; i++) { + for (int i = 0; i < g_Dbs.num_of_RPR; i++) { int rand_num = rand() % 100; if (0 != winfo->superTblInfo->disorderRatio && rand_num < winfo->superTblInfo->disorderRatio) { @@ -4066,7 +4104,7 @@ void callBack(void *param, TAOS_RES *res, int code) { } } - if (winfo->superTblInfo->insertInterval) { + if (g_Dbs.insert_interval) { winfo->st = taosGetTimestampMs(); } taos_query_a(winfo->taos, buffer, callBack, winfo); @@ -4083,7 +4121,7 @@ void *asyncWrite(void *sarg) { winfo->et = 0; winfo->lastTs = winfo->start_time; - if (winfo->superTblInfo->insertInterval) { + if (g_Dbs.insert_interval) { winfo->st = taosGetTimestampMs(); } taos_query_a(winfo->taos, "show databases", callBack, winfo); @@ -4136,7 +4174,11 @@ void startMultiThreadInsertData(int threads, char* db_name, char* precision, SSu if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) { start_time = taosGetTimestamp(timePrec); } else { - (void)taosParseTime(superTblInfo->startTimestamp, &start_time, strlen(superTblInfo->startTimestamp), timePrec, 0); + (void)taosParseTime( + superTblInfo->startTimestamp, + &start_time, + strlen(superTblInfo->startTimestamp), + timePrec, 0); } double start = getCurrentTime(); @@ -4153,7 +4195,9 @@ void startMultiThreadInsertData(int threads, char* db_name, char* precision, SSu if (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5)) { //t_info->taos = taos; - t_info->taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, db_name, g_Dbs.port); + t_info->taos = taos_connect( + g_Dbs.host, g_Dbs.user, + g_Dbs.password, db_name, g_Dbs.port); if (NULL == t_info->taos) { printf("connect to server fail from insert sub thread, reason: %s\n", taos_errstr(NULL)); exit(-1); @@ -4173,7 +4217,6 @@ void startMultiThreadInsertData(int threads, char* db_name, char* precision, SSu } tsem_init(&(t_info->lock_sem), 0, 0); - if (SYNC == g_Dbs.queryMode) { pthread_create(pids + i, NULL, syncWrite, t_info); } else { @@ -4217,7 +4260,6 @@ void startMultiThreadInsertData(int threads, char* db_name, char* precision, SSu fprintf(g_fpOfInsertResult, "Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s) into %s.%s. %2.f records/second\n\n", t, superTblInfo->totalRowsInserted, superTblInfo->totalAffectedRows, threads, db_name, superTblInfo->sTblName, superTblInfo->totalRowsInserted / t); - printf("insert delay, avg: %10.6fms, max: %10.6fms, min: %10.6fms\n\n", avgDelay/1000.0, (double)maxDelay/1000.0, (double)minDelay/1000.0); fprintf(g_fpOfInsertResult, "insert delay, avg:%10.6fms, max: %10.6fms, min: %10.6fms\n\n", @@ -4406,10 +4448,13 @@ int insertTestProcess() { createChildTables(); end = getCurrentTime(); if (g_totalChildTables > 0) { - printf("Spent %.4f seconds to create %d tables with %d thread(s)\n\n", end - start, g_totalChildTables, g_Dbs.threadCount); - fprintf(g_fpOfInsertResult, "Spent %.4f seconds to create %d tables with %d thread(s)\n\n", end - start, g_totalChildTables, g_Dbs.threadCount); + printf("Spent %.4f seconds to create %d tables with %d thread(s)\n\n", + end - start, g_totalChildTables, g_Dbs.threadCount); + fprintf(g_fpOfInsertResult, + "Spent %.4f seconds to create %d tables with %d thread(s)\n\n", + end - start, g_totalChildTables, g_Dbs.threadCount); } - + taosMsleep(1000); // create sub threads for inserting data @@ -4420,11 +4465,15 @@ int insertTestProcess() { if (0 == g_Dbs.db[i].superTbls[j].insertRows) { continue; } - startMultiThreadInsertData(g_Dbs.threadCount, g_Dbs.db[i].dbName, g_Dbs.db[i].dbCfg.precision, superTblInfo); + startMultiThreadInsertData( + g_Dbs.threadCount, + g_Dbs.db[i].dbName, + g_Dbs.db[i].dbCfg.precision, + superTblInfo); } } //end = getCurrentTime(); - + //int64_t totalRowsInserted = 0; //int64_t totalAffectedRows = 0; //for (int i = 0; i < g_Dbs.dbCount; i++) { @@ -4443,7 +4492,12 @@ int insertTestProcess() { //rInfo->do_aggreFunc = g_Dbs.do_aggreFunc; //rInfo->nrecords_per_table = g_Dbs.db[0].superTbls[0].insertRows; rInfo->superTblInfo = &g_Dbs.db[0].superTbls[0]; - rInfo->taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, g_Dbs.db[0].dbName, g_Dbs.port); + rInfo->taos = taos_connect( + g_Dbs.host, + g_Dbs.user, + g_Dbs.password, + g_Dbs.db[0].dbName, + g_Dbs.port); strcpy(rInfo->tb_prefix, g_Dbs.db[0].superTbls[0].childTblPrefix); strcpy(rInfo->fp, g_Dbs.resultFile); @@ -4486,12 +4540,15 @@ void *superQueryProcess(void *sarg) { } selectAndGetResult(winfo->taos, g_queryInfo.superQueryInfo.sql[i], tmpFile); int64_t t2 = taosGetTimestampUs(); - printf("=[taosc] thread[%"PRId64"] complete one sql, Spent %f s\n", taosGetSelfPthreadId(), (t2 - t1)/1000000.0); + printf("=[taosc] thread[%"PRId64"] complete one sql, Spent %f s\n", + taosGetSelfPthreadId(), (t2 - t1)/1000000.0); } else { int64_t t1 = taosGetTimestampUs(); - int retCode = postProceSql(g_queryInfo.host, g_queryInfo.port, g_queryInfo.superQueryInfo.sql[i]); + int retCode = postProceSql(g_queryInfo.host, + g_queryInfo.port, g_queryInfo.superQueryInfo.sql[i]); int64_t t2 = taosGetTimestampUs(); - printf("=[restful] thread[%"PRId64"] complete one sql, Spent %f s\n", taosGetSelfPthreadId(), (t2 - t1)/1000000.0); + printf("=[restful] thread[%"PRId64"] complete one sql, Spent %f s\n", + taosGetSelfPthreadId(), (t2 - t1)/1000000.0); if (0 != retCode) { printf("====restful return fail, threadID[%d]\n", winfo->threadID); @@ -4500,7 +4557,8 @@ void *superQueryProcess(void *sarg) { } } et = taosGetTimestampMs(); - printf("==thread[%"PRId64"] complete all sqls to specify tables once queries duration:%.6fs\n\n", taosGetSelfPthreadId(), (double)(et - st)/1000.0); + printf("==thread[%"PRId64"] complete all sqls to specify tables once queries duration:%.6fs\n\n", + taosGetSelfPthreadId(), (double)(et - st)/1000.0); } return NULL; } @@ -4508,7 +4566,9 @@ void *superQueryProcess(void *sarg) { void replaceSubTblName(char* inSql, char* outSql, int tblIndex) { char sourceString[32] = "xxxx"; char subTblName[MAX_TB_NAME_SIZE*3]; - sprintf(subTblName, "%s.%s", g_queryInfo.dbName, g_queryInfo.subQueryInfo.childTblName + tblIndex*TSDB_TABLE_NAME_LEN); + sprintf(subTblName, "%s.%s", + g_queryInfo.dbName, + g_queryInfo.subQueryInfo.childTblName + tblIndex*TSDB_TABLE_NAME_LEN); //printf("inSql: %s\n", inSql); @@ -4543,27 +4603,41 @@ void *subQueryProcess(void *sarg) { replaceSubTblName(g_queryInfo.subQueryInfo.sql[i], sqlstr, i); char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; if (g_queryInfo.subQueryInfo.result[i][0] != 0) { - sprintf(tmpFile, "%s-%d", g_queryInfo.subQueryInfo.result[i], winfo->threadID); + sprintf(tmpFile, "%s-%d", + g_queryInfo.subQueryInfo.result[i], + winfo->threadID); } selectAndGetResult(winfo->taos, sqlstr, tmpFile); } } et = taosGetTimestampMs(); - printf("####thread[%"PRId64"] complete all sqls to allocate all sub-tables[%d - %d] once queries duration:%.4fs\n\n", taosGetSelfPthreadId(), winfo->start_table_id, winfo->end_table_id, (double)(et - st)/1000.0); + printf("####thread[%"PRId64"] complete all sqls to allocate all sub-tables[%d - %d] once queries duration:%.4fs\n\n", + taosGetSelfPthreadId(), + winfo->start_table_id, + winfo->end_table_id, + (double)(et - st)/1000.0); } return NULL; } int queryTestProcess() { TAOS * taos = NULL; - taos = taos_connect(g_queryInfo.host, g_queryInfo.user, g_queryInfo.password, NULL, g_queryInfo.port); + taos = taos_connect(g_queryInfo.host, + g_queryInfo.user, + g_queryInfo.password, + NULL, + g_queryInfo.port); if (taos == NULL) { fprintf(stderr, "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL)); exit(-1); } if (0 != g_queryInfo.subQueryInfo.sqlCount) { - (void)getAllChildNameOfSuperTable(taos, g_queryInfo.dbName, g_queryInfo.subQueryInfo.sTblName, &g_queryInfo.subQueryInfo.childTblName, &g_queryInfo.subQueryInfo.childTblCount); + (void)getAllChildNameOfSuperTable(taos, + g_queryInfo.dbName, + g_queryInfo.subQueryInfo.sTblName, + &g_queryInfo.subQueryInfo.childTblName, + &g_queryInfo.subQueryInfo.childTblCount); } printfQueryMeta(); @@ -4683,9 +4757,14 @@ static TAOS_SUB* subscribeImpl(TAOS *taos, char *sql, char* topic, char* resultF TAOS_SUB* tsub = NULL; if (g_queryInfo.superQueryInfo.subscribeMode) { - tsub = taos_subscribe(taos, g_queryInfo.superQueryInfo.subscribeRestart, topic, sql, subscribe_callback, (void*)resultFileName, g_queryInfo.superQueryInfo.subscribeInterval); + tsub = taos_subscribe(taos, + g_queryInfo.superQueryInfo.subscribeRestart, + topic, sql, subscribe_callback, (void*)resultFileName, + g_queryInfo.superQueryInfo.subscribeInterval); } else { - tsub = taos_subscribe(taos, g_queryInfo.superQueryInfo.subscribeRestart, topic, sql, NULL, NULL, 0); + tsub = taos_subscribe(taos, + g_queryInfo.superQueryInfo.subscribeRestart, + topic, sql, NULL, NULL, 0); } if (tsub == NULL) { @@ -4837,7 +4916,11 @@ int subscribeTestProcess() { } TAOS * taos = NULL; - taos = taos_connect(g_queryInfo.host, g_queryInfo.user, g_queryInfo.password, g_queryInfo.dbName, g_queryInfo.port); + taos = taos_connect(g_queryInfo.host, + g_queryInfo.user, + g_queryInfo.password, + g_queryInfo.dbName, + g_queryInfo.port); if (taos == NULL) { fprintf(stderr, "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL)); exit(-1); @@ -4935,23 +5018,23 @@ int subscribeTestProcess() { void initOfInsertMeta() { memset(&g_Dbs, 0, sizeof(SDbs)); - // set default values - tstrncpy(g_Dbs.host, "127.0.0.1", MAX_DB_NAME_SIZE); - g_Dbs.port = 6030; - tstrncpy(g_Dbs.user, TSDB_DEFAULT_USER, MAX_DB_NAME_SIZE); - tstrncpy(g_Dbs.password, TSDB_DEFAULT_PASS, MAX_DB_NAME_SIZE); - g_Dbs.threadCount = 2; - g_Dbs.use_metric = true; + // set default values + tstrncpy(g_Dbs.host, "127.0.0.1", MAX_DB_NAME_SIZE); + g_Dbs.port = 6030; + tstrncpy(g_Dbs.user, TSDB_DEFAULT_USER, MAX_DB_NAME_SIZE); + tstrncpy(g_Dbs.password, TSDB_DEFAULT_PASS, MAX_DB_NAME_SIZE); + g_Dbs.threadCount = 2; + g_Dbs.use_metric = true; } void initOfQueryMeta() { memset(&g_queryInfo, 0, sizeof(SQueryMetaInfo)); - // set default values - tstrncpy(g_queryInfo.host, "127.0.0.1", MAX_DB_NAME_SIZE); - g_queryInfo.port = 6030; - tstrncpy(g_queryInfo.user, TSDB_DEFAULT_USER, MAX_DB_NAME_SIZE); - tstrncpy(g_queryInfo.password, TSDB_DEFAULT_PASS, MAX_DB_NAME_SIZE); + // set default values + tstrncpy(g_queryInfo.host, "127.0.0.1", MAX_DB_NAME_SIZE); + g_queryInfo.port = 6030; + tstrncpy(g_queryInfo.user, TSDB_DEFAULT_USER, MAX_DB_NAME_SIZE); + tstrncpy(g_queryInfo.password, TSDB_DEFAULT_PASS, MAX_DB_NAME_SIZE); } void setParaFromArg(){ @@ -4995,8 +5078,6 @@ void setParaFromArg(){ g_Dbs.db[0].superTbls[0].autoCreateTable = PRE_CREATE_SUBTBL; g_Dbs.db[0].superTbls[0].superTblExists = TBL_NO_EXISTS; g_Dbs.db[0].superTbls[0].childTblExists = TBL_NO_EXISTS; - g_Dbs.db[0].superTbls[0].insertInterval = 0; - g_Dbs.db[0].superTbls[0].numRecPerReq = 0; g_Dbs.db[0].superTbls[0].disorderRange = g_args.disorderRange; g_Dbs.db[0].superTbls[0].disorderRatio = g_args.disorderRatio; tstrncpy(g_Dbs.db[0].superTbls[0].childTblPrefix, @@ -5145,10 +5226,12 @@ int main(int argc, char *argv[]) { if (g_args.metaFile) { initOfInsertMeta(); initOfQueryMeta(); + if (false == getInfoFromJsonFile(g_args.metaFile)) { printf("Failed to read %s\n", g_args.metaFile); return 1; } + if (INSERT_MODE == g_jsonType) { if (g_Dbs.cfgDir[0]) taos_options(TSDB_OPTION_CONFIGDIR, g_Dbs.cfgDir); (void)insertTestProcess(); From 8e4ca52588f34579d8ae748a8cd702f0feef15d6 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 4 Mar 2021 17:14:38 +0800 Subject: [PATCH 37/66] db/walLevel: fix test cases to allow wal set to 0 --- tests/script/general/db/alter_option.sim | 2 +- tests/script/general/parser/create_db.sim | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/script/general/db/alter_option.sim b/tests/script/general/db/alter_option.sim index 9d4dedfce0..170ba21c28 100644 --- a/tests/script/general/db/alter_option.sim +++ b/tests/script/general/db/alter_option.sim @@ -209,7 +209,7 @@ sql alter database db wal 1 sql alter database db wal 2 sql alter database db wal 1 sql alter database db wal 2 -sql_error alter database db wal 0 +sql alter database db wal 0 sql_error alter database db wal 3 sql_error alter database db wal 4 sql_error alter database db wal -1 diff --git a/tests/script/general/parser/create_db.sim b/tests/script/general/parser/create_db.sim index 6cf08a5ac4..ea1cc17a6a 100644 --- a/tests/script/general/parser/create_db.sim +++ b/tests/script/general/parser/create_db.sim @@ -169,8 +169,8 @@ sql_error create database $db cache 0 sql_error create database $db ctime 29 sql_error create database $db ctime 40961 -# wal {1, 2} -sql_error create database $db wal 0 +# wal {0, 2} +#sql_error create database $db wal 0 sql_error create database $db wal -1 sql_error create database $db wal 3 From 1734b1d197c9b0efa3cf3ab7ca907a0f2ad04ed6 Mon Sep 17 00:00:00 2001 From: zyyang Date: Thu, 4 Mar 2021 17:18:52 +0800 Subject: [PATCH 38/66] [TD-3173]: java taosdemo can execute one sql statement by commandline --- .../taosdemo/TaosDemoApplication.java | 22 +++++++++--- .../components/JdbcTaosdemoConfig.java | 7 ++-- .../taosdemo/service/SqlExecuteor.java | 36 +++++++++++++++++++ .../com/taosdata/taosdemo/utils/Printer.java | 27 ++++++++++++++ 4 files changed, 85 insertions(+), 7 deletions(-) create mode 100644 tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SqlExecuteor.java create mode 100644 tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/utils/Printer.java diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/TaosDemoApplication.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/TaosDemoApplication.java index 19f47e13d6..6bd3523fc5 100644 --- a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/TaosDemoApplication.java +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/TaosDemoApplication.java @@ -3,15 +3,17 @@ package com.taosdata.taosdemo; import com.taosdata.taosdemo.components.DataSourceFactory; import com.taosdata.taosdemo.components.JdbcTaosdemoConfig; import com.taosdata.taosdemo.domain.SuperTableMeta; -import com.taosdata.taosdemo.service.DatabaseService; -import com.taosdata.taosdemo.service.QueryService; -import com.taosdata.taosdemo.service.SubTableService; -import com.taosdata.taosdemo.service.SuperTableService; +import com.taosdata.taosdemo.service.*; import com.taosdata.taosdemo.service.data.SuperTableMetaGenerator; +import com.taosdata.taosdemo.utils.Printer; import org.apache.log4j.Logger; import javax.sql.DataSource; import java.io.IOException; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; import java.time.Duration; import java.time.Instant; import java.util.Arrays; @@ -32,6 +34,17 @@ public class TaosDemoApplication { } // 初始化 final DataSource dataSource = DataSourceFactory.getInstance(config.host, config.port, config.user, config.password); + if (config.executeSql != null && !config.executeSql.isEmpty() && !config.executeSql.replaceAll("\\s", "").isEmpty()) { + Thread task = new Thread(new SqlExecuteor(dataSource, config.executeSql)); + task.start(); + try { + task.join(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + return; + } + final DatabaseService databaseService = new DatabaseService(dataSource); final SuperTableService superTableService = new SuperTableService(dataSource); final SubTableService subTableService = new SubTableService(dataSource); @@ -96,7 +109,6 @@ public class TaosDemoApplication { // 查询 - /**********************************************************************************/ // 删除表 if (config.dropTable) { diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/components/JdbcTaosdemoConfig.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/components/JdbcTaosdemoConfig.java index 971c10dee2..974a2755a5 100644 --- a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/components/JdbcTaosdemoConfig.java +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/components/JdbcTaosdemoConfig.java @@ -42,7 +42,7 @@ public final class JdbcTaosdemoConfig { public int rate = 10; public long range = 1000l; // select task - + public String executeSql; // drop task public boolean dropTable = false; @@ -89,7 +89,7 @@ public final class JdbcTaosdemoConfig { System.out.println("-rate The proportion of data out of order. effective only if order is 1. min 0, max 100, default is 10"); System.out.println("-range The range of data out of order. effective only if order is 1. default is 1000 ms"); // query task -// System.out.println("-sqlFile The select sql file"); + System.out.println("-executeSql execute a specific sql."); // drop task System.out.println("-dropTable Drop data before quit. Default is false"); System.out.println("--help Give this help list"); @@ -207,6 +207,9 @@ public final class JdbcTaosdemoConfig { range = Integer.parseInt(args[++i]); } // select task + if ("-executeSql".equals(args[i]) && i < args.length - 1) { + executeSql = args[++i]; + } // drop task if ("-dropTable".equals(args[i]) && i < args.length - 1) { diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SqlExecuteor.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SqlExecuteor.java new file mode 100644 index 0000000000..2016768605 --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SqlExecuteor.java @@ -0,0 +1,36 @@ +package com.taosdata.taosdemo.service; + +import com.taosdata.taosdemo.utils.Printer; + +import javax.sql.DataSource; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; + +public class SqlExecuteor implements Runnable { + private final DataSource dataSource; + private final String sql; + + public SqlExecuteor(DataSource dataSource, String sql) { + this.dataSource = dataSource; + this.sql = sql; + } + + @Override + public void run() { + try (Connection conn = dataSource.getConnection(); Statement stmt = conn.createStatement()) { + long start = System.currentTimeMillis(); + boolean execute = stmt.execute(sql); + long end = System.currentTimeMillis(); + if (execute) { + ResultSet rs = stmt.getResultSet(); + Printer.printResult(rs); + } else { + Printer.printSql(sql, true, (end - start)); + } + } catch (SQLException e) { + e.printStackTrace(); + } + } +} diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/utils/Printer.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/utils/Printer.java new file mode 100644 index 0000000000..a4627463ec --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/utils/Printer.java @@ -0,0 +1,27 @@ +package com.taosdata.taosdemo.utils; + +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; + +public class Printer { + + public static void printResult(ResultSet resultSet) throws SQLException { + ResultSetMetaData metaData = resultSet.getMetaData(); + while (resultSet.next()) { + for (int i = 1; i <= metaData.getColumnCount(); i++) { + String columnLabel = metaData.getColumnLabel(i); + String value = resultSet.getString(i); + System.out.printf("%s: %s\t", columnLabel, value); + } + System.out.println(); + } + } + + public static void printSql(String sql, boolean succeed, long cost) { + System.out.println("[ " + (succeed ? "OK" : "ERROR!") + " ] time cost: " + cost + " ms, execute statement ====> " + sql); + } + + private Printer() { + } +} From 5785339857d1b892260484bc65a5cc2debc6af10 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 4 Mar 2021 20:02:33 +0800 Subject: [PATCH 39/66] [TD-3147] : support insert interval. support no-stb in middle. --- src/kit/taosdemo/taosdemo.c | 466 +++++++++++++++++++++++------------- 1 file changed, 297 insertions(+), 169 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index de726ae816..d7725a807c 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -172,6 +172,7 @@ typedef struct { /* Used by main to communicate with parse_opt. */ typedef struct SArguments_S { char * metaFile; + int test_mode; char * host; uint16_t port; char * user; @@ -323,9 +324,6 @@ typedef struct SDbs_S { int dbCount; SDataBase db[MAX_DB_COUNT]; - int insert_interval; - int num_of_RPR; - // statistics int64_t totalRowsInserted; int64_t totalAffectedRows; @@ -473,7 +471,9 @@ double randdouble[MAX_PREPARED_RAND]; char *aggreFunc[] = {"*", "count(*)", "avg(col0)", "sum(col0)", "max(col0)", "min(col0)", "first(col0)", "last(col0)"}; -SArguments g_args = {NULL, +SArguments g_args = { + NULL, // metaFile + 0, // test_mode "127.0.0.1", // host 6030, // port "root", // user @@ -519,7 +519,6 @@ SArguments g_args = {NULL, }; -static int g_jsonType = 0; static SDbs g_Dbs; static int g_totalChildTables = 0; @@ -729,6 +728,7 @@ void parse_args(int argc, char *argv[], SArguments *arguments) { if (arguments->debug_print) { printf("###################################################################\n"); + printf("# meta file: %s\n", arguments->metaFile); printf("# Server IP: %s:%hu\n", arguments->host == NULL ? "localhost" : arguments->host, arguments->port ); @@ -793,6 +793,7 @@ static int queryDbExec(TAOS *taos, char *command, int type) { } if (code != 0) { + debugPrint("DEBUG %s() %d - command: %s\n", __func__, __LINE__, command); fprintf(stderr, "Failed to run %s, reason: %s\n", command, taos_errstr(res)); taos_free_result(res); //taos_close(taos); @@ -949,16 +950,37 @@ static void init_rand_data() { } } +#define SHOW_PARSE_RESULT_START() \ + do { if (g_args.metaFile) \ + printf("\033[1m\033[40;32m================ %s parse result START ================\033[0m\n", \ + g_args.metaFile); } while(0) + +#define SHOW_PARSE_RESULT_END() \ + do { if (g_args.metaFile) \ + printf("\033[1m\033[40;32m================ %s parse result END================\033[0m\n", \ + g_args.metaFile); } while(0) + +#define SHOW_PARSE_RESULT_START_TO_FILE(fp) \ + do { if (g_args.metaFile) \ + fprintf(fp, "\033[1m\033[40;32m================ %s parse result START ================\033[0m\n", \ + g_args.metaFile); } while(0) + +#define SHOW_PARSE_RESULT_END_TO_FILE(fp) \ + do { if (g_args.metaFile) \ + fprintf(fp, "\033[1m\033[40;32m================ %s parse result END================\033[0m\n", \ + g_args.metaFile); } while(0) + static int printfInsertMeta() { - printf("\033[1m\033[40;32m================ insert.json parse result START ================\033[0m\n"); + SHOW_PARSE_RESULT_START(); + printf("host: \033[33m%s:%u\033[0m\n", g_Dbs.host, g_Dbs.port); printf("user: \033[33m%s\033[0m\n", g_Dbs.user); printf("password: \033[33m%s\033[0m\n", g_Dbs.password); printf("resultFile: \033[33m%s\033[0m\n", g_Dbs.resultFile); printf("thread num of insert data: \033[33m%d\033[0m\n", g_Dbs.threadCount); printf("thread num of create table: \033[33m%d\033[0m\n", g_Dbs.threadCountByCreateTbl); - printf("insert interval: \033[33m%d\033[0m\n", g_Dbs.insert_interval); - printf("number of records per req: \033[33m%d\033[0m\n", g_Dbs.num_of_RPR); + printf("insert interval: \033[33m%d\033[0m\n", g_args.insert_interval); + printf("number of records per req: \033[33m%d\033[0m\n", g_args.num_of_RPR); printf("database count: \033[33m%d\033[0m\n", g_Dbs.dbCount); for (int i = 0; i < g_Dbs.dbCount; i++) { @@ -1007,11 +1029,13 @@ static int printfInsertMeta() { printf(" quorum: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.quorum); } if (g_Dbs.db[i].dbCfg.precision[0] != 0) { - if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2)) || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) { + if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2)) + || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) { printf(" precision: \033[33m%s\033[0m\n", g_Dbs.db[i].dbCfg.precision); } else { - printf(" precision error: \033[33m%s\033[0m\n", g_Dbs.db[i].dbCfg.precision); - return -1; + printf("\033[1m\033[40;31m precision error: %s\033[0m\n", + g_Dbs.db[i].dbCfg.precision); + return -1; } } @@ -1063,34 +1087,43 @@ static int printfInsertMeta() { printf(" columnCount: \033[33m%d\033[0m\n ", g_Dbs.db[i].superTbls[j].columnCount); for (int k = 0; k < g_Dbs.db[i].superTbls[j].columnCount; k++) { //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen); - if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType, "binary", 6)) || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType, "nchar", 5))) { - printf("column[\033[33m%d\033[0m]:\033[33m%s(%d)\033[0m ", k, g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen); + if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType, "binary", 6)) + || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType, "nchar", 5))) { + printf("column[\033[33m%d\033[0m]:\033[33m%s(%d)\033[0m ", k, + g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen); } else { - printf("column[%d]:\033[33m%s\033[0m ", k, g_Dbs.db[i].superTbls[j].columns[k].dataType); + printf("column[%d]:\033[33m%s\033[0m ", k, + g_Dbs.db[i].superTbls[j].columns[k].dataType); } } printf("\n"); - - printf(" tagCount: \033[33m%d\033[0m\n ", g_Dbs.db[i].superTbls[j].tagCount); + + printf(" tagCount: \033[33m%d\033[0m\n ", + g_Dbs.db[i].superTbls[j].tagCount); for (int k = 0; k < g_Dbs.db[i].superTbls[j].tagCount; k++) { //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen); - if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType, "binary", 6)) || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType, "nchar", 5))) { - printf("tag[%d]:\033[33m%s(%d)\033[0m ", k, g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen); + if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType, "binary", 6)) + || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType, "nchar", 5))) { + printf("tag[%d]:\033[33m%s(%d)\033[0m ", k, + g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen); } else { - printf("tag[%d]:\033[33m%s\033[0m ", k, g_Dbs.db[i].superTbls[j].tags[k].dataType); + printf("tag[%d]:\033[33m%s\033[0m ", k, + g_Dbs.db[i].superTbls[j].tags[k].dataType); } } printf("\n"); } printf("\n"); } - printf("\033[1m\033[40;32m================ insert.json parse result END================\033[0m\n"); + + SHOW_PARSE_RESULT_END(); return 0; } static void printfInsertMetaToFile(FILE* fp) { - fprintf(fp, "================ insert.json parse result START================\n"); + SHOW_PARSE_RESULT_START_TO_FILE(fp); + fprintf(fp, "host: %s:%u\n", g_Dbs.host, g_Dbs.port); fprintf(fp, "user: %s\n", g_Dbs.user); fprintf(fp, "password: %s\n", g_Dbs.password); @@ -1145,7 +1178,8 @@ static void printfInsertMetaToFile(FILE* fp) { fprintf(fp, " quorum: %d\n", g_Dbs.db[i].dbCfg.quorum); } if (g_Dbs.db[i].dbCfg.precision[0] != 0) { - if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2)) || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) { + if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2)) + || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) { fprintf(fp, " precision: %s\n", g_Dbs.db[i].dbCfg.precision); } else { fprintf(fp, " precision error: %s\n", g_Dbs.db[i].dbCfg.precision); @@ -1221,11 +1255,11 @@ static void printfInsertMetaToFile(FILE* fp) { } fprintf(fp, "\n"); } - fprintf(fp, "================ insert.json parse result END ================\n\n"); + SHOW_PARSE_RESULT_END_TO_FILE(fp); } static void printfQueryMeta() { - printf("\033[1m\033[40;32m================ query.json parse result ================\033[0m\n"); + SHOW_PARSE_RESULT_START(); printf("host: \033[33m%s:%u\033[0m\n", g_queryInfo.host, g_queryInfo.port); printf("user: \033[33m%s\033[0m\n", g_queryInfo.user); printf("password: \033[33m%s\033[0m\n", g_queryInfo.password); @@ -1237,14 +1271,13 @@ static void printfQueryMeta() { printf("concurrent: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.concurrent); printf("sqlCount: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.sqlCount); - if (SUBSCRIBE_MODE == g_jsonType) { + if (SUBSCRIBE_MODE == g_args.test_mode) { printf("mod: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.subscribeMode); printf("interval: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.subscribeInterval); printf("restart: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.subscribeRestart); printf("keepProgress: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.subscribeKeepProgress); } - for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { printf(" sql[%d]: \033[33m%s\033[0m\n", i, g_queryInfo.superQueryInfo.sql[i]); } @@ -1255,7 +1288,7 @@ static void printfQueryMeta() { printf("childTblCount: \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.childTblCount); printf("stable name: \033[33m%s\033[0m\n", g_queryInfo.subQueryInfo.sTblName); - if (SUBSCRIBE_MODE == g_jsonType) { + if (SUBSCRIBE_MODE == g_args.test_mode) { printf("mod: \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.subscribeMode); printf("interval: \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.subscribeInterval); printf("restart: \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.subscribeRestart); @@ -1267,7 +1300,8 @@ static void printfQueryMeta() { printf(" sql[%d]: \033[33m%s\033[0m\n", i, g_queryInfo.subQueryInfo.sql[i]); } printf("\n"); - printf("\033[1m\033[40;32m================ query.json parse result ================\033[0m\n"); + + SHOW_PARSE_RESULT_END(); } @@ -1434,7 +1468,9 @@ static int getDbFromServer(TAOS * taos, SDbInfo** dbInfos) { dbInfos[count]->comp = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_COMP_INDEX])); dbInfos[count]->cachelast = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_CACHELAST_INDEX])); - tstrncpy(dbInfos[count]->precision, (char *)row[TSDB_SHOW_DB_PRECISION_INDEX], fields[TSDB_SHOW_DB_PRECISION_INDEX].bytes); + tstrncpy(dbInfos[count]->precision, + (char *)row[TSDB_SHOW_DB_PRECISION_INDEX], + fields[TSDB_SHOW_DB_PRECISION_INDEX].bytes); dbInfos[count]->update = *((int8_t *)row[TSDB_SHOW_DB_UPDATE_INDEX]); tstrncpy(dbInfos[count]->status, (char *)row[TSDB_SHOW_DB_STATUS_INDEX], fields[TSDB_SHOW_DB_STATUS_INDEX].bytes); @@ -1982,6 +2018,7 @@ static int createSuperTable(TAOS * taos, char* dbName, SSuperTable* superTbls, superTbls->lenOfTagOfOneRow = lenOfTagOfOneRow; snprintf(command, BUFFER_SIZE, "create table if not exists %s.%s (ts timestamp%s) tags %s", dbName, superTbls->sTblName, cols, tags); + debugPrint("DEBUG %s() %d \n", __func__, __LINE__); if (0 != queryDbExec(taos, command, NO_INSERT_TYPE)) { return -1; } @@ -2005,6 +2042,7 @@ static int createDatabases() { for (int i = 0; i < g_Dbs.dbCount; i++) { if (g_Dbs.db[i].drop) { sprintf(command, "drop database if exists %s;", g_Dbs.db[i].dbName); + debugPrint("DEBUG %s() %d \n", __func__, __LINE__); if (0 != queryDbExec(taos, command, NO_INSERT_TYPE)) { taos_close(taos); return -1; @@ -2053,19 +2091,24 @@ static int createDatabases() { if (g_Dbs.db[i].dbCfg.fsync > 0) { dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "fsync %d ", g_Dbs.db[i].dbCfg.fsync); } - if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2)) || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) { - dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "precision \'%s\';", g_Dbs.db[i].dbCfg.precision); + if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2)) + || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) { + dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, + "precision \'%s\';", g_Dbs.db[i].dbCfg.precision); } + debugPrint("DEBUG %s() %d \n", __func__, __LINE__); if (0 != queryDbExec(taos, command, NO_INSERT_TYPE)) { taos_close(taos); return -1; } printf("\ncreate database %s success!\n\n", g_Dbs.db[i].dbName); + debugPrint("DEBUG %s() %d count:%d\n", __func__, __LINE__, g_Dbs.db[i].superTblCount); for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { // describe super table, if exists sprintf(command, "describe %s.%s;", g_Dbs.db[i].dbName, g_Dbs.db[i].superTbls[j].sTblName); + debugPrint("DEBUG %s() %d \n", __func__, __LINE__); if (0 != queryDbExec(taos, command, NO_INSERT_TYPE)) { g_Dbs.db[i].superTbls[j].superTblExists = TBL_NO_EXISTS; ret = createSuperTable(taos, g_Dbs.db[i].dbName, &g_Dbs.db[i].superTbls[j], g_Dbs.use_metric); @@ -2085,7 +2128,6 @@ static int createDatabases() { return 0; } - void * createTable(void *sarg) { threadInfo *winfo = (threadInfo *)sarg; @@ -2100,35 +2142,50 @@ void * createTable(void *sarg) //printf("Creating table from %d to %d\n", winfo->start_table_id, winfo->end_table_id); for (int i = winfo->start_table_id; i <= winfo->end_table_id; i++) { if (0 == g_Dbs.use_metric) { - snprintf(buffer, BUFFER_SIZE, "create table if not exists %s.%s%d %s;", winfo->db_name, superTblInfo->childTblPrefix, i, superTblInfo->colsOfCreatChildTable); + snprintf(buffer, BUFFER_SIZE, + "create table if not exists %s.%s%d %s;", + winfo->db_name, + superTblInfo->childTblPrefix, i, + superTblInfo->colsOfCreatChildTable); } else { if (0 == len) { batchNum = 0; memset(buffer, 0, superTblInfo->maxSqlLen); - len += snprintf(buffer + len, superTblInfo->maxSqlLen - len, "create table "); + len += snprintf(buffer + len, + superTblInfo->maxSqlLen - len, "create table "); } char* tagsValBuf = NULL; if (0 == superTblInfo->tagSource) { tagsValBuf = generateTagVaulesForStb(superTblInfo); } else { - tagsValBuf = getTagValueFromTagSample(superTblInfo, i % superTblInfo->tagSampleCount); + tagsValBuf = getTagValueFromTagSample( + superTblInfo, + i % superTblInfo->tagSampleCount); } if (NULL == tagsValBuf) { free(buffer); return NULL; } - len += snprintf(buffer + len, superTblInfo->maxSqlLen - len, "if not exists %s.%s%d using %s.%s tags %s ", winfo->db_name, superTblInfo->childTblPrefix, i, winfo->db_name, superTblInfo->sTblName, tagsValBuf); + len += snprintf(buffer + len, + superTblInfo->maxSqlLen - len, + "if not exists %s.%s%d using %s.%s tags %s ", + winfo->db_name, superTblInfo->childTblPrefix, + i, winfo->db_name, + superTblInfo->sTblName, tagsValBuf); free(tagsValBuf); batchNum++; - if ((batchNum < superTblInfo->batchCreateTableNum) && ((superTblInfo->maxSqlLen - len) >= (superTblInfo->lenOfTagOfOneRow + 256))) { + if ((batchNum < superTblInfo->batchCreateTableNum) + && ((superTblInfo->maxSqlLen - len) + >= (superTblInfo->lenOfTagOfOneRow + 256))) { continue; } } len = 0; + debugPrint("DEBUG %s() %d \n", __func__, __LINE__); if (0 != queryDbExec(winfo->taos, buffer, NO_INSERT_TYPE)){ free(buffer); return NULL; @@ -2136,20 +2193,24 @@ void * createTable(void *sarg) int64_t currentPrintTime = taosGetTimestampMs(); if (currentPrintTime - lastPrintTime > 30*1000) { - printf("thread[%d] already create %d - %d tables\n", winfo->threadID, winfo->start_table_id, i); + printf("thread[%d] already create %d - %d tables\n", + winfo->threadID, winfo->start_table_id, i); lastPrintTime = currentPrintTime; } } if (0 != len) { + debugPrint("DEBUG %s() %d \n", __func__, __LINE__); (void)queryDbExec(winfo->taos, buffer, NO_INSERT_TYPE); } - + free(buffer); return NULL; } -void startMultiThreadCreateChildTable(char* cols, int threads, int ntables, char* db_name, SSuperTable* superTblInfo) { +void startMultiThreadCreateChildTable( + char* cols, int threads, int ntables, + char* db_name, SSuperTable* superTblInfo) { pthread_t *pids = malloc(threads * sizeof(pthread_t)); threadInfo *infos = malloc(threads * sizeof(threadInfo)); @@ -2177,7 +2238,12 @@ void startMultiThreadCreateChildTable(char* cols, int threads, int ntables, char t_info->threadID = i; tstrncpy(t_info->db_name, db_name, MAX_DB_NAME_SIZE); t_info->superTblInfo = superTblInfo; - t_info->taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, db_name, g_Dbs.port); + t_info->taos = taos_connect( + g_Dbs.host, + g_Dbs.user, + g_Dbs.password, + db_name, + g_Dbs.port); t_info->start_table_id = last; t_info->end_table_id = i < b ? last + a : last + a - 1; last = t_info->end_table_id + 1; @@ -2204,10 +2270,15 @@ void startMultiThreadCreateChildTable(char* cols, int threads, int ntables, char static void createChildTables() { for (int i = 0; i < g_Dbs.dbCount; i++) { for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { - if ((AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) || (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists)) { + if ((AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) + || (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists)) { continue; } - startMultiThreadCreateChildTable(g_Dbs.db[i].superTbls[j].colsOfCreatChildTable, g_Dbs.threadCountByCreateTbl, g_Dbs.db[i].superTbls[j].childTblCount, g_Dbs.db[i].dbName, &(g_Dbs.db[i].superTbls[j])); + startMultiThreadCreateChildTable( + g_Dbs.db[i].superTbls[j].colsOfCreatChildTable, + g_Dbs.threadCountByCreateTbl, + g_Dbs.db[i].superTbls[j].childTblCount, + g_Dbs.db[i].dbName, &(g_Dbs.db[i].superTbls[j])); g_totalChildTables += g_Dbs.db[i].superTbls[j].childTblCount; } } @@ -2247,7 +2318,8 @@ int readTagFromCsvFileToMem(SSuperTable * superTblInfo) { FILE *fp = fopen(superTblInfo->tagsFile, "r"); if (fp == NULL) { - printf("Failed to open tags file: %s, reason:%s\n", superTblInfo->tagsFile, strerror(errno)); + printf("Failed to open tags file: %s, reason:%s\n", + superTblInfo->tagsFile, strerror(errno)); return -1; } @@ -2278,11 +2350,13 @@ int readTagFromCsvFileToMem(SSuperTable * superTblInfo) { count++; if (count >= tagCount - 1) { - char *tmp = realloc(tagDataBuf, (size_t)tagCount*1.5*superTblInfo->lenOfTagOfOneRow); + char *tmp = realloc(tagDataBuf, + (size_t)tagCount*1.5*superTblInfo->lenOfTagOfOneRow); if (tmp != NULL) { tagDataBuf = tmp; tagCount = (int)(tagCount*1.5); - memset(tagDataBuf + count*superTblInfo->lenOfTagOfOneRow, 0, (size_t)((tagCount-count)*superTblInfo->lenOfTagOfOneRow)); + memset(tagDataBuf + count*superTblInfo->lenOfTagOfOneRow, + 0, (size_t)((tagCount-count)*superTblInfo->lenOfTagOfOneRow)); } else { // exit, if allocate more memory failed printf("realloc fail for save tag val from %s\n", superTblInfo->tagsFile); @@ -2322,7 +2396,8 @@ int readSampleFromCsvFileToMem(FILE *fp, SSuperTable* superTblInfo, char* sample readLen = tgetline(&line, &n, fp); if (-1 == readLen) { if(0 != fseek(fp, 0, SEEK_SET)) { - printf("Failed to fseek file: %s, reason:%s\n", superTblInfo->sampleFile, strerror(errno)); + printf("Failed to fseek file: %s, reason:%s\n", + superTblInfo->sampleFile, strerror(errno)); return -1; } continue; @@ -2337,7 +2412,8 @@ int readSampleFromCsvFileToMem(FILE *fp, SSuperTable* superTblInfo, char* sample } if (readLen > superTblInfo->lenOfOneRow) { - printf("sample row len[%d] overflow define schema len[%d], so discard this row\n", (int32_t)readLen, superTblInfo->lenOfOneRow); + printf("sample row len[%d] overflow define schema len[%d], so discard this row\n", + (int32_t)readLen, superTblInfo->lenOfOneRow); continue; } @@ -2383,14 +2459,15 @@ static bool getColumnAndTagTypeFromInsertJsonFile(cJSON* stbInfo, SSuperTable* s int columnSize = cJSON_GetArraySize(columns); if (columnSize > MAX_COLUMN_COUNT) { - printf("failed to read json, column size overflow, max column size is %d\n", MAX_COLUMN_COUNT); + printf("failed to read json, column size overflow, max column size is %d\n", + MAX_COLUMN_COUNT); goto PARSE_OVER; } int count = 1; int index = 0; StrColumn columnCase; - + //superTbls->columnCount = columnSize; for (int k = 0; k < columnSize; ++k) { cJSON* column = cJSON_GetArrayItem(columns, k); @@ -2570,9 +2647,9 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { cJSON* insertInterval = cJSON_GetObjectItem(root, "insert_interval"); if (insertInterval && insertInterval->type == cJSON_Number) { - g_Dbs.insert_interval = insertInterval->valueint; + g_args.insert_interval = insertInterval->valueint; } else if (!insertInterval) { - g_Dbs.insert_interval = 0; + g_args.insert_interval = 0; } else { printf("failed to read json, insert_interval not found"); goto PARSE_OVER; @@ -2580,16 +2657,18 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { cJSON* numRecPerReq = cJSON_GetObjectItem(root, "num_of_records_per_req"); if (numRecPerReq && numRecPerReq->type == cJSON_Number) { - g_Dbs.num_of_RPR = numRecPerReq->valueint; + g_args.num_of_RPR = numRecPerReq->valueint; } else if (!numRecPerReq) { - g_Dbs.num_of_RPR = 0; + g_args.num_of_RPR = 100; } else { printf("failed to read json, num_of_records_per_req not found"); goto PARSE_OVER; } cJSON *answerPrompt = cJSON_GetObjectItem(root, "confirm_parameter_prompt"); // yes, no, - if (answerPrompt && answerPrompt->type == cJSON_String && answerPrompt->valuestring != NULL) { + if (answerPrompt + && answerPrompt->type == cJSON_String + && answerPrompt->valuestring != NULL) { if (0 == strncasecmp(answerPrompt->valuestring, "yes", 3)) { g_args.answer_yes = false; } else if (0 == strncasecmp(answerPrompt->valuestring, "no", 2)) { @@ -2834,7 +2913,9 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { tstrncpy(g_Dbs.db[i].superTbls[j].childTblPrefix, prefix->valuestring, MAX_DB_NAME_SIZE); cJSON *autoCreateTbl = cJSON_GetObjectItem(stbInfo, "auto_create_table"); // yes, no, null - if (autoCreateTbl && autoCreateTbl->type == cJSON_String && autoCreateTbl->valuestring != NULL) { + if (autoCreateTbl + && autoCreateTbl->type == cJSON_String + && autoCreateTbl->valuestring != NULL) { if (0 == strncasecmp(autoCreateTbl->valuestring, "yes", 3)) { g_Dbs.db[i].superTbls[j].autoCreateTable = AUTO_CREATE_SUBTBL; } else if (0 == strncasecmp(autoCreateTbl->valuestring, "no", 2)) { @@ -2860,7 +2941,9 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } cJSON *childTblExists = cJSON_GetObjectItem(stbInfo, "child_table_exists"); // yes, no - if (childTblExists && childTblExists->type == cJSON_String && childTblExists->valuestring != NULL) { + if (childTblExists + && childTblExists->type == cJSON_String + && childTblExists->valuestring != NULL) { if (0 == strncasecmp(childTblExists->valuestring, "yes", 3)) { g_Dbs.db[i].superTbls[j].childTblExists = TBL_ALREADY_EXISTS; } else if (0 == strncasecmp(childTblExists->valuestring, "no", 2)) { @@ -2883,8 +2966,10 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { g_Dbs.db[i].superTbls[j].childTblCount = count->valueint; cJSON *dataSource = cJSON_GetObjectItem(stbInfo, "data_source"); - if (dataSource && dataSource->type == cJSON_String && dataSource->valuestring != NULL) { - tstrncpy(g_Dbs.db[i].superTbls[j].dataSource, dataSource->valuestring, MAX_DB_NAME_SIZE); + if (dataSource && dataSource->type == cJSON_String + && dataSource->valuestring != NULL) { + tstrncpy(g_Dbs.db[i].superTbls[j].dataSource, + dataSource->valuestring, MAX_DB_NAME_SIZE); } else if (!dataSource) { tstrncpy(g_Dbs.db[i].superTbls[j].dataSource, "rand", MAX_DB_NAME_SIZE); } else { @@ -2893,8 +2978,10 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } cJSON *insertMode = cJSON_GetObjectItem(stbInfo, "insert_mode"); // taosc , restful - if (insertMode && insertMode->type == cJSON_String && insertMode->valuestring != NULL) { - tstrncpy(g_Dbs.db[i].superTbls[j].insertMode, insertMode->valuestring, MAX_DB_NAME_SIZE); + if (insertMode && insertMode->type == cJSON_String + && insertMode->valuestring != NULL) { + tstrncpy(g_Dbs.db[i].superTbls[j].insertMode, + insertMode->valuestring, MAX_DB_NAME_SIZE); } else if (!insertMode) { tstrncpy(g_Dbs.db[i].superTbls[j].insertMode, "taosc", MAX_DB_NAME_SIZE); } else { @@ -2937,7 +3024,8 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { cJSON *sampleFormat = cJSON_GetObjectItem(stbInfo, "sample_format"); if (sampleFormat && sampleFormat->type == cJSON_String && sampleFormat->valuestring != NULL) { - tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat, sampleFormat->valuestring, MAX_DB_NAME_SIZE); + tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat, + sampleFormat->valuestring, MAX_DB_NAME_SIZE); } else if (!sampleFormat) { tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat, "csv", MAX_DB_NAME_SIZE); } else { @@ -2947,7 +3035,8 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { cJSON *sampleFile = cJSON_GetObjectItem(stbInfo, "sample_file"); if (sampleFile && sampleFile->type == cJSON_String && sampleFile->valuestring != NULL) { - tstrncpy(g_Dbs.db[i].superTbls[j].sampleFile, sampleFile->valuestring, MAX_FILE_NAME_LEN); + tstrncpy(g_Dbs.db[i].superTbls[j].sampleFile, + sampleFile->valuestring, MAX_FILE_NAME_LEN); } else if (!sampleFile) { memset(g_Dbs.db[i].superTbls[j].sampleFile, 0, MAX_FILE_NAME_LEN); } else { @@ -2957,7 +3046,8 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { cJSON *tagsFile = cJSON_GetObjectItem(stbInfo, "tags_file"); if (tagsFile && tagsFile->type == cJSON_String && tagsFile->valuestring != NULL) { - tstrncpy(g_Dbs.db[i].superTbls[j].tagsFile, tagsFile->valuestring, MAX_FILE_NAME_LEN); + tstrncpy(g_Dbs.db[i].superTbls[j].tagsFile, + tagsFile->valuestring, MAX_FILE_NAME_LEN); if (0 == g_Dbs.db[i].superTbls[j].tagsFile[0]) { g_Dbs.db[i].superTbls[j].tagSource = 0; } else { @@ -2987,8 +3077,11 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { goto PARSE_OVER; } - cJSON *multiThreadWriteOneTbl = cJSON_GetObjectItem(stbInfo, "multi_thread_write_one_tbl"); // no , yes - if (multiThreadWriteOneTbl && multiThreadWriteOneTbl->type == cJSON_String && multiThreadWriteOneTbl->valuestring != NULL) { + cJSON *multiThreadWriteOneTbl = + cJSON_GetObjectItem(stbInfo, "multi_thread_write_one_tbl"); // no , yes + if (multiThreadWriteOneTbl + && multiThreadWriteOneTbl->type == cJSON_String + && multiThreadWriteOneTbl->valuestring != NULL) { if (0 == strncasecmp(multiThreadWriteOneTbl->valuestring, "yes", 3)) { g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 1; } else { @@ -3055,7 +3148,8 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { goto PARSE_OVER; } - if (NO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable || (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists)) { + if (NO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable + || (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists)) { continue; } @@ -3115,7 +3209,8 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { } cJSON *answerPrompt = cJSON_GetObjectItem(root, "confirm_parameter_prompt"); // yes, no, - if (answerPrompt && answerPrompt->type == cJSON_String && answerPrompt->valuestring != NULL) { + if (answerPrompt && answerPrompt->type == cJSON_String + && answerPrompt->valuestring != NULL) { if (0 == strncasecmp(answerPrompt->valuestring, "yes", 3)) { g_args.answer_yes = false; } else if (0 == strncasecmp(answerPrompt->valuestring, "no", 2)) { @@ -3209,7 +3304,9 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { } cJSON* keepProgress = cJSON_GetObjectItem(superQuery, "keepProgress"); - if (keepProgress && keepProgress->type == cJSON_String && keepProgress->valuestring != NULL) { + if (keepProgress + && keepProgress->type == cJSON_String + && keepProgress->valuestring != NULL) { if (0 == strcmp("yes", keepProgress->valuestring)) { g_queryInfo.superQueryInfo.subscribeKeepProgress = 1; } else if (0 == strcmp("no", keepProgress->valuestring)) { @@ -3400,6 +3497,8 @@ PARSE_OVER: } static bool getInfoFromJsonFile(char* file) { + debugPrint("DEBUG - %s %d %s\n", __func__, __LINE__, file); + FILE *fp = fopen(file, "r"); if (!fp) { printf("failed to read %s, reason:%s\n", file, strerror(errno)); @@ -3427,27 +3526,27 @@ static bool getInfoFromJsonFile(char* file) { cJSON* filetype = cJSON_GetObjectItem(root, "filetype"); if (filetype && filetype->type == cJSON_String && filetype->valuestring != NULL) { if (0 == strcasecmp("insert", filetype->valuestring)) { - g_jsonType = INSERT_MODE; + g_args.test_mode = INSERT_MODE; } else if (0 == strcasecmp("query", filetype->valuestring)) { - g_jsonType = QUERY_MODE; + g_args.test_mode = QUERY_MODE; } else if (0 == strcasecmp("subscribe", filetype->valuestring)) { - g_jsonType = SUBSCRIBE_MODE; + g_args.test_mode = SUBSCRIBE_MODE; } else { printf("failed to read json, filetype not support\n"); goto PARSE_OVER; } } else if (!filetype) { - g_jsonType = INSERT_MODE; + g_args.test_mode = INSERT_MODE; } else { printf("failed to read json, filetype not found\n"); goto PARSE_OVER; } - if (INSERT_MODE == g_jsonType) { + if (INSERT_MODE == g_args.test_mode) { ret = getMetaFromInsertJsonFile(root); - } else if (QUERY_MODE == g_jsonType) { + } else if (QUERY_MODE == g_args.test_mode) { ret = getMetaFromQueryJsonFile(root); - } else if (SUBSCRIBE_MODE == g_jsonType) { + } else if (SUBSCRIBE_MODE == g_args.test_mode) { ret = getMetaFromQueryJsonFile(root); } else { printf("input json file type error! please input correct file type: insert or query or subscribe\n"); @@ -3593,11 +3692,11 @@ void syncWriteForNumberOfTblInOneSql( int64_t st = 0; int64_t et = 0; for (int i = 0; i < superTblInfo->insertRows;) { - if (g_Dbs.insert_interval && (g_Dbs.insert_interval > (et - st))) { - taosMsleep(g_Dbs.insert_interval - (et - st)); // ms + if (g_args.insert_interval && (g_args.insert_interval > (et - st))) { + taosMsleep(g_args.insert_interval - (et - st)); // ms } - if (g_Dbs.insert_interval) { + if (g_args.insert_interval) { st = taosGetTimestampMs(); } @@ -3605,7 +3704,7 @@ void syncWriteForNumberOfTblInOneSql( for (int tID = winfo->start_table_id; tID <= winfo->end_table_id; ) { int inserted = i; - for (int k = 0; k < g_Dbs.num_of_RPR;) { + for (int k = 0; k < g_args.num_of_RPR;) { int len = 0; memset(buffer, 0, superTblInfo->maxSqlLen); char *pstr = buffer; @@ -3746,6 +3845,7 @@ send_to_server: int64_t endTs; startTs = taosGetTimestampUs(); + debugPrint("DEBUG %s() %d \n", __func__, __LINE__); int affectedRows = queryDbExec( winfo->taos, buffer, INSERT_TYPE); if (0 > affectedRows) { @@ -3795,7 +3895,7 @@ send_to_server: } } - if (g_Dbs.insert_interval) { + if (g_args.insert_interval) { et = taosGetTimestampMs(); } //printf("========loop %d childTables duration:%"PRId64 "========inserted rows:%d\n", winfo->end_table_id - winfo->start_table_id, et - st, i); @@ -3821,7 +3921,7 @@ void *syncWrite(void *sarg) { uint64_t totalRowsInserted = 0; uint64_t totalAffectedRows = 0; uint64_t lastPrintTime = taosGetTimestampMs(); - + threadInfo *winfo = (threadInfo *)sarg; SSuperTable* superTblInfo = winfo->superTblInfo; @@ -3880,12 +3980,12 @@ void *syncWrite(void *sarg) { uint64_t et = 0; for (int i = 0; i < superTblInfo->insertRows;) { - if (i > 0 && g_Dbs.insert_interval - && (g_Dbs.insert_interval > (et - st) )) { - taosMsleep(g_Dbs.insert_interval - (et - st)); // ms + if (i > 0 && g_args.insert_interval + && (g_args.insert_interval > (et - st) )) { + taosMsleep(g_args.insert_interval - (et - st)); // ms } - if (g_Dbs.insert_interval) { + if (g_args.insert_interval) { st = taosGetTimestampMs(); } @@ -3939,7 +4039,7 @@ void *syncWrite(void *sarg) { tID); } - for (k = 0; k < g_Dbs.num_of_RPR;) { + for (k = 0; k < g_args.num_of_RPR;) { int retLen = 0; if (0 == strncasecmp(superTblInfo->dataSource, "sample", strlen("sample"))) { retLen = getRowDataFromSample( @@ -3991,6 +4091,7 @@ void *syncWrite(void *sarg) { int64_t endTs; startTs = taosGetTimestampUs(); + debugPrint("DEBUG %s() %d \n", __func__, __LINE__); int affectedRows = queryDbExec(winfo->taos, buffer, INSERT_TYPE); if (0 > affectedRows){ goto free_and_statistics_2; @@ -4038,7 +4139,7 @@ void *syncWrite(void *sarg) { } } - if (g_Dbs.insert_interval) { + if (g_args.insert_interval) { et = taosGetTimestampMs(); } //printf("========loop %d childTables duration:%"PRId64 "========inserted rows:%d\n", winfo->end_table_id - winfo->start_table_id, et - st, i); @@ -4062,7 +4163,7 @@ free_and_statistics_2: void callBack(void *param, TAOS_RES *res, int code) { threadInfo* winfo = (threadInfo*)param; - if (g_Dbs.insert_interval) { + if (g_args.insert_interval) { winfo->et = taosGetTimestampMs(); if (winfo->et - winfo->st < 1000) { taosMsleep(1000 - (winfo->et - winfo->st)); // ms @@ -4085,7 +4186,7 @@ void callBack(void *param, TAOS_RES *res, int code) { return; } - for (int i = 0; i < g_Dbs.num_of_RPR; i++) { + for (int i = 0; i < g_args.num_of_RPR; i++) { int rand_num = rand() % 100; if (0 != winfo->superTblInfo->disorderRatio && rand_num < winfo->superTblInfo->disorderRatio) { @@ -4104,7 +4205,7 @@ void callBack(void *param, TAOS_RES *res, int code) { } } - if (g_Dbs.insert_interval) { + if (g_args.insert_interval) { winfo->st = taosGetTimestampMs(); } taos_query_a(winfo->taos, buffer, callBack, winfo); @@ -4121,7 +4222,7 @@ void *asyncWrite(void *sarg) { winfo->et = 0; winfo->lastTs = winfo->start_time; - if (g_Dbs.insert_interval) { + if (g_args.insert_interval) { winfo->st = taosGetTimestampMs(); } taos_query_a(winfo->taos, "show databases", callBack, winfo); @@ -4131,23 +4232,29 @@ void *asyncWrite(void *sarg) { return NULL; } -void startMultiThreadInsertData(int threads, char* db_name, char* precision, SSuperTable* superTblInfo) { +void startMultiThreadInsertData(int threads, char* db_name, char* precision, + SSuperTable* superTblInfo) { pthread_t *pids = malloc(threads * sizeof(pthread_t)); threadInfo *infos = malloc(threads * sizeof(threadInfo)); memset(pids, 0, threads * sizeof(pthread_t)); memset(infos, 0, threads * sizeof(threadInfo)); - int ntables = superTblInfo->childTblCount; - int a = ntables / threads; - if (a < 1) { - threads = ntables; - a = 1; - } + int ntables = 0; + if (superTblInfo) + ntables = superTblInfo->childTblCount; + else + ntables = g_args.num_of_tables; - int b = 0; - if (threads != 0) { - b = ntables % threads; - } + int a = ntables / threads; + if (a < 1) { + threads = ntables; + a = 1; + } + + int b = 0; + if (threads != 0) { + b = ntables % threads; + } //TAOS* taos; //if (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5)) { @@ -4256,21 +4363,25 @@ void startMultiThreadInsertData(int threads, char* db_name, char* precision, SSu double end = getCurrentTime(); double t = end - start; printf("Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s) into %s.%s. %2.f records/second\n\n", - t, superTblInfo->totalRowsInserted, superTblInfo->totalAffectedRows, threads, db_name, superTblInfo->sTblName, superTblInfo->totalRowsInserted / t); + t, superTblInfo->totalRowsInserted, + superTblInfo->totalAffectedRows, + threads, db_name, superTblInfo->sTblName, + superTblInfo->totalRowsInserted / t); fprintf(g_fpOfInsertResult, "Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s) into %s.%s. %2.f records/second\n\n", - t, superTblInfo->totalRowsInserted, superTblInfo->totalAffectedRows, threads, db_name, superTblInfo->sTblName, superTblInfo->totalRowsInserted / t); + t, superTblInfo->totalRowsInserted, + superTblInfo->totalAffectedRows, + threads, db_name, superTblInfo->sTblName, + superTblInfo->totalRowsInserted / t); printf("insert delay, avg: %10.6fms, max: %10.6fms, min: %10.6fms\n\n", avgDelay/1000.0, (double)maxDelay/1000.0, (double)minDelay/1000.0); fprintf(g_fpOfInsertResult, "insert delay, avg:%10.6fms, max: %10.6fms, min: %10.6fms\n\n", avgDelay/1000.0, (double)maxDelay/1000.0, (double)minDelay/1000.0); - //taos_close(taos); free(pids); free(infos); - } @@ -4413,6 +4524,8 @@ void *readMetric(void *sarg) { int insertTestProcess() { + debugPrint("DEBUG - %d result file: %s\n", __LINE__, g_Dbs.resultFile); + g_fpOfInsertResult = fopen(g_Dbs.resultFile, "a"); if (NULL == g_fpOfInsertResult) { fprintf(stderr, "Failed to open %s for save result\n", g_Dbs.resultFile); @@ -4422,6 +4535,7 @@ int insertTestProcess() { setupForAnsiEscape(); int ret = printfInsertMeta(); resetAfterAnsiEscape(); + if (ret == -1) exit(EXIT_FAILURE); @@ -4439,7 +4553,7 @@ int insertTestProcess() { // pretreatement prePareSampleData(); - + double start; double end; @@ -4459,19 +4573,27 @@ int insertTestProcess() { // create sub threads for inserting data //start = getCurrentTime(); - for (int i = 0; i < g_Dbs.dbCount; i++) { - for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { - SSuperTable* superTblInfo = &g_Dbs.db[i].superTbls[j]; - if (0 == g_Dbs.db[i].superTbls[j].insertRows) { - continue; - } - startMultiThreadInsertData( - g_Dbs.threadCount, - g_Dbs.db[i].dbName, - g_Dbs.db[i].dbCfg.precision, - superTblInfo); + for (int i = 0; i < g_Dbs.dbCount; i++) { + if (g_Dbs.db[i].superTblCount > 0) { + for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { + SSuperTable* superTblInfo = &g_Dbs.db[i].superTbls[j]; + if (0 == g_Dbs.db[i].superTbls[j].insertRows) { + continue; + } + startMultiThreadInsertData( + g_Dbs.threadCount, + g_Dbs.db[i].dbName, + g_Dbs.db[i].dbCfg.precision, + superTblInfo); + } + } else { + startMultiThreadInsertData( + g_Dbs.threadCount, + g_Dbs.db[i].dbName, + g_Dbs.db[i].dbCfg.precision, + NULL); + } } - } //end = getCurrentTime(); //int64_t totalRowsInserted = 0; @@ -4671,6 +4793,7 @@ int queryTestProcess() { char sqlStr[MAX_TB_NAME_SIZE*2]; sprintf(sqlStr, "use %s", g_queryInfo.dbName); + debugPrint("DEBUG %s() %d \n", __func__, __LINE__); (void)queryDbExec(t_info->taos, sqlStr, NO_INSERT_TYPE); } else { t_info->taos = NULL; @@ -4781,6 +4904,7 @@ void *subSubscribeProcess(void *sarg) { char sqlStr[MAX_TB_NAME_SIZE*2]; sprintf(sqlStr, "use %s", g_queryInfo.dbName); + debugPrint("DEBUG %s() %d \n", __func__, __LINE__); if (0 != queryDbExec(winfo->taos, sqlStr, NO_INSERT_TYPE)){ return NULL; } @@ -4846,6 +4970,7 @@ void *superSubscribeProcess(void *sarg) { char sqlStr[MAX_TB_NAME_SIZE*2]; sprintf(sqlStr, "use %s", g_queryInfo.dbName); + debugPrint("DEBUG %s() %d \n", __func__, __LINE__); if (0 != queryDbExec(winfo->taos, sqlStr, NO_INSERT_TYPE)) { return NULL; } @@ -5058,7 +5183,7 @@ void setParaFromArg(){ g_Dbs.dbCount = 1; g_Dbs.db[0].drop = 1; - + tstrncpy(g_Dbs.db[0].dbName, g_args.database, MAX_DB_NAME_SIZE); g_Dbs.db[0].dbCfg.replica = g_args.replica; tstrncpy(g_Dbs.db[0].dbCfg.precision, "ms", MAX_DB_NAME_SIZE); @@ -5068,30 +5193,6 @@ void setParaFromArg(){ g_Dbs.use_metric = g_args.use_metric; g_Dbs.insert_only = g_args.insert_only; - g_Dbs.db[0].superTblCount = 1; - tstrncpy(g_Dbs.db[0].superTbls[0].sTblName, "meters", MAX_TB_NAME_SIZE); - g_Dbs.db[0].superTbls[0].childTblCount = g_args.num_of_tables; - g_Dbs.threadCount = g_args.num_of_threads; - g_Dbs.threadCountByCreateTbl = 1; - g_Dbs.queryMode = g_args.mode; - - g_Dbs.db[0].superTbls[0].autoCreateTable = PRE_CREATE_SUBTBL; - g_Dbs.db[0].superTbls[0].superTblExists = TBL_NO_EXISTS; - g_Dbs.db[0].superTbls[0].childTblExists = TBL_NO_EXISTS; - g_Dbs.db[0].superTbls[0].disorderRange = g_args.disorderRange; - g_Dbs.db[0].superTbls[0].disorderRatio = g_args.disorderRatio; - tstrncpy(g_Dbs.db[0].superTbls[0].childTblPrefix, - g_args.tb_prefix, MAX_TB_NAME_SIZE); - tstrncpy(g_Dbs.db[0].superTbls[0].dataSource, "rand", MAX_TB_NAME_SIZE); - tstrncpy(g_Dbs.db[0].superTbls[0].insertMode, "taosc", MAX_TB_NAME_SIZE); - tstrncpy(g_Dbs.db[0].superTbls[0].startTimestamp, - "2017-07-14 10:40:00.000", MAX_TB_NAME_SIZE); - g_Dbs.db[0].superTbls[0].timeStampStep = 10; - - // g_args.num_of_RPR; - g_Dbs.db[0].superTbls[0].insertRows = g_args.num_of_DPT; - g_Dbs.db[0].superTbls[0].maxSqlLen = TSDB_PAYLOAD_SIZE; - g_Dbs.do_aggreFunc = true; char dataString[STRING_LEN]; @@ -5105,29 +5206,52 @@ void setParaFromArg(){ g_Dbs.do_aggreFunc = false; } - g_Dbs.db[0].superTbls[0].columnCount = 0; - for (int i = 0; i < MAX_NUM_DATATYPE; i++) { - if (data_type[i] == NULL) { - break; - } + if (g_args.use_metric) { + g_Dbs.db[0].superTblCount = 1; + tstrncpy(g_Dbs.db[0].superTbls[0].sTblName, "meters", MAX_TB_NAME_SIZE); + g_Dbs.db[0].superTbls[0].childTblCount = g_args.num_of_tables; + g_Dbs.threadCount = g_args.num_of_threads; + g_Dbs.threadCountByCreateTbl = 1; + g_Dbs.queryMode = g_args.mode; + + g_Dbs.db[0].superTbls[0].autoCreateTable = PRE_CREATE_SUBTBL; + g_Dbs.db[0].superTbls[0].superTblExists = TBL_NO_EXISTS; + g_Dbs.db[0].superTbls[0].childTblExists = TBL_NO_EXISTS; + g_Dbs.db[0].superTbls[0].disorderRange = g_args.disorderRange; + g_Dbs.db[0].superTbls[0].disorderRatio = g_args.disorderRatio; + tstrncpy(g_Dbs.db[0].superTbls[0].childTblPrefix, + g_args.tb_prefix, MAX_TB_NAME_SIZE); + tstrncpy(g_Dbs.db[0].superTbls[0].dataSource, "rand", MAX_TB_NAME_SIZE); + tstrncpy(g_Dbs.db[0].superTbls[0].insertMode, "taosc", MAX_TB_NAME_SIZE); + tstrncpy(g_Dbs.db[0].superTbls[0].startTimestamp, + "2017-07-14 10:40:00.000", MAX_TB_NAME_SIZE); + g_Dbs.db[0].superTbls[0].timeStampStep = 10; + + g_Dbs.db[0].superTbls[0].insertRows = g_args.num_of_DPT; + g_Dbs.db[0].superTbls[0].maxSqlLen = TSDB_PAYLOAD_SIZE; - tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, - data_type[i], MAX_TB_NAME_SIZE); - g_Dbs.db[0].superTbls[0].columns[i].dataLen = g_args.len_of_binary; - g_Dbs.db[0].superTbls[0].columnCount++; - } - - if (g_Dbs.db[0].superTbls[0].columnCount > g_args.num_of_CPR) { - g_Dbs.db[0].superTbls[0].columnCount = g_args.num_of_CPR; - } else { - for (int i = g_Dbs.db[0].superTbls[0].columnCount; i < g_args.num_of_CPR; i++) { - tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, "INT", MAX_TB_NAME_SIZE); - g_Dbs.db[0].superTbls[0].columns[i].dataLen = 0; + g_Dbs.db[0].superTbls[0].columnCount = 0; + for (int i = 0; i < MAX_NUM_DATATYPE; i++) { + if (data_type[i] == NULL) { + break; + } + + tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, + data_type[i], MAX_TB_NAME_SIZE); + g_Dbs.db[0].superTbls[0].columns[i].dataLen = g_args.len_of_binary; g_Dbs.db[0].superTbls[0].columnCount++; } - } + + if (g_Dbs.db[0].superTbls[0].columnCount > g_args.num_of_CPR) { + g_Dbs.db[0].superTbls[0].columnCount = g_args.num_of_CPR; + } else { + for (int i = g_Dbs.db[0].superTbls[0].columnCount; i < g_args.num_of_CPR; i++) { + tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, "INT", MAX_TB_NAME_SIZE); + g_Dbs.db[0].superTbls[0].columns[i].dataLen = 0; + g_Dbs.db[0].superTbls[0].columnCount++; + } + } - if (g_Dbs.use_metric) { tstrncpy(g_Dbs.db[0].superTbls[0].tags[0].dataType, "INT", MAX_TB_NAME_SIZE); g_Dbs.db[0].superTbls[0].tags[0].dataLen = 0; @@ -5137,6 +5261,7 @@ void setParaFromArg(){ } else { g_Dbs.db[0].superTbls[0].tagCount = 0; } + } /* Function to do regular expression check */ @@ -5206,6 +5331,7 @@ void querySqlFile(TAOS* taos, char* sqlFile) } memcpy(cmd + cmd_len, line, read_len); + debugPrint("DEBUG %s() %d \n", __func__, __LINE__); queryDbExec(taos, cmd, NO_INSERT_TYPE); memset(cmd, 0, MAX_SQL_SIZE); cmd_len = 0; @@ -5223,6 +5349,8 @@ void querySqlFile(TAOS* taos, char* sqlFile) int main(int argc, char *argv[]) { parse_args(argc, argv, &g_args); + debugPrint("DEBUG - meta file: %s\n", g_args.metaFile); + if (g_args.metaFile) { initOfInsertMeta(); initOfQueryMeta(); @@ -5232,14 +5360,14 @@ int main(int argc, char *argv[]) { return 1; } - if (INSERT_MODE == g_jsonType) { + if (INSERT_MODE == g_args.test_mode) { if (g_Dbs.cfgDir[0]) taos_options(TSDB_OPTION_CONFIGDIR, g_Dbs.cfgDir); (void)insertTestProcess(); - } else if (QUERY_MODE == g_jsonType) { + } else if (QUERY_MODE == g_args.test_mode) { if (g_queryInfo.cfgDir[0]) taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir); (void)queryTestProcess(); - } else if (SUBSCRIBE_MODE == g_jsonType) { + } else if (SUBSCRIBE_MODE == g_args.test_mode) { if (g_queryInfo.cfgDir[0]) taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir); (void)subscribeTestProcess(); @@ -5248,7 +5376,7 @@ int main(int argc, char *argv[]) { } } else { memset(&g_Dbs, 0, sizeof(SDbs)); - g_jsonType = INSERT_MODE; + g_args.test_mode = INSERT_MODE; setParaFromArg(); if (NULL != g_args.sqlFile) { @@ -5262,7 +5390,7 @@ int main(int argc, char *argv[]) { taos_close(qtaos); return 0; } - + (void)insertTestProcess(); if (g_Dbs.insert_only) return 0; From 17dc360f6476a877bc2f6547db78b08027721c74 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 4 Mar 2021 20:03:12 +0800 Subject: [PATCH 40/66] [TD-3147] : support insert interval. support no-stb in middle. --- src/kit/taosdemo/insert.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kit/taosdemo/insert.json b/src/kit/taosdemo/insert.json index c532448384..5276b9fb61 100644 --- a/src/kit/taosdemo/insert.json +++ b/src/kit/taosdemo/insert.json @@ -13,7 +13,7 @@ "num_of_records_per_req": 100, "databases": [{ "dbinfo": { - "name": "dbx", + "name": "db", "drop": "yes", "replica": 1, "days": 10, From 471dc330934b79ff049785bd2c9ca57fa5bbe2ee Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 5 Mar 2021 00:21:00 +0800 Subject: [PATCH 41/66] [TD-3147] : support insert interval. refactor. --- src/kit/taosdemo/taosdemo.c | 313 +++++++++++++++++++++--------------- 1 file changed, 179 insertions(+), 134 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index d7725a807c..5eab2fde63 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -458,9 +458,9 @@ void resetAfterAnsiEscape(void) { } #endif -static int createDatabases(); -static void createChildTables(); -static int queryDbExec(TAOS *taos, char *command, int type); +int createDatabases(); +void createChildTables(); +int queryDbExec(TAOS *taos, char *command, int type); /* ************ Global variables ************ */ @@ -774,7 +774,7 @@ void tmfree(char *buf) { } } -static int queryDbExec(TAOS *taos, char *command, int type) { +int queryDbExec(TAOS *taos, char *command, int type) { int i; TAOS_RES *res = NULL; int32_t code = -1; @@ -1921,10 +1921,14 @@ static int createSuperTable(TAOS * taos, char* dbName, SSuperTable* superTbls, char* dataType = superTbls->columns[colIndex].dataType; if (strcasecmp(dataType, "BINARY") == 0) { - len += snprintf(cols + len, STRING_LEN - len, ", col%d %s(%d)", colIndex, "BINARY", superTbls->columns[colIndex].dataLen); + len += snprintf(cols + len, STRING_LEN - len, + ", col%d %s(%d)", colIndex, "BINARY", + superTbls->columns[colIndex].dataLen); lenOfOneRow += superTbls->columns[colIndex].dataLen + 3; } else if (strcasecmp(dataType, "NCHAR") == 0) { - len += snprintf(cols + len, STRING_LEN - len, ", col%d %s(%d)", colIndex, "NCHAR", superTbls->columns[colIndex].dataLen); + len += snprintf(cols + len, STRING_LEN - len, + ", col%d %s(%d)", colIndex, "NCHAR", + superTbls->columns[colIndex].dataLen); lenOfOneRow += superTbls->columns[colIndex].dataLen + 3; } else if (strcasecmp(dataType, "INT") == 0) { len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "INT"); @@ -2028,65 +2032,77 @@ static int createSuperTable(TAOS * taos, char* dbName, SSuperTable* superTbls, } -static int createDatabases() { +int createDatabases() { TAOS * taos = NULL; int ret = 0; taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, NULL, g_Dbs.port); if (taos == NULL) { fprintf(stderr, "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL)); - exit(-1); + return -1; } char command[BUFFER_SIZE] = "\0"; - for (int i = 0; i < g_Dbs.dbCount; i++) { if (g_Dbs.db[i].drop) { sprintf(command, "drop database if exists %s;", g_Dbs.db[i].dbName); - debugPrint("DEBUG %s() %d \n", __func__, __LINE__); + debugPrint("DEBUG %s() %d \n", __func__, __LINE__); if (0 != queryDbExec(taos, command, NO_INSERT_TYPE)) { taos_close(taos); return -1; } } - + int dataLen = 0; - dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "create database if not exists %s ", g_Dbs.db[i].dbName); + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, "create database if not exists %s ", g_Dbs.db[i].dbName); if (g_Dbs.db[i].dbCfg.blocks > 0) { - dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "blocks %d ", g_Dbs.db[i].dbCfg.blocks); + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, "blocks %d ", g_Dbs.db[i].dbCfg.blocks); } if (g_Dbs.db[i].dbCfg.cache > 0) { - dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "cache %d ", g_Dbs.db[i].dbCfg.cache); + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, "cache %d ", g_Dbs.db[i].dbCfg.cache); } if (g_Dbs.db[i].dbCfg.days > 0) { - dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "days %d ", g_Dbs.db[i].dbCfg.days); + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, "days %d ", g_Dbs.db[i].dbCfg.days); } if (g_Dbs.db[i].dbCfg.keep > 0) { - dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "keep %d ", g_Dbs.db[i].dbCfg.keep); + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, "keep %d ", g_Dbs.db[i].dbCfg.keep); } if (g_Dbs.db[i].dbCfg.replica > 0) { - dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "replica %d ", g_Dbs.db[i].dbCfg.replica); + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, "replica %d ", g_Dbs.db[i].dbCfg.replica); } if (g_Dbs.db[i].dbCfg.update > 0) { - dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "update %d ", g_Dbs.db[i].dbCfg.update); + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, "update %d ", g_Dbs.db[i].dbCfg.update); } //if (g_Dbs.db[i].dbCfg.maxtablesPerVnode > 0) { - // dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "tables %d ", g_Dbs.db[i].dbCfg.maxtablesPerVnode); + // dataLen += snprintf(command + dataLen, + // BUFFER_SIZE - dataLen, "tables %d ", g_Dbs.db[i].dbCfg.maxtablesPerVnode); //} if (g_Dbs.db[i].dbCfg.minRows > 0) { - dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "minrows %d ", g_Dbs.db[i].dbCfg.minRows); + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, "minrows %d ", g_Dbs.db[i].dbCfg.minRows); } if (g_Dbs.db[i].dbCfg.maxRows > 0) { - dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "maxrows %d ", g_Dbs.db[i].dbCfg.maxRows); + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, "maxrows %d ", g_Dbs.db[i].dbCfg.maxRows); } if (g_Dbs.db[i].dbCfg.comp > 0) { - dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "comp %d ", g_Dbs.db[i].dbCfg.comp); + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, "comp %d ", g_Dbs.db[i].dbCfg.comp); } if (g_Dbs.db[i].dbCfg.walLevel > 0) { - dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "wal %d ", g_Dbs.db[i].dbCfg.walLevel); + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, "wal %d ", g_Dbs.db[i].dbCfg.walLevel); } if (g_Dbs.db[i].dbCfg.cacheLast > 0) { - dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "cachelast %d ", g_Dbs.db[i].dbCfg.cacheLast); + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, "cachelast %d ", g_Dbs.db[i].dbCfg.cacheLast); } if (g_Dbs.db[i].dbCfg.fsync > 0) { dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "fsync %d ", g_Dbs.db[i].dbCfg.fsync); @@ -2100,6 +2116,7 @@ static int createDatabases() { debugPrint("DEBUG %s() %d \n", __func__, __LINE__); if (0 != queryDbExec(taos, command, NO_INSERT_TYPE)) { taos_close(taos); + printf("\ncreate database %s failed!\n\n", g_Dbs.db[i].dbName); return -1; } printf("\ncreate database %s success!\n\n", g_Dbs.db[i].dbName); @@ -2108,7 +2125,7 @@ static int createDatabases() { for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { // describe super table, if exists sprintf(command, "describe %s.%s;", g_Dbs.db[i].dbName, g_Dbs.db[i].superTbls[j].sTblName); - debugPrint("DEBUG %s() %d \n", __func__, __LINE__); + debugPrint("DEBUG %s() %d \n", __func__, __LINE__); if (0 != queryDbExec(taos, command, NO_INSERT_TYPE)) { g_Dbs.db[i].superTbls[j].superTblExists = TBL_NO_EXISTS; ret = createSuperTable(taos, g_Dbs.db[i].dbName, &g_Dbs.db[i].superTbls[j], g_Dbs.use_metric); @@ -2118,8 +2135,11 @@ static int createDatabases() { } if (0 != ret) { + printf("\ncreate super table %d failed!\n\n", j); taos_close(taos); return -1; + } else { + printf("\ncreate super table %d success!\n\n", j); } } } @@ -2135,14 +2155,24 @@ void * createTable(void *sarg) int64_t lastPrintTime = taosGetTimestampMs(); - char* buffer = calloc(superTblInfo->maxSqlLen, 1); + int buff_len; + if (superTblInfo) + buff_len = superTblInfo->maxSqlLen; + else + buff_len = BUFFER_SIZE; + + char *buffer = calloc(superTblInfo->maxSqlLen, 1); + if (buffer == NULL) { + fprintf(stderr, "Memory allocated failed!"); + exit(-1); + } int len = 0; int batchNum = 0; //printf("Creating table from %d to %d\n", winfo->start_table_id, winfo->end_table_id); for (int i = winfo->start_table_id; i <= winfo->end_table_id; i++) { if (0 == g_Dbs.use_metric) { - snprintf(buffer, BUFFER_SIZE, + snprintf(buffer, buff_len, "create table if not exists %s.%s%d %s;", winfo->db_name, superTblInfo->childTblPrefix, i, @@ -2150,11 +2180,11 @@ void * createTable(void *sarg) } else { if (0 == len) { batchNum = 0; - memset(buffer, 0, superTblInfo->maxSqlLen); + memset(buffer, 0, buff_len); len += snprintf(buffer + len, - superTblInfo->maxSqlLen - len, "create table "); + buff_len - len, "create table "); } - + char* tagsValBuf = NULL; if (0 == superTblInfo->tagSource) { tagsValBuf = generateTagVaulesForStb(superTblInfo); @@ -2208,7 +2238,7 @@ void * createTable(void *sarg) return NULL; } -void startMultiThreadCreateChildTable( +int startMultiThreadCreateChildTable( char* cols, int threads, int ntables, char* db_name, SSuperTable* superTblInfo) { pthread_t *pids = malloc(threads * sizeof(pthread_t)); @@ -2231,7 +2261,7 @@ void startMultiThreadCreateChildTable( int b = 0; b = ntables % threads; - + int last = 0; for (int i = 0; i < threads; i++) { threadInfo *t_info = infos + i; @@ -2244,6 +2274,10 @@ void startMultiThreadCreateChildTable( g_Dbs.password, db_name, g_Dbs.port); + if (t_info->taos == NULL) { + fprintf(stderr, "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL)); + return -1; + } t_info->start_table_id = last; t_info->end_table_id = i < b ? last + a : last + a - 1; last = t_info->end_table_id + 1; @@ -2264,23 +2298,35 @@ void startMultiThreadCreateChildTable( free(pids); free(infos); + + return 0; } -static void createChildTables() { +void createChildTables() { for (int i = 0; i < g_Dbs.dbCount; i++) { - for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { - if ((AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) + if (g_Dbs.db[i].superTblCount > 0) { + for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { + if ((AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) || (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists)) { - continue; - } - startMultiThreadCreateChildTable( + continue; + } + + startMultiThreadCreateChildTable( g_Dbs.db[i].superTbls[j].colsOfCreatChildTable, g_Dbs.threadCountByCreateTbl, g_Dbs.db[i].superTbls[j].childTblCount, g_Dbs.db[i].dbName, &(g_Dbs.db[i].superTbls[j])); - g_totalChildTables += g_Dbs.db[i].superTbls[j].childTblCount; - } + g_totalChildTables += g_Dbs.db[i].superTbls[j].childTblCount; + } + } else { + startMultiThreadCreateChildTable( + g_Dbs.db[i].superTbls[j].colsOfCreatChildTable, + g_Dbs.threadCountByCreateTbl, + g_args.num_of_DPT, + g_Dbs.db[i].dbName, + NULL); + } } } @@ -4524,14 +4570,6 @@ void *readMetric(void *sarg) { int insertTestProcess() { - debugPrint("DEBUG - %d result file: %s\n", __LINE__, g_Dbs.resultFile); - - g_fpOfInsertResult = fopen(g_Dbs.resultFile, "a"); - if (NULL == g_fpOfInsertResult) { - fprintf(stderr, "Failed to open %s for save result\n", g_Dbs.resultFile); - return 1; - }; - setupForAnsiEscape(); int ret = printfInsertMeta(); resetAfterAnsiEscape(); @@ -4539,7 +4577,14 @@ int insertTestProcess() { if (ret == -1) exit(EXIT_FAILURE); - printfInsertMetaToFile(g_fpOfInsertResult); + debugPrint("DEBUG - %d result file: %s\n", __LINE__, g_Dbs.resultFile); + g_fpOfInsertResult = fopen(g_Dbs.resultFile, "a"); + if (NULL == g_fpOfInsertResult) { + fprintf(stderr, "Failed to open %s for save result\n", g_Dbs.resultFile); + return -1; + } { + printfInsertMetaToFile(g_fpOfInsertResult); + } if (!g_args.answer_yes) { printf("Press enter key to continue\n\n"); @@ -4549,7 +4594,10 @@ int insertTestProcess() { init_rand_data(); // create database and super tables - (void)createDatabases(); + if( createDatabases() != 0) { + fclose(g_fpOfInsertResult); + return -1; + } // pretreatement prePareSampleData(); @@ -4561,6 +4609,7 @@ int insertTestProcess() { start = getCurrentTime(); createChildTables(); end = getCurrentTime(); + if (g_totalChildTables > 0) { printf("Spent %.4f seconds to create %d tables with %d thread(s)\n\n", end - start, g_totalChildTables, g_Dbs.threadCount); @@ -4570,7 +4619,6 @@ int insertTestProcess() { } taosMsleep(1000); - // create sub threads for inserting data //start = getCurrentTime(); for (int i = 0; i < g_Dbs.dbCount; i++) { @@ -4604,36 +4652,8 @@ int insertTestProcess() { // totalAffectedRows += g_Dbs.db[i].superTbls[j].totalAffectedRows; //} //printf("Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s)\n\n", end - start, totalRowsInserted, totalAffectedRows, g_Dbs.threadCount); - if (NULL == g_args.metaFile && false == g_Dbs.insert_only) { - // query data - pthread_t read_id; - threadInfo *rInfo = malloc(sizeof(threadInfo)); - rInfo->start_time = 1500000000000; // 2017-07-14 10:40:00.000 - rInfo->start_table_id = 0; - rInfo->end_table_id = g_Dbs.db[0].superTbls[0].childTblCount - 1; - //rInfo->do_aggreFunc = g_Dbs.do_aggreFunc; - //rInfo->nrecords_per_table = g_Dbs.db[0].superTbls[0].insertRows; - rInfo->superTblInfo = &g_Dbs.db[0].superTbls[0]; - rInfo->taos = taos_connect( - g_Dbs.host, - g_Dbs.user, - g_Dbs.password, - g_Dbs.db[0].dbName, - g_Dbs.port); - strcpy(rInfo->tb_prefix, g_Dbs.db[0].superTbls[0].childTblPrefix); - strcpy(rInfo->fp, g_Dbs.resultFile); - - if (!g_Dbs.use_metric) { - pthread_create(&read_id, NULL, readTable, rInfo); - } else { - pthread_create(&read_id, NULL, readMetric, rInfo); - } - pthread_join(read_id, NULL); - taos_close(rInfo->taos); - } - postFreeResource(); - + return 0; } @@ -5259,6 +5279,7 @@ void setParaFromArg(){ g_Dbs.db[0].superTbls[0].tags[1].dataLen = g_args.len_of_binary; g_Dbs.db[0].superTbls[0].tagCount = 2; } else { + g_Dbs.threadCountByCreateTbl = 1; g_Dbs.db[0].superTbls[0].tagCount = 0; } @@ -5306,7 +5327,7 @@ void querySqlFile(TAOS* taos, char* sqlFile) printf("failed to open file %s, reason:%s\n", sqlFile, strerror(errno)); return; } - + int read_len = 0; char * cmd = calloc(1, MAX_SQL_SIZE); size_t cmd_len = 0; @@ -5314,7 +5335,7 @@ void querySqlFile(TAOS* taos, char* sqlFile) size_t line_len = 0; double t = getCurrentTime(); - + while ((read_len = tgetline(&line, &line_len, fp)) != -1) { if (read_len >= MAX_SQL_SIZE) continue; line[--read_len] = '\0'; @@ -5346,6 +5367,78 @@ void querySqlFile(TAOS* taos, char* sqlFile) return; } + + +void testMetaFile() { + if (INSERT_MODE == g_args.test_mode) { + if (g_Dbs.cfgDir[0]) taos_options(TSDB_OPTION_CONFIGDIR, g_Dbs.cfgDir); + insertTestProcess(); + } else if (QUERY_MODE == g_args.test_mode) { + if (g_queryInfo.cfgDir[0]) + taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir); + (void)queryTestProcess(); + } else if (SUBSCRIBE_MODE == g_args.test_mode) { + if (g_queryInfo.cfgDir[0]) + taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir); + (void)subscribeTestProcess(); + } else { + ; + } +} + +void testCmdLine() { + + g_args.test_mode = INSERT_MODE; + insertTestProcess(); + + if (g_Dbs.insert_only) + return; + + // select + if (false == g_Dbs.insert_only) { + // query data + + pthread_t read_id; + threadInfo *rInfo = malloc(sizeof(threadInfo)); + rInfo->start_time = 1500000000000; // 2017-07-14 10:40:00.000 + rInfo->start_table_id = 0; + + //rInfo->do_aggreFunc = g_Dbs.do_aggreFunc; + if (g_args.use_metric) { + rInfo->end_table_id = g_Dbs.db[0].superTbls[0].childTblCount - 1; + rInfo->superTblInfo = &g_Dbs.db[0].superTbls[0]; + strcpy(rInfo->tb_prefix, + g_Dbs.db[0].superTbls[0].childTblPrefix); + } else { + rInfo->end_table_id = g_args.num_of_tables -1; + strcpy(rInfo->tb_prefix, g_args.tb_prefix); + } + + rInfo->taos = taos_connect( + g_Dbs.host, + g_Dbs.user, + g_Dbs.password, + g_Dbs.db[0].dbName, + g_Dbs.port); + if (rInfo->taos == NULL) { + fprintf(stderr, "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL)); + free(rInfo); + exit(-1); + } + + strcpy(rInfo->fp, g_Dbs.resultFile); + + if (!g_Dbs.use_metric) { + pthread_create(&read_id, NULL, readTable, rInfo); + } else { + pthread_create(&read_id, NULL, readMetric, rInfo); + } + pthread_join(read_id, NULL); + taos_close(rInfo->taos); + free(rInfo); + } +} + int main(int argc, char *argv[]) { parse_args(argc, argv, &g_args); @@ -5360,23 +5453,9 @@ int main(int argc, char *argv[]) { return 1; } - if (INSERT_MODE == g_args.test_mode) { - if (g_Dbs.cfgDir[0]) taos_options(TSDB_OPTION_CONFIGDIR, g_Dbs.cfgDir); - (void)insertTestProcess(); - } else if (QUERY_MODE == g_args.test_mode) { - if (g_queryInfo.cfgDir[0]) - taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir); - (void)queryTestProcess(); - } else if (SUBSCRIBE_MODE == g_args.test_mode) { - if (g_queryInfo.cfgDir[0]) - taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir); - (void)subscribeTestProcess(); - } else { - ; - } - } else { + testMetaFile(); + } else { memset(&g_Dbs, 0, sizeof(SDbs)); - g_args.test_mode = INSERT_MODE; setParaFromArg(); if (NULL != g_args.sqlFile) { @@ -5388,46 +5467,12 @@ int main(int argc, char *argv[]) { g_Dbs.port); querySqlFile(qtaos, g_args.sqlFile); taos_close(qtaos); - return 0; - } - (void)insertTestProcess(); - if (g_Dbs.insert_only) return 0; - - // select - if (false == g_Dbs.insert_only) { - // query data - - pthread_t read_id; - threadInfo *rInfo = malloc(sizeof(threadInfo)); - rInfo->start_time = 1500000000000; // 2017-07-14 10:40:00.000 - rInfo->start_table_id = 0; - rInfo->end_table_id = g_Dbs.db[0].superTbls[0].childTblCount - 1; - //rInfo->do_aggreFunc = g_Dbs.do_aggreFunc; - //rInfo->nrecords_per_table = g_Dbs.db[0].superTbls[0].insertRows; - rInfo->superTblInfo = &g_Dbs.db[0].superTbls[0]; - rInfo->taos = taos_connect( - g_Dbs.host, - g_Dbs.user, - g_Dbs.password, - g_Dbs.db[0].dbName, - g_Dbs.port); - strcpy(rInfo->tb_prefix, - g_Dbs.db[0].superTbls[0].childTblPrefix); - strcpy(rInfo->fp, g_Dbs.resultFile); - - if (!g_Dbs.use_metric) { - pthread_create(&read_id, NULL, readTable, rInfo); - } else { - pthread_create(&read_id, NULL, readMetric, rInfo); - } - pthread_join(read_id, NULL); - taos_close(rInfo->taos); - free(rInfo); + } else { + testCmdLine(); } } - taos_cleanup(); return 0; } From 179a4f5588ac984f6d95f87d312f96abd36c6186 Mon Sep 17 00:00:00 2001 From: zyyang Date: Fri, 5 Mar 2021 09:28:06 +0800 Subject: [PATCH 42/66] [TD-3142]: fix JDBC-RESTful back ResultSet with null Type --- .../jdbc/DatabaseMetaDataResultSet.java | 2 +- .../java/com/taosdata/jdbc/TSDBConstants.java | 68 +++++++++- .../com/taosdata/jdbc/TSDBErrorNumbers.java | 2 + .../taosdata/jdbc/TSDBResultSetMetaData.java | 28 ++-- .../taosdata/jdbc/TSDBResultSetWrapper.java | 4 +- .../taosdata/jdbc/rs/RestfulResultSet.java | 126 +++++++++++++----- .../jdbc/rs/RestfulResultSetMetaData.java | 82 +++++------- .../taosdata/jdbc/rs/RestfulStatement.java | 33 +++-- .../java/com/taosdata/jdbc/rs/SQLTest.java | 16 ++- .../taosdemo/TaosDemoApplication.java | 12 +- ...{SqlExecuteor.java => SqlExecuteTask.java} | 4 +- 11 files changed, 234 insertions(+), 143 deletions(-) rename tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/{SqlExecuteor.java => SqlExecuteTask.java} (89%) diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/DatabaseMetaDataResultSet.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/DatabaseMetaDataResultSet.java index 66dc07a634..f6a0fcca31 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/DatabaseMetaDataResultSet.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/DatabaseMetaDataResultSet.java @@ -308,7 +308,7 @@ public class DatabaseMetaDataResultSet implements ResultSet { return colMetaData.getColIndex() + 1; } } - throw new SQLException(TSDBConstants.INVALID_VARIABLES); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); } @Override diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java index 043db9bbd7..6179b47da1 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java @@ -14,16 +14,13 @@ *****************************************************************************/ package com.taosdata.jdbc; +import java.sql.SQLException; +import java.sql.Types; import java.util.HashMap; import java.util.Map; public abstract class TSDBConstants { - public static final String STATEMENT_CLOSED = "statement is closed"; - public static final String UNSUPPORTED_METHOD_EXCEPTION_MSG = "this operation is NOT supported currently!"; - public static final String INVALID_VARIABLES = "invalid variables"; - public static final String RESULT_SET_IS_CLOSED = "resultSet is closed"; - public static final String DEFAULT_PORT = "6200"; public static Map DATATYPE_MAP = null; @@ -77,8 +74,65 @@ public abstract class TSDBConstants { return WrapErrMsg("unkown error!"); } + public static int taosType2JdbcType(int taosType) throws SQLException { + switch (taosType) { + case TSDBConstants.TSDB_DATA_TYPE_NULL: + return Types.NULL; + case TSDBConstants.TSDB_DATA_TYPE_BOOL: + return Types.BOOLEAN; + case TSDBConstants.TSDB_DATA_TYPE_TINYINT: + return Types.TINYINT; + case TSDBConstants.TSDB_DATA_TYPE_SMALLINT: + return Types.SMALLINT; + case TSDBConstants.TSDB_DATA_TYPE_INT: + return Types.INTEGER; + case TSDBConstants.TSDB_DATA_TYPE_BIGINT: + return Types.BIGINT; + case TSDBConstants.TSDB_DATA_TYPE_FLOAT: + return Types.FLOAT; + case TSDBConstants.TSDB_DATA_TYPE_DOUBLE: + return Types.DOUBLE; + case TSDBConstants.TSDB_DATA_TYPE_BINARY: + return Types.BINARY; + case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP: + return Types.TIMESTAMP; + case TSDBConstants.TSDB_DATA_TYPE_NCHAR: + return Types.NCHAR; + } + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNKNOWN_SQL_TYPE_IN_TDENGINE); + } + + public static int jdbcType2TaosType(int jdbcType) throws SQLException { + switch (jdbcType){ + case Types.NULL: + return TSDBConstants.TSDB_DATA_TYPE_NULL; + case Types.BOOLEAN: + return TSDBConstants.TSDB_DATA_TYPE_BOOL; + case Types.TINYINT: + return TSDBConstants.TSDB_DATA_TYPE_TINYINT; + case Types.SMALLINT: + return TSDBConstants.TSDB_DATA_TYPE_SMALLINT; + case Types.INTEGER: + return TSDBConstants.TSDB_DATA_TYPE_INT; + case Types.BIGINT: + return TSDBConstants.TSDB_DATA_TYPE_BIGINT; + case Types.FLOAT: + return TSDBConstants.TSDB_DATA_TYPE_FLOAT; + case Types.DOUBLE: + return TSDBConstants.TSDB_DATA_TYPE_DOUBLE; + case Types.BINARY: + return TSDBConstants.TSDB_DATA_TYPE_BINARY; + case Types.TIMESTAMP: + return TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP; + case Types.NCHAR: + return TSDBConstants.TSDB_DATA_TYPE_NCHAR; + } + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNKNOWN_SQL_TYPE_IN_TDENGINE); + } + static { DATATYPE_MAP = new HashMap<>(); + DATATYPE_MAP.put(0, "NULL"); DATATYPE_MAP.put(1, "BOOL"); DATATYPE_MAP.put(2, "TINYINT"); DATATYPE_MAP.put(3, "SMALLINT"); @@ -90,4 +144,8 @@ public abstract class TSDBConstants { DATATYPE_MAP.put(9, "TIMESTAMP"); DATATYPE_MAP.put(10, "NCHAR"); } + + public static String jdbcType2TaosTypeName(int type) throws SQLException { + return DATATYPE_MAP.get(jdbcType2TaosType(type)); + } } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java index 3ae8696f7d..78e7aec79c 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java @@ -18,6 +18,7 @@ public class TSDBErrorNumbers { public static final int ERROR_INVALID_FOR_EXECUTE = 0x230c; //not a valid sql for execute: (SQL) public static final int ERROR_PARAMETER_INDEX_OUT_RANGE = 0x230d; // parameter index out of range public static final int ERROR_SQLCLIENT_EXCEPTION_ON_CONNECTION_CLOSED = 0x230e; // connection already closed + public static final int ERROR_UNKNOWN_SQL_TYPE_IN_TDENGINE = 0x230f; //unknown sql type in tdengine public static final int ERROR_UNKNOWN = 0x2350; //unknown error @@ -49,6 +50,7 @@ public class TSDBErrorNumbers { errorNumbers.add(ERROR_INVALID_FOR_EXECUTE); errorNumbers.add(ERROR_PARAMETER_INDEX_OUT_RANGE); errorNumbers.add(ERROR_SQLCLIENT_EXCEPTION_ON_CONNECTION_CLOSED); + errorNumbers.add(ERROR_UNKNOWN_SQL_TYPE_IN_TDENGINE); /*****************************************************/ errorNumbers.add(ERROR_SUBSCRIBE_FAILED); diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetMetaData.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetMetaData.java index 0c0071a949..e0b1c246cb 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetMetaData.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetMetaData.java @@ -20,7 +20,7 @@ import java.sql.Timestamp; import java.sql.Types; import java.util.List; -public class TSDBResultSetMetaData implements ResultSetMetaData { +public class TSDBResultSetMetaData extends WrapperImpl implements ResultSetMetaData { List colMetaDataList = null; @@ -28,14 +28,6 @@ public class TSDBResultSetMetaData implements ResultSetMetaData { this.colMetaDataList = metaDataList; } - public T unwrap(Class iface) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORTED_METHOD_EXCEPTION_MSG); - } - - public boolean isWrapperFor(Class iface) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORTED_METHOD_EXCEPTION_MSG); - } - public int getColumnCount() throws SQLException { return colMetaDataList.size(); } @@ -94,7 +86,7 @@ public class TSDBResultSetMetaData implements ResultSetMetaData { } public String getSchemaName(int column) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORTED_METHOD_EXCEPTION_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } public int getPrecision(int column) throws SQLException { @@ -125,18 +117,18 @@ public class TSDBResultSetMetaData implements ResultSetMetaData { } public String getTableName(int column) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORTED_METHOD_EXCEPTION_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } public String getCatalogName(int column) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORTED_METHOD_EXCEPTION_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } public int getColumnType(int column) throws SQLException { ColumnMetaData meta = this.colMetaDataList.get(column - 1); switch (meta.getColType()) { case TSDBConstants.TSDB_DATA_TYPE_BOOL: - return java.sql.Types.BIT; + return Types.BOOLEAN; case TSDBConstants.TSDB_DATA_TYPE_TINYINT: return java.sql.Types.TINYINT; case TSDBConstants.TSDB_DATA_TYPE_SMALLINT: @@ -150,13 +142,13 @@ public class TSDBResultSetMetaData implements ResultSetMetaData { case TSDBConstants.TSDB_DATA_TYPE_DOUBLE: return java.sql.Types.DOUBLE; case TSDBConstants.TSDB_DATA_TYPE_BINARY: - return java.sql.Types.CHAR; + return Types.BINARY; case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP: - return java.sql.Types.BIGINT; + return java.sql.Types.TIMESTAMP; case TSDBConstants.TSDB_DATA_TYPE_NCHAR: - return java.sql.Types.CHAR; + return Types.NCHAR; } - throw new SQLException(TSDBConstants.INVALID_VARIABLES); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); } public String getColumnTypeName(int column) throws SQLException { @@ -173,7 +165,7 @@ public class TSDBResultSetMetaData implements ResultSetMetaData { } public boolean isDefinitelyWritable(int column) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORTED_METHOD_EXCEPTION_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } public String getColumnClassName(int column) throws SQLException { diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetWrapper.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetWrapper.java index 98b823a3c1..48854e773f 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetWrapper.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetWrapper.java @@ -1153,11 +1153,11 @@ public class TSDBResultSetWrapper implements ResultSet { } public T getObject(int columnIndex, Class type) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORTED_METHOD_EXCEPTION_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } public T getObject(String columnLabel, Class type) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORTED_METHOD_EXCEPTION_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java index b7a0df7de7..0bdc33e973 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java @@ -18,10 +18,10 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { private final String database; private final Statement statement; // data - private ArrayList> resultSet = new ArrayList<>(); + private ArrayList> resultSet; // meta - private ArrayList columnNames = new ArrayList<>(); - private ArrayList columns = new ArrayList<>(); + private ArrayList columnNames; + private ArrayList columns; private RestfulResultSetMetaData metaData; /** @@ -29,11 +29,36 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { * * @param resultJson: 包含data信息的结果集，有sql返回的结果集 ***/ - public RestfulResultSet(String database, Statement statement, JSONObject resultJson) { + public RestfulResultSet(String database, Statement statement, JSONObject resultJson) throws SQLException { this.database = database; this.statement = statement; + // column metadata + JSONArray columnMeta = resultJson.getJSONArray("column_meta"); + columnNames = new ArrayList<>(); + columns = new ArrayList<>(); + for (int colIndex = 0; colIndex < columnMeta.size(); colIndex++) { + JSONArray col = columnMeta.getJSONArray(colIndex); + String col_name = col.getString(0); + int col_type = TSDBConstants.taosType2JdbcType(col.getInteger(1)); + int col_length = col.getInteger(2); + columnNames.add(col_name); + columns.add(new Field(col_name, col_type, col_length, "")); + } + this.metaData = new RestfulResultSetMetaData(this.database, columns, this); + // row data JSONArray data = resultJson.getJSONArray("data"); + resultSet = new ArrayList<>(); + for (int rowIndex = 0; rowIndex < data.size(); rowIndex++) { + ArrayList row = new ArrayList(); + JSONArray jsonRow = data.getJSONArray(rowIndex); + for (int colIndex = 0; colIndex < jsonRow.size(); colIndex++) { + row.add(parseColumnData(jsonRow, colIndex, columns.get(colIndex).type)); + } + resultSet.add(row); + } + + /* int columnIndex = 0; for (; columnIndex < data.size(); columnIndex++) { ArrayList oneRow = new ArrayList<>(); @@ -52,50 +77,77 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { columns.add(new Field(name, "", 0, "")); } this.metaData = new RestfulResultSetMetaData(this.database, columns, this); + */ } - /** - * 由多个resultSet的JSON构造结果集 - * - * @param resultJson: 包含data信息的结果集，有sql返回的结果集 - * @param fieldJson: 包含多个（最多2个）meta信息的结果集，有describe xxx - **/ - public RestfulResultSet(String database, Statement statement, JSONObject resultJson, List fieldJson) { - this(database, statement, resultJson); - ArrayList newColumns = new ArrayList<>(); - - for (Field column : columns) { - Field field = findField(column.name, fieldJson); - if (field != null) { - newColumns.add(field); - } else { - newColumns.add(column); - } + private Object parseColumnData(JSONArray row, int colIndex, int sqlType) { + switch (sqlType) { + case Types.NULL: + return null; + case Types.BOOLEAN: + return row.getBoolean(colIndex); + case Types.TINYINT: + case Types.SMALLINT: + return row.getShort(colIndex); + case Types.INTEGER: + return row.getInteger(colIndex); + case Types.BIGINT: + return row.getBigInteger(colIndex); + case Types.FLOAT: + return row.getFloat(colIndex); + case Types.DOUBLE: + return row.getDouble(colIndex); + case Types.TIMESTAMP: + return row.getTimestamp(colIndex); + case Types.BINARY: + case Types.NCHAR: + default: + return row.getString(colIndex); } - this.columns = newColumns; - this.metaData = new RestfulResultSetMetaData(this.database, this.columns, this); } - public Field findField(String columnName, List fieldJsonList) { - for (JSONObject fieldJSON : fieldJsonList) { - JSONArray fieldDataJson = fieldJSON.getJSONArray("data"); - for (int i = 0; i < fieldDataJson.size(); i++) { - JSONArray field = fieldDataJson.getJSONArray(i); - if (columnName.equalsIgnoreCase(field.getString(0))) { - return new Field(field.getString(0), field.getString(1), field.getInteger(2), field.getString(3)); - } - } - } - return null; - } +// /** +// * 由多个resultSet的JSON构造结果集 +// * +// * @param resultJson: 包含data信息的结果集，有sql返回的结果集 +// * @param fieldJson: 包含多个（最多2个）meta信息的结果集，有describe xxx +// **/ +// public RestfulResultSet(String database, Statement statement, JSONObject resultJson, List fieldJson) throws SQLException { +// this(database, statement, resultJson); +// ArrayList newColumns = new ArrayList<>(); +// +// for (Field column : columns) { +// Field field = findField(column.name, fieldJson); +// if (field != null) { +// newColumns.add(field); +// } else { +// newColumns.add(column); +// } +// } +// this.columns = newColumns; +// this.metaData = new RestfulResultSetMetaData(this.database, this.columns, this); +// } + +// public Field findField(String columnName, List fieldJsonList) { +// for (JSONObject fieldJSON : fieldJsonList) { +// JSONArray fieldDataJson = fieldJSON.getJSONArray("data"); +// for (int i = 0; i < fieldDataJson.size(); i++) { +// JSONArray field = fieldDataJson.getJSONArray(i); +// if (columnName.equalsIgnoreCase(field.getString(0))) { +// return new Field(field.getString(0), field.getString(1), field.getInteger(2), field.getString(3)); +// } +// } +// } +// return null; +// } public class Field { String name; - String type; + int type; int length; String note; - public Field(String name, String type, int length, String note) { + public Field(String name, int type, int length, String note) { this.name = name; this.type = type; this.length = length; diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSetMetaData.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSetMetaData.java index 44a02f486b..29ba13bec1 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSetMetaData.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSetMetaData.java @@ -5,6 +5,7 @@ import com.taosdata.jdbc.TSDBConstants; import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Timestamp; +import java.sql.Types; import java.util.ArrayList; public class RestfulResultSetMetaData implements ResultSetMetaData { @@ -53,14 +54,14 @@ public class RestfulResultSetMetaData implements ResultSetMetaData { @Override public boolean isSigned(int column) throws SQLException { - String type = this.fields.get(column - 1).type.toUpperCase(); + int type = this.fields.get(column - 1).type; switch (type) { - case "TINYINT": - case "SMALLINT": - case "INT": - case "BIGINT": - case "FLOAT": - case "DOUBLE": + case Types.TINYINT: + case Types.SMALLINT: + case Types.INTEGER: + case Types.BIGINT: + case Types.FLOAT: + case Types.DOUBLE: return true; default: return false; @@ -89,14 +90,14 @@ public class RestfulResultSetMetaData implements ResultSetMetaData { @Override public int getPrecision(int column) throws SQLException { - String type = this.fields.get(column - 1).type.toUpperCase(); + int type = this.fields.get(column - 1).type; switch (type) { - case "FLOAT": + case Types.FLOAT: return 5; - case "DOUBLE": + case Types.DOUBLE: return 9; - case "BINARY": - case "NCHAR": + case Types.BINARY: + case Types.NCHAR: return this.fields.get(column - 1).length; default: return 0; @@ -105,11 +106,11 @@ public class RestfulResultSetMetaData implements ResultSetMetaData { @Override public int getScale(int column) throws SQLException { - String type = this.fields.get(column - 1).type.toUpperCase(); + int type = this.fields.get(column - 1).type; switch (type) { - case "FLOAT": + case Types.FLOAT: return 5; - case "DOUBLE": + case Types.DOUBLE: return 9; default: return 0; @@ -128,36 +129,13 @@ public class RestfulResultSetMetaData implements ResultSetMetaData { @Override public int getColumnType(int column) throws SQLException { - String type = this.fields.get(column - 1).type.toUpperCase(); - switch (type) { - case "BOOL": - return java.sql.Types.BOOLEAN; - case "TINYINT": - return java.sql.Types.TINYINT; - case "SMALLINT": - return java.sql.Types.SMALLINT; - case "INT": - return java.sql.Types.INTEGER; - case "BIGINT": - return java.sql.Types.BIGINT; - case "FLOAT": - return java.sql.Types.FLOAT; - case "DOUBLE": - return java.sql.Types.DOUBLE; - case "BINARY": - return java.sql.Types.BINARY; - case "TIMESTAMP": - return java.sql.Types.TIMESTAMP; - case "NCHAR": - return java.sql.Types.NCHAR; - } - throw new SQLException(TSDBConstants.INVALID_VARIABLES); + return this.fields.get(column - 1).type; } @Override public String getColumnTypeName(int column) throws SQLException { - String type = fields.get(column - 1).type; - return type.toUpperCase(); + int type = fields.get(column - 1).type; + return TSDBConstants.jdbcType2TaosTypeName(type); } @Override @@ -177,26 +155,26 @@ public class RestfulResultSetMetaData implements ResultSetMetaData { @Override public String getColumnClassName(int column) throws SQLException { - String type = this.fields.get(column - 1).type; + int type = this.fields.get(column - 1).type; String columnClassName = ""; switch (type) { - case "BOOL": + case Types.BOOLEAN: return Boolean.class.getName(); - case "TINYINT": - case "SMALLINT": + case Types.TINYINT: + case Types.SMALLINT: return Short.class.getName(); - case "INT": + case Types.INTEGER: return Integer.class.getName(); - case "BIGINT": + case Types.BIGINT: return Long.class.getName(); - case "FLOAT": + case Types.FLOAT: return Float.class.getName(); - case "DOUBLE": + case Types.DOUBLE: return Double.class.getName(); - case "TIMESTAMP": + case Types.TIMESTAMP: return Timestamp.class.getName(); - case "BINARY": - case "NCHAR": + case Types.BINARY: + case Types.NCHAR: return String.class.getName(); } return columnClassName; diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java index 8d67586be2..d60940d877 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java @@ -151,22 +151,21 @@ public class RestfulStatement extends AbstractStatement { throw new SQLException(TSDBConstants.WrapErrMsg("SQL execution error: " + resultJson.getString("desc") + "\n" + "error code: " + resultJson.getString("code"))); } // parse table name from sql - String[] tableIdentifiers = parseTableIdentifier(sql); - if (tableIdentifiers != null) { - List fieldJsonList = new ArrayList<>(); - for (String tableIdentifier : tableIdentifiers) { - // field meta - String fields = HttpClientPoolUtil.execute(url, "DESCRIBE " + tableIdentifier); - JSONObject fieldJson = JSON.parseObject(fields); - if (fieldJson.getString("status").equals("error")) { - throw new SQLException(TSDBConstants.WrapErrMsg("SQL execution error: " + fieldJson.getString("desc") + "\n" + "error code: " + fieldJson.getString("code"))); - } - fieldJsonList.add(fieldJson); - } - this.resultSet = new RestfulResultSet(database, this, resultJson, fieldJsonList); - } else { - this.resultSet = new RestfulResultSet(database, this, resultJson); - } +// String[] tableIdentifiers = parseTableIdentifier(sql); +// if (tableIdentifiers != null) { +// List fieldJsonList = new ArrayList<>(); +// for (String tableIdentifier : tableIdentifiers) { +// String fields = HttpClientPoolUtil.execute(url, "DESCRIBE " + tableIdentifier); +// JSONObject fieldJson = JSON.parseObject(fields); +// if (fieldJson.getString("status").equals("error")) { +// throw new SQLException(TSDBConstants.WrapErrMsg("SQL execution error: " + fieldJson.getString("desc") + "\n" + "error code: " + fieldJson.getString("code"))); +// } +// fieldJsonList.add(fieldJson); +// } +// this.resultSet = new RestfulResultSet(database, this, resultJson, fieldJsonList); +// } else { + this.resultSet = new RestfulResultSet(database, this, resultJson); +// } this.affectedRows = 0; return resultSet; } @@ -201,7 +200,7 @@ public class RestfulStatement extends AbstractStatement { @Override public ResultSet getResultSet() throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); return resultSet; } diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/SQLTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/SQLTest.java index 313abfd278..0b246e74d7 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/SQLTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/SQLTest.java @@ -11,8 +11,8 @@ import java.sql.*; @FixMethodOrder(MethodSorters.NAME_ASCENDING) public class SQLTest { - private static final String host = "127.0.0.1"; - // private static final String host = "master"; +// private static final String host = "127.0.0.1"; + private static final String host = "master"; private static Connection connection; @Test @@ -323,6 +323,18 @@ public class SQLTest { SQLExecutor.executeQuery(connection, sql); } + @Test + public void testCase052() { + String sql = "select server_status()"; + SQLExecutor.executeQuery(connection, sql); + } + + @Test + public void testCase053() { + String sql = "select avg(cpu_taosd), avg(cpu_system), max(cpu_cores), avg(mem_taosd), avg(mem_system), max(mem_total), avg(disk_used), max(disk_total), avg(band_speed), avg(io_read), avg(io_write), sum(req_http), sum(req_select), sum(req_insert) from log.dn1 where ts> now - 60m and ts<= now interval(1m) fill(value, 0)"; + SQLExecutor.executeQuery(connection, sql); + } + @BeforeClass public static void before() throws ClassNotFoundException, SQLException { Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/TaosDemoApplication.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/TaosDemoApplication.java index 6bd3523fc5..c361df82b0 100644 --- a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/TaosDemoApplication.java +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/TaosDemoApplication.java @@ -3,17 +3,15 @@ package com.taosdata.taosdemo; import com.taosdata.taosdemo.components.DataSourceFactory; import com.taosdata.taosdemo.components.JdbcTaosdemoConfig; import com.taosdata.taosdemo.domain.SuperTableMeta; -import com.taosdata.taosdemo.service.*; +import com.taosdata.taosdemo.service.DatabaseService; +import com.taosdata.taosdemo.service.SqlExecuteTask; +import com.taosdata.taosdemo.service.SubTableService; +import com.taosdata.taosdemo.service.SuperTableService; import com.taosdata.taosdemo.service.data.SuperTableMetaGenerator; -import com.taosdata.taosdemo.utils.Printer; import org.apache.log4j.Logger; import javax.sql.DataSource; import java.io.IOException; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; import java.time.Duration; import java.time.Instant; import java.util.Arrays; @@ -35,7 +33,7 @@ public class TaosDemoApplication { // 初始化 final DataSource dataSource = DataSourceFactory.getInstance(config.host, config.port, config.user, config.password); if (config.executeSql != null && !config.executeSql.isEmpty() && !config.executeSql.replaceAll("\\s", "").isEmpty()) { - Thread task = new Thread(new SqlExecuteor(dataSource, config.executeSql)); + Thread task = new Thread(new SqlExecuteTask(dataSource, config.executeSql)); task.start(); try { task.join(); diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SqlExecuteor.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SqlExecuteTask.java similarity index 89% rename from tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SqlExecuteor.java rename to tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SqlExecuteTask.java index 2016768605..ff2e4d0af0 100644 --- a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SqlExecuteor.java +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SqlExecuteTask.java @@ -8,11 +8,11 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; -public class SqlExecuteor implements Runnable { +public class SqlExecuteTask implements Runnable { private final DataSource dataSource; private final String sql; - public SqlExecuteor(DataSource dataSource, String sql) { + public SqlExecuteTask(DataSource dataSource, String sql) { this.dataSource = dataSource; this.sql = sql; } From 83ba94b6edeb3e44cf72a528dfa9a5a2f999607c Mon Sep 17 00:00:00 2001 From: zyyang Date: Fri, 5 Mar 2021 10:18:57 +0800 Subject: [PATCH 43/66] change --- .../src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java | 2 +- .../src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java index 0bdc33e973..9d394b8b03 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java @@ -98,7 +98,7 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { case Types.DOUBLE: return row.getDouble(colIndex); case Types.TIMESTAMP: - return row.getTimestamp(colIndex); + return new Timestamp(row.getDate(colIndex).getTime()); case Types.BINARY: case Types.NCHAR: default: diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java index 185c0306f5..8c3d562001 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java @@ -9,8 +9,8 @@ import java.util.Random; @FixMethodOrder(MethodSorters.NAME_ASCENDING) public class RestfulJDBCTest { - private static final String host = "127.0.0.1"; - // private static final String host = "master"; +// private static final String host = "127.0.0.1"; + private static final String host = "master"; private static Connection connection; private Random random = new Random(System.currentTimeMillis()); From b16280f964c5e97146ad6a6aa8371be2d0efc0bb Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Fri, 5 Mar 2021 10:21:05 +0800 Subject: [PATCH 44/66] [TD-3024] : clarify mnode EP set file & port total number. --- documentation20/cn/03.architecture/docs.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation20/cn/03.architecture/docs.md b/documentation20/cn/03.architecture/docs.md index 34ffb5d944..dc710704e5 100644 --- a/documentation20/cn/03.architecture/docs.md +++ b/documentation20/cn/03.architecture/docs.md @@ -178,11 +178,11 @@ TDengine 分布式架构的逻辑结构图如下： **FQDN配置**：一个数据节点有一个或多个FQDN，可以在系统配置文件taos.cfg通过参数“fqdn"进行指定，如果没有指定，系统将自动获取计算机的hostname作为其FQDN。如果节点没有配置FQDN，可以直接将该节点的配置参数fqdn设置为它的IP地址。但不建议使用IP，因为IP地址可变，一旦变化，将让集群无法正常工作。一个数据节点的EP(End Point)由FQDN + Port组成。采用FQDN，需要保证DNS服务正常工作，或者在节点以及应用所在的节点配置好hosts文件。 -**端口配置：**一个数据节点对外的端口由TDengine的系统配置参数serverPort决定，对集群内部通讯的端口是serverPort+5。集群内数据节点之间的数据复制操作还占有一个TCP端口，是serverPort+10. 为支持多线程高效的处理UDP数据，每个对内和对外的UDP连接，都需要占用5个连续的端口。因此一个数据节点总的端口范围为serverPort到serverPort + 10，总共11个TCP/UDP端口。使用时，需要确保防火墙将这些端口打开。每个数据节点可以配置不同的serverPort。 +**端口配置：**一个数据节点对外的端口由TDengine的系统配置参数serverPort决定，对集群内部通讯的端口是serverPort+5。集群内数据节点之间的数据复制操作还占有一个TCP端口，是serverPort+10. 为支持多线程高效的处理UDP数据，每个对内和对外的UDP连接，都需要占用5个连续的端口。因此一个数据节点总的端口范围为serverPort到serverPort + 10，总共11个TCP/UDP端口。（另外还可能有 RESTful、Arbitrator 所使用的端口，那样的话就一共是 13 个。）使用时，需要确保防火墙将这些端口打开，以备使用。每个数据节点可以配置不同的serverPort。 **集群对外连接:** TDengine集群可以容纳单个、多个甚至几千个数据节点。应用只需要向集群中任何一个数据节点发起连接即可，连接需要提供的网络参数是一数据节点的End Point(FQDN加配置的端口号）。通过命令行CLI启动应用taos时，可以通过选项-h来指定数据节点的FQDN, -P来指定其配置的端口号，如果端口不配置，将采用TDengine的系统配置参数serverPort。 -**集群内部通讯**: 各个数据节点之间通过TCP/UDP进行连接。一个数据节点启动时，将获取mnode所在的dnode的EP信息，然后与系统中的mnode建立起连接，交换信息。获取mnode的EP信息有三步，1：检查mnodeEpList文件是否存在，如果不存在或不能正常打开获得mnode EP信息，进入第二步；2：检查系统配置文件taos.cfg, 获取节点配置参数firstEp, secondEp，（这两个参数指定的节点可以是不带mnode的普通节点，这样的话，节点被连接时会尝试重定向到mnode节点）如果不存在或者taos.cfg里没有这两个配置参数，或无效，进入第三步；3：将自己的EP设为mnode EP, 并独立运行起来。获取mnode EP列表后，数据节点发起连接，如果连接成功，则成功加入进工作的集群，如果不成功，则尝试mnode EP列表中的下一个。如果都尝试了，但连接都仍然失败，则休眠几秒后，再进行尝试。 +**集群内部通讯**: 各个数据节点之间通过TCP/UDP进行连接。一个数据节点启动时，将获取mnode所在的dnode的EP信息，然后与系统中的mnode建立起连接，交换信息。获取mnode的EP信息有三步，1：检查mnodeEpSet文件是否存在，如果不存在或不能正常打开获得mnode EP信息，进入第二步；2：检查系统配置文件taos.cfg, 获取节点配置参数firstEp, secondEp，（这两个参数指定的节点可以是不带mnode的普通节点，这样的话，节点被连接时会尝试重定向到mnode节点）如果不存在或者taos.cfg里没有这两个配置参数，或无效，进入第三步；3：将自己的EP设为mnode EP, 并独立运行起来。获取mnode EP列表后，数据节点发起连接，如果连接成功，则成功加入进工作的集群，如果不成功，则尝试mnode EP列表中的下一个。如果都尝试了，但连接都仍然失败，则休眠几秒后，再进行尝试。 **MNODE的选择:** TDengine逻辑上有管理节点，但没有单独的执行代码，服务器侧只有一套执行代码taosd。那么哪个数据节点会是管理节点呢？这是系统自动决定的，无需任何人工干预。原则如下：一个数据节点启动时，会检查自己的End Point, 并与获取的mnode EP List进行比对，如果在其中，该数据节点认为自己应该启动mnode模块，成为mnode。如果自己的EP不在mnode EP List里，则不启动mnode模块。在系统的运行过程中，由于负载均衡、宕机等原因，mnode有可能迁移至新的dnode，但一切都是透明的，无需人工干预，配置参数的修改，是mnode自己根据资源做出的决定。 From ee5dbf129eb6de34aca563cc5fc53072c5e9b112 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Fri, 5 Mar 2021 10:23:37 +0800 Subject: [PATCH 45/66] [TD-3155]: add test case --- tests/pytest/cluster/clusterSetup.py | 144 +++++++++----------------- tests/pytest/insert/metadataUpdate.py | 3 +- 2 files changed, 53 insertions(+), 94 deletions(-) diff --git a/tests/pytest/cluster/clusterSetup.py b/tests/pytest/cluster/clusterSetup.py index dbda5657b6..8a26427021 100644 --- a/tests/pytest/cluster/clusterSetup.py +++ b/tests/pytest/cluster/clusterSetup.py @@ -11,15 +11,9 @@ # -*- coding: utf-8 -*- -import os -import sys -sys.path.insert(0, os.getcwd()) from fabric import Connection -from util.sql import * -from util.log import * -import taos import random -import threading +import time import logging class Node: @@ -76,6 +70,19 @@ class Node: print("remove taosd error for node %d " % self.index) logging.exception(e) + def forceStopOneTaosd(self): + try: + self.conn.run("kill -9 $(ps -ax|grep taosd|awk '{print $1}')") + except Exception as e: + print("kill taosd error on node%d " % self.index) + + def startOneTaosd(self): + try: + self.conn.run("nohup taosd -c /etc/taos/ > /dev/null 2>&1 &") + except Exception as e: + print("start taosd error on node%d " % self.index) + logging.exception(e) + def installTaosd(self, packagePath): self.conn.put(packagePath, self.homeDir) self.conn.cd(self.homeDir) @@ -122,100 +129,51 @@ class Node: class Nodes: def __init__(self): - self.node1 = Node(1, 'root', '52.151.60.239', 'node1', 'r', '/root/') - self.node2 = Node(2, 'root', '52.183.32.246', 'node1', 'r', '/root/') - self.node3 = Node(3, 'root', '51.143.46.79', 'node1', 'r', '/root/') - self.node4 = Node(4, 'root', '52.183.2.76', 'node1', 'r', '/root/') - self.node5 = Node(5, 'root', '13.66.225.87', 'node1', 'r', '/root/') + self.tdnodes = [] + self.tdnodes.append(Node(0, 'root', '52.143.103.7', 'node1', 'a', '/root/')) + self.tdnodes.append(Node(1, 'root', '52.250.48.222', 'node2', 'a', '/root/')) + self.tdnodes.append(Node(2, 'root', '51.141.167.23', 'node3', 'a', '/root/')) + self.tdnodes.append(Node(3, 'root', '52.247.207.173', 'node4', 'a', '/root/')) + self.tdnodes.append(Node(4, 'root', '51.141.166.100', 'node5', 'a', '/root/')) + + def stopOneNode(self, index): + self.tdnodes[index].forceStopOneTaosd() + + def startOneNode(self, index): + self.tdnodes[index].startOneTaosd() def stopAllTaosd(self): - self.node1.stopTaosd() - self.node2.stopTaosd() - self.node3.stopTaosd() - + for i in range(len(self.tdnodes)): + self.tdnodes[i].stopTaosd() + def startAllTaosd(self): - self.node1.startTaosd() - self.node2.startTaosd() - self.node3.startTaosd() + for i in range(len(self.tdnodes)): + self.tdnodes[i].startTaosd() def restartAllTaosd(self): - self.node1.restartTaosd() - self.node2.restartTaosd() - self.node3.restartTaosd() + for i in range(len(self.tdnodes)): + self.tdnodes[i].restartTaosd() def addConfigs(self, configKey, configValue): - self.node1.configTaosd(configKey, configValue) - self.node2.configTaosd(configKey, configValue) - self.node3.configTaosd(configKey, configValue) + for i in range(len(self.tdnodes)): + self.tdnodes[i].configTaosd(configKey, configValue) - def removeConfigs(self, configKey, configValue): - self.node1.removeTaosConfig(configKey, configValue) - self.node2.removeTaosConfig(configKey, configValue) - self.node3.removeTaosConfig(configKey, configValue) + def removeConfigs(self, configKey, configValue): + for i in range(len(self.tdnodes)): + self.tdnodes[i].removeTaosConfig(configKey, configValue) def removeAllDataFiles(self): - self.node1.removeData() - self.node2.removeData() - self.node3.removeData() + for i in range(len(self.tdnodes)): + self.tdnodes[i].removeData() -class ClusterTest: - def __init__(self, hostName): - self.host = hostName - self.user = "root" - self.password = "taosdata" - self.config = "/etc/taos" - self.dbName = "mytest" - self.stbName = "meters" - self.numberOfThreads = 20 - self.numberOfTables = 10000 - self.numberOfRecords = 1000 - self.tbPrefix = "t" - self.ts = 1538548685000 - self.repeat = 1 - - def connectDB(self): - self.conn = taos.connect( - host=self.host, - user=self.user, - password=self.password, - config=self.config) - - def createSTable(self, replica): - cursor = self.conn.cursor() - tdLog.info("drop database if exists %s" % self.dbName) - cursor.execute("drop database if exists %s" % self.dbName) - tdLog.info("create database %s replica %d" % (self.dbName, replica)) - cursor.execute("create database %s replica %d" % (self.dbName, replica)) - tdLog.info("use %s" % self.dbName) - cursor.execute("use %s" % self.dbName) - tdLog.info("drop table if exists %s" % self.stbName) - cursor.execute("drop table if exists %s" % self.stbName) - tdLog.info("create table %s(ts timestamp, current float, voltage int, phase int) tags(id int)" % self.stbName) - cursor.execute("create table %s(ts timestamp, current float, voltage int, phase int) tags(id int)" % self.stbName) - cursor.close() - - def insertData(self, threadID): - print("Thread %d: starting" % threadID) - cursor = self.conn.cursor() - tablesPerThread = int(self.numberOfTables / self.numberOfThreads) - baseTableID = tablesPerThread * threadID - for i in range (tablesPerThread): - cursor.execute("create table %s%d using %s tags(%d)" % (self.tbPrefix, baseTableID + i, self.stbName, baseTableID + i)) - query = "insert into %s%d values" % (self.tbPrefix, baseTableID + i) - base = self.numberOfRecords * i - for j in range(self.numberOfRecords): - query += "(%d, %f, %d, %d)" % (self.ts + base + j, random.random(), random.randint(210, 230), random.randint(0, 10)) - cursor.execute(query) - cursor.close() - print("Thread %d: finishing" % threadID) - - def run(self): - threads = [] - tdLog.info("Inserting data") - for i in range(self.numberOfThreads): - thread = threading.Thread(target=self.insertData, args=(i,)) - threads.append(thread) - thread.start() - - for i in range(self.numberOfThreads): - threads[i].join() \ No newline at end of file +# kill taosd randomly every 10 mins +nodes = Nodes() +loop = 0 +while True: + loop = loop + 1 + index = random.randint(0, 4) + print("loop: %d, kill taosd on node%d" %(loop, index)) + nodes.stopOneNode(index) + time.sleep(60) + nodes.startOneNode(index) + time.sleep(600) \ No newline at end of file diff --git a/tests/pytest/insert/metadataUpdate.py b/tests/pytest/insert/metadataUpdate.py index c3ea5c6a7e..77795d13f1 100644 --- a/tests/pytest/insert/metadataUpdate.py +++ b/tests/pytest/insert/metadataUpdate.py @@ -54,10 +54,11 @@ class TDTestCase: p.terminate() tdSql.execute("insert into tb values(%d, 1, 2)" % (self.ts + 1)) + tdSql.execute("insert into tb(ts, col1, col2) values(%d, 1, 2)" % (self.ts + 2)) print("==============step2") tdSql.query("select * from tb") - tdSql.checkRows(2) + tdSql.checkRows(3) def stop(self): tdSql.close() From e5898c4a67324dc6b6ef448b774f08c0c78d1576 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Fri, 5 Mar 2021 10:23:50 +0800 Subject: [PATCH 46/66] fix bug --- src/os/inc/osDef.h | 14 ++++++++++++++ src/util/src/tcompare.c | 4 ++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/os/inc/osDef.h b/src/os/inc/osDef.h index cb91b0526b..bbe0f98ec0 100644 --- a/src/os/inc/osDef.h +++ b/src/os/inc/osDef.h @@ -83,6 +83,20 @@ extern "C" { } \ } while (0) +#define DEFAULT_DOUBLE_COMP(x, y) \ + do { \ + if (isnan(x) && isnan(y)) { return 0; } \ + if (isnan(x)) { return -1; } \ + if (isnan(y)) { return 1; } \ + if ((x) == (y)) { \ + return 0; \ + } else { \ + return (x) < (y) ? -1 : 1; \ + } \ + } while (0) + +#define DEFAULT_FLOAT_COMP(x, y) DEFAULT_DOUBLE_COMP(x, y) + #define ALIGN_NUM(n, align) (((n) + ((align)-1)) & (~((align)-1))) // align to 8bytes diff --git a/src/util/src/tcompare.c b/src/util/src/tcompare.c index cd3428ddc5..4d18ef14e2 100644 --- a/src/util/src/tcompare.c +++ b/src/util/src/tcompare.c @@ -392,8 +392,8 @@ __compar_fn_t getKeyComparFunc(int32_t keyType) { int32_t doCompare(const char* f1, const char* f2, int32_t type, size_t size) { switch (type) { case TSDB_DATA_TYPE_INT: DEFAULT_COMP(GET_INT32_VAL(f1), GET_INT32_VAL(f2)); - case TSDB_DATA_TYPE_DOUBLE: DEFAULT_COMP(GET_DOUBLE_VAL(f1), GET_DOUBLE_VAL(f2)); - case TSDB_DATA_TYPE_FLOAT: DEFAULT_COMP(GET_FLOAT_VAL(f1), GET_FLOAT_VAL(f2)); + case TSDB_DATA_TYPE_DOUBLE: DEFAULT_DOUBLE_COMP(GET_DOUBLE_VAL(f1), GET_DOUBLE_VAL(f2)); + case TSDB_DATA_TYPE_FLOAT: DEFAULT_FLOAT_COMP(GET_FLOAT_VAL(f1), GET_FLOAT_VAL(f2)); case TSDB_DATA_TYPE_BIGINT: DEFAULT_COMP(GET_INT64_VAL(f1), GET_INT64_VAL(f2)); case TSDB_DATA_TYPE_SMALLINT: DEFAULT_COMP(GET_INT16_VAL(f1), GET_INT16_VAL(f2)); case TSDB_DATA_TYPE_TINYINT: From dbfb10984af254e98f6ec437afcc740ad5e8e2d9 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Fri, 5 Mar 2021 10:43:08 +0800 Subject: [PATCH 47/66] fix bug --- src/query/src/qExtbuffer.c | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/src/query/src/qExtbuffer.c b/src/query/src/qExtbuffer.c index e4c62d90e3..a73f385282 100644 --- a/src/query/src/qExtbuffer.c +++ b/src/query/src/qExtbuffer.c @@ -362,20 +362,10 @@ static FORCE_INLINE int32_t columnValueAscendingComparator(char *f1, char *f2, i return (first < second) ? -1 : 1; }; case TSDB_DATA_TYPE_DOUBLE: { - double first = GET_DOUBLE_VAL(f1); - double second = GET_DOUBLE_VAL(f2); - if (first == second) { - return 0; - } - return (first < second) ? -1 : 1; + DEFAULT_DOUBLE_COMP(GET_DOUBLE_VAL(f1), GET_DOUBLE_VAL(f2)); }; case TSDB_DATA_TYPE_FLOAT: { - float first = GET_FLOAT_VAL(f1); - float second = GET_FLOAT_VAL(f2); - if (first == second) { - return 0; - } - return (first < second) ? -1 : 1; + DEFAULT_FLOAT_COMP(GET_FLOAT_VAL(f1), GET_FLOAT_VAL(f2)); }; case TSDB_DATA_TYPE_BIGINT: { int64_t first = *(int64_t *)f1; From e15501e4b6a57eb84ad0f84c16cbdc15d933e36c Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Fri, 5 Mar 2021 11:04:52 +0800 Subject: [PATCH 48/66] [TD-1977]: add query unit test into CI --- tests/test-all.sh | 49 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 43 insertions(+), 6 deletions(-) diff --git a/tests/test-all.sh b/tests/test-all.sh index 3ebdffebf2..a6ef13a04a 100755 --- a/tests/test-all.sh +++ b/tests/test-all.sh @@ -160,9 +160,10 @@ function runPyCaseOneByOnefq { totalFailed=0 totalPyFailed=0 totalJDBCFailed=0 +totalUnitTestFailed=0 corepath=`grep -oP '.*(?=core_)' /proc/sys/kernel/core_pattern||grep -oP '.*(?=core-)' /proc/sys/kernel/core_pattern` -if [ "$2" != "jdbc" ] && [ "$2" != "python" ]; then +if [ "$2" != "jdbc" ] && [ "$2" != "python" ] && [ "$2" != "unit" ]; then echo "### run TSIM test case ###" cd $tests_dir/script @@ -231,7 +232,7 @@ if [ "$2" != "jdbc" ] && [ "$2" != "python" ]; then fi fi -if [ "$2" != "sim" ] && [ "$2" != "jdbc" ] ; then +if [ "$2" != "sim" ] && [ "$2" != "jdbc" ] && [ "$2" != "unit" ]; then echo "### run Python test case ###" cd $tests_dir @@ -300,8 +301,8 @@ if [ "$2" != "sim" ] && [ "$2" != "jdbc" ] ; then fi -if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$1" == "full" ]; then - echo "### run JDBC test case ###" +if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "unit" ] && [ "$1" == "full" ]; then + echo "### run JDBC test cases ###" cd $tests_dir @@ -318,7 +319,7 @@ if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$1" == "full" ]; then nohup build/bin/taosd -c /etc/taos/ > /dev/null 2>&1 & sleep 30 - cd $tests_dir/../src/connector/jdbc + cd $tests_dir/../src/connector/jdbc mvn test > jdbc-out.log 2>&1 tail -n 20 jdbc-out.log @@ -343,4 +344,40 @@ if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$1" == "full" ]; then dohavecore 1 fi -exit $(($totalFailed + $totalPyFailed + $totalJDBCFailed)) +if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "jdbc" ] && [ "$1" == "full" ]; then + echo "### run Unit tests ###" + + stopTaosd + cd $tests_dir + + if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then + cd ../../ + else + cd ../ + fi + + pwd + cd debug/build/bin + nohup ./taosd -c /etc/taos/ > /dev/null 2>&1 & + sleep 30 + + pwd + ./queryTest > unittest-out.log 2>&1 + tail -n 20 unittest-out.log + + totalUnitTests=`grep "Running" unittest-out.log | awk '{print $3}'` + totalUnitSuccess=`grep 'PASSED' unittest-out.log | awk '{print $4}'` + totalUnitFailed=`expr $totalUnitTests - $totalUnitSuccess` + + if [ "$totalUnitSuccess" -gt "0" ]; then + echo -e "\n${GREEN} ### Total $totalUnitSuccess Unit test succeed! ### ${NC}" + fi + + if [ "$totalUnitFailed" -ne "0" ]; then + echo -e "\n${RED} ### Total $totalUnitFailed Unit test failed! ### ${NC}" + fi + dohavecore 1 +fi + + +exit $(($totalFailed + $totalPyFailed + $totalJDBCFailed + $totalUnitFailed)) From d148af163d8ac5660298b157f9244cb4904e8301 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 5 Mar 2021 11:15:15 +0800 Subject: [PATCH 49/66] [td-225]update the unit test. --- src/query/tests/percentileTest.cpp | 4 ++-- src/query/tests/unitTest.cpp | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/query/tests/percentileTest.cpp b/src/query/tests/percentileTest.cpp index f2b457e7dd..104bfb3c06 100644 --- a/src/query/tests/percentileTest.cpp +++ b/src/query/tests/percentileTest.cpp @@ -48,7 +48,7 @@ tMemBucket *createUnsignedDataBucket(int32_t start, int32_t end, int32_t type) { uint64_t k = i; int32_t ret = tMemBucketPut(pBucket, &k, 1); if (ret != 0) { - printf("value out of range:%f", k); + printf("value out of range:%" PRId64, k); } } @@ -245,7 +245,7 @@ void unsignedDataTest() { } // namespace TEST(testCase, percentileTest) { - // qsortTest(); +// qsortTest(); intDataTest(); bigintDataTest(); doubleDataTest(); diff --git a/src/query/tests/unitTest.cpp b/src/query/tests/unitTest.cpp index a8500364dc..3406d83090 100644 --- a/src/query/tests/unitTest.cpp +++ b/src/query/tests/unitTest.cpp @@ -227,10 +227,10 @@ TEST(testCase, db_table_name) { EXPECT_EQ(testValidateName(t60_1), TSDB_CODE_TSC_INVALID_SQL); char t61[] = "' ABC '"; - EXPECT_EQ(testValidateName(t61), TSDB_CODE_SUCCESS); + EXPECT_EQ(testValidateName(t61), TSDB_CODE_TSC_INVALID_SQL); char t61_1[] = "' ABC '"; - EXPECT_EQ(testValidateName(t61_1), TSDB_CODE_SUCCESS); + EXPECT_EQ(testValidateName(t61_1), TSDB_CODE_TSC_INVALID_SQL); char t62[] = " ABC . def "; EXPECT_EQ(testValidateName(t62), TSDB_CODE_TSC_INVALID_SQL); @@ -249,13 +249,13 @@ TEST(testCase, db_table_name) { EXPECT_EQ(testValidateName(t65), TSDB_CODE_TSC_INVALID_SQL); char t66[] = "' ABC '.' DEF '"; - EXPECT_EQ(testValidateName(t66), TSDB_CODE_SUCCESS); + EXPECT_EQ(testValidateName(t66), TSDB_CODE_TSC_INVALID_SQL); char t67[] = "abc . ' DEF '"; EXPECT_EQ(testValidateName(t67), TSDB_CODE_TSC_INVALID_SQL); char t68[] = "' abc '.' DEF '"; - EXPECT_EQ(testValidateName(t68), TSDB_CODE_SUCCESS); + EXPECT_EQ(testValidateName(t68), TSDB_CODE_TSC_INVALID_SQL); // do not use key words char t69[] = "table.'DEF'"; @@ -265,7 +265,7 @@ TEST(testCase, db_table_name) { EXPECT_EQ(testValidateName(t70), TSDB_CODE_TSC_INVALID_SQL); char t71[] = "'_abXYZ1234 '.' deFF '"; - EXPECT_EQ(testValidateName(t71), TSDB_CODE_SUCCESS); + EXPECT_EQ(testValidateName(t71), TSDB_CODE_TSC_INVALID_SQL); char t72[] = "'_abDEF&^%1234'.' DIef'"; EXPECT_EQ(testValidateName(t72), TSDB_CODE_TSC_INVALID_SQL); From adf9d96f7377d7cf2b8e556943b2a4435801fe62 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Fri, 5 Mar 2021 11:29:31 +0800 Subject: [PATCH 50/66] fix bug --- src/query/src/qParserImpl.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/query/src/qParserImpl.c b/src/query/src/qParserImpl.c index 2416250dce..5937fdb68f 100644 --- a/src/query/src/qParserImpl.c +++ b/src/query/src/qParserImpl.c @@ -58,6 +58,15 @@ SSqlInfo qSQLParse(const char *pStr) { sqlInfo.valid = false; goto abort_parse; } + + case TK_HEX: + case TK_OCT: + case TK_BIN:{ + snprintf(sqlInfo.msg, tListLen(sqlInfo.msg), "unsupported token: \"%s\"", t0.z); + sqlInfo.valid = false; + goto abort_parse; + } + default: Parse(pParser, t0.type, t0, &sqlInfo); if (sqlInfo.valid == false) { From 94e364b962ff02a5e69144a8bab976b073b37a75 Mon Sep 17 00:00:00 2001 From: Xiaxin Li Date: Fri, 5 Mar 2021 11:41:32 +0800 Subject: [PATCH 51/66] Update docs.md Modify the chapter order --- documentation20/cn/00.index/docs.md | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/documentation20/cn/00.index/docs.md b/documentation20/cn/00.index/docs.md index 7e5f98d909..b4dbc30cda 100644 --- a/documentation20/cn/00.index/docs.md +++ b/documentation20/cn/00.index/docs.md @@ -31,6 +31,20 @@ TDengine是一个高效的存储、查询、分析时序大数据的平台，专 * [创建超级表](/model#create-stable)：为同一类型的数据采集点创建一个超级表 * [创建表](/model#create-table)：使用超级表做模板，为每一个具体的数据采集点单独建表 +## [TAOS SQL](/taos-sql) + +* [支持的数据类型](/taos-sql#data-type)：支持时间戳、整型、浮点型、布尔型、字符型等多种数据类型 +* [数据库管理](/taos-sql#management)：添加、删除、查看数据库 +* [表管理](/taos-sql#table)：添加、删除、查看、修改表 +* [超级表管理](/taos-sql#super-table)：添加、删除、查看、修改超级表 +* [标签管理](/taos-sql#tags)：增加、删除、修改标签 +* [数据写入](/taos-sql#insert)：支持单表单条、多条、多表多条写入，支持历史数据写入 +* [数据查询](/taos-sql#select)：支持时间段、值过滤、排序、查询结果手动分页等 +* [SQL函数](/taos-sql#functions)：支持各种聚合函数、选择函数、计算函数，如avg, min, diff等 +* [时间维度聚合](/taos-sql#aggregation)：将表中数据按照时间段进行切割后聚合，降维处理 +* [边界限制](/taos-sql#limitation)：库、表、SQL等边界限制条件 +* [错误码](/taos-sql/error-code)：TDengine 2.0 错误码以及对应的十进制码 + ## [高效写入数据](/insert) * [SQL写入](/insert#sql)：使用SQL insert命令向一张或多张表写入单条或多条记录 @@ -94,20 +108,6 @@ TDengine是一个高效的存储、查询、分析时序大数据的平台，专 * [文件目录结构](/administrator#directories)：TDengine数据文件、配置文件等所在目录 * [参数限制与保留关键字](/administrator#keywords)：TDengine的参数限制与保留关键字列表 -## [TAOS SQL](/taos-sql) - -* [支持的数据类型](/taos-sql#data-type)：支持时间戳、整型、浮点型、布尔型、字符型等多种数据类型 -* [数据库管理](/taos-sql#management)：添加、删除、查看数据库 -* [表管理](/taos-sql#table)：添加、删除、查看、修改表 -* [超级表管理](/taos-sql#super-table)：添加、删除、查看、修改超级表 -* [标签管理](/taos-sql#tags)：增加、删除、修改标签 -* [数据写入](/taos-sql#insert)：支持单表单条、多条、多表多条写入，支持历史数据写入 -* [数据查询](/taos-sql#select)：支持时间段、值过滤、排序、查询结果手动分页等 -* [SQL函数](/taos-sql#functions)：支持各种聚合函数、选择函数、计算函数，如avg, min, diff等 -* [时间维度聚合](/taos-sql#aggregation)：将表中数据按照时间段进行切割后聚合，降维处理 -* [边界限制](/taos-sql#limitation)：库、表、SQL等边界限制条件 -* [错误码](/taos-sql/error-code)：TDengine 2.0 错误码以及对应的十进制码 - ## TDengine的技术设计 * [系统模块](/architecture/taosd)：taosd的功能和模块划分 From 0a5afb30eea779c1ab586181d2b204641d2dcde6 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 5 Mar 2021 11:50:37 +0800 Subject: [PATCH 52/66] [TD-3171]: response error code if any --- src/mnode/src/mnodeTable.c | 8 ++++++-- src/mnode/src/mnodeVgroup.c | 3 +++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index b1ae118b4b..1378847d0b 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -832,12 +832,13 @@ static int32_t mnodeProcessBatchCreateTableMsg(SMnodeMsg *pMsg) { return code; } else if (code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { ++pMsg->pBatchMasterMsg->received; + pMsg->pBatchMasterMsg->code = code; mnodeDestroySubMsg(pMsg); } if (pMsg->pBatchMasterMsg->successed + pMsg->pBatchMasterMsg->received >= pMsg->pBatchMasterMsg->expected) { - dnodeSendRpcMWriteRsp(pMsg->pBatchMasterMsg, TSDB_CODE_SUCCESS); + dnodeSendRpcMWriteRsp(pMsg->pBatchMasterMsg, pMsg->pBatchMasterMsg->code); } return TSDB_CODE_MND_ACTION_IN_PROGRESS; @@ -1908,7 +1909,8 @@ static int32_t mnodeDoCreateChildTableCb(SMnodeMsg *pMsg, int32_t code) { sdbDeleteRow(&desc); if (pMsg->pBatchMasterMsg) { - ++pMsg->pBatchMasterMsg->successed; + ++pMsg->pBatchMasterMsg->received; + pMsg->pBatchMasterMsg->code = code; if (pMsg->pBatchMasterMsg->successed + pMsg->pBatchMasterMsg->received >= pMsg->pBatchMasterMsg->expected) { dnodeSendRpcMWriteRsp(pMsg->pBatchMasterMsg, code); @@ -2690,6 +2692,7 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { if (pMsg->pBatchMasterMsg) { ++pMsg->pBatchMasterMsg->received; + pMsg->pBatchMasterMsg->code = code; if (pMsg->pBatchMasterMsg->successed + pMsg->pBatchMasterMsg->received >= pMsg->pBatchMasterMsg->expected) { dnodeSendRpcMWriteRsp(pMsg->pBatchMasterMsg, code); @@ -2728,6 +2731,7 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { if (pMsg->pBatchMasterMsg) { ++pMsg->pBatchMasterMsg->received; + pMsg->pBatchMasterMsg->code = rpcMsg->code; if (pMsg->pBatchMasterMsg->successed + pMsg->pBatchMasterMsg->received >= pMsg->pBatchMasterMsg->expected) { dnodeSendRpcMWriteRsp(pMsg->pBatchMasterMsg, rpcMsg->code); diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index 67ee11640b..98eab3f1ed 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -537,6 +537,7 @@ static int32_t mnodeCreateVgroupCb(SMnodeMsg *pMsg, int32_t code) { if (pMsg->pBatchMasterMsg) { ++pMsg->pBatchMasterMsg->received; + pMsg->pBatchMasterMsg->code = pMsg->code; if (pMsg->pBatchMasterMsg->successed + pMsg->pBatchMasterMsg->received >= pMsg->pBatchMasterMsg->expected) { dnodeSendRpcMWriteRsp(pMsg->pBatchMasterMsg, pMsg->code); @@ -1002,6 +1003,7 @@ static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) { if (mnodeMsg->pBatchMasterMsg) { ++mnodeMsg->pBatchMasterMsg->received; + mnodeMsg->pBatchMasterMsg->code = code; if (mnodeMsg->pBatchMasterMsg->successed + mnodeMsg->pBatchMasterMsg->received >= mnodeMsg->pBatchMasterMsg->expected) { dnodeSendRpcMWriteRsp(mnodeMsg->pBatchMasterMsg, code); @@ -1024,6 +1026,7 @@ static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) { if (mnodeMsg->pBatchMasterMsg) { ++mnodeMsg->pBatchMasterMsg->received; + mnodeMsg->pBatchMasterMsg->code = mnodeMsg->code; if (mnodeMsg->pBatchMasterMsg->successed + mnodeMsg->pBatchMasterMsg->received >= mnodeMsg->pBatchMasterMsg->expected) { dnodeSendRpcMWriteRsp(mnodeMsg->pBatchMasterMsg, mnodeMsg->code); From b44bff0c6aff9922938e0ca41ef13fcbfbfea621 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 5 Mar 2021 11:50:51 +0800 Subject: [PATCH 53/66] [TD-3177] : fix demo.c checking result error mistake. --- .gitignore | 1 - tests/examples/c/demo.c | 9 ++++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index b400d719cc..1ff1108056 100644 --- a/.gitignore +++ b/.gitignore @@ -27,7 +27,6 @@ tests/hdfs/ nmake/ sln/ hdfs/ -c/ taoshebei/ taosdalipu/ Target/ diff --git a/tests/examples/c/demo.c b/tests/examples/c/demo.c index 3853d81fb2..e074e64966 100644 --- a/tests/examples/c/demo.c +++ b/tests/examples/c/demo.c @@ -92,15 +92,14 @@ void Test(TAOS *taos, char *qstr, int index) { // printf("insert row: %i, reason:%s\n", i, taos_errstr(taos)); // } TAOS_RES *result1 = taos_query(taos, qstr); - if (result1) { - printf("insert row: %i\n", i); - } else { - printf("failed to insert row: %i, reason:%s\n", i, "null result"/*taos_errstr(result)*/); + if (result1 == NULL || taos_errno(result1) != 0) { + printf("failed to insert row, reason:%s\n", taos_errstr(result1)); taos_free_result(result1); exit(1); + } else { + printf("insert row: %i\n", i); } taos_free_result(result1); - } printf("success to insert rows, total %d rows\n", i); From 494abf094e9732f1c8c0744811e9982180a69cc8 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Fri, 5 Mar 2021 13:10:48 +0800 Subject: [PATCH 54/66] Fix some issues --- Jenkinsfile | 2 ++ tests/Jenkinsfile | 6 ++++++ tests/test-all.sh | 2 +- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 976812bd0a..8d2429c137 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -227,6 +227,8 @@ pipeline { ./test-all.sh p4 cd ${WKC}/tests ./test-all.sh full jdbc + cd ${WKC}/tests + ./test-all.sh full unit date''' } } diff --git a/tests/Jenkinsfile b/tests/Jenkinsfile index 2f8b0de09d..7cdcfb2e24 100644 --- a/tests/Jenkinsfile +++ b/tests/Jenkinsfile @@ -55,9 +55,15 @@ pipeline { sh ''' cd ${WKC}/tests ./test-all.sh b1 + date''' + sh ''' cd ${WKC}/tests ./test-all.sh full jdbc date''' + sh ''' + cd ${WKC}/tests + ./test-all.sh full unit + date''' } } diff --git a/tests/test-all.sh b/tests/test-all.sh index a6ef13a04a..4f7afe7d17 100755 --- a/tests/test-all.sh +++ b/tests/test-all.sh @@ -160,7 +160,7 @@ function runPyCaseOneByOnefq { totalFailed=0 totalPyFailed=0 totalJDBCFailed=0 -totalUnitTestFailed=0 +totalUnitFailed=0 corepath=`grep -oP '.*(?=core_)' /proc/sys/kernel/core_pattern||grep -oP '.*(?=core-)' /proc/sys/kernel/core_pattern` if [ "$2" != "jdbc" ] && [ "$2" != "python" ] && [ "$2" != "unit" ]; then From 47e3591d507b296cf90639ee367b70a820b529c8 Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Fri, 5 Mar 2021 13:35:24 +0800 Subject: [PATCH 55/66] [TD-3040] : clarify return value about function taos_query(). --- documentation20/cn/08.connector/docs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation20/cn/08.connector/docs.md b/documentation20/cn/08.connector/docs.md index 9331d6e9cf..70403382b7 100644 --- a/documentation20/cn/08.connector/docs.md +++ b/documentation20/cn/08.connector/docs.md @@ -209,7 +209,7 @@ C/C++的API类似于MySQL的C API。应用程序使用时，需要包含TDengine - `TAOS_RES* taos_query(TAOS *taos, const char *sql)` - 该API用来执行SQL语句，可以是DQL、DML或DDL语句。 其中的`taos`参数是通过`taos_connect`获得的指针。返回值 NULL 表示失败。 + 该API用来执行SQL语句，可以是DQL、DML或DDL语句。 其中的`taos`参数是通过`taos_connect`获得的指针。不能通过返回值是否是 NULL 来判断执行结果是否失败，而是需要用`taos_errno`函数解析结果集中的错误代码来进行判断。 - `int taos_result_precision(TAOS_RES *res)` From f76225443ea3eb9e42640c155eee0cf457aae471 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 5 Mar 2021 14:44:54 +0800 Subject: [PATCH 56/66] rpc: fix deadlock when tsched queue is full and session limit reached --- src/rpc/src/rpcMain.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 6d34c9fb15..cae227cbdb 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -1281,7 +1281,7 @@ static void rpcSendReqToServer(SRpcInfo *pRpc, SRpcReqContext *pContext) { SRpcConn *pConn = rpcSetupConnToServer(pContext); if (pConn == NULL) { pContext->code = terrno; - taosTmrStart(rpcProcessConnError, 0, pContext, pRpc->tmrCtrl); + taosTmrStart(rpcProcessConnError, 1, pContext, pRpc->tmrCtrl); return; } From 7ffbb65588a312a43c6e8121d59a510ffa9a1112 Mon Sep 17 00:00:00 2001 From: zyyang Date: Fri, 5 Mar 2021 15:21:23 +0800 Subject: [PATCH 57/66] modified the springboot demo --- tests/examples/JDBC/springbootdemo/pom.xml | 2 ++ .../SpringbootdemoApplication.java | 2 +- .../controller/WeatherController.java | 2 +- .../springbootdemo/dao/WeatherMapper.java | 2 +- .../example/springbootdemo/dao/WeatherMapper.xml | 12 +++++++++--- .../example/springbootdemo/domain/Weather.java | 8 ++++---- .../springbootdemo/service/WeatherService.java | 2 +- .../src/main/resources/application.properties | 16 ++++++++-------- 8 files changed, 27 insertions(+), 19 deletions(-) diff --git a/tests/examples/JDBC/springbootdemo/pom.xml b/tests/examples/JDBC/springbootdemo/pom.xml index 4b7272323d..bd5f7efbc0 100644 --- a/tests/examples/JDBC/springbootdemo/pom.xml +++ b/tests/examples/JDBC/springbootdemo/pom.xml @@ -64,6 +64,8 @@ com.taosdata.jdbc taos-jdbcdriver 2.0.20 + + diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/SpringbootdemoApplication.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/SpringbootdemoApplication.java index 8066126d62..8f30c29946 100644 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/SpringbootdemoApplication.java +++ b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/SpringbootdemoApplication.java @@ -10,4 +10,4 @@ public class SpringbootdemoApplication { public static void main(String[] args) { SpringApplication.run(SpringbootdemoApplication.class, args); } -} +} \ No newline at end of file diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/controller/WeatherController.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/controller/WeatherController.java index 5871d80afe..c153e27701 100644 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/controller/WeatherController.java +++ b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/controller/WeatherController.java @@ -60,7 +60,7 @@ public class WeatherController { } @GetMapping("/avg") - public Map avg() { + public List avg() { return weatherService.avg(); } diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.java index 6cb290bc38..ad6733558a 100644 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.java +++ b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.java @@ -24,6 +24,6 @@ public interface WeatherMapper { List getSubTables(); - Map avg(); + List avg(); } diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.xml b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.xml index b4a36d0e19..2d3e054065 100644 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.xml +++ b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.xml @@ -5,7 +5,7 @@ - + @@ -18,7 +18,7 @@ - create table if not exists test.weather(ts timestamp, temperature float, humidity int) tags(location nchar(64), groupId int) + create table if not exists test.weather(ts timestamp, temperature float, humidity float) tags(location nchar(64), groupId int) @@ -47,7 +47,13 @@ select count(*) from test.weather - select avg(temperature), avg(humidity)from test.weather interval(1m) diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/domain/Weather.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/domain/Weather.java index c57eb26ca9..255b2cdca9 100644 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/domain/Weather.java +++ b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/domain/Weather.java @@ -9,14 +9,14 @@ public class Weather { @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss.SSS", timezone = "GMT+8") private Timestamp ts; private float temperature; - private int humidity; + private float humidity; private String location; private int groupId; public Weather() { } - public Weather(Timestamp ts, float temperature, int humidity) { + public Weather(Timestamp ts, float temperature, float humidity) { this.ts = ts; this.temperature = temperature; this.humidity = humidity; @@ -38,11 +38,11 @@ public class Weather { this.temperature = temperature; } - public int getHumidity() { + public float getHumidity() { return humidity; } - public void setHumidity(int humidity) { + public void setHumidity(float humidity) { this.humidity = humidity; } diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/service/WeatherService.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/service/WeatherService.java index dfd7ce23fd..0aef828e1c 100644 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/service/WeatherService.java +++ b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/service/WeatherService.java @@ -55,7 +55,7 @@ public class WeatherService { return weatherMapper.getSubTables(); } - public Map avg() { + public List avg() { return weatherMapper.avg(); } } diff --git a/tests/examples/JDBC/springbootdemo/src/main/resources/application.properties b/tests/examples/JDBC/springbootdemo/src/main/resources/application.properties index 6b1689bb3e..4d7e64d105 100644 --- a/tests/examples/JDBC/springbootdemo/src/main/resources/application.properties +++ b/tests/examples/JDBC/springbootdemo/src/main/resources/application.properties @@ -1,15 +1,15 @@ # datasource config - JDBC-JNI -spring.datasource.driver-class-name=com.taosdata.jdbc.TSDBDriver -spring.datasource.url=jdbc:TAOS://127.0.0.1:6030/test?timezone=UTC-8&charset=UTF-8&locale=en_US.UTF-8 -spring.datasource.username=root -spring.datasource.password=taosdata - -# datasource config - JDBC-RESTful -#spring.datasource.driver-class-name=com.taosdata.jdbc.rs.RestfulDriver -#spring.datasource.url=jdbc:TAOS-RS://master:6041/test?timezone=UTC-8&charset=UTF-8&locale=en_US.UTF-8 +#spring.datasource.driver-class-name=com.taosdata.jdbc.TSDBDriver +#spring.datasource.url=jdbc:TAOS://127.0.0.1:6030/test?timezone=UTC-8&charset=UTF-8&locale=en_US.UTF-8 #spring.datasource.username=root #spring.datasource.password=taosdata +# datasource config - JDBC-RESTful +spring.datasource.driver-class-name=com.taosdata.jdbc.rs.RestfulDriver +spring.datasource.url=jdbc:TAOS-RS://master:6041/test?timezone=UTC-8&charset=UTF-8&locale=en_US.UTF-8 +spring.datasource.username=root +spring.datasource.password=taosdata + spring.datasource.druid.initial-size=5 spring.datasource.druid.min-idle=5 spring.datasource.druid.max-active=5 From 6a5831c4389ebba315a048517999fa982ff01e73 Mon Sep 17 00:00:00 2001 From: zyyang Date: Fri, 5 Mar 2021 15:33:11 +0800 Subject: [PATCH 58/66] change --- .../jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribe.java | 5 ++--- .../jdbc/src/test/java/com/taosdata/jdbc/ResultSetTest.java | 1 - .../jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java | 5 ++--- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribe.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribe.java index c21a058ba2..dd0d0d5b7b 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribe.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribe.java @@ -14,12 +14,11 @@ *****************************************************************************/ package com.taosdata.jdbc; -import javax.management.OperationsException; import java.sql.SQLException; public class TSDBSubscribe { - private TSDBJNIConnector connecter = null; - private long id = 0; + private final TSDBJNIConnector connecter; + private final long id; TSDBSubscribe(TSDBJNIConnector connecter, long id) throws SQLException { if (null != connecter) { diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ResultSetTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ResultSetTest.java index fb0053cb4b..87348f9026 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ResultSetTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ResultSetTest.java @@ -13,7 +13,6 @@ import java.util.HashMap; import java.util.Properties; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; public class ResultSetTest { static Connection connection; diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java index 685957d60a..8d76bffa20 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java @@ -48,7 +48,6 @@ public class SubscribeTest { @Test public void subscribe() { try { - String rawSql = "select * from " + dbName + "." + tName + ";"; System.out.println(rawSql); TSDBSubscribe subscribe = ((TSDBConnection) connection).subscribe(topic, rawSql, false); @@ -67,10 +66,10 @@ public class SubscribeTest { if (a >= 2) { break; } -// resSet.close(); + resSet.close(); } - subscribe.close(true); + subscribe.close(false); } catch (Exception e) { e.printStackTrace(); } From d8c02c10db3b860a10bf2eee0b5bc4b3a8c660cf Mon Sep 17 00:00:00 2001 From: zyyang Date: Fri, 5 Mar 2021 16:12:26 +0800 Subject: [PATCH 59/66] change --- .../jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java index 8d76bffa20..1e05d62287 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java @@ -69,7 +69,7 @@ public class SubscribeTest { resSet.close(); } - subscribe.close(false); + subscribe.close(true); } catch (Exception e) { e.printStackTrace(); } From 3b9eb5ea9accf5f96fe1df94a3eab7530c40a896 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 5 Mar 2021 16:48:26 +0800 Subject: [PATCH 60/66] [TD-3147] : support insert interval. use seperate func for normal table write. --- src/kit/taosdemo/taosdemo.c | 342 ++++++++++++++++++++++++++++-------- 1 file changed, 271 insertions(+), 71 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 5eab2fde63..80cbecb96b 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -80,7 +80,6 @@ extern char configDir[]; #define OPT_ABORT 1 /* –abort */ #define STRING_LEN 60000 #define MAX_PREPARED_RAND 1000000 -//#define MAX_SQL_SIZE 65536 #define MAX_FILE_NAME_LEN 256 #define MAX_SAMPLES_ONCE_FROM_FILE 10000 @@ -240,7 +239,7 @@ typedef struct SSuperTable_S { StrColumn tags[MAX_TAG_COUNT]; char* childTblName; - char* colsOfCreatChildTable; + char* colsOfCreateChildTable; int lenOfOneRow; int lenOfTagOfOneRow; @@ -458,9 +457,9 @@ void resetAfterAnsiEscape(void) { } #endif -int createDatabases(); -void createChildTables(); -int queryDbExec(TAOS *taos, char *command, int type); +static int createDatabases(); +static void createChildTables(); +static int queryDbExec(TAOS *taos, char *command, int type); /* ************ Global variables ************ */ @@ -638,6 +637,7 @@ void parse_args(int argc, char *argv[], SArguments *arguments) { sptr = arguments->datatype; ++i; if (strstr(argv[i], ",") == NULL) { + // only one col if (strcasecmp(argv[i], "INT") && strcasecmp(argv[i], "FLOAT") && strcasecmp(argv[i], "TINYINT") @@ -653,6 +653,7 @@ void parse_args(int argc, char *argv[], SArguments *arguments) { } sptr[0] = argv[i]; } else { + // more than one col int index = 0; char *dupstr = strdup(argv[i]); char *running = dupstr; @@ -675,6 +676,7 @@ void parse_args(int argc, char *argv[], SArguments *arguments) { token = strsep(&running, ","); if (index >= MAX_NUM_DATATYPE) break; } + sptr[index] = NULL; } } else if (strcmp(argv[i], "-w") == 0) { arguments->len_of_binary = atoi(argv[++i]); @@ -735,6 +737,15 @@ void parse_args(int argc, char *argv[], SArguments *arguments) { printf("# User: %s\n", arguments->user); printf("# Password: %s\n", arguments->password); printf("# Use metric: %s\n", arguments->use_metric ? "true" : "false"); + if (*(arguments->datatype)) { + printf("# Specified data type: "); + for (int i = 0; i < MAX_NUM_DATATYPE; i++) + if (arguments->datatype[i]) + printf("%s,", arguments->datatype[i]); + else + break; + printf("\n"); + } printf("# Insertion interval: %d\n", arguments->insert_interval); printf("# Number of Columns per record: %d\n", arguments->num_of_RPR); printf("# Number of Threads: %d\n", arguments->num_of_threads); @@ -774,7 +785,7 @@ void tmfree(char *buf) { } } -int queryDbExec(TAOS *taos, char *command, int type) { +static int queryDbExec(TAOS *taos, char *command, int type) { int i; TAOS_RES *res = NULL; int32_t code = -1; @@ -784,7 +795,7 @@ int queryDbExec(TAOS *taos, char *command, int type) { taos_free_result(res); res = NULL; } - + res = taos_query(taos, command); code = taos_errno(res); if (0 == code) { @@ -793,7 +804,7 @@ int queryDbExec(TAOS *taos, char *command, int type) { } if (code != 0) { - debugPrint("DEBUG %s() %d - command: %s\n", __func__, __LINE__, command); + debugPrint("DEBUG %s() LN%d - command: %s\n", __func__, __LINE__, command); fprintf(stderr, "Failed to run %s, reason: %s\n", command, taos_errstr(res)); taos_free_result(res); //taos_close(taos); @@ -1965,13 +1976,14 @@ static int createSuperTable(TAOS * taos, char* dbName, SSuperTable* superTbls, //printf("%s.%s column count:%d, column length:%d\n\n", g_Dbs.db[i].dbName, g_Dbs.db[i].superTbls[j].sTblName, g_Dbs.db[i].superTbls[j].columnCount, lenOfOneRow); // save for creating child table - superTbls->colsOfCreatChildTable = (char*)calloc(len+20, 1); - if (NULL == superTbls->colsOfCreatChildTable) { + superTbls->colsOfCreateChildTable = (char*)calloc(len+20, 1); + if (NULL == superTbls->colsOfCreateChildTable) { printf("Failed when calloc, size:%d", len+1); taos_close(taos); exit(-1); } - snprintf(superTbls->colsOfCreatChildTable, len+20, "(ts timestamp%s)", cols); + snprintf(superTbls->colsOfCreateChildTable, len+20, "(ts timestamp%s)", cols); + debugPrint("DEBUG - %s() LN%d: %s\n", __func__, __LINE__, superTbls->colsOfCreateChildTable); if (use_metric) { char tags[STRING_LEN] = "\0"; @@ -2020,19 +2032,23 @@ static int createSuperTable(TAOS * taos, char* dbName, SSuperTable* superTbls, len += snprintf(tags + len, STRING_LEN - len, ")"); superTbls->lenOfTagOfOneRow = lenOfTagOfOneRow; - - snprintf(command, BUFFER_SIZE, "create table if not exists %s.%s (ts timestamp%s) tags %s", dbName, superTbls->sTblName, cols, tags); - debugPrint("DEBUG %s() %d \n", __func__, __LINE__); + + snprintf(command, BUFFER_SIZE, + "create table if not exists %s.%s (ts timestamp%s) tags %s", + dbName, superTbls->sTblName, cols, tags); + debugPrint("DEBUG - %s() LN%d: %s\n", __func__, __LINE__, command); + if (0 != queryDbExec(taos, command, NO_INSERT_TYPE)) { - return -1; + fprintf(stderr, "create supertable %s failed!\n\n", superTbls->sTblName); + return -1; } - printf("\ncreate supertable %s success!\n\n", superTbls->sTblName); + debugPrint("DEBUG - create supertable %s success!\n\n", superTbls->sTblName); } return 0; } -int createDatabases() { +static int createDatabases() { TAOS * taos = NULL; int ret = 0; taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, NULL, g_Dbs.port); @@ -2112,7 +2128,7 @@ int createDatabases() { dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "precision \'%s\';", g_Dbs.db[i].dbCfg.precision); } - + debugPrint("DEBUG %s() %d \n", __func__, __LINE__); if (0 != queryDbExec(taos, command, NO_INSERT_TYPE)) { taos_close(taos); @@ -2138,8 +2154,6 @@ int createDatabases() { printf("\ncreate super table %d failed!\n\n", j); taos_close(taos); return -1; - } else { - printf("\ncreate super table %d success!\n\n", j); } } } @@ -2148,7 +2162,7 @@ int createDatabases() { return 0; } -void * createTable(void *sarg) +static void* createTable(void *sarg) { threadInfo *winfo = (threadInfo *)sarg; SSuperTable* superTblInfo = winfo->superTblInfo; @@ -2161,7 +2175,7 @@ void * createTable(void *sarg) else buff_len = BUFFER_SIZE; - char *buffer = calloc(superTblInfo->maxSqlLen, 1); + char *buffer = calloc(buff_len, 1); if (buffer == NULL) { fprintf(stderr, "Memory allocated failed!"); exit(-1); @@ -2175,8 +2189,8 @@ void * createTable(void *sarg) snprintf(buffer, buff_len, "create table if not exists %s.%s%d %s;", winfo->db_name, - superTblInfo->childTblPrefix, i, - superTblInfo->colsOfCreatChildTable); + g_args.tb_prefix, i, + winfo->cols); } else { if (0 == len) { batchNum = 0; @@ -2215,7 +2229,7 @@ void * createTable(void *sarg) } len = 0; - debugPrint("DEBUG %s() %d \n", __func__, __LINE__); + debugPrint("DEBUG %s() LN%d %s\n", __func__, __LINE__, buffer); if (0 != queryDbExec(winfo->taos, buffer, NO_INSERT_TYPE)){ free(buffer); return NULL; @@ -2303,25 +2317,50 @@ int startMultiThreadCreateChildTable( } -void createChildTables() { +static void createChildTables() { + char tblColsBuf[MAX_SQL_SIZE]; + int len; + for (int i = 0; i < g_Dbs.dbCount; i++) { if (g_Dbs.db[i].superTblCount > 0) { + // with super table for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { if ((AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) || (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists)) { continue; } + debugPrint("DEBUG - %s() LN%d: %s\n", __func__, __LINE__, + g_Dbs.db[i].superTbls[j].colsOfCreateChildTable); startMultiThreadCreateChildTable( - g_Dbs.db[i].superTbls[j].colsOfCreatChildTable, + g_Dbs.db[i].superTbls[j].colsOfCreateChildTable, g_Dbs.threadCountByCreateTbl, g_Dbs.db[i].superTbls[j].childTblCount, g_Dbs.db[i].dbName, &(g_Dbs.db[i].superTbls[j])); g_totalChildTables += g_Dbs.db[i].superTbls[j].childTblCount; } } else { + // normal table + len = snprintf(tblColsBuf, MAX_SQL_SIZE, "(TS TIMESTAMP"); + for (int i = 0; i < MAX_COLUMN_COUNT; i++) { + if (g_args.datatype[i]) { + if ((strncasecmp(g_args.datatype[i], "BINARY", strlen("BINARY")) == 0) + || (strncasecmp(g_args.datatype[i], "NCHAR", strlen("NCHAR")) == 0)) { + len = snprintf(tblColsBuf + len, MAX_SQL_SIZE, ", COL%d %s(60)", i, g_args.datatype[i]); + } else { + len = snprintf(tblColsBuf + len, MAX_SQL_SIZE, ", COL%d %s", i, g_args.datatype[i]); + } + len = strlen(tblColsBuf); + } else { + len = snprintf(tblColsBuf + len, MAX_SQL_SIZE, ")"); + break; + } + } + + debugPrint("DEBUG - %s() LN%d: %s\n", __func__, __LINE__, + tblColsBuf); startMultiThreadCreateChildTable( - g_Dbs.db[i].superTbls[j].colsOfCreatChildTable, + tblColsBuf, g_Dbs.threadCountByCreateTbl, g_args.num_of_DPT, g_Dbs.db[i].dbName, @@ -2361,7 +2400,7 @@ int readTagFromCsvFileToMem(SSuperTable * superTblInfo) { size_t n = 0; ssize_t readLen = 0; char * line = NULL; - + FILE *fp = fopen(superTblInfo->tagsFile, "r"); if (fp == NULL) { printf("Failed to open tags file: %s, reason:%s\n", @@ -2373,7 +2412,7 @@ int readTagFromCsvFileToMem(SSuperTable * superTblInfo) { free(superTblInfo->tagDataBuf); superTblInfo->tagDataBuf = NULL; } - + int tagCount = 10000; int count = 0; char* tagDataBuf = calloc(1, superTblInfo->lenOfTagOfOneRow * tagCount); @@ -3612,7 +3651,7 @@ void prePareSampleData() { //if (0 == strncasecmp(g_Dbs.db[i].superTbls[j].dataSource, "sample", 6)) { // readSampleFromFileToMem(&g_Dbs.db[i].superTbls[j]); //} - + if (g_Dbs.db[i].superTbls[j].tagsFile[0] != 0) { (void)readTagFromCsvFileToMem(&g_Dbs.db[i].superTbls[j]); } @@ -3624,9 +3663,9 @@ void postFreeResource() { tmfclose(g_fpOfInsertResult); for (int i = 0; i < g_Dbs.dbCount; i++) { for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { - if (0 != g_Dbs.db[i].superTbls[j].colsOfCreatChildTable) { - free(g_Dbs.db[i].superTbls[j].colsOfCreatChildTable); - g_Dbs.db[i].superTbls[j].colsOfCreatChildTable = NULL; + if (0 != g_Dbs.db[i].superTbls[j].colsOfCreateChildTable) { + free(g_Dbs.db[i].superTbls[j].colsOfCreateChildTable); + g_Dbs.db[i].superTbls[j].colsOfCreateChildTable = NULL; } if (0 != g_Dbs.db[i].superTbls[j].sampleDataBuf) { free(g_Dbs.db[i].superTbls[j].sampleDataBuf); @@ -3704,11 +3743,11 @@ int generateRowData(char* dataBuf, int maxLen, int64_t timestamp, SSuperTable* } dataLen -= 2; dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, ")"); - + return dataLen; } -void syncWriteForNumberOfTblInOneSql( +static void syncWriteForNumberOfTblInOneSql( threadInfo *winfo, FILE *fp, char* sampleDataBuf) { SSuperTable* superTblInfo = winfo->superTblInfo; @@ -3731,7 +3770,7 @@ void syncWriteForNumberOfTblInOneSql( numberOfTblInOneSql = tbls; } - int64_t time_counter = winfo->start_time; + uint64_t time_counter = winfo->start_time; int64_t tmp_time; int sampleUsePos; @@ -3955,6 +3994,64 @@ send_to_server: return; } +int32_t generateData(char *res, char **data_type, + int num_of_cols, int64_t timestamp, int len_of_binary) { + memset(res, 0, MAX_DATA_SIZE); + char *pstr = res; + pstr += sprintf(pstr, "(%" PRId64, timestamp); + int c = 0; + + for (; c < MAX_NUM_DATATYPE; c++) { + if (data_type[c] == NULL) { + break; + } + } + + if (0 == c) { + perror("data type error!"); + exit(-1); + } + + for (int i = 0; i < num_of_cols; i++) { + if (strcasecmp(data_type[i % c], "tinyint") == 0) { + pstr += sprintf(pstr, ", %d", rand_tinyint() ); + } else if (strcasecmp(data_type[i % c], "smallint") == 0) { + pstr += sprintf(pstr, ", %d", rand_smallint()); + } else if (strcasecmp(data_type[i % c], "int") == 0) { + pstr += sprintf(pstr, ", %d", rand_int()); + } else if (strcasecmp(data_type[i % c], "bigint") == 0) { + pstr += sprintf(pstr, ", %" PRId64, rand_bigint()); + } else if (strcasecmp(data_type[i % c], "float") == 0) { + pstr += sprintf(pstr, ", %10.4f", rand_float()); + } else if (strcasecmp(data_type[i % c], "double") == 0) { + double t = rand_double(); + pstr += sprintf(pstr, ", %20.8f", t); + } else if (strcasecmp(data_type[i % c], "bool") == 0) { + bool b = rand() & 1; + pstr += sprintf(pstr, ", %s", b ? "true" : "false"); + } else if (strcasecmp(data_type[i % c], "binary") == 0) { + char *s = malloc(len_of_binary); + rand_string(s, len_of_binary); + pstr += sprintf(pstr, ", \"%s\"", s); + free(s); + }else if (strcasecmp(data_type[i % c], "nchar") == 0) { + char *s = malloc(len_of_binary); + rand_string(s, len_of_binary); + pstr += sprintf(pstr, ", \"%s\"", s); + free(s); + } + + if (pstr - res > MAX_DATA_SIZE) { + perror("column length too long, abort"); + exit(-1); + } + } + + pstr += sprintf(pstr, ")"); + + return (int32_t)(pstr - res); +} + // sync insertion /* 1 thread: 100 tables * 2000 rows/s @@ -3963,7 +4060,88 @@ send_to_server: 2 taosinsertdata , 1 thread: 10 tables * 20000 rows/s */ -void *syncWrite(void *sarg) { +static void* syncWrite(void *sarg) { + + threadInfo *winfo = (threadInfo *)sarg; + + char buffer[BUFFER_SIZE] = "\0"; + char data[MAX_DATA_SIZE]; + char **data_type = g_args.datatype; + int len_of_binary = g_args.len_of_binary; + + int ncols_per_record = 1; // count first col ts + for (int i = 0; i < MAX_COLUMN_COUNT; i ++) { + if (NULL == g_args.datatype[i]) + break; + else + ncols_per_record ++; + } + + srand((uint32_t)time(NULL)); + int64_t time_counter = winfo->start_time; + + for (int i = 0; i < g_args.num_of_DPT;) { + for (int tID = winfo->start_table_id; tID <= winfo->end_table_id; tID++) { + int inserted = i; + int64_t tmp_time = time_counter; + + char *pstr = buffer; + pstr += sprintf(pstr, "insert into %s.%s%d values", winfo->db_name, g_args.tb_prefix, tID); + int k; + for (k = 0; k < g_args.num_of_RPR;) { + int rand_num = rand() % 100; + int len = -1; + + if ((g_args.disorderRatio != 0) && (rand_num < g_args.disorderRange)) { + + int64_t d = tmp_time - rand() % 1000000 + rand_num; + len = generateData(data, data_type, ncols_per_record, d, len_of_binary); + } else { + len = generateData(data, data_type, ncols_per_record, tmp_time += 1000, len_of_binary); + } + + //assert(len + pstr - buffer < BUFFER_SIZE); + if (len + pstr - buffer >= BUFFER_SIZE) { // too long + break; + } + + pstr += sprintf(pstr, " %s", data); + inserted++; + k++; + + if (inserted >= g_args.num_of_DPT) + break; + } + + /* puts(buffer); */ + int64_t startTs; + int64_t endTs; + startTs = taosGetTimestampUs(); + //queryDB(winfo->taos, buffer); + debugPrint("DEBUG - %s() LN%d %s\n", __func__, __LINE__, buffer); + int affectedRows = queryDbExec(winfo->taos, buffer, 1); + + if (0 <= affectedRows){ + endTs = taosGetTimestampUs(); + int64_t delay = endTs - startTs; + if (delay > winfo->maxDelay) winfo->maxDelay = delay; + if (delay < winfo->minDelay) winfo->minDelay = delay; + winfo->cntDelay++; + winfo->totalDelay += delay; + //winfo->avgDelay = (double)winfo->totalDelay / winfo->cntDelay; + } + + if (tID == winfo->end_table_id) { + i = inserted; + time_counter = tmp_time; + } + } + } + return NULL; +} + + +static void* syncWriteWithStb(void *sarg) { uint64_t totalRowsInserted = 0; uint64_t totalAffectedRows = 0; uint64_t lastPrintTime = taosGetTimestampMs(); @@ -4137,7 +4315,7 @@ void *syncWrite(void *sarg) { int64_t endTs; startTs = taosGetTimestampUs(); - debugPrint("DEBUG %s() %d \n", __func__, __LINE__); + debugPrint("DEBUG %s() LN%d %s\n", __func__, __LINE__, buffer); int affectedRows = queryDbExec(winfo->taos, buffer, INSERT_TYPE); if (0 > affectedRows){ goto free_and_statistics_2; @@ -4280,10 +4458,11 @@ void *asyncWrite(void *sarg) { void startMultiThreadInsertData(int threads, char* db_name, char* precision, SSuperTable* superTblInfo) { - pthread_t *pids = malloc(threads * sizeof(pthread_t)); - threadInfo *infos = malloc(threads * sizeof(threadInfo)); - memset(pids, 0, threads * sizeof(pthread_t)); - memset(infos, 0, threads * sizeof(threadInfo)); + + pthread_t *pids = malloc(threads * sizeof(pthread_t)); + threadInfo *infos = malloc(threads * sizeof(threadInfo)); + memset(pids, 0, threads * sizeof(pthread_t)); + memset(infos, 0, threads * sizeof(threadInfo)); int ntables = 0; if (superTblInfo) @@ -4323,15 +4502,20 @@ void startMultiThreadInsertData(int threads, char* db_name, char* precision, } } - int64_t start_time; - if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) { - start_time = taosGetTimestamp(timePrec); - } else { - (void)taosParseTime( + + int64_t start_time; + if (superTblInfo) { + if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) { + start_time = taosGetTimestamp(timePrec); + } else { + taosParseTime( superTblInfo->startTimestamp, &start_time, strlen(superTblInfo->startTimestamp), timePrec, 0); + } + } else { + start_time = 1500000000000; } double start = getCurrentTime(); @@ -4346,7 +4530,7 @@ void startMultiThreadInsertData(int threads, char* db_name, char* precision, t_info->start_time = start_time; t_info->minDelay = INT16_MAX; - if (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5)) { + if ((NULL == superTblInfo) || (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5))) { //t_info->taos = taos; t_info->taos = taos_connect( g_Dbs.host, g_Dbs.user, @@ -4359,7 +4543,7 @@ void startMultiThreadInsertData(int threads, char* db_name, char* precision, t_info->taos = NULL; } - if (0 == superTblInfo->multiThreadWriteOneTbl) { + if ((NULL == superTblInfo) || (0 == superTblInfo->multiThreadWriteOneTbl)) { t_info->start_table_id = last; t_info->end_table_id = i < b ? last + a : last + a - 1; last = t_info->end_table_id + 1; @@ -4371,7 +4555,11 @@ void startMultiThreadInsertData(int threads, char* db_name, char* precision, tsem_init(&(t_info->lock_sem), 0, 0); if (SYNC == g_Dbs.queryMode) { - pthread_create(pids + i, NULL, syncWrite, t_info); + if (superTblInfo) { + pthread_create(pids + i, NULL, syncWriteWithStb, t_info); + } else { + pthread_create(pids + i, NULL, syncWrite, t_info); + } } else { pthread_create(pids + i, NULL, asyncWrite, t_info); } @@ -4393,13 +4581,15 @@ void startMultiThreadInsertData(int threads, char* db_name, char* precision, tsem_destroy(&(t_info->lock_sem)); taos_close(t_info->taos); - superTblInfo->totalAffectedRows += t_info->totalAffectedRows; - superTblInfo->totalRowsInserted += t_info->totalRowsInserted; + if (superTblInfo) { + superTblInfo->totalAffectedRows += t_info->totalAffectedRows; + superTblInfo->totalRowsInserted += t_info->totalRowsInserted; - totalDelay += t_info->totalDelay; - cntDelay += t_info->cntDelay; - if (t_info->maxDelay > maxDelay) maxDelay = t_info->maxDelay; - if (t_info->minDelay < minDelay) minDelay = t_info->minDelay; + totalDelay += t_info->totalDelay; + cntDelay += t_info->cntDelay; + if (t_info->maxDelay > maxDelay) maxDelay = t_info->maxDelay; + if (t_info->minDelay < minDelay) minDelay = t_info->minDelay; + } } cntDelay -= 1; @@ -4408,16 +4598,19 @@ void startMultiThreadInsertData(int threads, char* db_name, char* precision, double end = getCurrentTime(); double t = end - start; - printf("Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s) into %s.%s. %2.f records/second\n\n", + + if (superTblInfo) { + printf("Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s) into %s.%s. %2.f records/second\n\n", t, superTblInfo->totalRowsInserted, superTblInfo->totalAffectedRows, threads, db_name, superTblInfo->sTblName, superTblInfo->totalRowsInserted / t); - fprintf(g_fpOfInsertResult, "Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s) into %s.%s. %2.f records/second\n\n", + fprintf(g_fpOfInsertResult, "Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s) into %s.%s. %2.f records/second\n\n", t, superTblInfo->totalRowsInserted, superTblInfo->totalAffectedRows, threads, db_name, superTblInfo->sTblName, superTblInfo->totalRowsInserted / t); + } printf("insert delay, avg: %10.6fms, max: %10.6fms, min: %10.6fms\n\n", avgDelay/1000.0, (double)maxDelay/1000.0, (double)minDelay/1000.0); @@ -4436,7 +4629,7 @@ void *readTable(void *sarg) { threadInfo *rinfo = (threadInfo *)sarg; TAOS *taos = rinfo->taos; char command[BUFFER_SIZE] = "\0"; - int64_t sTime = rinfo->start_time; + uint64_t sTime = rinfo->start_time; char *tb_prefix = rinfo->tb_prefix; FILE *fp = fopen(rinfo->fp, "a"); if (NULL == fp) { @@ -4444,7 +4637,13 @@ void *readTable(void *sarg) { return NULL; } - int num_of_DPT = rinfo->superTblInfo->insertRows; // nrecords_per_table; + int num_of_DPT; + if (rinfo->superTblInfo) { + num_of_DPT = rinfo->superTblInfo->insertRows; // nrecords_per_table; + } else { + num_of_DPT = g_args.num_of_DPT; + } + int num_of_tables = rinfo->end_table_id - rinfo->start_table_id + 1; int totalData = num_of_DPT * num_of_tables; bool do_aggreFunc = g_Dbs.do_aggreFunc; @@ -4594,7 +4793,7 @@ int insertTestProcess() { init_rand_data(); // create database and super tables - if( createDatabases() != 0) { + if(createDatabases() != 0) { fclose(g_fpOfInsertResult); return -1; } @@ -4762,7 +4961,7 @@ void *subQueryProcess(void *sarg) { return NULL; } -int queryTestProcess() { +static int queryTestProcess() { TAOS * taos = NULL; taos = taos_connect(g_queryInfo.host, g_queryInfo.user, @@ -5052,7 +5251,7 @@ void *superSubscribeProcess(void *sarg) { return NULL; } -int subscribeTestProcess() { +static int subscribeTestProcess() { printfQueryMeta(); if (!g_args.answer_yes) { @@ -5201,6 +5400,9 @@ void setParaFromArg(){ g_Dbs.port = g_args.port; } + g_Dbs.threadCount = g_args.num_of_threads; + g_Dbs.threadCountByCreateTbl = g_args.num_of_threads; + g_Dbs.dbCount = 1; g_Dbs.db[0].drop = 1; @@ -5352,7 +5554,7 @@ void querySqlFile(TAOS* taos, char* sqlFile) } memcpy(cmd + cmd_len, line, read_len); - debugPrint("DEBUG %s() %d \n", __func__, __LINE__); + debugPrint("DEBUG %s() LN%d %s\n", __func__, __LINE__, cmd); queryDbExec(taos, cmd, NO_INSERT_TYPE); memset(cmd, 0, MAX_SQL_SIZE); cmd_len = 0; @@ -5367,26 +5569,24 @@ void querySqlFile(TAOS* taos, char* sqlFile) return; } - - -void testMetaFile() { +static void testMetaFile() { if (INSERT_MODE == g_args.test_mode) { if (g_Dbs.cfgDir[0]) taos_options(TSDB_OPTION_CONFIGDIR, g_Dbs.cfgDir); insertTestProcess(); } else if (QUERY_MODE == g_args.test_mode) { if (g_queryInfo.cfgDir[0]) taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir); - (void)queryTestProcess(); + queryTestProcess(); } else if (SUBSCRIBE_MODE == g_args.test_mode) { if (g_queryInfo.cfgDir[0]) taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir); - (void)subscribeTestProcess(); + subscribeTestProcess(); } else { ; } } -void testCmdLine() { +static void testCmdLine() { g_args.test_mode = INSERT_MODE; insertTestProcess(); From a886b8e4472e7c637510cabd4f9d18d3849c3069 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Fri, 5 Mar 2021 16:51:04 +0800 Subject: [PATCH 61/66] update test case, reduce case time --- tests/pytest/insert/metadataUpdate.py | 5 ++--- tests/pytest/tools/taosdemoTest.py | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/pytest/insert/metadataUpdate.py b/tests/pytest/insert/metadataUpdate.py index 77795d13f1..76b9a3ae8f 100644 --- a/tests/pytest/insert/metadataUpdate.py +++ b/tests/pytest/insert/metadataUpdate.py @@ -52,13 +52,12 @@ class TDTestCase: p.start() p.join() p.terminate() - - tdSql.execute("insert into tb values(%d, 1, 2)" % (self.ts + 1)) + tdSql.execute("insert into tb(ts, col1, col2) values(%d, 1, 2)" % (self.ts + 2)) print("==============step2") tdSql.query("select * from tb") - tdSql.checkRows(3) + tdSql.checkRows(2) def stop(self): tdSql.close() diff --git a/tests/pytest/tools/taosdemoTest.py b/tests/pytest/tools/taosdemoTest.py index 1cb2f71d8f..c450570d24 100644 --- a/tests/pytest/tools/taosdemoTest.py +++ b/tests/pytest/tools/taosdemoTest.py @@ -24,7 +24,7 @@ class TDTestCase: tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) - self.numberOfTables = 10000 + self.numberOfTables = 1000 self.numberOfRecords = 100 def getBuildPath(self): From ff370c2471e49a23c1115a4e68dd37d95e70fe66 Mon Sep 17 00:00:00 2001 From: zyyang Date: Fri, 5 Mar 2021 17:05:22 +0800 Subject: [PATCH 62/66] change --- cmake/install.inc | 2 +- src/connector/jdbc/CMakeLists.txt | 2 +- src/connector/jdbc/deploy-pom.xml | 2 +- src/connector/jdbc/pom.xml | 2 +- .../java/com/taosdata/jdbc/SubscribeTest.java | 38 +++++++++---------- .../com/taosdata/jdbc/rs/RestfulJDBCTest.java | 4 +- .../java/com/taosdata/jdbc/rs/SQLTest.java | 4 +- 7 files changed, 27 insertions(+), 27 deletions(-) diff --git a/cmake/install.inc b/cmake/install.inc index 0dda0f6821..0ea79589ca 100755 --- a/cmake/install.inc +++ b/cmake/install.inc @@ -32,7 +32,7 @@ ELSEIF (TD_WINDOWS) #INSTALL(TARGETS taos RUNTIME DESTINATION driver) #INSTALL(TARGETS shell RUNTIME DESTINATION .) IF (TD_MVN_INSTALLED) - INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.20-dist.jar DESTINATION connector/jdbc) + INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.21-dist.jar DESTINATION connector/jdbc) ENDIF () ELSEIF (TD_DARWIN) SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh") diff --git a/src/connector/jdbc/CMakeLists.txt b/src/connector/jdbc/CMakeLists.txt index cec8197849..3c50ac566b 100644 --- a/src/connector/jdbc/CMakeLists.txt +++ b/src/connector/jdbc/CMakeLists.txt @@ -8,7 +8,7 @@ IF (TD_MVN_INSTALLED) ADD_CUSTOM_COMMAND(OUTPUT ${JDBC_CMD_NAME} POST_BUILD COMMAND mvn -Dmaven.test.skip=true install -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml - COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.20-dist.jar ${LIBRARY_OUTPUT_PATH} + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.21-dist.jar ${LIBRARY_OUTPUT_PATH} COMMAND mvn -Dmaven.test.skip=true clean -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml COMMENT "build jdbc driver") ADD_CUSTOM_TARGET(${JDBC_TARGET_NAME} ALL WORKING_DIRECTORY ${EXECUTABLE_OUTPUT_PATH} DEPENDS ${JDBC_CMD_NAME}) diff --git a/src/connector/jdbc/deploy-pom.xml b/src/connector/jdbc/deploy-pom.xml index 999e11357d..1c24b621ef 100755 --- a/src/connector/jdbc/deploy-pom.xml +++ b/src/connector/jdbc/deploy-pom.xml @@ -5,7 +5,7 @@ com.taosdata.jdbc taos-jdbcdriver - 2.0.20 + 2.0.21 jar JDBCDriver diff --git a/src/connector/jdbc/pom.xml b/src/connector/jdbc/pom.xml index 25ed3d22f2..7a421eff22 100755 --- a/src/connector/jdbc/pom.xml +++ b/src/connector/jdbc/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.taosdata.jdbc taos-jdbcdriver - 2.0.20 + 2.0.21 jar JDBCDriver https://github.com/taosdata/TDengine/tree/master/src/connector/jdbc diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java index 1e05d62287..11c3de3052 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java @@ -50,26 +50,26 @@ public class SubscribeTest { try { String rawSql = "select * from " + dbName + "." + tName + ";"; System.out.println(rawSql); - TSDBSubscribe subscribe = ((TSDBConnection) connection).subscribe(topic, rawSql, false); +// TSDBSubscribe subscribe = ((TSDBConnection) connection).subscribe(topic, rawSql, false); - int a = 0; - while (true) { - TimeUnit.MILLISECONDS.sleep(1000); - TSDBResultSet resSet = subscribe.consume(); - while (resSet.next()) { - for (int i = 1; i <= resSet.getMetaData().getColumnCount(); i++) { - System.out.printf(i + ": " + resSet.getString(i) + "\t"); - } - System.out.println("\n======" + a + "=========="); - } - a++; - if (a >= 2) { - break; - } - resSet.close(); - } - - subscribe.close(true); +// int a = 0; +// while (true) { +// TimeUnit.MILLISECONDS.sleep(1000); +// TSDBResultSet resSet = subscribe.consume(); +// while (resSet.next()) { +// for (int i = 1; i <= resSet.getMetaData().getColumnCount(); i++) { +// System.out.printf(i + ": " + resSet.getString(i) + "\t"); +// } +// System.out.println("\n======" + a + "=========="); +// } +// a++; +// if (a >= 2) { +// break; +// } +// resSet.close(); +// } +// +// subscribe.close(true); } catch (Exception e) { e.printStackTrace(); } diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java index 8c3d562001..451f5d4916 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java @@ -9,8 +9,8 @@ import java.util.Random; @FixMethodOrder(MethodSorters.NAME_ASCENDING) public class RestfulJDBCTest { -// private static final String host = "127.0.0.1"; - private static final String host = "master"; + private static final String host = "127.0.0.1"; +// private static final String host = "master"; private static Connection connection; private Random random = new Random(System.currentTimeMillis()); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/SQLTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/SQLTest.java index 0b246e74d7..fe4d04775d 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/SQLTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/SQLTest.java @@ -11,8 +11,8 @@ import java.sql.*; @FixMethodOrder(MethodSorters.NAME_ASCENDING) public class SQLTest { -// private static final String host = "127.0.0.1"; - private static final String host = "master"; + private static final String host = "127.0.0.1"; +// private static final String host = "master"; private static Connection connection; @Test From 60be6be07f476bf1ceff7b9e0a0842b34b67d759 Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Fri, 5 Mar 2021 18:01:39 +0800 Subject: [PATCH 63/66] [TD-3158] : add column_meta in RESTful result. --- documentation20/cn/08.connector/docs.md | 36 ++++++++++++++++++------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/documentation20/cn/08.connector/docs.md b/documentation20/cn/08.connector/docs.md index 70403382b7..fb50e20e51 100644 --- a/documentation20/cn/08.connector/docs.md +++ b/documentation20/cn/08.connector/docs.md @@ -591,7 +591,8 @@ curl -u username:password -d '' :/rest/sql ```json { "status": "succ", - "head": ["Time Stamp","current", …], + "head": ["ts","current", …], + "column_meta": [["ts",9,8],["current",6,4], …], "data": [ ["2018-10-03 14:38:05.000", 10.3, …], ["2018-10-03 14:38:15.000", 12.6, …] @@ -602,10 +603,23 @@ curl -u username:password -d '' :/rest/sql 说明： -- status: 告知操作结果是成功还是失败 -- head: 表的定义，如果不返回结果集，仅有一列“affected_rows” -- data: 具体返回的数据，一排一排的呈现，如果不返回结果集，仅[[affected_rows]] -- rows: 表明总共多少行数据 +- status: 告知操作结果是成功还是失败。 +- head: 表的定义，如果不返回结果集，则仅有一列“affected_rows”。（从 2.0.17 版本开始，建议不要依赖 head 返回值来判断数据列类型，而推荐使用 column_meta。在未来版本中，有可能会从返回值中去掉 head 这一项。） +- column_meta: 从 2.0.17 版本开始，返回值中增加这一项来说明 data 里每一列的数据类型。具体每个列会用三个值来说明，分别为：列名、列类型、类型长度。例如`["current",6,4]`表示列名为“current”；列类型为 6，也即 float 类型；类型长度为 4，也即对应 4 个字节表示的 float。如果列类型为 binary 或 nchar，则类型长度表示该列最多可以保存的内容长度，而不是本次返回值中的具体数据长度。当列类型是 nchar 的时候，其类型长度表示可以保存的 unicode 字符数量，而不是 bytes。 +- data: 具体返回的数据，一行一行的呈现，如果不返回结果集，那么就仅有[[affected_rows]]。data 中每一行的数据列顺序，与 column_meta 中描述数据列的顺序完全一致。 +- rows: 表明总共多少行数据。 + +column_meta 中的列类型说明： +* 1：BOOL +* 2：TINYINT +* 3：SMALLINT +* 4：INT +* 5：BIGINT +* 6：FLOAT +* 7：DOUBLE +* 8：BINARY +* 9：TIMESTAMP +* 10：NCHAR ### 自定义授权码 @@ -651,7 +665,8 @@ curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.d1001 ```json { "status": "succ", - "head": ["Time Stamp","current","voltage","phase"], + "head": ["ts","current","voltage","phase"], + "column_meta": [["ts",9,8],["current",6,4],["voltage",4,4],["phase",6,4]], "data": [ ["2018-10-03 14:38:05.000",10.3,219,0.31], ["2018-10-03 14:38:15.000",12.6,218,0.33] @@ -671,8 +686,9 @@ curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'create database demo' 19 { "status": "succ", "head": ["affected_rows"], + "column_meta": [["affected_rows",4,4]], "data": [[1]], - "rows": 1, + "rows": 1 } ``` @@ -691,7 +707,8 @@ curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.d1001 ```json { "status": "succ", - "head": ["column1","column2","column3"], + "head": ["ts","current","voltage","phase"], + "column_meta": [["ts",9,8],["current",6,4],["voltage",4,4],["phase",6,4]], "data": [ [1538548685000,10.3,219,0.31], [1538548695000,12.6,218,0.33] @@ -712,7 +729,8 @@ HTTP请求URL采用`sqlutc`时，返回结果集的时间戳将采用UTC时间 ```json { "status": "succ", - "head": ["column1","column2","column3"], + "head": ["ts","current","voltage","phase"], + "column_meta": [["ts",9,8],["current",6,4],["voltage",4,4],["phase",6,4]], "data": [ ["2018-10-03T14:38:05.000+0800",10.3,219,0.31], ["2018-10-03T14:38:15.000+0800",12.6,218,0.33] From 42e8a73f91bf39ecd9c3260442972dee079575e2 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 5 Mar 2021 18:26:24 +0800 Subject: [PATCH 64/66] [TD-3147] : support insert interval. stb case passed. --- src/kit/taosdemo/taosdemo.c | 171 +++++++++++++++++++++++------------- 1 file changed, 112 insertions(+), 59 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 80cbecb96b..30370cd797 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -2061,7 +2061,7 @@ static int createDatabases() { for (int i = 0; i < g_Dbs.dbCount; i++) { if (g_Dbs.db[i].drop) { sprintf(command, "drop database if exists %s;", g_Dbs.db[i].dbName); - debugPrint("DEBUG %s() %d \n", __func__, __LINE__); + debugPrint("DEBUG %s() %d command: %s\n", __func__, __LINE__, command); if (0 != queryDbExec(taos, command, NO_INSERT_TYPE)) { taos_close(taos); return -1; @@ -2129,7 +2129,7 @@ static int createDatabases() { "precision \'%s\';", g_Dbs.db[i].dbCfg.precision); } - debugPrint("DEBUG %s() %d \n", __func__, __LINE__); + debugPrint("DEBUG %s() %d command: %s\n", __func__, __LINE__, command); if (0 != queryDbExec(taos, command, NO_INSERT_TYPE)) { taos_close(taos); printf("\ncreate database %s failed!\n\n", g_Dbs.db[i].dbName); @@ -2141,7 +2141,7 @@ static int createDatabases() { for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { // describe super table, if exists sprintf(command, "describe %s.%s;", g_Dbs.db[i].dbName, g_Dbs.db[i].superTbls[j].sTblName); - debugPrint("DEBUG %s() %d \n", __func__, __LINE__); + debugPrint("DEBUG %s() %d command: %s\n", __func__, __LINE__, command); if (0 != queryDbExec(taos, command, NO_INSERT_TYPE)) { g_Dbs.db[i].superTbls[j].superTblExists = TBL_NO_EXISTS; ret = createSuperTable(taos, g_Dbs.db[i].dbName, &g_Dbs.db[i].superTbls[j], g_Dbs.use_metric); @@ -2244,7 +2244,7 @@ static void* createTable(void *sarg) } if (0 != len) { - debugPrint("DEBUG %s() %d \n", __func__, __LINE__); + debugPrint("DEBUG %s() %d buffer: %s\n", __func__, __LINE__, buffer); (void)queryDbExec(winfo->taos, buffer, NO_INSERT_TYPE); } @@ -3777,14 +3777,6 @@ static void syncWriteForNumberOfTblInOneSql( int64_t st = 0; int64_t et = 0; for (int i = 0; i < superTblInfo->insertRows;) { - if (g_args.insert_interval && (g_args.insert_interval > (et - st))) { - taosMsleep(g_args.insert_interval - (et - st)); // ms - } - - if (g_args.insert_interval) { - st = taosGetTimestampMs(); - } - int32_t tbl_id = 0; for (int tID = winfo->start_table_id; tID <= winfo->end_table_id; ) { int inserted = i; @@ -3921,6 +3913,16 @@ static void syncWriteForNumberOfTblInOneSql( inserted += superTblInfo->rowsPerTbl; send_to_server: + if (g_args.insert_interval && (g_args.insert_interval > (et - st))) { + int sleep_time = g_args.insert_interval - (et -st); + debugPrint("DEBUG sleep: %d ms\n", sleep_time); + taosMsleep(sleep_time); // ms + } + + if (g_args.insert_interval) { + st = taosGetTimestampMs(); + } + if (0 == strncasecmp(superTblInfo->insertMode, "taosc", strlen("taosc"))) { @@ -3930,9 +3932,10 @@ send_to_server: int64_t endTs; startTs = taosGetTimestampUs(); - debugPrint("DEBUG %s() %d \n", __func__, __LINE__); + debugPrint("DEBUG %s() LN%d buff: %s\n", __func__, __LINE__, buffer); int affectedRows = queryDbExec( winfo->taos, buffer, INSERT_TYPE); + if (0 > affectedRows) { goto free_and_statistics; } else { @@ -3967,7 +3970,10 @@ send_to_server: goto free_and_statistics; } } - + if (g_args.insert_interval) { + et = taosGetTimestampMs(); + } + break; } @@ -3980,13 +3986,10 @@ send_to_server: } } - if (g_args.insert_interval) { - et = taosGetTimestampMs(); - } //printf("========loop %d childTables duration:%"PRId64 "========inserted rows:%d\n", winfo->end_table_id - winfo->start_table_id, et - st, i); } - free_and_statistics: +free_and_statistics: tmfree(buffer); winfo->totalRowsInserted = totalRowsInserted; winfo->totalAffectedRows = totalAffectedRows; @@ -4080,24 +4083,33 @@ static void* syncWrite(void *sarg) { srand((uint32_t)time(NULL)); int64_t time_counter = winfo->start_time; + uint64_t st = 0; + uint64_t et = 0; + for (int i = 0; i < g_args.num_of_DPT;) { + for (int tID = winfo->start_table_id; tID <= winfo->end_table_id; tID++) { int inserted = i; int64_t tmp_time = time_counter; char *pstr = buffer; - pstr += sprintf(pstr, "insert into %s.%s%d values", winfo->db_name, g_args.tb_prefix, tID); + pstr += sprintf(pstr, + "insert into %s.%s%d values", + winfo->db_name, g_args.tb_prefix, tID); int k; for (k = 0; k < g_args.num_of_RPR;) { int rand_num = rand() % 100; int len = -1; - if ((g_args.disorderRatio != 0) && (rand_num < g_args.disorderRange)) { + if ((g_args.disorderRatio != 0) + && (rand_num < g_args.disorderRange)) { int64_t d = tmp_time - rand() % 1000000 + rand_num; - len = generateData(data, data_type, ncols_per_record, d, len_of_binary); + len = generateData(data, data_type, + ncols_per_record, d, len_of_binary); } else { - len = generateData(data, data_type, ncols_per_record, tmp_time += 1000, len_of_binary); + len = generateData(data, data_type, + ncols_per_record, tmp_time += 1000, len_of_binary); } //assert(len + pstr - buffer < BUFFER_SIZE); @@ -4118,24 +4130,41 @@ static void* syncWrite(void *sarg) { int64_t endTs; startTs = taosGetTimestampUs(); //queryDB(winfo->taos, buffer); + if (i > 0 && g_args.insert_interval + && (g_args.insert_interval > (et - st) )) { + int sleep_time = g_args.insert_interval - (et -st); + debugPrint("DEBUG sleep: %d ms\n", sleep_time); + taosMsleep(sleep_time); // ms + } + + if (g_args.insert_interval) { + st = taosGetTimestampMs(); + } debugPrint("DEBUG - %s() LN%d %s\n", __func__, __LINE__, buffer); int affectedRows = queryDbExec(winfo->taos, buffer, 1); if (0 <= affectedRows){ endTs = taosGetTimestampUs(); int64_t delay = endTs - startTs; - if (delay > winfo->maxDelay) winfo->maxDelay = delay; - if (delay < winfo->minDelay) winfo->minDelay = delay; + if (delay > winfo->maxDelay) + winfo->maxDelay = delay; + if (delay < winfo->minDelay) + winfo->minDelay = delay; winfo->cntDelay++; winfo->totalDelay += delay; //winfo->avgDelay = (double)winfo->totalDelay / winfo->cntDelay; - } + } + + if (g_args.insert_interval) { + et = taosGetTimestampMs(); + } if (tID == winfo->end_table_id) { i = inserted; time_counter = tmp_time; } } + } return NULL; } @@ -4199,19 +4228,12 @@ static void* syncWriteWithStb(void *sarg) { return NULL; } - uint64_t time_counter = winfo->start_time; + int64_t time_counter = winfo->start_time; uint64_t st = 0; uint64_t et = 0; + debugPrint("DEBUG - %s() LN%d insertRows=%ld\n", __func__, __LINE__, superTblInfo->insertRows); for (int i = 0; i < superTblInfo->insertRows;) { - if (i > 0 && g_args.insert_interval - && (g_args.insert_interval > (et - st) )) { - taosMsleep(g_args.insert_interval - (et - st)); // ms - } - - if (g_args.insert_interval) { - st = taosGetTimestampMs(); - } for (uint32_t tID = winfo->start_table_id; tID <= winfo->end_table_id; tID++) { uint64_t inserted = i; @@ -4219,8 +4241,8 @@ static void* syncWriteWithStb(void *sarg) { int sampleUsePos = samplePos; int k = 0; - while (1) - { + debugPrint("DEBUG - %s() LN%d num_of_RPR=%d\n", __func__, __LINE__, g_args.num_of_RPR); + for (k = 0; k < g_args.num_of_RPR;) { int len = 0; memset(buffer, 0, superTblInfo->maxSqlLen); char *pstr = buffer; @@ -4263,9 +4285,8 @@ static void* syncWriteWithStb(void *sarg) { tID); } - for (k = 0; k < g_args.num_of_RPR;) { - int retLen = 0; - if (0 == strncasecmp(superTblInfo->dataSource, "sample", strlen("sample"))) { + int retLen = 0; + if (0 == strncasecmp(superTblInfo->dataSource, "sample", strlen("sample"))) { retLen = getRowDataFromSample( pstr + len, superTblInfo->maxSqlLen - len, @@ -4277,7 +4298,7 @@ static void* syncWriteWithStb(void *sarg) { if (retLen < 0) { goto free_and_statistics_2; } - } else if (0 == strncasecmp(superTblInfo->dataSource, "rand", strlen("rand"))) { + } else if (0 == strncasecmp(superTblInfo->dataSource, "rand", strlen("rand"))) { int rand_num = rand_tinyint() % 100; if (0 != superTblInfo->disorderRatio && rand_num < superTblInfo->disorderRatio) { @@ -4287,7 +4308,7 @@ static void* syncWriteWithStb(void *sarg) { superTblInfo->maxSqlLen - len, d, superTblInfo); //printf("disorder rows, rand_num:%d, last ts:%"PRId64" current ts:%"PRId64"\n", rand_num, tmp_time, d); - } else { + } else { retLen = generateRowData( pstr + len, superTblInfo->maxSqlLen - len, @@ -4297,17 +4318,31 @@ static void* syncWriteWithStb(void *sarg) { if (retLen < 0) { goto free_and_statistics_2; } - } - len += retLen; - inserted++; - k++; - totalRowsInserted++; + } +/* len += retLen; +*/ + inserted++; + k++; + totalRowsInserted++; - if (inserted >= superTblInfo->insertRows + debugPrint("DEBUG %s() LN%d inserted=%ld k=%d totalRowsInserted=%ld superTblInfo->insertRows=%ld\n", __func__, __LINE__, inserted, k, totalRowsInserted, superTblInfo->insertRows); + if (inserted > superTblInfo->insertRows) + break; +/* if (inserted >= superTblInfo->insertRows || (superTblInfo->maxSqlLen - len) < (superTblInfo->lenOfOneRow + 128)) break; +*/ + if (i > 0 && g_args.insert_interval + && (g_args.insert_interval > (et - st) )) { + int sleep_time = g_args.insert_interval - (et -st); + debugPrint("DEBUG sleep: %d ms\n", sleep_time); + taosMsleep(sleep_time); // ms } - + + if (g_args.insert_interval) { + st = taosGetTimestampMs(); + } + if (0 == strncasecmp(superTblInfo->insertMode, "taosc", strlen("taosc"))) { //printf("===== sql: %s \n\n", buffer); //int64_t t1 = taosGetTimestampMs(); @@ -4317,6 +4352,7 @@ static void* syncWriteWithStb(void *sarg) { debugPrint("DEBUG %s() LN%d %s\n", __func__, __LINE__, buffer); int affectedRows = queryDbExec(winfo->taos, buffer, INSERT_TYPE); + if (0 > affectedRows){ goto free_and_statistics_2; } else { @@ -4351,6 +4387,23 @@ static void* syncWriteWithStb(void *sarg) { goto free_and_statistics_2; } } + if (g_args.insert_interval) { + et = taosGetTimestampMs(); + } +/* + if (loop_cnt) { + loop_cnt--; + if ((1 == loop_cnt) && (0 != nrecords_last_req)) { + nrecords_cur_req = nrecords_last_req; + } else if (0 == loop_cnt){ + nrecords_cur_req = nrecords_no_last_req; + loop_cnt = loop_cnt_orig; + break; + } + } else { + break; + } + */ } if (tID == winfo->end_table_id) { @@ -4358,14 +4411,12 @@ static void* syncWriteWithStb(void *sarg) { superTblInfo->dataSource, "sample", strlen("sample"))) { samplePos = sampleUsePos; } + i = inserted; time_counter = tmp_time; } } - if (g_args.insert_interval) { - et = taosGetTimestampMs(); - } //printf("========loop %d childTables duration:%"PRId64 "========inserted rows:%d\n", winfo->end_table_id - winfo->start_table_id, et - st, i); } @@ -4502,7 +4553,6 @@ void startMultiThreadInsertData(int threads, char* db_name, char* precision, } } - int64_t start_time; if (superTblInfo) { if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) { @@ -4530,20 +4580,23 @@ void startMultiThreadInsertData(int threads, char* db_name, char* precision, t_info->start_time = start_time; t_info->minDelay = INT16_MAX; - if ((NULL == superTblInfo) || (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5))) { + if ((NULL == superTblInfo) || + (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5))) { //t_info->taos = taos; t_info->taos = taos_connect( g_Dbs.host, g_Dbs.user, g_Dbs.password, db_name, g_Dbs.port); if (NULL == t_info->taos) { - printf("connect to server fail from insert sub thread, reason: %s\n", taos_errstr(NULL)); + printf("connect to server fail from insert sub thread, reason: %s\n", + taos_errstr(NULL)); exit(-1); } } else { t_info->taos = NULL; } - if ((NULL == superTblInfo) || (0 == superTblInfo->multiThreadWriteOneTbl)) { + if ((NULL == superTblInfo) + || (0 == superTblInfo->multiThreadWriteOneTbl)) { t_info->start_table_id = last; t_info->end_table_id = i < b ? last + a : last + a - 1; last = t_info->end_table_id + 1; @@ -5012,7 +5065,7 @@ static int queryTestProcess() { char sqlStr[MAX_TB_NAME_SIZE*2]; sprintf(sqlStr, "use %s", g_queryInfo.dbName); - debugPrint("DEBUG %s() %d \n", __func__, __LINE__); + debugPrint("DEBUG %s() %d sqlStr: %s\n", __func__, __LINE__, sqlStr); (void)queryDbExec(t_info->taos, sqlStr, NO_INSERT_TYPE); } else { t_info->taos = NULL; @@ -5123,7 +5176,7 @@ void *subSubscribeProcess(void *sarg) { char sqlStr[MAX_TB_NAME_SIZE*2]; sprintf(sqlStr, "use %s", g_queryInfo.dbName); - debugPrint("DEBUG %s() %d \n", __func__, __LINE__); + debugPrint("DEBUG %s() %d sqlStr: %s\n", __func__, __LINE__, sqlStr); if (0 != queryDbExec(winfo->taos, sqlStr, NO_INSERT_TYPE)){ return NULL; } @@ -5189,7 +5242,7 @@ void *superSubscribeProcess(void *sarg) { char sqlStr[MAX_TB_NAME_SIZE*2]; sprintf(sqlStr, "use %s", g_queryInfo.dbName); - debugPrint("DEBUG %s() %d \n", __func__, __LINE__); + debugPrint("DEBUG %s() %d sqlStr: %s\n", __func__, __LINE__, sqlStr); if (0 != queryDbExec(winfo->taos, sqlStr, NO_INSERT_TYPE)) { return NULL; } @@ -5554,7 +5607,7 @@ void querySqlFile(TAOS* taos, char* sqlFile) } memcpy(cmd + cmd_len, line, read_len); - debugPrint("DEBUG %s() LN%d %s\n", __func__, __LINE__, cmd); + debugPrint("DEBUG %s() LN%d cmd: %s\n", __func__, __LINE__, cmd); queryDbExec(taos, cmd, NO_INSERT_TYPE); memset(cmd, 0, MAX_SQL_SIZE); cmd_len = 0; From a7d6df3dbab0ef59210fc55456728f2d1c4aff13 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 6 Mar 2021 13:28:44 +0800 Subject: [PATCH 65/66] [TD-3147] : support insert interval. fix compile incompatible issue. --- src/kit/taosdemo/taosdemo.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 30370cd797..de63e785c1 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -4232,7 +4232,9 @@ static void* syncWriteWithStb(void *sarg) { uint64_t st = 0; uint64_t et = 0; - debugPrint("DEBUG - %s() LN%d insertRows=%ld\n", __func__, __LINE__, superTblInfo->insertRows); + debugPrint("DEBUG - %s() LN%d insertRows=%"PRId64"\n", __func__, __LINE__, superTblInfo->insertRows); + //printf("disorder rows, rand_num:%d, last ts:%"PRId64" current ts:%"PRId64"\n", rand_num, tmp_time, d); + for (int i = 0; i < superTblInfo->insertRows;) { for (uint32_t tID = winfo->start_table_id; tID <= winfo->end_table_id; tID++) { @@ -4325,7 +4327,6 @@ static void* syncWriteWithStb(void *sarg) { k++; totalRowsInserted++; - debugPrint("DEBUG %s() LN%d inserted=%ld k=%d totalRowsInserted=%ld superTblInfo->insertRows=%ld\n", __func__, __LINE__, inserted, k, totalRowsInserted, superTblInfo->insertRows); if (inserted > superTblInfo->insertRows) break; /* if (inserted >= superTblInfo->insertRows From a2a1417a1b2c2ba70f203de09a24277f4c341327 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 6 Mar 2021 14:33:54 +0800 Subject: [PATCH 66/66] [TD-3147] : support insert interval. cleanup --- src/kit/taosdemo/taosdemo.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index de63e785c1..0353fcabd3 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -4233,7 +4233,6 @@ static void* syncWriteWithStb(void *sarg) { uint64_t et = 0; debugPrint("DEBUG - %s() LN%d insertRows=%"PRId64"\n", __func__, __LINE__, superTblInfo->insertRows); - //printf("disorder rows, rand_num:%d, last ts:%"PRId64" current ts:%"PRId64"\n", rand_num, tmp_time, d); for (int i = 0; i < superTblInfo->insertRows;) {