From e5c2b6aa10ab3fe032567afb1e3ba027b2a55b21 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 12 Jan 2021 22:27:43 +0800 Subject: [PATCH 001/147] TD-1207 --- src/common/inc/tdataformat.h | 4 +- src/common/inc/tglobal.h | 4 +- src/common/src/tglobal.c | 8 ++-- src/kit/shell/src/shellMain.c | 2 +- src/mnode/CMakeLists.txt | 2 +- src/mnode/src/mnodeAcct.c | 8 ++-- src/mnode/src/mnodeCluster.c | 10 ++--- src/mnode/src/mnodeDb.c | 8 ++-- src/mnode/src/mnodeDnode.c | 14 +++---- src/mnode/src/mnodeMnode.c | 6 +-- src/mnode/src/mnodeSdb.c | 22 +++++----- src/mnode/src/mnodeShow.c | 2 +- src/mnode/src/mnodeTable.c | 64 ++++++++++++++--------------- src/mnode/src/mnodeUser.c | 12 +++--- src/mnode/src/mnodeVgroup.c | 8 ++-- src/os/inc/osFile.h | 2 +- src/os/inc/osSocket.h | 1 + src/os/src/detail/osFile.c | 2 +- src/os/src/detail/osSocket.c | 10 +++++ src/os/src/windows/wFile.c | 2 +- src/os/src/windows/wSocket.c | 1 + src/plugins/http/CMakeLists.txt | 2 +- src/plugins/http/inc/httpInt.h | 4 +- src/plugins/http/src/httpGcJson.c | 2 +- src/plugins/http/src/httpGzip.c | 4 +- src/plugins/http/src/httpJson.c | 12 +++--- src/plugins/http/src/httpParser.c | 14 +++---- src/plugins/http/src/httpQueue.c | 4 +- src/plugins/http/src/httpServer.c | 22 +++------- src/plugins/http/src/httpTgHandle.c | 2 +- src/plugins/http/src/httpUtil.c | 4 +- src/plugins/monitor/CMakeLists.txt | 2 +- src/plugins/monitor/src/monMain.c | 8 ++-- src/rpc/src/rpcTcp.c | 2 +- src/sync/CMakeLists.txt | 2 +- src/sync/inc/syncInt.h | 4 +- src/sync/inc/syncTcp.h | 4 +- src/sync/src/syncArbitrator.c | 25 +++++++---- src/sync/src/syncMain.c | 14 +++---- src/sync/src/syncRestore.c | 6 +-- src/sync/src/syncRetrieve.c | 11 +++-- src/sync/src/syncTcp.c | 30 +++++++++----- src/util/inc/tsocket.h | 8 ++++ src/vnode/src/vnodeCfg.c | 26 ++++++------ src/wal/CMakeLists.txt | 2 +- src/wal/src/walMgmt.c | 2 +- src/wal/src/walWrite.c | 4 +- 47 files changed, 218 insertions(+), 194 deletions(-) diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index 8d4949d9b4..f1bf89aed4 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -29,7 +29,7 @@ extern "C" { #define STR_TO_VARSTR(x, str) \ do { \ - VarDataLenT __len = strlen(str); \ + VarDataLenT __len = (int32_t)strlen(str); \ *(VarDataLenT *)(x) = __len; \ memcpy(varDataVal(x), (str), __len); \ } while (0); @@ -42,7 +42,7 @@ extern "C" { #define STR_WITH_SIZE_TO_VARSTR(x, str, _size) \ do { \ - *(VarDataLenT *)(x) = (_size); \ + *(VarDataLenT *)(x) = (int32_t)(_size); \ memcpy(varDataVal(x), (str), (_size)); \ } while (0); diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index f6a08f8198..bf0d4e61df 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -89,8 +89,8 @@ extern int32_t tsMinRowsInFileBlock; extern int32_t tsMaxRowsInFileBlock; extern int16_t tsCommitTime; // seconds extern int32_t tsTimePrecision; -extern int16_t tsCompression; -extern int16_t tsWAL; +extern int8_t tsCompression; +extern int8_t tsWAL; extern int32_t tsFsyncPeriod; extern int32_t tsReplications; extern int32_t tsQuorum; diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 551fb96195..a7826f91ad 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -122,8 +122,8 @@ int32_t tsMinRowsInFileBlock = TSDB_DEFAULT_MIN_ROW_FBLOCK; int32_t tsMaxRowsInFileBlock = TSDB_DEFAULT_MAX_ROW_FBLOCK; int16_t tsCommitTime = TSDB_DEFAULT_COMMIT_TIME; // seconds int32_t tsTimePrecision = TSDB_DEFAULT_PRECISION; -int16_t tsCompression = TSDB_DEFAULT_COMP_LEVEL; -int16_t tsWAL = TSDB_DEFAULT_WAL_LEVEL; +int8_t tsCompression = TSDB_DEFAULT_COMP_LEVEL; +int8_t tsWAL = TSDB_DEFAULT_WAL_LEVEL; int32_t tsFsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD; int32_t tsReplications = TSDB_DEFAULT_DB_REPLICA_OPTION; int32_t tsQuorum = TSDB_DEFAULT_DB_QUORUM_OPTION; @@ -769,7 +769,7 @@ static void doInitGlobalConfig(void) { cfg.option = "comp"; cfg.ptr = &tsCompression; - cfg.valType = TAOS_CFG_VTYPE_INT16; + cfg.valType = TAOS_CFG_VTYPE_INT8; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; cfg.minValue = TSDB_MIN_COMP_LEVEL; cfg.maxValue = TSDB_MAX_COMP_LEVEL; @@ -779,7 +779,7 @@ static void doInitGlobalConfig(void) { cfg.option = "walLevel"; cfg.ptr = &tsWAL; - cfg.valType = TAOS_CFG_VTYPE_INT16; + cfg.valType = TAOS_CFG_VTYPE_INT8; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; cfg.minValue = TSDB_MIN_WAL_LEVEL; cfg.maxValue = TSDB_MAX_WAL_LEVEL; diff --git a/src/kit/shell/src/shellMain.c b/src/kit/shell/src/shellMain.c index 041ad71ccb..496cef41ba 100644 --- a/src/kit/shell/src/shellMain.c +++ b/src/kit/shell/src/shellMain.c @@ -21,7 +21,7 @@ pthread_t pid; static tsem_t cancelSem; -void shellQueryInterruptHandler(int signum) { +void shellQueryInterruptHandler(int32_t signum) { tsem_post(&cancelSem); } diff --git a/src/mnode/CMakeLists.txt b/src/mnode/CMakeLists.txt index ff5c9335b6..6e6d515de4 100644 --- a/src/mnode/CMakeLists.txt +++ b/src/mnode/CMakeLists.txt @@ -1,7 +1,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) -IF (TD_LINUX) +IF (TD_LINUX OR TD_WINDOWS) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/dnode/inc) diff --git a/src/mnode/src/mnodeAcct.c b/src/mnode/src/mnodeAcct.c index 6fba05674f..afe474df6b 100644 --- a/src/mnode/src/mnodeAcct.c +++ b/src/mnode/src/mnodeAcct.c @@ -81,7 +81,7 @@ static int32_t mnodeAcctActionDecode(SSdbRow *pRow) { } static int32_t mnodeAcctActionRestored() { - int32_t numOfRows = sdbGetNumOfRows(tsAcctSdb); + int64_t numOfRows = sdbGetNumOfRows(tsAcctSdb); if (numOfRows <= 0 && dnodeIsFirstDeploy()) { mInfo("dnode first deploy, create root acct"); int32_t code = mnodeCreateRootAcct(); @@ -97,14 +97,14 @@ static int32_t mnodeAcctActionRestored() { int32_t mnodeInitAccts() { SAcctObj tObj; - tsAcctUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; + tsAcctUpdateSize = (int32_t)((int8_t *)tObj.updateEnd - (int8_t *)&tObj); SSdbTableDesc desc = { .id = SDB_TABLE_ACCOUNT, .name = "accounts", .hashSessions = TSDB_DEFAULT_ACCOUNTS_HASH_SIZE, .maxRowSize = tsAcctUpdateSize, - .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, + .refCountPos = (int32_t)((int8_t *)(&tObj.refCount) - (int8_t *)&tObj), .keyType = SDB_KEY_STRING, .fpInsert = mnodeAcctActionInsert, .fpDelete = mnodeAcctActionDelete, @@ -206,7 +206,7 @@ void mnodeDropUserFromAcct(SAcctObj *pAcct, SUserObj *pUser) { } static int32_t mnodeCreateRootAcct() { - int32_t numOfAccts = sdbGetNumOfRows(tsAcctSdb); + int64_t numOfAccts = sdbGetNumOfRows(tsAcctSdb); if (numOfAccts != 0) return TSDB_CODE_SUCCESS; SAcctObj *pAcct = malloc(sizeof(SAcctObj)); diff --git a/src/mnode/src/mnodeCluster.c b/src/mnode/src/mnodeCluster.c index a35e304810..0931f31012 100644 --- a/src/mnode/src/mnodeCluster.c +++ b/src/mnode/src/mnodeCluster.c @@ -68,7 +68,7 @@ static int32_t mnodeClusterActionDecode(SSdbRow *pRow) { } static int32_t mnodeClusterActionRestored() { - int32_t numOfRows = sdbGetNumOfRows(tsClusterSdb); + int64_t numOfRows = sdbGetNumOfRows(tsClusterSdb); if (numOfRows <= 0 && dnodeIsFirstDeploy()) { mInfo("dnode first deploy, create cluster"); int32_t code = mnodeCreateCluster(); @@ -84,14 +84,14 @@ static int32_t mnodeClusterActionRestored() { int32_t mnodeInitCluster() { SClusterObj tObj; - tsClusterUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; + tsClusterUpdateSize = (int32_t)((int8_t *)tObj.updateEnd - (int8_t *)&tObj); SSdbTableDesc desc = { .id = SDB_TABLE_CLUSTER, .name = "cluster", .hashSessions = TSDB_DEFAULT_CLUSTER_HASH_SIZE, .maxRowSize = tsClusterUpdateSize, - .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, + .refCountPos = (int32_t)((int8_t *)(&tObj.refCount) - (int8_t *)&tObj), .keyType = SDB_KEY_STRING, .fpInsert = mnodeClusterActionInsert, .fpDelete = mnodeClusterActionDelete, @@ -139,7 +139,7 @@ void mnodeDecClusterRef(SClusterObj *pCluster) { } static int32_t mnodeCreateCluster() { - int32_t numOfClusters = sdbGetNumOfRows(tsClusterSdb); + int64_t numOfClusters = sdbGetNumOfRows(tsClusterSdb); if (numOfClusters != 0) return TSDB_CODE_SUCCESS; SClusterObj *pCluster = malloc(sizeof(SClusterObj)); @@ -226,7 +226,7 @@ static int32_t mnodeRetrieveClusters(SShowObj *pShow, char *data, int32_t rows, cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int32_t *) pWrite = pCluster->createdTime; + *(int64_t *) pWrite = pCluster->createdTime; cols++; mnodeDecClusterRef(pCluster); diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index 8a03b1cd0e..aa402b257a 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -143,14 +143,14 @@ static int32_t mnodeDbActionRestored() { int32_t mnodeInitDbs() { SDbObj tObj; - tsDbUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; + tsDbUpdateSize = (int32_t)((int8_t *)tObj.updateEnd - (int8_t *)&tObj); SSdbTableDesc desc = { .id = SDB_TABLE_DB, .name = "dbs", .hashSessions = TSDB_DEFAULT_DBS_HASH_SIZE, .maxRowSize = tsDbUpdateSize, - .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, + .refCountPos = (int32_t)((int8_t *)(&tObj.refCount) - (int8_t *)&tObj), .keyType = SDB_KEY_STRING, .fpInsert = mnodeDbActionInsert, .fpDelete = mnodeDbActionDelete, @@ -192,11 +192,11 @@ SDbObj *mnodeGetDb(char *db) { } void mnodeIncDbRef(SDbObj *pDb) { - return sdbIncRef(tsDbSdb, pDb); + sdbIncRef(tsDbSdb, pDb); } void mnodeDecDbRef(SDbObj *pDb) { - return sdbDecRef(tsDbSdb, pDb); + sdbDecRef(tsDbSdb, pDb); } SDbObj *mnodeGetDbByTableId(char *tableId) { diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index 792e41dd5b..01034b170f 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -148,7 +148,7 @@ static int32_t mnodeDnodeActionDecode(SSdbRow *pRow) { } static int32_t mnodeDnodeActionRestored() { - int32_t numOfRows = sdbGetNumOfRows(tsDnodeSdb); + int64_t numOfRows = sdbGetNumOfRows(tsDnodeSdb); if (numOfRows <= 0 && dnodeIsFirstDeploy()) { mInfo("dnode first deploy, create dnode:%s", tsLocalEp); mnodeCreateDnode(tsLocalEp, NULL); @@ -165,7 +165,7 @@ static int32_t mnodeDnodeActionRestored() { int32_t mnodeInitDnodes() { SDnodeObj tObj; - tsDnodeUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; + tsDnodeUpdateSize = (int32_t)((int8_t *)tObj.updateEnd - (int8_t *)&tObj); pthread_mutex_init(&tsDnodeEpsMutex, NULL); SSdbTableDesc desc = { @@ -173,7 +173,7 @@ int32_t mnodeInitDnodes() { .name = "dnodes", .hashSessions = TSDB_DEFAULT_DNODES_HASH_SIZE, .maxRowSize = tsDnodeUpdateSize, - .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, + .refCountPos = (int32_t)((int8_t *)(&tObj.refCount) - (int8_t *)&tObj), .keyType = SDB_KEY_AUTO, .fpInsert = mnodeDnodeActionInsert, .fpDelete = mnodeDnodeActionDelete, @@ -227,7 +227,7 @@ void mnodeCancelGetNextDnode(void *pIter) { } int32_t mnodeGetDnodesNum() { - return sdbGetNumOfRows(tsDnodeSdb); + return (int32_t)sdbGetNumOfRows(tsDnodeSdb); } int32_t mnodeGetOnlinDnodesCpuCoreNum() { @@ -407,7 +407,7 @@ static int32_t mnodeCheckClusterCfgPara(const SClusterCfg *clusterCfg) { int64_t checkTime = 0; char timestr[32] = "1970-01-01 00:00:00.00"; - (void)taosParseTime(timestr, &checkTime, strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); + (void)taosParseTime(timestr, &checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); if ((0 != strncasecmp(clusterCfg->timezone, tsTimezone, strlen(tsTimezone))) && (checkTime != clusterCfg->checkTime)) { mError("\"timezone\"[%s - %s] [%" PRId64 " - %" PRId64 "] cfg parameters inconsistent", clusterCfg->timezone, @@ -638,9 +638,9 @@ static int32_t mnodeCreateDnode(char *ep, SMnodeMsg *pMsg) { char *temp = strchr(dnodeEp, ':'); if (!temp) { - int len = strlen(dnodeEp); + int32_t len = (int32_t)strlen(dnodeEp); if (dnodeEp[len - 1] == ';') dnodeEp[len - 1] = 0; - len = strlen(dnodeEp); + len = (int32_t)strlen(dnodeEp); snprintf(dnodeEp + len, TSDB_EP_LEN - len, ":%d", tsServerPort); } ep = dnodeEp; diff --git a/src/mnode/src/mnodeMnode.c b/src/mnode/src/mnodeMnode.c index 3ea41c41c6..49473d3e06 100644 --- a/src/mnode/src/mnodeMnode.c +++ b/src/mnode/src/mnodeMnode.c @@ -136,14 +136,14 @@ int32_t mnodeInitMnodes() { mnodeMnodeInitLock(); SMnodeObj tObj; - tsMnodeUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; + tsMnodeUpdateSize = (int32_t)((int8_t *)tObj.updateEnd - (int8_t *)&tObj); SSdbTableDesc desc = { .id = SDB_TABLE_MNODE, .name = "mnodes", .hashSessions = TSDB_DEFAULT_MNODES_HASH_SIZE, .maxRowSize = tsMnodeUpdateSize, - .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, + .refCountPos = (int32_t)((int8_t *)(&tObj.refCount) - (int8_t *)&tObj), .keyType = SDB_KEY_INT, .fpInsert = mnodeMnodeActionInsert, .fpDelete = mnodeMnodeActionDelete, @@ -176,7 +176,7 @@ void mnodeCleanupMnodes() { } int32_t mnodeGetMnodesNum() { - return sdbGetNumOfRows(tsMnodeSdb); + return (int32_t)sdbGetNumOfRows(tsMnodeSdb); } void *mnodeGetMnode(int32_t mnodeId) { diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index ae495108b3..17cfd3e9d4 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -207,7 +207,7 @@ static void sdbRestoreTables() { (*pTable->fpRestored)(); } - totalRows += pTable->numOfRows; + totalRows += (int32_t)pTable->numOfRows; numOfTables++; sdbInfo("vgId:1, sdb:%s is checked, rows:%" PRId64, pTable->name, pTable->numOfRows); } @@ -475,7 +475,7 @@ void sdbIncRef(void *tparam, void *pRow) { if (pRow == NULL || tparam == NULL) return; SSdbTable *pTable = tparam; - int32_t * pRefCount = (int32_t *)(pRow + pTable->refCountPos); + int32_t * pRefCount = (int32_t *)((char *)pRow + pTable->refCountPos); int32_t refCount = atomic_add_fetch_32(pRefCount, 1); sdbTrace("vgId:1, sdb:%s, inc ref to row:%p:%s:%d", pTable->name, pRow, sdbGetRowStr(pTable, pRow), refCount); } @@ -484,11 +484,11 @@ void sdbDecRef(void *tparam, void *pRow) { if (pRow == NULL || tparam == NULL) return; SSdbTable *pTable = tparam; - int32_t * pRefCount = (int32_t *)(pRow + pTable->refCountPos); + int32_t * pRefCount = (int32_t *)((char *)pRow + pTable->refCountPos); int32_t refCount = atomic_sub_fetch_32(pRefCount, 1); sdbTrace("vgId:1, sdb:%s, dec ref to row:%p:%s:%d", pTable->name, pRow, sdbGetRowStr(pTable, pRow), refCount); - int32_t *updateEnd = pRow + pTable->refCountPos - 4; + int32_t *updateEnd = (int32_t *)((char *)pRow + pTable->refCountPos - 4); if (refCount <= 0 && *updateEnd) { sdbTrace("vgId:1, sdb:%s, row:%p:%s:%d destroyed", pTable->name, pRow, sdbGetRowStr(pTable, pRow), refCount); SSdbRow row = {.pObj = pRow}; @@ -501,7 +501,7 @@ static void *sdbGetRowMeta(SSdbTable *pTable, void *key) { int32_t keySize = sizeof(int32_t); if (pTable->keyType == SDB_KEY_STRING || pTable->keyType == SDB_KEY_VAR_STRING) { - keySize = strlen((char *)key); + keySize = (int32_t)strlen((char *)key); } void **ppRow = (void **)taosHashGet(pTable->iHandle, key, keySize); @@ -534,7 +534,7 @@ static int32_t sdbInsertHash(SSdbTable *pTable, SSdbRow *pRow) { int32_t keySize = sizeof(int32_t); if (pTable->keyType == SDB_KEY_STRING || pTable->keyType == SDB_KEY_VAR_STRING) { - keySize = strlen((char *)key); + keySize = (int32_t)strlen((char *)key); } pthread_mutex_lock(&pTable->mutex); @@ -564,7 +564,7 @@ static int32_t sdbInsertHash(SSdbTable *pTable, SSdbRow *pRow) { } static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbRow *pRow) { - int32_t *updateEnd = pRow->pObj + pTable->refCountPos - 4; + int32_t *updateEnd = (int32_t *)((char*)pRow->pObj + pTable->refCountPos - 4); bool set = atomic_val_compare_exchange_32(updateEnd, 0, 1) == 0; if (!set) { sdbError("vgId:1, sdb:%s, failed to delete key:%s from hash, for it already removed", pTable->name, @@ -577,7 +577,7 @@ static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbRow *pRow) { void * key = sdbGetObjKey(pTable, pRow->pObj); int32_t keySize = sizeof(int32_t); if (pTable->keyType == SDB_KEY_STRING || pTable->keyType == SDB_KEY_VAR_STRING) { - keySize = strlen((char *)key); + keySize = (int32_t)strlen((char *)key); } pthread_mutex_lock(&pTable->mutex); @@ -764,7 +764,7 @@ bool sdbCheckRowDeleted(void *tparam, void *pRow) { SSdbTable *pTable = tparam; if (pTable == NULL) return false; - int32_t *updateEnd = pRow + pTable->refCountPos - 4; + int32_t *updateEnd = (int32_t *)((char*)pRow + pTable->refCountPos - 4); return atomic_val_compare_exchange_32(updateEnd, 1, 1) == 1; } @@ -942,14 +942,14 @@ static int32_t sdbInitWorker() { static void sdbCleanupWorker() { for (int32_t i = 0; i < tsSdbPool.num; ++i) { SSdbWorker *pWorker = tsSdbPool.worker + i; - if (pWorker->thread) { + if (taosCheckPthreadValid(pWorker->thread)) { taosQsetThreadResume(tsSdbWQset); } } for (int32_t i = 0; i < tsSdbPool.num; ++i) { SSdbWorker *pWorker = tsSdbPool.worker + i; - if (pWorker->thread) { + if (taosCheckPthreadValid(pWorker->thread)) { pthread_join(pWorker->thread, NULL); } } diff --git a/src/mnode/src/mnodeShow.c b/src/mnode/src/mnodeShow.c index 6b9f0e26a7..0377df97fd 100644 --- a/src/mnode/src/mnodeShow.c +++ b/src/mnode/src/mnodeShow.c @@ -218,7 +218,7 @@ static int32_t mnodeProcessRetrieveMsg(SMnodeMsg *pMsg) { } pRsp->numOfRows = htonl(rowsRead); - pRsp->precision = htonl(TSDB_TIME_PRECISION_MILLI); // millisecond time precision + pRsp->precision = (int16_t)htonl(TSDB_TIME_PRECISION_MILLI); // millisecond time precision pMsg->rpcRsp.rsp = pRsp; pMsg->rpcRsp.len = size; diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index de37b09345..a0c8d88c51 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -222,23 +222,23 @@ static int32_t mnodeChildTableActionEncode(SSdbRow *pRow) { SCTableObj *pTable = pRow->pObj; assert(pTable != NULL && pRow->rowData != NULL); - int32_t len = strlen(pTable->info.tableId); + int32_t len = (int32_t)strlen(pTable->info.tableId); if (len >= TSDB_TABLE_FNAME_LEN) return TSDB_CODE_MND_INVALID_TABLE_ID; memcpy(pRow->rowData, pTable->info.tableId, len); - memset(pRow->rowData + len, 0, 1); + memset((char *)pRow->rowData + len, 0, 1); len++; - memcpy(pRow->rowData + len, (char*)pTable + sizeof(char *), tsChildTableUpdateSize); + memcpy((char *)pRow->rowData + len, (char *)pTable + sizeof(char *), tsChildTableUpdateSize); len += tsChildTableUpdateSize; if (pTable->info.type != TSDB_CHILD_TABLE) { int32_t schemaSize = pTable->numOfColumns * sizeof(SSchema); - memcpy(pRow->rowData + len, pTable->schema, schemaSize); + memcpy((char *)pRow->rowData + len, pTable->schema, schemaSize); len += schemaSize; if (pTable->sqlLen != 0) { - memcpy(pRow->rowData + len, pTable->sql, pTable->sqlLen); + memcpy((char *)pRow->rowData + len, pTable->sql, pTable->sqlLen); len += pTable->sqlLen; } } @@ -253,7 +253,7 @@ static int32_t mnodeChildTableActionDecode(SSdbRow *pRow) { SCTableObj *pTable = calloc(1, sizeof(SCTableObj)); if (pTable == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; - int32_t len = strlen(pRow->rowData); + int32_t len = (int32_t)strlen(pRow->rowData); if (len >= TSDB_TABLE_FNAME_LEN) { free(pTable); return TSDB_CODE_MND_INVALID_TABLE_ID; @@ -261,7 +261,7 @@ static int32_t mnodeChildTableActionDecode(SSdbRow *pRow) { pTable->info.tableId = strdup(pRow->rowData); len++; - memcpy((char*)pTable + sizeof(char *), pRow->rowData + len, tsChildTableUpdateSize); + memcpy((char *)pTable + sizeof(char *), (char *)pRow->rowData + len, tsChildTableUpdateSize); len += tsChildTableUpdateSize; if (pTable->info.type != TSDB_CHILD_TABLE) { @@ -271,7 +271,7 @@ static int32_t mnodeChildTableActionDecode(SSdbRow *pRow) { mnodeDestroyChildTable(pTable); return TSDB_CODE_MND_INVALID_TABLE_TYPE; } - memcpy(pTable->schema, pRow->rowData + len, schemaSize); + memcpy(pTable->schema, (char *)pRow->rowData + len, schemaSize); len += schemaSize; if (pTable->sqlLen != 0) { @@ -280,7 +280,7 @@ static int32_t mnodeChildTableActionDecode(SSdbRow *pRow) { mnodeDestroyChildTable(pTable); return TSDB_CODE_MND_OUT_OF_MEMORY; } - memcpy(pTable->sql, pRow->rowData + len, pTable->sqlLen); + memcpy(pTable->sql, (char *)pRow->rowData + len, pTable->sqlLen); } } @@ -352,14 +352,14 @@ static int32_t mnodeChildTableActionRestored() { static int32_t mnodeInitChildTables() { SCTableObj tObj; - tsChildTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj.info.type; + tsChildTableUpdateSize = (int32_t)((int8_t *)tObj.updateEnd - (int8_t *)&tObj.info.type); SSdbTableDesc desc = { .id = SDB_TABLE_CTABLE, .name = "ctables", .hashSessions = TSDB_DEFAULT_CTABLES_HASH_SIZE, .maxRowSize = sizeof(SCTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16) + TSDB_TABLE_FNAME_LEN + TSDB_CQ_SQL_SIZE, - .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, + .refCountPos = (int32_t)((int8_t *)(&tObj.refCount) - (int8_t *)&tObj), .keyType = SDB_KEY_VAR_STRING, .fpInsert = mnodeChildTableActionInsert, .fpDelete = mnodeChildTableActionDelete, @@ -501,18 +501,18 @@ static int32_t mnodeSuperTableActionEncode(SSdbRow *pRow) { SSTableObj *pStable = pRow->pObj; assert(pRow->pObj != NULL && pRow->rowData != NULL); - int32_t len = strlen(pStable->info.tableId); + int32_t len = (int32_t)strlen(pStable->info.tableId); if (len >= TSDB_TABLE_FNAME_LEN) len = TSDB_CODE_MND_INVALID_TABLE_ID; memcpy(pRow->rowData, pStable->info.tableId, len); - memset(pRow->rowData + len, 0, 1); + memset((char *)pRow->rowData + len, 0, 1); len++; - memcpy(pRow->rowData + len, (char*)pStable + sizeof(char *), tsSuperTableUpdateSize); + memcpy((char *)pRow->rowData + len, (char *)pStable + sizeof(char *), tsSuperTableUpdateSize); len += tsSuperTableUpdateSize; int32_t schemaSize = sizeof(SSchema) * (pStable->numOfColumns + pStable->numOfTags); - memcpy(pRow->rowData + len, pStable->schema, schemaSize); + memcpy((char *)pRow->rowData + len, pStable->schema, schemaSize); len += schemaSize; pRow->rowSize = len; @@ -525,7 +525,7 @@ static int32_t mnodeSuperTableActionDecode(SSdbRow *pRow) { SSTableObj *pStable = (SSTableObj *) calloc(1, sizeof(SSTableObj)); if (pStable == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; - int32_t len = strlen(pRow->rowData); + int32_t len = (int32_t)strlen(pRow->rowData); if (len >= TSDB_TABLE_FNAME_LEN){ free(pStable); return TSDB_CODE_MND_INVALID_TABLE_ID; @@ -533,7 +533,7 @@ static int32_t mnodeSuperTableActionDecode(SSdbRow *pRow) { pStable->info.tableId = strdup(pRow->rowData); len++; - memcpy((char*)pStable + sizeof(char *), pRow->rowData + len, tsSuperTableUpdateSize); + memcpy((char *)pStable + sizeof(char *), (char *)pRow->rowData + len, tsSuperTableUpdateSize); len += tsSuperTableUpdateSize; int32_t schemaSize = sizeof(SSchema) * (pStable->numOfColumns + pStable->numOfTags); @@ -543,7 +543,7 @@ static int32_t mnodeSuperTableActionDecode(SSdbRow *pRow) { return TSDB_CODE_MND_NOT_SUPER_TABLE; } - memcpy(pStable->schema, pRow->rowData + len, schemaSize); + memcpy(pStable->schema, (char *)pRow->rowData + len, schemaSize); pRow->pObj = pStable; @@ -556,14 +556,14 @@ static int32_t mnodeSuperTableActionRestored() { static int32_t mnodeInitSuperTables() { SSTableObj tObj; - tsSuperTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj.info.type; + tsSuperTableUpdateSize = (int32_t)((int8_t *)tObj.updateEnd - (int8_t *)&tObj.info.type); SSdbTableDesc desc = { .id = SDB_TABLE_STABLE, .name = "stables", .hashSessions = TSDB_DEFAULT_STABLES_HASH_SIZE, .maxRowSize = sizeof(SSTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16) + TSDB_TABLE_FNAME_LEN, - .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, + .refCountPos = (int32_t)((int8_t *)(&tObj.refCount) - (int8_t *)&tObj), .keyType = SDB_KEY_VAR_STRING, .fpInsert = mnodeSuperTableActionInsert, .fpDelete = mnodeSuperTableActionDelete, @@ -1266,7 +1266,7 @@ static int32_t mnodeModifySuperTableTagName(SMnodeMsg *pMsg, char *oldTagName, c } // int32_t rowSize = 0; - uint32_t len = strlen(newTagName); + uint32_t len = (int32_t)strlen(newTagName); if (len >= TSDB_COL_NAME_LEN) { return TSDB_CODE_MND_COL_NAME_TOO_LONG; } @@ -1429,7 +1429,7 @@ static int32_t mnodeChangeSuperTableColumn(SMnodeMsg *pMsg, char *oldName, char } // int32_t rowSize = 0; - uint32_t len = strlen(newName); + uint32_t len = (uint32_t)strlen(newName); if (len >= TSDB_COL_NAME_LEN) { return TSDB_CODE_MND_COL_NAME_TOO_LONG; } @@ -1534,7 +1534,7 @@ int32_t mnodeRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, tstrncpy(prefix, pDb->name, 64); strcat(prefix, TS_PATH_DELIMITER); - prefixLen = strlen(prefix); + prefixLen = (int32_t)strlen(prefix); SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; char stableName[TSDB_TABLE_NAME_LEN] = {0}; @@ -1559,7 +1559,7 @@ int32_t mnodeRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - int16_t len = strnlen(stableName, TSDB_TABLE_NAME_LEN - 1); + int16_t len = (int16_t)strnlen(stableName, TSDB_TABLE_NAME_LEN - 1); *(int16_t*) pWrite = len; pWrite += sizeof(int16_t); // todo refactor @@ -1602,7 +1602,7 @@ void mnodeDropAllSuperTables(SDbObj *pDropDb) { char prefix[64] = {0}; tstrncpy(prefix, pDropDb->name, 64); strcat(prefix, TS_PATH_DELIMITER); - int32_t prefixLen = strlen(prefix); + int32_t prefixLen = (int32_t)strlen(prefix); mInfo("db:%s, all super tables will be dropped from sdb", pDropDb->name); @@ -1755,9 +1755,9 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) { rpcFreeCont(pRsp); return TSDB_CODE_MND_INVALID_TABLE_NAME; } else { - pRsp->numOfTables = htonl(pRsp->numOfTables); + pRsp->numOfTables = (int32_t)htonl(pRsp->numOfTables); pMsg->rpcRsp.rsp = pRsp; - pMsg->rpcRsp.len = msg - (char *)pRsp; + pMsg->rpcRsp.len = (int32_t)((char *)msg - (char *)pRsp); return TSDB_CODE_SUCCESS; } @@ -2030,7 +2030,7 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) { static int32_t mnodeProcessCreateChildTableMsg(SMnodeMsg *pMsg) { //SCMCreateTableMsg* p1 = pMsg->rpcMsg.pCont; // there are several tables here. - SCreateTableMsg* pCreate = (SCreateTableMsg*)(pMsg->rpcMsg.pCont + sizeof(SCMCreateTableMsg)); + SCreateTableMsg* pCreate = (SCreateTableMsg*)((char *)pMsg->rpcMsg.pCont + sizeof(SCMCreateTableMsg)); int32_t code = grantCheck(TSDB_GRANT_TIMESERIES); if (code != TSDB_CODE_SUCCESS) { @@ -2287,7 +2287,7 @@ static int32_t mnodeChangeNormalTableColumn(SMnodeMsg *pMsg, char *oldName, char } // int32_t rowSize = 0; - uint32_t len = strlen(newName); + uint32_t len = (uint32_t)strlen(newName); if (len >= TSDB_COL_NAME_LEN) { return TSDB_CODE_MND_COL_NAME_TOO_LONG; } @@ -2491,7 +2491,7 @@ void mnodeDropAllChildTables(SDbObj *pDropDb) { char prefix[64] = {0}; tstrncpy(prefix, pDropDb->name, 64); strcat(prefix, TS_PATH_DELIMITER); - int32_t prefixLen = strlen(prefix); + int32_t prefixLen = (int32_t)strlen(prefix); mInfo("db:%s, all child tables will be dropped from sdb", pDropDb->name); @@ -2907,7 +2907,7 @@ static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; char prefix[64] = {0}; - int32_t prefixLen = tableIdPrefix(pDb->name, prefix, 64); + int32_t prefixLen = (int32_t)tableIdPrefix(pDb->name, prefix, 64); char* pattern = NULL; if (pShow->payloadLen > 0) { @@ -3143,7 +3143,7 @@ static int32_t mnodeRetrieveStreamTables(SShowObj *pShow, char *data, int32_t ro char prefix[64] = {0}; tstrncpy(prefix, pDb->name, 64); strcat(prefix, TS_PATH_DELIMITER); - int32_t prefixLen = strlen(prefix); + int32_t prefixLen = (int32_t)strlen(prefix); while (numOfRows < rows) { pShow->pIter = mnodeGetNextChildTable(pShow->pIter, &pTable); diff --git a/src/mnode/src/mnodeUser.c b/src/mnode/src/mnodeUser.c index fb26086d04..3a699757d9 100644 --- a/src/mnode/src/mnodeUser.c +++ b/src/mnode/src/mnodeUser.c @@ -128,7 +128,7 @@ static void mnodePrintUserAuth() { } static int32_t mnodeUserActionRestored() { - int32_t numOfRows = sdbGetNumOfRows(tsUserSdb); + int64_t numOfRows = sdbGetNumOfRows(tsUserSdb); if (numOfRows <= 0 && dnodeIsFirstDeploy()) { mInfo("dnode first deploy, create root user"); SAcctObj *pAcct = mnodeGetAcct(TSDB_DEFAULT_USER); @@ -148,14 +148,14 @@ static int32_t mnodeUserActionRestored() { int32_t mnodeInitUsers() { SUserObj tObj; - tsUserUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; + tsUserUpdateSize = (int32_t)((int8_t *)tObj.updateEnd - (int8_t *)&tObj); SSdbTableDesc desc = { .id = SDB_TABLE_USER, .name = "users", .hashSessions = TSDB_DEFAULT_USERS_HASH_SIZE, .maxRowSize = tsUserUpdateSize, - .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, + .refCountPos = (int32_t)((int8_t *)(&tObj.refCount) - (int8_t *)&tObj), .keyType = SDB_KEY_STRING, .fpInsert = mnodeUserActionInsert, .fpDelete = mnodeUserActionDelete, @@ -204,11 +204,11 @@ void mnodeCancelGetNextUser(void *pIter) { } void mnodeIncUserRef(SUserObj *pUser) { - return sdbIncRef(tsUserSdb, pUser); + sdbIncRef(tsUserSdb, pUser); } void mnodeDecUserRef(SUserObj *pUser) { - return sdbDecRef(tsUserSdb, pUser); + sdbDecRef(tsUserSdb, pUser); } static int32_t mnodeUpdateUser(SUserObj *pUser, void *pMsg) { @@ -561,7 +561,7 @@ static int32_t mnodeProcessDropUserMsg(SMnodeMsg *pMsg) { void mnodeDropAllUsers(SAcctObj *pAcct) { void * pIter = NULL; int32_t numOfUsers = 0; - int32_t acctNameLen = strlen(pAcct->user); + int32_t acctNameLen = (int32_t)strlen(pAcct->user); SUserObj *pUser = NULL; while (1) { diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index 827be0687d..bd89f2a267 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -206,14 +206,14 @@ static int32_t mnodeVgroupActionRestored() { int32_t mnodeInitVgroups() { SVgObj tObj; - tsVgUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; + tsVgUpdateSize = (int32_t)((int8_t *)tObj.updateEnd - (int8_t *)&tObj); SSdbTableDesc desc = { .id = SDB_TABLE_VGROUP, .name = "vgroups", .hashSessions = TSDB_DEFAULT_VGROUPS_HASH_SIZE, .maxRowSize = tsVgUpdateSize, - .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, + .refCountPos = (int32_t)((int8_t *)(&tObj.refCount) - (int8_t *)&tObj), .keyType = SDB_KEY_AUTO, .fpInsert = mnodeVgroupActionInsert, .fpDelete = mnodeVgroupActionDelete, @@ -245,11 +245,11 @@ int32_t mnodeInitVgroups() { } void mnodeIncVgroupRef(SVgObj *pVgroup) { - return sdbIncRef(tsVgroupSdb, pVgroup); + sdbIncRef(tsVgroupSdb, pVgroup); } void mnodeDecVgroupRef(SVgObj *pVgroup) { - return sdbDecRef(tsVgroupSdb, pVgroup); + sdbDecRef(tsVgroupSdb, pVgroup); } SVgObj *mnodeGetVgroup(int32_t vgId) { diff --git a/src/os/inc/osFile.h b/src/os/inc/osFile.h index c9b3b9cd76..91ac5c9958 100644 --- a/src/os/inc/osFile.h +++ b/src/os/inc/osFile.h @@ -37,7 +37,7 @@ int32_t taosRenameFile(char *fullPath, char *suffix, char delimiter, char **dstP } // TAOS_OS_FUNC_FILE_SENDIFLE -int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t *offset, int64_t size); +int64_t taosSendFile(SOCKET dfd, int32_t sfd, int64_t *offset, int64_t size); int64_t taosFSendFile(FILE *outfile, FILE *infile, int64_t *offset, int64_t size); #ifdef TAOS_RANDOM_FILE_FAIL diff --git a/src/os/inc/osSocket.h b/src/os/inc/osSocket.h index 13d3fa4079..4e2671ce2b 100644 --- a/src/os/inc/osSocket.h +++ b/src/os/inc/osSocket.h @@ -61,6 +61,7 @@ extern "C" { int32_t taosSetNonblocking(SOCKET sock, int32_t on); void taosIgnSIGPIPE(); void taosBlockSIGPIPE(); +void taosSetMaskSIGPIPE(); // TAOS_OS_FUNC_SOCKET_SETSOCKETOPT int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen); diff --git a/src/os/src/detail/osFile.c b/src/os/src/detail/osFile.c index 2e6886aa21..6ba38635f4 100644 --- a/src/os/src/detail/osFile.c +++ b/src/os/src/detail/osFile.c @@ -121,7 +121,7 @@ int64_t taosLSeekImp(int32_t fd, int64_t offset, int32_t whence) { #ifndef TAOS_OS_FUNC_FILE_SENDIFLE -int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t *offset, int64_t size) { +int64_t taosSendFile(SOCKET dfd, int32_t sfd, int64_t *offset, int64_t size) { int64_t leftbytes = size; int64_t sentbytes; diff --git a/src/os/src/detail/osSocket.c b/src/os/src/detail/osSocket.c index 729471247f..d03e8086bf 100644 --- a/src/os/src/detail/osSocket.c +++ b/src/os/src/detail/osSocket.c @@ -53,6 +53,16 @@ void taosBlockSIGPIPE() { } } +void taosSetMaskSIGPIPE() { + sigset_t signal_mask; + sigemptyset(&signal_mask); + sigaddset(&signal_mask, SIGPIPE); + int32_t rc = pthread_sigmask(SIG_SETMASK, &signal_mask, NULL); + if (rc != 0) { + uError("failed to setmask SIGPIPE"); + } +} + #endif #ifndef TAOS_OS_FUNC_SOCKET_SETSOCKETOPT diff --git a/src/os/src/windows/wFile.c b/src/os/src/windows/wFile.c index 2204135ae6..a127c17ce7 100644 --- a/src/os/src/windows/wFile.c +++ b/src/os/src/windows/wFile.c @@ -78,7 +78,7 @@ int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t co return writeLen; } -int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t* offset, int64_t size) { +int64_t taosSendFile(SOCKET dfd, int32_t sfd, int64_t* offset, int64_t size) { uError("taosSendFile no implemented yet"); return 0; } diff --git a/src/os/src/windows/wSocket.c b/src/os/src/windows/wSocket.c index 9697c5e65f..4e6ee15e77 100644 --- a/src/os/src/windows/wSocket.c +++ b/src/os/src/windows/wSocket.c @@ -48,6 +48,7 @@ int32_t taosSetNonblocking(SOCKET sock, int32_t on) { void taosIgnSIGPIPE() {} void taosBlockSIGPIPE() {} +void taosSetMaskSIGPIPE() {} int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen) { if (level == SOL_SOCKET && optname == TCP_KEEPCNT) { diff --git a/src/plugins/http/CMakeLists.txt b/src/plugins/http/CMakeLists.txt index 5d8b52986a..86d857876d 100644 --- a/src/plugins/http/CMakeLists.txt +++ b/src/plugins/http/CMakeLists.txt @@ -9,7 +9,7 @@ INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) INCLUDE_DIRECTORIES(inc) AUX_SOURCE_DIRECTORY(src SRC) -IF (TD_LINUX) +IF (TD_LINUX OR TD_WINDOWS) ADD_LIBRARY(http ${SRC}) TARGET_LINK_LIBRARIES(http z) diff --git a/src/plugins/http/inc/httpInt.h b/src/plugins/http/inc/httpInt.h index ebdfabf310..ada67efbfd 100644 --- a/src/plugins/http/inc/httpInt.h +++ b/src/plugins/http/inc/httpInt.h @@ -166,7 +166,7 @@ typedef struct HttpThread { HttpContext * pHead; pthread_mutex_t threadMutex; bool stop; - int32_t pollFd; + SOCKET pollFd; int32_t numOfContexts; int32_t threadId; char label[HTTP_LABEL_SIZE]; @@ -177,7 +177,7 @@ typedef struct HttpServer { char label[HTTP_LABEL_SIZE]; uint32_t serverIp; uint16_t serverPort; - int32_t fd; + SOCKET fd; int32_t numOfThreads; int32_t methodScannerLen; int32_t requestNum; diff --git a/src/plugins/http/src/httpGcJson.c b/src/plugins/http/src/httpGcJson.c index 2c9eca11de..3a053afd87 100644 --- a/src/plugins/http/src/httpGcJson.c +++ b/src/plugins/http/src/httpGcJson.c @@ -158,7 +158,7 @@ bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, if (row[i]!= NULL){ len += snprintf(target + len, HTTP_GC_TARGET_SIZE - len, "%s:", fields[i].name); memcpy(target + len, (char *) row[i], length[i]); - len = strlen(target); + len = (int32_t)strlen(target); } break; default: diff --git a/src/plugins/http/src/httpGzip.c b/src/plugins/http/src/httpGzip.c index 54f900c755..d94e3b1235 100644 --- a/src/plugins/http/src/httpGzip.c +++ b/src/plugins/http/src/httpGzip.c @@ -133,7 +133,7 @@ int32_t ehttp_gzip_write(ehttp_gzip_t *gzip, const char *buf, int32_t len) { if (ret!=Z_STREAM_END) continue; } - int32_t len = gzip->gzip->next_out - (z_const Bytef*)gzip->chunk; + int32_t len = (int32_t)(gzip->gzip->next_out - (z_const Bytef*)gzip->chunk); gzip->gzip->next_out[0] = '\0'; gzip->callbacks.on_data(gzip, gzip->arg, gzip->chunk, len); @@ -155,7 +155,7 @@ int32_t ehttp_gzip_finish(ehttp_gzip_t *gzip) { if (ret != Z_STREAM_END) return -1; - int32_t len = gzip->gzip->next_out - (z_const Bytef*)gzip->chunk; + int32_t len = (int32_t)(gzip->gzip->next_out - (z_const Bytef*)gzip->chunk); gzip->gzip->next_out[0] = '\0'; gzip->callbacks.on_data(gzip, gzip->arg, gzip->chunk, len); diff --git a/src/plugins/http/src/httpJson.c b/src/plugins/http/src/httpJson.c index 1aa6cfac4b..132553ffe1 100644 --- a/src/plugins/http/src/httpJson.c +++ b/src/plugins/http/src/httpJson.c @@ -93,7 +93,7 @@ int32_t httpWriteBufNoTrace(struct HttpContext *pContext, const char *buf, int32 int32_t httpWriteJsonBufBody(JsonBuf* buf, bool isTheLast) { int32_t remain = 0; char sLen[24]; - uint64_t srcLen = (uint64_t) (buf->lst - buf->buf); + int32_t srcLen = (int32_t) (buf->lst - buf->buf); if (buf->pContext->fd <= 0) { httpTrace("context:%p, fd:%d, write json body error", buf->pContext, buf->pContext->fd); @@ -113,11 +113,11 @@ int32_t httpWriteJsonBufBody(JsonBuf* buf, bool isTheLast) { httpTrace("context:%p, fd:%d, no data need dump", buf->pContext, buf->pContext->fd); return 0; // there is no data to dump. } else { - int32_t len = sprintf(sLen, "%" PRIx64 "\r\n", srcLen); - httpTrace("context:%p, fd:%d, write body, chunkSize:%" PRIu64 ", response:\n%s", buf->pContext, buf->pContext->fd, + int32_t len = sprintf(sLen, "%d\r\n", srcLen); + httpTrace("context:%p, fd:%d, write body, chunkSize:%d, response:\n%s", buf->pContext, buf->pContext->fd, srcLen, buf->buf); httpWriteBufNoTrace(buf->pContext, sLen, len); - remain = httpWriteBufNoTrace(buf->pContext, buf->buf, (int32_t)srcLen); + remain = httpWriteBufNoTrace(buf->pContext, buf->buf, srcLen); } } else { char compressBuf[JSON_BUFFER_SIZE] = {0}; @@ -126,7 +126,7 @@ int32_t httpWriteJsonBufBody(JsonBuf* buf, bool isTheLast) { if (ret == 0) { if (compressBufLen > 0) { int32_t len = sprintf(sLen, "%x\r\n", compressBufLen); - httpTrace("context:%p, fd:%d, write body, chunkSize:%" PRIu64 ", compressSize:%d, last:%d, response:\n%s", + httpTrace("context:%p, fd:%d, write body, chunkSize:%d, compressSize:%d, last:%d, response:\n%s", buf->pContext, buf->pContext->fd, srcLen, compressBufLen, isTheLast, buf->buf); httpWriteBufNoTrace(buf->pContext, sLen, len); remain = httpWriteBufNoTrace(buf->pContext, (const char*)compressBuf, compressBufLen); @@ -136,7 +136,7 @@ int32_t httpWriteJsonBufBody(JsonBuf* buf, bool isTheLast) { remain = 0; // there is no data to dump. } } else { - httpError("context:%p, fd:%d, failed to compress data, chunkSize:%" PRIu64 ", last:%d, error:%d, response:\n%s", + httpError("context:%p, fd:%d, failed to compress data, chunkSize:%d, last:%d, error:%d, response:\n%s", buf->pContext, buf->pContext->fd, srcLen, isTheLast, ret, buf->buf); remain = 0; } diff --git a/src/plugins/http/src/httpParser.c b/src/plugins/http/src/httpParser.c index b844834537..331ca079fa 100644 --- a/src/plugins/http/src/httpParser.c +++ b/src/plugins/http/src/httpParser.c @@ -153,7 +153,7 @@ static int32_t httpOnRequestLine(HttpParser *pParser, char *method, char *target for (int32_t i = 0; i < HTTP_MAX_URL; i++) { char *pSeek = strchr(pStart, '/'); if (pSeek == NULL) { - (void)httpAppendString(pParser->path + i, pStart, strlen(pStart)); + (void)httpAppendString(pParser->path + i, pStart, (int32_t)strlen(pStart)); break; } else { (void)httpAppendString(pParser->path + i, pStart, (int32_t)(pSeek - pStart)); @@ -285,7 +285,7 @@ static int32_t httpOnParseHeaderField(HttpParser *parser, const char *key, const free(t); free(s); httpTrace("context:%p, fd:%d, basic auth:%s", pContext, pContext->fd, parser->authContent); - int32_t ok = httpParseBasicAuthToken(pContext, parser->authContent, strlen(parser->authContent)); + int32_t ok = httpParseBasicAuthToken(pContext, parser->authContent, (int32_t)strlen(parser->authContent)); if (ok != 0) { httpOnError(parser, 0, TSDB_CODE_HTTP_INVALID_BASIC_AUTH); return -1; @@ -299,7 +299,7 @@ static int32_t httpOnParseHeaderField(HttpParser *parser, const char *key, const free(t); free(s); httpTrace("context:%p, fd:%d, taosd auth:%s", pContext, pContext->fd, parser->authContent); - int32_t ok = httpParseTaosdAuthToken(pContext, parser->authContent, strlen(parser->authContent)); + int32_t ok = httpParseTaosdAuthToken(pContext, parser->authContent, (int32_t)strlen(parser->authContent)); if (ok != 0) { httpOnError(parser, 0, TSDB_CODE_HTTP_INVALID_TAOSD_AUTH); return -1; @@ -524,14 +524,14 @@ char *httpDecodeUrl(const char *enc) { int32_t hex, cnt; int32_t n = sscanf(p+1, "%2x%n", &hex, &cnt); if (n!=1 && cnt !=2) { ok = 0; break; } - if (httpAppendString(&str, enc, p-enc)) { ok = 0; break; } + if (httpAppendString(&str, enc, (int32_t)(p-enc))) { ok = 0; break; } char c = (char)hex; if (httpAppendString(&str, &c, 1)) { ok = 0; break; } enc = p+3; } char *dec = NULL; if (ok && *enc) { - if (httpAppendString(&str, enc, strlen(enc))) { ok = 0; } + if (httpAppendString(&str, enc, (int32_t)strlen(enc))) { ok = 0; } } if (ok) { dec = str.str; @@ -667,7 +667,7 @@ static int32_t httpParserOnVersion(HttpParser *parser, HTTP_PARSER_STATE state, int32_t ok = 0; do { const char *prefix = "HTTP/1."; - int32_t len = strlen(prefix); + int32_t len = (int32_t)strlen(prefix); if (parser->str.pos < len) { if (prefix[parser->str.pos] != c) { httpError("context:%p, fd:%d, parser state:%d, unexpected char:[%c]%02x", pContext, pContext->fd, state, c, c); @@ -811,7 +811,7 @@ static int32_t httpParserOnCrlf(HttpParser *parser, HTTP_PARSER_STATE state, con int32_t ok = 0; do { const char *s = "\r\n"; - int32_t len = strlen(s); + int32_t len = (int32_t)strlen(s); if (s[parser->str.pos] != c) { httpError("context:%p, fd:%d, parser state:%d, unexpected char:[%c]%02x", pContext, pContext->fd, state, c, c); ok = -1; diff --git a/src/plugins/http/src/httpQueue.c b/src/plugins/http/src/httpQueue.c index 1c039abb4d..7ae54717f3 100644 --- a/src/plugins/http/src/httpQueue.c +++ b/src/plugins/http/src/httpQueue.c @@ -134,14 +134,14 @@ void httpCleanupResultQueue() { for (int32_t i = 0; i < tsHttpPool.num; ++i) { SHttpWorker *pWorker = tsHttpPool.httpWorker + i; - if (pWorker->thread) { + if (taosCheckPthreadValid(pWorker->thread)) { taosQsetThreadResume(tsHttpQset); } } for (int32_t i = 0; i < tsHttpPool.num; ++i) { SHttpWorker *pWorker = tsHttpPool.httpWorker + i; - if (pWorker->thread) { + if (taosCheckPthreadValid(pWorker->thread)) { pthread_join(pWorker->thread, NULL); } } diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index 4896d50c6c..ee6addd6fa 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -25,10 +25,6 @@ #include "httpResp.h" #include "httpUtil.h" -#ifndef EPOLLWAKEUP - #define EPOLLWAKEUP (1u << 29) -#endif - static bool httpReadData(HttpContext *pContext); static void httpStopThread(HttpThread* pThread) { @@ -49,10 +45,10 @@ static void httpStopThread(HttpThread* pThread) { pthread_join(pThread->thread, NULL); if (fd != -1) { - close(fd); + taosCloseSocket(fd); } - close(pThread->pollFd); + taosCloseSocket(pThread->pollFd); pthread_mutex_destroy(&(pThread->threadMutex)); } @@ -77,10 +73,7 @@ static void httpProcessHttpData(void *param) { HttpContext *pContext; int32_t fdNum; - sigset_t set; - sigemptyset(&set); - sigaddset(&set, SIGPIPE); - pthread_sigmask(SIG_SETMASK, &set, NULL); + taosSetMaskSIGPIPE(); while (1) { struct epoll_event events[HTTP_MAX_EVENTS]; @@ -162,10 +155,7 @@ static void *httpAcceptHttpConnection(void *arg) { HttpContext * pContext = NULL; int32_t totalFds = 0; - sigset_t set; - sigemptyset(&set); - sigaddset(&set, SIGPIPE); - pthread_sigmask(SIG_SETMASK, &set, NULL); + taosSetMaskSIGPIPE(); pServer->fd = taosOpenTcpServerSocket(pServer->serverIp, pServer->serverPort); @@ -242,7 +232,7 @@ static void *httpAcceptHttpConnection(void *arg) { threadId = threadId % pServer->numOfThreads; } - close(pServer->fd); + taosCloseSocket(pServer->fd); return NULL; } @@ -265,7 +255,7 @@ bool httpInitConnect() { return false; } - pThread->pollFd = epoll_create(HTTP_MAX_EVENTS); // size does not matter + pThread->pollFd = (SOCKET)epoll_create(HTTP_MAX_EVENTS); // size does not matter if (pThread->pollFd < 0) { httpError("http thread:%s, failed to create HTTP epoll", pThread->label); pthread_mutex_destroy(&(pThread->threadMutex)); diff --git a/src/plugins/http/src/httpTgHandle.c b/src/plugins/http/src/httpTgHandle.c index e2b57b87bb..f9c8860afe 100644 --- a/src/plugins/http/src/httpTgHandle.c +++ b/src/plugins/http/src/httpTgHandle.c @@ -276,7 +276,7 @@ int32_t tgReadSchema(char *fileName) { rewind(fp); char * content = (char *)calloc(contentSize + 1, 1); - int32_t result = fread(content, 1, contentSize, fp); + int32_t result = (int32_t)fread(content, 1, contentSize, fp); if (result != contentSize) { httpError("failed to read telegraf schema file:%s", fileName); diff --git a/src/plugins/http/src/httpUtil.c b/src/plugins/http/src/httpUtil.c index 39168ee96d..51333d2118 100644 --- a/src/plugins/http/src/httpUtil.c +++ b/src/plugins/http/src/httpUtil.c @@ -388,7 +388,7 @@ int32_t httpGzipDeCompress(char *srcData, int32_t nSrcData, char *destData, int3 if (inflateEnd(&gzipStream) != Z_OK) { return -4; } - *nDestData = gzipStream.total_out; + *nDestData = (int32_t)gzipStream.total_out; return 0; } @@ -417,7 +417,7 @@ int32_t httpGzipCompress(HttpContext *pContext, char *srcData, int32_t nSrcData, return -1; } - int32_t cacheLen = pContext->gzipStream.total_out - lastTotalLen; + int32_t cacheLen = (int32_t)(pContext->gzipStream.total_out - lastTotalLen); if (cacheLen >= *nDestData) { return -2; } diff --git a/src/plugins/monitor/CMakeLists.txt b/src/plugins/monitor/CMakeLists.txt index edea2187ea..c9314fe1cb 100644 --- a/src/plugins/monitor/CMakeLists.txt +++ b/src/plugins/monitor/CMakeLists.txt @@ -6,7 +6,7 @@ INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) AUX_SOURCE_DIRECTORY(./src SRC) -IF (TD_LINUX) +IF (TD_LINUX OR TD_WINDOWS) ADD_LIBRARY(monitor ${SRC}) IF (TD_SOMODE_STATIC) diff --git a/src/plugins/monitor/src/monMain.c b/src/plugins/monitor/src/monMain.c index 9443b1ce12..85d28bf0a5 100644 --- a/src/plugins/monitor/src/monMain.c +++ b/src/plugins/monitor/src/monMain.c @@ -79,8 +79,8 @@ int32_t monInitSystem() { strcpy(tsMonitor.ep, tsLocalEp); } - int len = strlen(tsMonitor.ep); - for (int i = 0; i < len; ++i) { + int32_t len = (int32_t)strlen(tsMonitor.ep); + for (int32_t i = 0; i < len; ++i) { if (tsMonitor.ep[i] == ':' || tsMonitor.ep[i] == '-' || tsMonitor.ep[i] == '.') { tsMonitor.ep[i] = '_'; } @@ -148,7 +148,7 @@ static void *monThreadFunc(void *param) { } if (tsMonitor.state == MON_STATE_NOT_INIT) { - int code = 0; + int32_t code = 0; for (; tsMonitor.cmdIndex < MON_CMD_MAX; ++tsMonitor.cmdIndex) { monBuildMonitorSql(tsMonitor.sql, tsMonitor.cmdIndex); @@ -330,7 +330,7 @@ static void monSaveSystemInfo() { pos += monBuildReqSql(sql + pos); void *res = taos_query(tsMonitor.conn, tsMonitor.sql); - int code = taos_errno(res); + int32_t code = taos_errno(res); taos_free_result(res); if (code != 0) { diff --git a/src/rpc/src/rpcTcp.c b/src/rpc/src/rpcTcp.c index 178b96c423..d11de3bc58 100644 --- a/src/rpc/src/rpcTcp.c +++ b/src/rpc/src/rpcTcp.c @@ -132,7 +132,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread break; } - pThreadObj->pollFd = (int64_t)epoll_create(10); // size does not matter + pThreadObj->pollFd = (SOCKET)epoll_create(10); // size does not matter if (pThreadObj->pollFd < 0) { tError("%s failed to create TCP epoll", label); code = -1; diff --git a/src/sync/CMakeLists.txt b/src/sync/CMakeLists.txt index aa38a56f38..7a0ea85867 100644 --- a/src/sync/CMakeLists.txt +++ b/src/sync/CMakeLists.txt @@ -4,7 +4,7 @@ PROJECT(TDengine) INCLUDE_DIRECTORIES(inc) AUX_SOURCE_DIRECTORY(src SRC) -IF (TD_LINUX) +IF (TD_LINUX OR TD_WINDOWS) LIST(REMOVE_ITEM SRC src/syncArbitrator.c) ADD_LIBRARY(sync ${SRC}) TARGET_LINK_LIBRARIES(sync tutil pthread common) diff --git a/src/sync/inc/syncInt.h b/src/sync/inc/syncInt.h index eef687d647..e43140d4e6 100644 --- a/src/sync/inc/syncInt.h +++ b/src/sync/inc/syncInt.h @@ -82,8 +82,8 @@ typedef struct SsyncPeer { uint64_t sversion; // track the peer version in retrieve process uint64_t lastFileVer; // track the file version while retrieve uint64_t lastWalVer; // track the wal version while retrieve - int32_t syncFd; - int32_t peerFd; // forward FD + SOCKET syncFd; + SOCKET peerFd; // forward FD int32_t numOfRetrieves; // number of retrieves tried int32_t fileChanged; // a flag to indicate file is changed during retrieving process int32_t refCount; diff --git a/src/sync/inc/syncTcp.h b/src/sync/inc/syncTcp.h index d4674fee6b..b322c3440c 100644 --- a/src/sync/inc/syncTcp.h +++ b/src/sync/inc/syncTcp.h @@ -27,12 +27,12 @@ typedef struct { int32_t bufferSize; void (*processBrokenLink)(int64_t handleId); int32_t (*processIncomingMsg)(int64_t handleId, void *buffer); - void (*processIncomingConn)(int32_t fd, uint32_t ip); + void (*processIncomingConn)(SOCKET fd, uint32_t ip); } SPoolInfo; void *syncOpenTcpThreadPool(SPoolInfo *pInfo); void syncCloseTcpThreadPool(void *); -void *syncAllocateTcpConn(void *, int64_t rid, int32_t connFd); +void *syncAllocateTcpConn(void *, int64_t rid, SOCKET connFd); void syncFreeTcpConn(void *); #ifdef __cplusplus diff --git a/src/sync/src/syncArbitrator.c b/src/sync/src/syncArbitrator.c index fed0774346..187e7a3b46 100644 --- a/src/sync/src/syncArbitrator.c +++ b/src/sync/src/syncArbitrator.c @@ -27,8 +27,12 @@ #include "syncInt.h" #include "syncTcp.h" -static void arbSignalHandler(int32_t signum, siginfo_t *sigInfo, void *context); -static void arbProcessIncommingConnection(int32_t connFd, uint32_t sourceIp); +#ifndef SIGHUP + #define SIGHUP SIGTERM +#endif + +static void arbSignalHandler(int32_t signum); +static void arbProcessIncommingConnection(SOCKET connFd, uint32_t sourceIp); static void arbProcessBrokenLink(int64_t rid); static int32_t arbProcessPeerMsg(int64_t rid, void *buffer); static tsem_t tsArbSem; @@ -36,7 +40,7 @@ static void * tsArbTcpPool; typedef struct { char id[TSDB_EP_LEN + 24]; - int32_t nodeFd; + SOCKET nodeFd; void * pConn; } SNodeConn; @@ -70,8 +74,9 @@ int32_t main(int32_t argc, char *argv[]) { /* Set termination handler. */ struct sigaction act = {{0}}; - act.sa_flags = SA_SIGINFO; - act.sa_sigaction = arbSignalHandler; + memset(&act, 0, sizeof(struct sigaction)); + + act.sa_handler = arbSignalHandler; sigaction(SIGTERM, &act, NULL); sigaction(SIGHUP, &act, NULL); sigaction(SIGINT, &act, NULL); @@ -103,12 +108,11 @@ int32_t main(int32_t argc, char *argv[]) { syncCloseTcpThreadPool(tsArbTcpPool); sInfo("TAOS arbitrator is shut down"); - closelog(); return 0; } -static void arbProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { +static void arbProcessIncommingConnection(SOCKET connFd, uint32_t sourceIp) { char ipstr[24]; tinet_ntoa(ipstr, sourceIp); sDebug("peer TCP connection from ip:%s", ipstr); @@ -172,15 +176,18 @@ static int32_t arbProcessPeerMsg(int64_t rid, void *buffer) { return 0; } -static void arbSignalHandler(int32_t signum, siginfo_t *sigInfo, void *context) { +static void arbSignalHandler(int32_t signum) { struct sigaction act = {{0}}; act.sa_handler = SIG_IGN; sigaction(SIGTERM, &act, NULL); sigaction(SIGHUP, &act, NULL); sigaction(SIGINT, &act, NULL); +#ifndef WINDOWS sInfo("shut down signal is %d, sender PID:%d", signum, sigInfo->si_pid); - +#else + sInfo("shut down signal is %d", signum); +#endif // inform main thread to exit tsem_post(&tsArbSem); } diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index 98100fbdd8..b43f5824be 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -45,7 +45,7 @@ static void syncCheckPeerConnection(void *param, void *tmrId); static int32_t syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack, int8_t type, uint16_t tranId); static void syncProcessBrokenLink(int64_t rid); static int32_t syncProcessPeerMsg(int64_t rid, void *buffer); -static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp); +static void syncProcessIncommingConnection(SOCKET connFd, uint32_t sourceIp); static void syncRemovePeer(SSyncPeer *pPeer); static void syncAddArbitrator(SSyncNode *pNode); static void syncFreeNode(void *); @@ -544,7 +544,7 @@ static void syncClosePeerConn(SSyncPeer *pPeer) { sDebug("%s, pfd:%d sfd:%d will be closed", pPeer->id, pPeer->peerFd, pPeer->syncFd); taosTmrStopA(&pPeer->timer); - taosClose(pPeer->syncFd); + taosCloseSocket(pPeer->syncFd); if (pPeer->peerFd >= 0) { pPeer->peerFd = -1; void *pConn = pPeer->pConn; @@ -869,7 +869,7 @@ static void syncProcessSyncRequest(char *msg, SSyncPeer *pPeer) { if (nodeRole != TAOS_SYNC_ROLE_MASTER) { sError("%s, I am not master anymore", pPeer->id); - taosClose(pPeer->syncFd); + taosCloseSocket(pPeer->syncFd); return; } @@ -1114,7 +1114,7 @@ static void syncSetupPeerConnection(SSyncPeer *pPeer) { return; } - int32_t connFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0); + SOCKET connFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0); if (connFd < 0) { sDebug("%s, failed to open tcp socket since %s", pPeer->id, strerror(errno)); taosTmrReset(syncCheckPeerConnection, SYNC_CHECK_INTERVAL, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer); @@ -1132,7 +1132,7 @@ static void syncSetupPeerConnection(SSyncPeer *pPeer) { if (pPeer->isArb) tsArbOnline = 1; } else { sDebug("%s, failed to setup peer connection to server since %s, try later", pPeer->id, strerror(errno)); - taosClose(connFd); + taosCloseSocket(connFd); taosTmrReset(syncCheckPeerConnection, SYNC_CHECK_INTERVAL, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer); } } @@ -1171,7 +1171,7 @@ static void syncCreateRestoreDataThread(SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; nodeSStatus = TAOS_SYNC_STATUS_INIT; sError("%s, failed to create sync restore thread, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); - taosClose(pPeer->syncFd); + taosCloseSocket(pPeer->syncFd); syncReleasePeer(pPeer); } else { sInfo("%s, sync restore thread:0x%08" PRIx64 " create successfully, rid:%" PRId64, pPeer->id, @@ -1179,7 +1179,7 @@ static void syncCreateRestoreDataThread(SSyncPeer *pPeer) { } } -static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { +static void syncProcessIncommingConnection(SOCKET connFd, uint32_t sourceIp) { char ipstr[24]; int32_t i; diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c index 78520c6608..bd4f9bd3e1 100644 --- a/src/sync/src/syncRestore.c +++ b/src/sync/src/syncRestore.c @@ -365,7 +365,7 @@ void *syncRestoreData(void *param) { SSyncNode *pNode = pPeer->pSyncNode; taosBlockSIGPIPE(); - __sync_fetch_and_add(&tsSyncNum, 1); + atomic_add_fetch_32(&tsSyncNum, 1); sInfo("%s, start to restore data, sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); (*pNode->notifyRole)(pNode->vgId, TAOS_SYNC_ROLE_SYNCING); @@ -390,9 +390,9 @@ void *syncRestoreData(void *param) { nodeSStatus = TAOS_SYNC_STATUS_INIT; sInfo("%s, restore data over, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); - taosClose(pPeer->syncFd); + taosCloseSocket(pPeer->syncFd); syncCloseRecvBuffer(pNode); - __sync_fetch_and_sub(&tsSyncNum, 1); + atomic_sub_fetch_32(&tsSyncNum, 1); // The ref is obtained in both the create thread and the current thread, so it is released twice syncReleasePeer(pPeer); diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c index 153886102e..9ff7955351 100644 --- a/src/sync/src/syncRetrieve.c +++ b/src/sync/src/syncRetrieve.c @@ -14,7 +14,6 @@ */ #define _DEFAULT_SOURCE -#include #include "os.h" #include "taoserror.h" #include "tlog.h" @@ -160,7 +159,7 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) { break; } - ret = taosSendFile(pPeer->syncFd, sfd, NULL, fileInfo.size); + ret = (int32_t)taosSendFile(pPeer->syncFd, sfd, NULL, fileInfo.size); close(sfd); if (ret < 0) { code = -1; @@ -228,7 +227,7 @@ static int32_t syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversi return -1; } - int32_t code = taosLSeek(sfd, offset, SEEK_SET); + int32_t code = (int32_t)taosLSeek(sfd, offset, SEEK_SET); if (code < 0) { sError("%s, failed to seek %" PRId64 " in wal:%s for retrieve since:%s", pPeer->id, offset, name, tstrerror(errno)); close(sfd); @@ -322,7 +321,7 @@ static int32_t syncProcessLastWal(SSyncPeer *pPeer, char *wname, int64_t index) // if all data are read out, and no update if (bytes == 0 && !walModified) { // wal not closed, it means some data not flushed to disk, wait for a while - usleep(10000); + taosMsleep(10); } // if bytes > 0, file is updated, or fversion is not reached but file still open, read again @@ -384,7 +383,7 @@ static int32_t syncRetrieveWal(SSyncPeer *pPeer) { break; } - code = taosSendFile(pPeer->syncFd, sfd, NULL, size); + code = (int32_t)taosSendFile(pPeer->syncFd, sfd, NULL, size); close(sfd); if (code < 0) { sError("%s, failed to send wal:%s for retrieve since %s, code:0x%x", pPeer->id, fname, strerror(errno), code); @@ -501,7 +500,7 @@ void *syncRetrieveData(void *param) { } pPeer->fileChanged = 0; - taosClose(pPeer->syncFd); + taosCloseSocket(pPeer->syncFd); // The ref is obtained in both the create thread and the current thread, so it is released twice sInfo("%s, sync retrieve data over, sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); diff --git a/src/sync/src/syncTcp.c b/src/sync/src/syncTcp.c index 4744666737..829c9ceec6 100644 --- a/src/sync/src/syncTcp.c +++ b/src/sync/src/syncTcp.c @@ -24,10 +24,18 @@ #include "syncInt.h" #include "syncTcp.h" +#ifdef WINDOWS +#include "wepoll.h" +#endif + +#ifndef EPOLLWAKEUP + #define EPOLLWAKEUP (1u << 29) +#endif + typedef struct SThreadObj { pthread_t thread; bool stop; - int32_t pollFd; + SOCKET pollFd; int32_t numOfFds; struct SPoolObj *pPool; } SThreadObj; @@ -37,13 +45,13 @@ typedef struct SPoolObj { SThreadObj **pThread; pthread_t thread; int32_t nextId; - int32_t acceptFd; // FD for accept new connection + SOCKET acceptFd; // FD for accept new connection } SPoolObj; typedef struct { SThreadObj *pThread; int64_t handleId; - int32_t fd; + SOCKET fd; int32_t closedByApp; } SConnObj; @@ -82,7 +90,7 @@ void *syncOpenTcpThreadPool(SPoolInfo *pInfo) { pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); if (pthread_create(&(pPool->thread), &thattr, (void *)syncAcceptPeerTcpConnection, pPool) != 0) { sError("failed to create accept thread for TCP server since %s", strerror(errno)); - close(pPool->acceptFd); + taosCloseSocket(pPool->acceptFd); tfree(pPool->pThread); tfree(pPool); return NULL; @@ -112,7 +120,7 @@ void syncCloseTcpThreadPool(void *param) { tfree(pPool); } -void *syncAllocateTcpConn(void *param, int64_t rid, int32_t connFd) { +void *syncAllocateTcpConn(void *param, int64_t rid, SOCKET connFd) { struct epoll_event event; SPoolObj *pPool = param; @@ -169,7 +177,7 @@ static void taosProcessBrokenLink(SConnObj *pConn) { pThread->numOfFds--; epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, pConn->fd, NULL); sDebug("%p fd:%d is removed from epoll thread, num:%d", pThread, pConn->fd, pThread->numOfFds); - taosClose(pConn->fd); + taosCloseSocket(pConn->fd); tfree(pConn); } @@ -233,7 +241,7 @@ static void *syncProcessTcpData(void *param) { sDebug("%p TCP epoll thread exits", pThread); - close(pThread->pollFd); + taosCloseSocket(pThread->pollFd); tfree(pThread); tfree(buffer); return NULL; @@ -248,7 +256,7 @@ static void *syncAcceptPeerTcpConnection(void *argv) { while (1) { struct sockaddr_in clientAddr; socklen_t addrlen = sizeof(clientAddr); - int32_t connFd = accept(pPool->acceptFd, (struct sockaddr *)&clientAddr, &addrlen); + SOCKET connFd = accept(pPool->acceptFd, (struct sockaddr *)&clientAddr, &addrlen); if (connFd < 0) { if (errno == EINVAL) { sDebug("%p TCP server accept is exiting...", pPool); @@ -264,7 +272,7 @@ static void *syncAcceptPeerTcpConnection(void *argv) { (*pInfo->processIncomingConn)(connFd, clientAddr.sin_addr.s_addr); } - taosClose(pPool->acceptFd); + taosCloseSocket(pPool->acceptFd); return NULL; } @@ -277,7 +285,7 @@ static SThreadObj *syncGetTcpThread(SPoolObj *pPool) { if (pThread == NULL) return NULL; pThread->pPool = pPool; - pThread->pollFd = epoll_create(10); // size does not matter + pThread->pollFd = (SOCKET)epoll_create(10); // size does not matter if (pThread->pollFd < 0) { tfree(pThread); return NULL; @@ -290,7 +298,7 @@ static SThreadObj *syncGetTcpThread(SPoolObj *pPool) { pthread_attr_destroy(&thattr); if (ret != 0) { - close(pThread->pollFd); + taosCloseSocket(pThread->pollFd); tfree(pThread); return NULL; } diff --git a/src/util/inc/tsocket.h b/src/util/inc/tsocket.h index a339955cc0..6b449d6bc9 100644 --- a/src/util/inc/tsocket.h +++ b/src/util/inc/tsocket.h @@ -20,6 +20,14 @@ extern "C" { #endif +#ifdef WINDOWS +#include "wepoll.h" +#endif + +#ifndef EPOLLWAKEUP + #define EPOLLWAKEUP (1u << 29) +#endif + int32_t taosReadn(SOCKET sock, char *buffer, int32_t len); int32_t taosWriteMsg(SOCKET fd, void *ptr, int32_t nbytes); int32_t taosReadMsg(SOCKET fd, void *ptr, int32_t nbytes); diff --git a/src/vnode/src/vnodeCfg.c b/src/vnode/src/vnodeCfg.c index 0b32f97939..1ea774afae 100644 --- a/src/vnode/src/vnodeCfg.c +++ b/src/vnode/src/vnodeCfg.c @@ -78,7 +78,7 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) { goto PARSE_VCFG_ERROR; } - len = fread(content, 1, maxLen, fp); + len = (int32_t)fread(content, 1, maxLen, fp); if (len <= 0) { vError("vgId:%d, failed to read %s, content is null", pVnode->vgId, file); goto PARSE_VCFG_ERROR; @@ -103,14 +103,14 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) { vError("vgId:%d, failed to read %s, cfgVersion not found", pVnode->vgId, file); goto PARSE_VCFG_ERROR; } - vnodeMsg.cfg.dbCfgVersion = dbCfgVersion->valueint; + vnodeMsg.cfg.dbCfgVersion = (int32_t)dbCfgVersion->valueint; cJSON *vgCfgVersion = cJSON_GetObjectItem(root, "vgCfgVersion"); if (!vgCfgVersion || vgCfgVersion->type != cJSON_Number) { vError("vgId:%d, failed to read %s, vgCfgVersion not found", pVnode->vgId, file); vnodeMsg.cfg.vgCfgVersion = 0; } else { - vnodeMsg.cfg.vgCfgVersion = vgCfgVersion->valueint; + vnodeMsg.cfg.vgCfgVersion = (int32_t)vgCfgVersion->valueint; } cJSON *cacheBlockSize = cJSON_GetObjectItem(root, "cacheBlockSize"); @@ -118,56 +118,56 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) { vError("vgId:%d, failed to read %s, cacheBlockSize not found", pVnode->vgId, file); goto PARSE_VCFG_ERROR; } - vnodeMsg.cfg.cacheBlockSize = cacheBlockSize->valueint; + vnodeMsg.cfg.cacheBlockSize = (int32_t)cacheBlockSize->valueint; cJSON *totalBlocks = cJSON_GetObjectItem(root, "totalBlocks"); if (!totalBlocks || totalBlocks->type != cJSON_Number) { vError("vgId:%d, failed to read %s, totalBlocks not found", pVnode->vgId, file); goto PARSE_VCFG_ERROR; } - vnodeMsg.cfg.totalBlocks = totalBlocks->valueint; + vnodeMsg.cfg.totalBlocks = (int32_t)totalBlocks->valueint; cJSON *daysPerFile = cJSON_GetObjectItem(root, "daysPerFile"); if (!daysPerFile || daysPerFile->type != cJSON_Number) { vError("vgId:%d, failed to read %s, daysPerFile not found", pVnode->vgId, file); goto PARSE_VCFG_ERROR; } - vnodeMsg.cfg.daysPerFile = daysPerFile->valueint; + vnodeMsg.cfg.daysPerFile = (int32_t)daysPerFile->valueint; cJSON *daysToKeep = cJSON_GetObjectItem(root, "daysToKeep"); if (!daysToKeep || daysToKeep->type != cJSON_Number) { vError("vgId:%d, failed to read %s, daysToKeep not found", pVnode->vgId, file); goto PARSE_VCFG_ERROR; } - vnodeMsg.cfg.daysToKeep = daysToKeep->valueint; + vnodeMsg.cfg.daysToKeep = (int32_t)daysToKeep->valueint; cJSON *daysToKeep1 = cJSON_GetObjectItem(root, "daysToKeep1"); if (!daysToKeep1 || daysToKeep1->type != cJSON_Number) { vError("vgId:%d, failed to read %s, daysToKeep1 not found", pVnode->vgId, file); goto PARSE_VCFG_ERROR; } - vnodeMsg.cfg.daysToKeep1 = daysToKeep1->valueint; + vnodeMsg.cfg.daysToKeep1 = (int32_t)daysToKeep1->valueint; cJSON *daysToKeep2 = cJSON_GetObjectItem(root, "daysToKeep2"); if (!daysToKeep2 || daysToKeep2->type != cJSON_Number) { vError("vgId:%d, failed to read %s, daysToKeep2 not found", pVnode->vgId, file); goto PARSE_VCFG_ERROR; } - vnodeMsg.cfg.daysToKeep2 = daysToKeep2->valueint; + vnodeMsg.cfg.daysToKeep2 = (int32_t)daysToKeep2->valueint; cJSON *minRowsPerFileBlock = cJSON_GetObjectItem(root, "minRowsPerFileBlock"); if (!minRowsPerFileBlock || minRowsPerFileBlock->type != cJSON_Number) { vError("vgId:%d, failed to read %s, minRowsPerFileBlock not found", pVnode->vgId, file); goto PARSE_VCFG_ERROR; } - vnodeMsg.cfg.minRowsPerFileBlock = minRowsPerFileBlock->valueint; + vnodeMsg.cfg.minRowsPerFileBlock = (int32_t)minRowsPerFileBlock->valueint; cJSON *maxRowsPerFileBlock = cJSON_GetObjectItem(root, "maxRowsPerFileBlock"); if (!maxRowsPerFileBlock || maxRowsPerFileBlock->type != cJSON_Number) { vError("vgId:%d, failed to read %s, maxRowsPerFileBlock not found", pVnode->vgId, file); goto PARSE_VCFG_ERROR; } - vnodeMsg.cfg.maxRowsPerFileBlock = maxRowsPerFileBlock->valueint; + vnodeMsg.cfg.maxRowsPerFileBlock = (int32_t)maxRowsPerFileBlock->valueint; cJSON *precision = cJSON_GetObjectItem(root, "precision"); if (!precision || precision->type != cJSON_Number) { @@ -195,7 +195,7 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) { vError("vgId:%d, failed to read %s, fsyncPeriod not found", pVnode->vgId, file); goto PARSE_VCFG_ERROR; } - vnodeMsg.cfg.fsyncPeriod = fsyncPeriod->valueint; + vnodeMsg.cfg.fsyncPeriod = (int32_t)fsyncPeriod->valueint; cJSON *wals = cJSON_GetObjectItem(root, "wals"); if (!wals || wals->type != cJSON_Number) { @@ -258,7 +258,7 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) { vError("vgId:%d, failed to read %s, nodeId not found", pVnode->vgId, file); goto PARSE_VCFG_ERROR; } - node->nodeId = nodeId->valueint; + node->nodeId = (int32_t)nodeId->valueint; cJSON *nodeEp = cJSON_GetObjectItem(nodeInfo, "nodeEp"); if (!nodeEp || nodeEp->type != cJSON_String || nodeEp->valuestring == NULL) { diff --git a/src/wal/CMakeLists.txt b/src/wal/CMakeLists.txt index 359e09287a..2125e4f33c 100644 --- a/src/wal/CMakeLists.txt +++ b/src/wal/CMakeLists.txt @@ -4,7 +4,7 @@ PROJECT(TDengine) INCLUDE_DIRECTORIES(inc) AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR}/src SRC) -IF (TD_LINUX) +IF (TD_LINUX OR TD_WINDOWS) ADD_LIBRARY(twal ${SRC}) TARGET_LINK_LIBRARIES(twal tutil common) ADD_SUBDIRECTORY(test) diff --git a/src/wal/src/walMgmt.c b/src/wal/src/walMgmt.c index 72ea239817..62b066500c 100644 --- a/src/wal/src/walMgmt.c +++ b/src/wal/src/walMgmt.c @@ -210,7 +210,7 @@ static int32_t walCreateThread() { static void walStopThread() { tsWal.stop = 1; - if (tsWal.thread) { + if (taosCheckPthreadValid(tsWal.thread)) { pthread_join(tsWal.thread, NULL); } diff --git a/src/wal/src/walWrite.c b/src/wal/src/walWrite.c index e67127d6e4..0eda6ff786 100644 --- a/src/wal/src/walWrite.c +++ b/src/wal/src/walWrite.c @@ -272,7 +272,7 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch SWalHead *pHead = buffer; while (1) { - int32_t ret = tfRead(tfd, pHead, sizeof(SWalHead)); + int32_t ret = (int32_t)tfRead(tfd, pHead, sizeof(SWalHead)); if (ret == 0) break; if (ret < 0) { @@ -307,7 +307,7 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch } } - ret = tfRead(tfd, pHead->cont, pHead->len); + ret = (int32_t)tfRead(tfd, pHead->cont, pHead->len); if (ret < 0) { wError("vgId:%d, file:%s, failed to read wal body since %s", pWal->vgId, name, strerror(errno)); code = TAOS_SYSTEM_ERROR(errno); From f4539411dd6063d1c66b64451b75d288cbe1e79a Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 14 Jan 2021 17:25:32 +0800 Subject: [PATCH 002/147] compile errors --- src/os/inc/osFile.h | 2 ++ src/sync/src/syncArbitrator.c | 5 +---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/os/inc/osFile.h b/src/os/inc/osFile.h index 91ac5c9958..19cc78472c 100644 --- a/src/os/inc/osFile.h +++ b/src/os/inc/osFile.h @@ -20,6 +20,8 @@ extern "C" { #endif +#include "osSocket.h" + int64_t taosReadImp(int32_t fd, void *buf, int64_t count); int64_t taosWriteImp(int32_t fd, void *buf, int64_t count); int64_t taosLSeekImp(int32_t fd, int64_t offset, int32_t whence); diff --git a/src/sync/src/syncArbitrator.c b/src/sync/src/syncArbitrator.c index 187e7a3b46..8c0d02bedc 100644 --- a/src/sync/src/syncArbitrator.c +++ b/src/sync/src/syncArbitrator.c @@ -183,11 +183,8 @@ static void arbSignalHandler(int32_t signum) { sigaction(SIGHUP, &act, NULL); sigaction(SIGINT, &act, NULL); -#ifndef WINDOWS - sInfo("shut down signal is %d, sender PID:%d", signum, sigInfo->si_pid); -#else sInfo("shut down signal is %d", signum); -#endif + // inform main thread to exit tsem_post(&tsArbSem); } From ac49a78d42aacb612580357f53d5604b66c17b20 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 14 Jan 2021 22:36:15 +0800 Subject: [PATCH 003/147] TD-1207 --- CMakeLists.txt | 7 +++- src/balance/CMakeLists.txt | 2 +- src/balance/inc/bnThread.h | 2 +- src/balance/src/bnMain.c | 2 +- src/balance/src/bnScore.c | 10 ++--- src/balance/src/bnThread.c | 2 +- src/cq/CMakeLists.txt | 2 +- src/cq/src/cqMain.c | 2 +- src/cq/test/CMakeLists.txt | 2 +- src/dnode/CMakeLists.txt | 4 +- src/dnode/src/dnodeCfg.c | 4 +- src/dnode/src/dnodeCheck.c | 40 +++++++++---------- src/dnode/src/dnodeEps.c | 4 +- src/dnode/src/dnodeMInfos.c | 16 ++++---- src/dnode/src/dnodeMPeer.c | 4 +- src/dnode/src/dnodeMRead.c | 6 +-- src/dnode/src/dnodeMWrite.c | 4 +- src/dnode/src/dnodePeer.c | 5 ++- src/dnode/src/dnodeShell.c | 2 +- src/dnode/src/dnodeStep.c | 2 +- src/dnode/src/dnodeSystem.c | 71 ++++++++++++++++++++-------------- src/dnode/src/dnodeTelemetry.c | 23 ++++++----- src/dnode/src/dnodeVWrite.c | 4 +- src/dnode/src/dnodeVnodes.c | 4 +- src/mnode/src/mnodeDnode.c | 2 +- src/mnode/src/mnodeTable.c | 1 + src/os/inc/osWindows.h | 6 +++ src/os/src/windows/wSysLog.c | 19 +++++++++ src/os/src/windows/wSysinfo.c | 7 +++- src/plugins/CMakeLists.txt | 2 +- src/sync/src/syncArbitrator.c | 12 +++--- src/vnode/CMakeLists.txt | 3 +- src/vnode/src/vnodeVersion.c | 2 +- src/vnode/src/vnodeWorker.c | 4 +- src/vnode/src/vnodeWrite.c | 2 +- 35 files changed, 168 insertions(+), 116 deletions(-) create mode 100644 src/os/src/windows/wSysLog.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 0c474a355d..7ac06c165d 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,7 +13,7 @@ ENDIF () SET(TD_ACCOUNT FALSE) SET(TD_ADMIN FALSE) SET(TD_GRANT FALSE) -SET(TD_MQTT TRUE) +SET(TD_MQTT FALSE) SET(TD_TSDB_PLUGINS FALSE) SET(TD_COVER FALSE) @@ -29,6 +29,11 @@ MESSAGE(STATUS "Community directory: " ${TD_COMMUNITY_DIR}) INCLUDE(cmake/input.inc) INCLUDE(cmake/platform.inc) + +IF (TD_WINDOWS) + SET(TD_SOMODE_STATIC TRUE) +ENDIF () + INCLUDE(cmake/define.inc) INCLUDE(cmake/env.inc) INCLUDE(cmake/version.inc) diff --git a/src/balance/CMakeLists.txt b/src/balance/CMakeLists.txt index fdc43ea32f..3c93f63d1a 100644 --- a/src/balance/CMakeLists.txt +++ b/src/balance/CMakeLists.txt @@ -8,6 +8,6 @@ INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/inc) INCLUDE_DIRECTORIES(inc) AUX_SOURCE_DIRECTORY(src SRC) -IF (TD_LINUX) +IF (TD_LINUX OR TD_WINDOWS) ADD_LIBRARY(balance ${SRC}) ENDIF () diff --git a/src/balance/inc/bnThread.h b/src/balance/inc/bnThread.h index 8f54b66028..74a761299d 100644 --- a/src/balance/inc/bnThread.h +++ b/src/balance/inc/bnThread.h @@ -24,7 +24,7 @@ extern "C" { int32_t bnInitThread(); void bnCleanupThread(); void bnNotify(); -void bnStartTimer(int64_t mseconds); +void bnStartTimer(int32_t mseconds); #ifdef __cplusplus } diff --git a/src/balance/src/bnMain.c b/src/balance/src/bnMain.c index 3e1d5eda76..236b22afaf 100644 --- a/src/balance/src/bnMain.c +++ b/src/balance/src/bnMain.c @@ -30,7 +30,7 @@ #include "mnodeVgroup.h" extern int64_t tsDnodeRid; -extern int64_t tsSdbRid; +extern int32_t tsSdbRid; static SBnMgmt tsBnMgmt; static void bnMonitorDnodeModule(); diff --git a/src/balance/src/bnScore.c b/src/balance/src/bnScore.c index cbc2c62184..7d94df1c23 100644 --- a/src/balance/src/bnScore.c +++ b/src/balance/src/bnScore.c @@ -271,23 +271,23 @@ static int32_t bnRetrieveScores(SShowObj *pShow, char *data, int32_t rows, void cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(float *)pWrite = systemScore; + *(float *)pWrite = (float)systemScore; cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(float *)pWrite = pDnode->customScore; + *(float *)pWrite = (float)pDnode->customScore; cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(float *)pWrite = (int32_t)moduleScore; + *(float *)pWrite = (float)moduleScore; cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(float *)pWrite = (int32_t)vnodeScore; + *(float *)pWrite = (float)vnodeScore; cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(float *)pWrite = (int32_t)(vnodeScore + moduleScore + pDnode->customScore + systemScore); + *(float *)pWrite = (float)(vnodeScore + moduleScore + pDnode->customScore + systemScore); cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; diff --git a/src/balance/src/bnThread.c b/src/balance/src/bnThread.c index 84f8694fca..caf33061d1 100644 --- a/src/balance/src/bnThread.c +++ b/src/balance/src/bnThread.c @@ -119,7 +119,7 @@ static void bnProcessTimer(void *handle, void *tmrId) { } } -void bnStartTimer(int64_t mseconds) { +void bnStartTimer(int32_t mseconds) { if (tsBnThread.stop) return; bool updateSoon = (mseconds != -1); diff --git a/src/cq/CMakeLists.txt b/src/cq/CMakeLists.txt index 9da831c9c1..dd84e96ecf 100644 --- a/src/cq/CMakeLists.txt +++ b/src/cq/CMakeLists.txt @@ -6,7 +6,7 @@ INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR}/src SRC) -IF (TD_LINUX) +IF (TD_LINUX OR TD_WINDOWS) ADD_LIBRARY(tcq ${SRC}) IF (TD_SOMODE_STATIC) TARGET_LINK_LIBRARIES(tcq tutil common taos_static) diff --git a/src/cq/src/cqMain.c b/src/cq/src/cqMain.c index 0fe5ea78d4..5f1fecc494 100644 --- a/src/cq/src/cqMain.c +++ b/src/cq/src/cqMain.c @@ -343,7 +343,7 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) { char buf[TSDB_MAX_NCHAR_LEN]; int32_t len = taos_fetch_lengths(tres)[i]; taosMbsToUcs4(val, len, buf, sizeof(buf), &len); - memcpy(val + sizeof(VarDataLenT), buf, len); + memcpy((char *)val + sizeof(VarDataLenT), buf, len); varDataLen(val) = len; } tdAppendColVal(trow, val, c->type, c->bytes, c->offset); diff --git a/src/cq/test/CMakeLists.txt b/src/cq/test/CMakeLists.txt index 87a8705fcd..fc3a1ea93a 100644 --- a/src/cq/test/CMakeLists.txt +++ b/src/cq/test/CMakeLists.txt @@ -3,4 +3,4 @@ PROJECT(TDengine) LIST(APPEND CQTEST_SRC ./cqtest.c) ADD_EXECUTABLE(cqtest ${CQTEST_SRC}) -TARGET_LINK_LIBRARIES(cqtest tcq) +TARGET_LINK_LIBRARIES(cqtest tcq taos_static) diff --git a/src/dnode/CMakeLists.txt b/src/dnode/CMakeLists.txt index 699ca00a25..59a8b2b486 100644 --- a/src/dnode/CMakeLists.txt +++ b/src/dnode/CMakeLists.txt @@ -10,7 +10,7 @@ INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/inc) INCLUDE_DIRECTORIES(inc) AUX_SOURCE_DIRECTORY(src SRC) -IF (TD_LINUX) +IF (TD_LINUX OR TD_WINDOWS) ADD_EXECUTABLE(taosd ${SRC}) TARGET_LINK_LIBRARIES(taosd mnode monitor http tsdb twal vnode cJson lz4 balance sync) @@ -28,7 +28,7 @@ IF (TD_LINUX) TARGET_LINK_LIBRARIES(taosd grant) ENDIF () - IF (TD_MQTT) + IF (TD_LINUX AND TD_MQTT) TARGET_LINK_LIBRARIES(taosd mqtt) ENDIF () diff --git a/src/dnode/src/dnodeCfg.c b/src/dnode/src/dnodeCfg.c index f495dbe285..e008d2fa0d 100644 --- a/src/dnode/src/dnodeCfg.c +++ b/src/dnode/src/dnodeCfg.c @@ -97,7 +97,7 @@ static int32_t dnodeReadCfg() { goto PARSE_CFG_OVER; } - len = fread(content, 1, maxLen, fp); + len = (int32_t)fread(content, 1, maxLen, fp); if (len <= 0) { dError("failed to read %s, content is null", file); goto PARSE_CFG_OVER; @@ -115,7 +115,7 @@ static int32_t dnodeReadCfg() { dError("failed to read %s, dnodeId not found", file); goto PARSE_CFG_OVER; } - cfg.dnodeId = dnodeId->valueint; + cfg.dnodeId = (int32_t)dnodeId->valueint; cJSON *clusterId = cJSON_GetObjectItem(root, "clusterId"); if (!clusterId || clusterId->type != cJSON_String) { diff --git a/src/dnode/src/dnodeCheck.c b/src/dnode/src/dnodeCheck.c index be26bb967b..8955fb5643 100644 --- a/src/dnode/src/dnodeCheck.c +++ b/src/dnode/src/dnodeCheck.c @@ -29,8 +29,8 @@ typedef struct { static SCheckItem tsCheckItem[TSDB_CHECK_ITEM_MAX] = {{0}}; int64_t tsMinFreeMemSizeForStart = 0; -static int bindTcpPort(int port) { - int serverSocket; +static int32_t bindTcpPort(int32_t port) { + SOCKET serverSocket; struct sockaddr_in server_addr; if ((serverSocket = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0) { @@ -45,22 +45,22 @@ static int bindTcpPort(int port) { if (bind(serverSocket, (struct sockaddr *)&server_addr, sizeof(server_addr)) < 0) { dError("port:%d tcp bind() fail: %s", port, strerror(errno)); - close(serverSocket); + taosCloseSocket(serverSocket); return -1; } if (listen(serverSocket, 5) < 0) { dError("port:%d listen() fail: %s", port, strerror(errno)); - close(serverSocket); + taosCloseSocket(serverSocket); return -1; } - close(serverSocket); + taosCloseSocket(serverSocket); return 0; } -static int bindUdpPort(int port) { - int serverSocket; +static int32_t bindUdpPort(int32_t port) { + SOCKET serverSocket; struct sockaddr_in server_addr; if ((serverSocket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0) { @@ -75,19 +75,19 @@ static int bindUdpPort(int port) { if (bind(serverSocket, (struct sockaddr *)&server_addr, sizeof(server_addr)) < 0) { dError("port:%d udp bind() fail: %s", port, strerror(errno)); - close(serverSocket); + taosCloseSocket(serverSocket); return -1; } - close(serverSocket); + taosCloseSocket(serverSocket); return 0; } -static int dnodeCheckNetwork() { - int ret; - int startPort = tsServerPort; +static int32_t dnodeCheckNetwork() { + int32_t ret; + int32_t startPort = tsServerPort; - for (int port = startPort; port < startPort + 12; port++) { + for (int32_t port = startPort; port < startPort + 12; port++) { ret = bindTcpPort(port); if (0 != ret) { dError("failed to tcp bind port %d, quit", port); @@ -103,7 +103,7 @@ static int dnodeCheckNetwork() { return 0; } -static int dnodeCheckMem() { +static int32_t dnodeCheckMem() { float memoryUsedMB; float memoryAvailMB; if (true != taosGetSysMemory(&memoryUsedMB)) { @@ -121,12 +121,12 @@ static int dnodeCheckMem() { return 0; } -static int dnodeCheckCpu() { +static int32_t dnodeCheckCpu() { // TODO: return 0; } -static int dnodeCheckDisk() { +static int32_t dnodeCheckDisk() { taosGetDisk(); if (tsAvailDataDirGB < tsMinimalDataDirGB) { @@ -147,24 +147,24 @@ static int dnodeCheckDisk() { return 0; } -static int dnodeCheckOs() { +static int32_t dnodeCheckOs() { // TODO: return 0; } -static int dnodeCheckAccess() { +static int32_t dnodeCheckAccess() { // TODO: return 0; } -static int dnodeCheckVersion() { +static int32_t dnodeCheckVersion() { // TODO: return 0; } -static int dnodeCheckDatafile() { +static int32_t dnodeCheckDatafile() { // TODO: return 0; diff --git a/src/dnode/src/dnodeEps.c b/src/dnode/src/dnodeEps.c index e7dc7efeb2..1e05c696ce 100644 --- a/src/dnode/src/dnodeEps.c +++ b/src/dnode/src/dnodeEps.c @@ -152,7 +152,7 @@ static int32_t dnodeReadEps() { goto PRASE_EPS_OVER; } - len = fread(content, 1, maxLen, fp); + len = (int32_t)fread(content, 1, maxLen, fp); if (len <= 0) { dError("failed to read %s, content is null", file); goto PRASE_EPS_OVER; @@ -199,7 +199,7 @@ static int32_t dnodeReadEps() { dError("failed to read %s, dnodeId not found", file); goto PRASE_EPS_OVER; } - ep->dnodeId = dnodeId->valueint; + ep->dnodeId = (int32_t)dnodeId->valueint; cJSON *dnodeFqdn = cJSON_GetObjectItem(dnodeInfo, "dnodeFqdn"); if (!dnodeFqdn || dnodeFqdn->type != cJSON_String || dnodeFqdn->valuestring == NULL) { diff --git a/src/dnode/src/dnodeMInfos.c b/src/dnode/src/dnodeMInfos.c index dc89487f8b..884924f113 100644 --- a/src/dnode/src/dnodeMInfos.c +++ b/src/dnode/src/dnodeMInfos.c @@ -80,7 +80,7 @@ void dnodeUpdateEpSetForPeer(SRpcEpSet *ep) { pthread_mutex_lock(&tsMInfosMutex); dInfo("minfos is changed, numOfEps:%d inUse:%d", ep->numOfEps, ep->inUse); - for (int i = 0; i < ep->numOfEps; ++i) { + for (int32_t i = 0; i < ep->numOfEps; ++i) { ep->port[i] -= TSDB_PORT_DNODEDNODE; dInfo("minfo:%d %s:%u", i, ep->fqdn[i], ep->port[i]); } @@ -108,7 +108,7 @@ void dnodeGetMInfos(SMInfos *pMinfos) { void dnodeGetEpSetForPeer(SRpcEpSet *epSet) { pthread_mutex_lock(&tsMInfosMutex); *epSet = tsMEpSet; - for (int i = 0; i < epSet->numOfEps; ++i) { + for (int32_t i = 0; i < epSet->numOfEps; ++i) { epSet->port[i] += TSDB_PORT_DNODEDNODE; } pthread_mutex_unlock(&tsMInfosMutex); @@ -171,7 +171,7 @@ static int32_t dnodeReadMInfos() { goto PARSE_MINFOS_OVER; } - len = fread(content, 1, maxLen, fp); + len = (int32_t)fread(content, 1, maxLen, fp); if (len <= 0) { dError("failed to read %s, content is null", file); goto PARSE_MINFOS_OVER; @@ -189,14 +189,14 @@ static int32_t dnodeReadMInfos() { dError("failed to read mnodeEpSet.json, inUse not found"); goto PARSE_MINFOS_OVER; } - tsMInfos.inUse = inUse->valueint; + tsMInfos.inUse = (int8_t)inUse->valueint; cJSON *nodeNum = cJSON_GetObjectItem(root, "nodeNum"); if (!nodeNum || nodeNum->type != cJSON_Number) { dError("failed to read mnodeEpSet.json, nodeNum not found"); goto PARSE_MINFOS_OVER; } - minfos.mnodeNum = nodeNum->valueint; + minfos.mnodeNum = (int8_t)nodeNum->valueint; cJSON *nodeInfos = cJSON_GetObjectItem(root, "nodeInfos"); if (!nodeInfos || nodeInfos->type != cJSON_Array) { @@ -204,13 +204,13 @@ static int32_t dnodeReadMInfos() { goto PARSE_MINFOS_OVER; } - int size = cJSON_GetArraySize(nodeInfos); + int32_t size = cJSON_GetArraySize(nodeInfos); if (size != minfos.mnodeNum) { dError("failed to read mnodeEpSet.json, nodeInfos size not matched"); goto PARSE_MINFOS_OVER; } - for (int i = 0; i < size; ++i) { + for (int32_t i = 0; i < size; ++i) { cJSON *nodeInfo = cJSON_GetArrayItem(nodeInfos, i); if (nodeInfo == NULL) continue; @@ -227,7 +227,7 @@ static int32_t dnodeReadMInfos() { } SMInfo *pMinfo = &minfos.mnodeInfos[i]; - pMinfo->mnodeId = nodeId->valueint; + pMinfo->mnodeId = (int32_t)nodeId->valueint; tstrncpy(pMinfo->mnodeEp, nodeEp->valuestring, TSDB_EP_LEN); bool changed = dnodeCheckEpChanged(pMinfo->mnodeId, pMinfo->mnodeEp); diff --git a/src/dnode/src/dnodeMPeer.c b/src/dnode/src/dnodeMPeer.c index 0863666f76..e4942c49aa 100644 --- a/src/dnode/src/dnodeMPeer.c +++ b/src/dnode/src/dnodeMPeer.c @@ -60,7 +60,7 @@ int32_t dnodeInitMPeer() { void dnodeCleanupMPeer() { for (int32_t i = 0; i < tsMPeerWP.maxNum; ++i) { SMPeerWorker *pWorker = tsMPeerWP.worker + i; - if (pWorker->thread) { + if (taosCheckPthreadValid(pWorker->thread)) { taosQsetThreadResume(tsMPeerQset); } dDebug("dnode mpeer worker:%d is closed", i); @@ -69,7 +69,7 @@ void dnodeCleanupMPeer() { for (int32_t i = 0; i < tsMPeerWP.maxNum; ++i) { SMPeerWorker *pWorker = tsMPeerWP.worker + i; dDebug("dnode mpeer worker:%d start to join", i); - if (pWorker->thread) { + if (taosCheckPthreadValid(pWorker->thread)) { pthread_join(pWorker->thread, NULL); } dDebug("dnode mpeer worker:%d join success", i); diff --git a/src/dnode/src/dnodeMRead.c b/src/dnode/src/dnodeMRead.c index 9027c346f5..90332e6783 100644 --- a/src/dnode/src/dnodeMRead.c +++ b/src/dnode/src/dnodeMRead.c @@ -40,7 +40,7 @@ static void *dnodeProcessMReadQueue(void *param); int32_t dnodeInitMRead() { tsMReadQset = taosOpenQset(); - tsMReadWP.maxNum = tsNumOfCores * tsNumOfThreadsPerCore / 2; + tsMReadWP.maxNum = (int32_t)(tsNumOfCores * tsNumOfThreadsPerCore / 2); tsMReadWP.maxNum = MAX(2, tsMReadWP.maxNum); tsMReadWP.maxNum = MIN(4, tsMReadWP.maxNum); tsMReadWP.curNum = 0; @@ -60,7 +60,7 @@ int32_t dnodeInitMRead() { void dnodeCleanupMRead() { for (int32_t i = 0; i < tsMReadWP.maxNum; ++i) { SMReadWorker *pWorker = tsMReadWP.worker + i; - if (pWorker->thread) { + if (taosCheckPthreadValid(pWorker->thread)) { taosQsetThreadResume(tsMReadQset); } dDebug("dnode mread worker:%d is closed", i); @@ -69,7 +69,7 @@ void dnodeCleanupMRead() { for (int32_t i = 0; i < tsMReadWP.maxNum; ++i) { SMReadWorker *pWorker = tsMReadWP.worker + i; dDebug("dnode mread worker:%d start to join", i); - if (pWorker->thread) { + if (taosCheckPthreadValid(pWorker->thread)) { pthread_join(pWorker->thread, NULL); } dDebug("dnode mread worker:%d start to join", i); diff --git a/src/dnode/src/dnodeMWrite.c b/src/dnode/src/dnodeMWrite.c index 8c9e22ef4b..218d487473 100644 --- a/src/dnode/src/dnodeMWrite.c +++ b/src/dnode/src/dnodeMWrite.c @@ -60,7 +60,7 @@ int32_t dnodeInitMWrite() { void dnodeCleanupMWrite() { for (int32_t i = 0; i < tsMWriteWP.maxNum; ++i) { SMWriteWorker *pWorker = tsMWriteWP.worker + i; - if (pWorker->thread) { + if (taosCheckPthreadValid(pWorker->thread)) { taosQsetThreadResume(tsMWriteQset); } dDebug("dnode mwrite worker:%d is closed", i); @@ -69,7 +69,7 @@ void dnodeCleanupMWrite() { for (int32_t i = 0; i < tsMWriteWP.maxNum; ++i) { SMWriteWorker *pWorker = tsMWriteWP.worker + i; dDebug("dnode mwrite worker:%d start to join", i); - if (pWorker->thread) { + if (taosCheckPthreadValid(pWorker->thread)) { pthread_join(pWorker->thread, NULL); } dDebug("dnode mwrite worker:%d join success", i); diff --git a/src/dnode/src/dnodePeer.c b/src/dnode/src/dnodePeer.c index 5ee10abc30..79c60874f9 100644 --- a/src/dnode/src/dnodePeer.c +++ b/src/dnode/src/dnodePeer.c @@ -90,7 +90,10 @@ static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { }; if (pMsg->pCont == NULL) return; - if (pMsg->msgType == TSDB_MSG_TYPE_NETWORK_TEST) return dnodeSendStartupStep(pMsg); + if (pMsg->msgType == TSDB_MSG_TYPE_NETWORK_TEST) { + dnodeSendStartupStep(pMsg); + return; + } if (dnodeGetRunStatus() != TSDB_RUN_STATUS_RUNING) { rspMsg.code = TSDB_CODE_APP_NOT_READY; diff --git a/src/dnode/src/dnodeShell.c b/src/dnode/src/dnodeShell.c index 79cc70005b..b80f75e963 100644 --- a/src/dnode/src/dnodeShell.c +++ b/src/dnode/src/dnodeShell.c @@ -70,7 +70,7 @@ int32_t dnodeInitShell() { dnodeProcessShellMsgFp[TSDB_MSG_TYPE_NETWORK_TEST] = dnodeSendStartupStep; - int32_t numOfThreads = (tsNumOfCores * tsNumOfThreadsPerCore) / 2.0; + int32_t numOfThreads = (int32_t)((tsNumOfCores * tsNumOfThreadsPerCore) / 2.0); if (numOfThreads < 1) { numOfThreads = 1; } diff --git a/src/dnode/src/dnodeStep.c b/src/dnode/src/dnodeStep.c index 2354b1d5a3..8878299d94 100644 --- a/src/dnode/src/dnodeStep.c +++ b/src/dnode/src/dnodeStep.c @@ -70,5 +70,5 @@ int32_t dnodeStepInit(SStep *pSteps, int32_t stepSize) { } void dnodeStepCleanup(SStep *pSteps, int32_t stepSize) { - return taosStepCleanupImp(pSteps, stepSize - 1); + taosStepCleanupImp(pSteps, stepSize - 1); } \ No newline at end of file diff --git a/src/dnode/src/dnodeSystem.c b/src/dnode/src/dnodeSystem.c index a4d7e791e6..6c28552e1e 100644 --- a/src/dnode/src/dnodeSystem.c +++ b/src/dnode/src/dnodeSystem.c @@ -19,9 +19,43 @@ #include "tconfig.h" #include "dnodeMain.h" -static void signal_handler(int32_t signum, siginfo_t *sigInfo, void *context); static tsem_t exitSem; +#ifdef WINDOWS +static void signal_handler(int32_t signum) { + dInfo("shut down signal is %d", signum); +#else +static void signal_handler(int32_t signum, siginfo_t *sigInfo, void *context) { + if (signum == SIGUSR1) { + taosCfgDynamicOptions("debugFlag 143"); + return; + } + if (signum == SIGUSR2) { + taosCfgDynamicOptions("resetlog"); + return; + } + dInfo("shut down signal is %d, sender PID:%d cmdline:%s", signum, sigInfo->si_pid, taosGetCmdlineByPID(sigInfo->si_pid)); +#endif + + syslog(LOG_INFO, "Shut down signal is %d", signum); + syslog(LOG_INFO, "Shutting down TDengine service..."); + + // protect the application from receive another signal + struct sigaction act = {{0}}; + act.sa_handler = SIG_IGN; + sigaction(SIGTERM, &act, NULL); + sigaction(SIGINT, &act, NULL); + +#ifndef WINDOWS + sigaction(SIGHUP, &act, NULL); + sigaction(SIGUSR1, &act, NULL); + sigaction(SIGUSR2, &act, NULL); +#endif + + // inform main thread to exit + tsem_post(&exitSem); +} + int32_t main(int32_t argc, char *argv[]) { int dump_config = 0; @@ -113,6 +147,8 @@ int32_t main(int32_t argc, char *argv[]) { /* Set termination handler. */ struct sigaction act = {{0}}; + +#ifndef WINDOWS act.sa_flags = SA_SIGINFO; act.sa_sigaction = signal_handler; sigaction(SIGTERM, &act, NULL); @@ -120,6 +156,11 @@ int32_t main(int32_t argc, char *argv[]) { sigaction(SIGINT, &act, NULL); sigaction(SIGUSR1, &act, NULL); sigaction(SIGUSR2, &act, NULL); +#else + act.sa_handler = signal_handler; + sigaction(SIGTERM, &act, NULL); + sigaction(SIGINT, &act, NULL); +#endif // Open /var/log/syslog file to record information. openlog("TDengine:", LOG_PID | LOG_CONS | LOG_NDELAY, LOG_LOCAL1); @@ -146,31 +187,3 @@ int32_t main(int32_t argc, char *argv[]) { closelog(); return EXIT_SUCCESS; } - -static void signal_handler(int32_t signum, siginfo_t *sigInfo, void *context) { - if (signum == SIGUSR1) { - taosCfgDynamicOptions("debugFlag 143"); - return; - } - if (signum == SIGUSR2) { - taosCfgDynamicOptions("resetlog"); - return; - } - - syslog(LOG_INFO, "Shut down signal is %d", signum); - syslog(LOG_INFO, "Shutting down TDengine service..."); - // clean the system. - dInfo("shut down signal is %d, sender PID:%d cmdline:%s", signum, sigInfo->si_pid, taosGetCmdlineByPID(sigInfo->si_pid)); - - // protect the application from receive another signal - struct sigaction act = {{0}}; - act.sa_handler = SIG_IGN; - sigaction(SIGTERM, &act, NULL); - sigaction(SIGHUP, &act, NULL); - sigaction(SIGINT, &act, NULL); - sigaction(SIGUSR1, &act, NULL); - sigaction(SIGUSR2, &act, NULL); - - // inform main thread to exit - tsem_post(&exitSem); -} diff --git a/src/dnode/src/dnodeTelemetry.c b/src/dnode/src/dnodeTelemetry.c index ff9598ecc5..9291831884 100644 --- a/src/dnode/src/dnodeTelemetry.c +++ b/src/dnode/src/dnodeTelemetry.c @@ -93,14 +93,14 @@ static void addStringField(SBufferWriter* bw, const char* k, const char* v) { static void addCpuInfo(SBufferWriter* bw) { char * line = NULL; size_t size = 0; - int done = 0; + int32_t done = 0; FILE* fp = fopen("/proc/cpuinfo", "r"); if (fp == NULL) { return; } - while (done != 3 && (size = getline(&line, &size, fp)) != -1) { + while (done != 3 && (size = tgetline(&line, &size, fp)) != -1) { line[size - 1] = '\0'; if (((done&1) == 0) && strncmp(line, "model name", 10) == 0) { const char* v = strchr(line, ':') + 2; @@ -129,7 +129,7 @@ static void addOsInfo(SBufferWriter* bw) { return; } - while ((size = getline(&line, &size, fp)) != -1) { + while ((size = tgetline(&line, &size, fp)) != -1) { line[size - 1] = '\0'; if (strncmp(line, "PRETTY_NAME", 11) == 0) { const char* p = strchr(line, '=') + 1; @@ -155,7 +155,7 @@ static void addMemoryInfo(SBufferWriter* bw) { return; } - while ((size = getline(&line, &size, fp)) != -1) { + while ((size = tgetline(&line, &size, fp)) != -1) { line[size - 1] = '\0'; if (strncmp(line, "MemTotal", 8) == 0) { const char* p = strchr(line, ':') + 1; @@ -200,7 +200,7 @@ static void sendTelemetryReport() { dTrace("failed to get IP address of " TELEMETRY_SERVER ", reason:%s", strerror(errno)); return; } - int fd = taosOpenTcpClientSocket(ip, TELEMETRY_PORT, 0); + SOCKET fd = taosOpenTcpClientSocket(ip, TELEMETRY_PORT, 0); if (fd < 0) { dTrace("failed to create socket for telemetry, reason:%s", strerror(errno)); return; @@ -222,10 +222,10 @@ static void sendTelemetryReport() { "Content-Type: application/json\n" "Content-Length: "; - taosWriteSocket(fd, header, strlen(header)); - int contLen = tbufTell(&bw) - 1; + taosWriteSocket(fd, header, (int32_t)strlen(header)); + int32_t contLen = (int32_t)(tbufTell(&bw) - 1); sprintf(buf, "%d\n\n", contLen); - taosWriteSocket(fd, buf, strlen(buf)); + taosWriteSocket(fd, buf, (int32_t)strlen(buf)); taosWriteSocket(fd, tbufGetData(&bw, false), contLen); tbufCloseWriter(&bw); @@ -258,7 +258,7 @@ static void* telemetryThread(void* param) { } static void dnodeGetEmail(char* filepath) { - int fd = open(filepath, O_RDONLY); + int32_t fd = open(filepath, O_RDONLY); if (fd < 0) { return; } @@ -267,10 +267,9 @@ static void dnodeGetEmail(char* filepath) { dError("failed to read %d bytes from file %s since %s", TSDB_FQDN_LEN, filepath, strerror(errno)); } - close(fd); + taosClose(fd); } - int32_t dnodeInitTelemetry() { if (!tsEnableTelemetryReporting) { return 0; @@ -303,7 +302,7 @@ void dnodeCleanupTelemetry() { return; } - if (tsTelemetryThread) { + if (taosCheckPthreadValid(tsTelemetryThread)) { tsem_post(&tsExitSem); pthread_join(tsTelemetryThread, NULL); tsem_destroy(&tsExitSem); diff --git a/src/dnode/src/dnodeVWrite.c b/src/dnode/src/dnodeVWrite.c index ab5a4ccaad..93d1611ebc 100644 --- a/src/dnode/src/dnodeVWrite.c +++ b/src/dnode/src/dnodeVWrite.c @@ -52,14 +52,14 @@ int32_t dnodeInitVWrite() { void dnodeCleanupVWrite() { for (int32_t i = 0; i < tsVWriteWP.max; ++i) { SVWriteWorker *pWorker = tsVWriteWP.worker + i; - if (pWorker->thread) { + if (taosCheckPthreadValid(pWorker->thread)) { taosQsetThreadResume(pWorker->qset); } } for (int32_t i = 0; i < tsVWriteWP.max; ++i) { SVWriteWorker *pWorker = tsVWriteWP.worker + i; - if (pWorker->thread) { + if (taosCheckPthreadValid(pWorker->thread)) { pthread_join(pWorker->thread, NULL); taosFreeQall(pWorker->qall); taosCloseQset(pWorker->qset); diff --git a/src/dnode/src/dnodeVnodes.c b/src/dnode/src/dnodeVnodes.c index c62d5a8207..9f32541612 100644 --- a/src/dnode/src/dnodeVnodes.c +++ b/src/dnode/src/dnodeVnodes.c @@ -157,7 +157,7 @@ int32_t dnodeInitVnodes() { int32_t failedVnodes = 0; for (int32_t t = 0; t < threadNum; ++t) { SOpenVnodeThread *pThread = &threads[t]; - if (pThread->vnodeNum > 0 && pThread->thread) { + if (pThread->vnodeNum > 0 && taosCheckPthreadValid(pThread->thread)) { pthread_join(pThread->thread, NULL); } openVnodes += pThread->opened; @@ -260,7 +260,7 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) { tstrncpy(pStatus->clusterCfg.timezone, tsTimezone, 64); pStatus->clusterCfg.checkTime = 0; char timestr[32] = "1970-01-01 00:00:00.00"; - (void)taosParseTime(timestr, &pStatus->clusterCfg.checkTime, strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); + (void)taosParseTime(timestr, &pStatus->clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); tstrncpy(pStatus->clusterCfg.locale, tsLocale, TSDB_LOCALE_LEN); tstrncpy(pStatus->clusterCfg.charset, tsCharset, TSDB_LOCALE_LEN); diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index 01034b170f..d76ecd9ba0 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -39,7 +39,7 @@ #include "mnodeCluster.h" int32_t tsAccessSquence = 0; -int64_t tsDnodeRid = -1; +int64_t tsDnodeRid = -1; static void * tsDnodeSdb = NULL; static int32_t tsDnodeUpdateSize = 0; extern void * tsMnodeSdb; diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index a0c8d88c51..a520a7d52d 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -841,6 +841,7 @@ static int32_t mnodeProcessBatchCreateTableMsg(SMnodeMsg *pMsg) { return TSDB_CODE_MND_ACTION_IN_PROGRESS; } else { // batch master replay, reprocess the whole batch assert(0); + return TSDB_CODE_MND_MSG_NOT_PROCESSED; } } } diff --git a/src/os/inc/osWindows.h b/src/os/inc/osWindows.h index 5a1e642572..396577a5d7 100644 --- a/src/os/inc/osWindows.h +++ b/src/os/inc/osWindows.h @@ -208,6 +208,12 @@ typedef struct { int wordexp(const char *words, wordexp_t *pwordexp, int flags); void wordfree(wordexp_t *pwordexp); +#define openlog(a, b, c) +#define closelog() +#define LOG_ERR 0 +#define LOG_INFO 1 +void syslog(int unused, const char *format, ...); + #define TAOS_OS_FUNC_ATOMIC #define atomic_load_8(ptr) (*(char volatile*)(ptr)) #define atomic_load_16(ptr) (*(short volatile*)(ptr)) diff --git a/src/os/src/windows/wSysLog.c b/src/os/src/windows/wSysLog.c new file mode 100644 index 0000000000..866cacbaba --- /dev/null +++ b/src/os/src/windows/wSysLog.c @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" + +void syslog(int unused, const char *format, ...) {} \ No newline at end of file diff --git a/src/os/src/windows/wSysinfo.c b/src/os/src/windows/wSysinfo.c index 1bfee25c4a..b3fd4c28b9 100644 --- a/src/os/src/windows/wSysinfo.c +++ b/src/os/src/windows/wSysinfo.c @@ -235,4 +235,9 @@ LONG WINAPI FlCrashDump(PEXCEPTION_POINTERS ep) { return EXCEPTION_CONTINUE_SEARCH; } -void taosSetCoreDump() { SetUnhandledExceptionFilter(&FlCrashDump); } \ No newline at end of file +void taosSetCoreDump() { SetUnhandledExceptionFilter(&FlCrashDump); } + +bool taosGetSystemUid(char *uid) { + sprintf(uid, "uid_not_implemented_yet"); + return true; +} \ No newline at end of file diff --git a/src/plugins/CMakeLists.txt b/src/plugins/CMakeLists.txt index d03717b6e1..e66997dc8e 100644 --- a/src/plugins/CMakeLists.txt +++ b/src/plugins/CMakeLists.txt @@ -3,6 +3,6 @@ PROJECT(TDengine) ADD_SUBDIRECTORY(monitor) ADD_SUBDIRECTORY(http) -IF (TD_MQTT) +IF (TD_LINUX AND TD_MQTT) ADD_SUBDIRECTORY(mqtt) ENDIF () \ No newline at end of file diff --git a/src/sync/src/syncArbitrator.c b/src/sync/src/syncArbitrator.c index 8c0d02bedc..dbcc4b40a6 100644 --- a/src/sync/src/syncArbitrator.c +++ b/src/sync/src/syncArbitrator.c @@ -27,10 +27,6 @@ #include "syncInt.h" #include "syncTcp.h" -#ifndef SIGHUP - #define SIGHUP SIGTERM -#endif - static void arbSignalHandler(int32_t signum); static void arbProcessIncommingConnection(SOCKET connFd, uint32_t sourceIp); static void arbProcessBrokenLink(int64_t rid); @@ -78,8 +74,10 @@ int32_t main(int32_t argc, char *argv[]) { act.sa_handler = arbSignalHandler; sigaction(SIGTERM, &act, NULL); - sigaction(SIGHUP, &act, NULL); sigaction(SIGINT, &act, NULL); +#ifndef WINDOWS + sigaction(SIGHUP, &act, NULL); +#endif tsAsyncLog = 0; strcat(arbLogPath, "/arbitrator.log"); @@ -180,8 +178,10 @@ static void arbSignalHandler(int32_t signum) { struct sigaction act = {{0}}; act.sa_handler = SIG_IGN; sigaction(SIGTERM, &act, NULL); - sigaction(SIGHUP, &act, NULL); sigaction(SIGINT, &act, NULL); +#ifndef WINDOWS + sigaction(SIGHUP, &act, NULL); +#endif sInfo("shut down signal is %d", signum); diff --git a/src/vnode/CMakeLists.txt b/src/vnode/CMakeLists.txt index c953883361..202f11e03e 100644 --- a/src/vnode/CMakeLists.txt +++ b/src/vnode/CMakeLists.txt @@ -1,6 +1,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 3.5) PROJECT(TDengine) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/tsdb/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/dnode/inc) @@ -9,7 +10,7 @@ INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/inc) INCLUDE_DIRECTORIES(inc) AUX_SOURCE_DIRECTORY(src SRC) -IF (TD_LINUX) +IF (TD_LINUX OR TD_WINDOWS) ADD_LIBRARY(vnode ${SRC}) TARGET_LINK_LIBRARIES(vnode tsdb tcq) ENDIF () diff --git a/src/vnode/src/vnodeVersion.c b/src/vnode/src/vnodeVersion.c index fb3b3ebd9e..68fa32b2de 100644 --- a/src/vnode/src/vnodeVersion.c +++ b/src/vnode/src/vnodeVersion.c @@ -41,7 +41,7 @@ int32_t vnodeReadVersion(SVnodeObj *pVnode) { goto PARSE_VER_ERROR; } - len = fread(content, 1, maxLen, fp); + len = (int32_t)fread(content, 1, maxLen, fp); if (len <= 0) { vError("vgId:%d, failed to read %s, content is null", pVnode->vgId, file); goto PARSE_VER_ERROR; diff --git a/src/vnode/src/vnodeWorker.c b/src/vnode/src/vnodeWorker.c index d6053cf18e..6fb79d10fe 100644 --- a/src/vnode/src/vnodeWorker.c +++ b/src/vnode/src/vnodeWorker.c @@ -109,7 +109,7 @@ static void vnodeStopMWorker() { void vnodeCleanupMWorker() { for (int32_t i = 0; i < tsVMWorkerPool.maxNum; ++i) { SVMWorker *pWorker = tsVMWorkerPool.worker + i; - if (pWorker->thread) { + if (taosCheckPthreadValid(pWorker->thread)) { taosQsetThreadResume(tsVMWorkerQset); } vDebug("vmworker:%d is closed", i); @@ -118,7 +118,7 @@ void vnodeCleanupMWorker() { for (int32_t i = 0; i < tsVMWorkerPool.maxNum; ++i) { SVMWorker *pWorker = tsVMWorkerPool.worker + i; vDebug("vmworker:%d start to join", i); - if (pWorker->thread) { + if (taosCheckPthreadValid(pWorker->thread)) { pthread_join(pWorker->thread, NULL); } vDebug("vmworker:%d join success", i); diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index 4b9f59279c..90b1e91979 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -333,7 +333,7 @@ static int32_t vnodePerformFlowCtrl(SVWriteMsg *pWrite) { if (pVnode->queuedWMsg < MAX_QUEUED_MSG_NUM && pVnode->flowctrlLevel <= 0) return 0; if (tsEnableFlowCtrl == 0) { - int32_t ms = pow(2, pVnode->flowctrlLevel + 2); + int32_t ms = (int32_t)pow(2, pVnode->flowctrlLevel + 2); if (ms > 100) ms = 100; vTrace("vgId:%d, msg:%p, app:%p, perform flowctrl for %d ms", pVnode->vgId, pWrite, pWrite->rpcMsg.ahandle, ms); taosMsleep(ms); From 8a9ed8e2efcc751a7efc2e3a4f4d391058b7da13 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 15 Jan 2021 18:14:50 +0800 Subject: [PATCH 004/147] TD-1207 --- src/balance/src/bnThread.c | 4 +- src/os/src/detail/osSysinfo.c | 117 +++++++++++++++++----------------- 2 files changed, 62 insertions(+), 59 deletions(-) diff --git a/src/balance/src/bnThread.c b/src/balance/src/bnThread.c index caf33061d1..c6a16da430 100644 --- a/src/balance/src/bnThread.c +++ b/src/balance/src/bnThread.c @@ -124,8 +124,8 @@ void bnStartTimer(int32_t mseconds) { bool updateSoon = (mseconds != -1); if (updateSoon) { - mTrace("balance function will be called after %" PRId64 " ms", mseconds); - taosTmrReset(bnProcessTimer, mseconds, (void *)mseconds, tsMnodeTmr, &tsBnThread.timer); + mTrace("balance function will be called after %d ms", mseconds); + taosTmrReset(bnProcessTimer, mseconds, (void *)(int64_t)mseconds, tsMnodeTmr, &tsBnThread.timer); } else { taosTmrReset(bnProcessTimer, tsStatusInterval * 1000, NULL, tsMnodeTmr, &tsBnThread.timer); } diff --git a/src/os/src/detail/osSysinfo.c b/src/os/src/detail/osSysinfo.c index b0ca6139ed..2ba0dd6917 100644 --- a/src/os/src/detail/osSysinfo.c +++ b/src/os/src/detail/osSysinfo.c @@ -45,6 +45,21 @@ static char tsProcMemFile[25] = {0}; static char tsProcIOFile[25] = {0}; static float tsPageSizeKB = 0; +static void taosGetProcInfos() { + tsPageSize = sysconf(_SC_PAGESIZE); + tsOpenMax = sysconf(_SC_OPEN_MAX); + tsStreamMax = sysconf(_SC_STREAM_MAX); + + tsProcId = (pid_t)syscall(SYS_gettid); + tsPageSizeKB = (float)(sysconf(_SC_PAGESIZE)) / 1024; + + snprintf(tsProcMemFile, 25, "/proc/%d/status", tsProcId); + snprintf(tsProcCpuFile, 25, "/proc/%d/stat", tsProcId); + snprintf(tsProcIOFile, 25, "/proc/%d/io", tsProcId); +} + +static int32_t taosGetTotalMemory() { return (int32_t)((float)sysconf(_SC_PHYS_PAGES) * tsPageSizeKB / 1024); } + bool taosGetSysMemory(float *memoryUsedMB) { float memoryAvailMB = (float)sysconf(_SC_AVPHYS_PAGES) * tsPageSizeKB / 1024; *memoryUsedMB = (float)tsTotalMemoryMB - memoryAvailMB; @@ -105,7 +120,8 @@ static bool taosGetSysCpuInfo(SysCpuInfo *cpuInfo) { } char cpu[10] = {0}; - sscanf(line, "%s %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64, cpu, &cpuInfo->user, &cpuInfo->nice, &cpuInfo->system, &cpuInfo->idle); + sscanf(line, "%s %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64, cpu, &cpuInfo->user, &cpuInfo->nice, &cpuInfo->system, + &cpuInfo->idle); tfree(line); fclose(fp); @@ -131,7 +147,8 @@ static bool taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { for (int i = 0, blank = 0; line[i] != 0; ++i) { if (line[i] == ' ') blank++; if (blank == PROCESS_ITEM) { - sscanf(line + i + 1, "%" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64, &cpuInfo->utime, &cpuInfo->stime, &cpuInfo->cutime, &cpuInfo->cstime); + sscanf(line + i + 1, "%" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64, &cpuInfo->utime, &cpuInfo->stime, + &cpuInfo->cutime, &cpuInfo->cstime); break; } } @@ -162,12 +179,12 @@ static void taosGetSystemTimezone() { char buf[68] = {0}; if (f != NULL) { int len = fread(buf, 64, 1, f); - if(len < 64 && ferror(f)) { + if (len < 64 && ferror(f)) { fclose(f); uError("read /etc/timezone error, reason:%s", strerror(errno)); return; } - + fclose(f); buf[sizeof(buf) - 1] = 0; @@ -258,6 +275,8 @@ static void taosGetSystemLocale() { // get and set default locale } } +static int32_t taosGetCpuCores() { return (int32_t)sysconf(_SC_NPROCESSORS_ONLN); } + bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) { static uint64_t lastSysUsed = 0; static uint64_t lastSysTotal = 0; @@ -300,11 +319,9 @@ bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) { bool taosGetDisk() { struct statvfs info; const double unit = 1024 * 1024 * 1024; - + if (tscEmbedded) { if (statvfs(tsDataDir, &info)) { - //tsTotalDataDirGB = 0; - //tsAvailDataDirGB = 0; uError("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno)); return false; } else { @@ -314,8 +331,6 @@ bool taosGetDisk() { } if (statvfs(tsLogDir, &info)) { - //tsTotalLogDirGB = 0; - //tsAvailLogDirGB = 0; uError("failed to get disk size, logDir:%s errno:%s", tsLogDir, strerror(errno)); return false; } else { @@ -324,8 +339,6 @@ bool taosGetDisk() { } if (statvfs("/tmp", &info)) { - //tsTotalTmpDirGB = 0; - //tsAvailTmpDirectorySpace = 0; uError("failed to get disk size, tmpDir:/tmp errno:%s", strerror(errno)); return false; } else { @@ -344,13 +357,12 @@ static bool taosGetCardInfo(int64_t *bytes) { return false; } - size_t len = 2048; char * line = calloc(1, len); while (!feof(fp)) { memset(line, 0, len); - + int64_t rbytes = 0; int64_t rpackts = 0; int64_t tbytes = 0; @@ -465,7 +477,7 @@ bool taosGetProcIO(float *readKB, float *writeKB) { static int64_t lastReadbyte = -1; static int64_t lastWritebyte = -1; - int64_t curReadbyte = 0; + int64_t curReadbyte = 0; int64_t curWritebyte = 0; if (!taosReadProcIO(&curReadbyte, &curWritebyte)) { @@ -490,18 +502,10 @@ bool taosGetProcIO(float *readKB, float *writeKB) { } void taosGetSystemInfo() { - tsNumOfCores = (int32_t)sysconf(_SC_NPROCESSORS_ONLN); - tsPageSize = sysconf(_SC_PAGESIZE); - tsOpenMax = sysconf(_SC_OPEN_MAX); - tsStreamMax = sysconf(_SC_STREAM_MAX); + taosGetProcInfos(); - tsProcId = (pid_t)syscall(SYS_gettid); - tsPageSizeKB = (float)(sysconf(_SC_PAGESIZE)) / 1024; - tsTotalMemoryMB = (int32_t)((float)sysconf(_SC_PHYS_PAGES) * tsPageSizeKB / 1024); - - snprintf(tsProcMemFile, 25, "/proc/%d/status", tsProcId); - snprintf(tsProcCpuFile, 25, "/proc/%d/stat", tsProcId); - snprintf(tsProcIOFile, 25, "/proc/%d/io", tsProcId); + tsNumOfCores = taosGetCpuCores(); + tsTotalMemoryMB = taosGetTotalMemory(); float tmp1, tmp2; taosGetSysMemory(&tmp1); @@ -573,16 +577,16 @@ void taosSetCoreDump() { if (0 == tsEnableCoreFile) { return; } - + // 1. set ulimit -c unlimited struct rlimit rlim; struct rlimit rlim_new; if (getrlimit(RLIMIT_CORE, &rlim) == 0) { - #ifndef _ALPINE +#ifndef _ALPINE uInfo("the old unlimited para: rlim_cur=%" PRIu64 ", rlim_max=%" PRIu64, rlim.rlim_cur, rlim.rlim_max); - #else +#else uInfo("the old unlimited para: rlim_cur=%llu, rlim_max=%llu", rlim.rlim_cur, rlim.rlim_max); - #endif +#endif rlim_new.rlim_cur = RLIM_INFINITY; rlim_new.rlim_max = RLIM_INFINITY; if (setrlimit(RLIMIT_CORE, &rlim_new) != 0) { @@ -594,57 +598,56 @@ void taosSetCoreDump() { } if (getrlimit(RLIMIT_CORE, &rlim) == 0) { - #ifndef _ALPINE +#ifndef _ALPINE uInfo("the new unlimited para: rlim_cur=%" PRIu64 ", rlim_max=%" PRIu64, rlim.rlim_cur, rlim.rlim_max); - #else +#else uInfo("the new unlimited para: rlim_cur=%llu, rlim_max=%llu", rlim.rlim_cur, rlim.rlim_max); - #endif +#endif } #ifndef _TD_ARM_ // 2. set the path for saving core file struct __sysctl_args args; - int old_usespid = 0; - size_t old_len = 0; - int new_usespid = 1; - size_t new_len = sizeof(new_usespid); - + + int old_usespid = 0; + size_t old_len = 0; + int new_usespid = 1; + size_t new_len = sizeof(new_usespid); + int name[] = {CTL_KERN, KERN_CORE_USES_PID}; - + memset(&args, 0, sizeof(struct __sysctl_args)); - args.name = name; - args.nlen = sizeof(name)/sizeof(name[0]); - args.oldval = &old_usespid; + args.name = name; + args.nlen = sizeof(name) / sizeof(name[0]); + args.oldval = &old_usespid; args.oldlenp = &old_len; - args.newval = &new_usespid; - args.newlen = new_len; - + args.newval = &new_usespid; + args.newlen = new_len; + old_len = sizeof(old_usespid); - + if (syscall(SYS__sysctl, &args) == -1) { - uInfo("_sysctl(kern_core_uses_pid) set fail: %s", strerror(errno)); + uInfo("_sysctl(kern_core_uses_pid) set fail: %s", strerror(errno)); } - + uInfo("The old core_uses_pid[%" PRIu64 "]: %d", old_len, old_usespid); - old_usespid = 0; - old_len = 0; + old_len = 0; memset(&args, 0, sizeof(struct __sysctl_args)); - args.name = name; - args.nlen = sizeof(name)/sizeof(name[0]); - args.oldval = &old_usespid; + args.name = name; + args.nlen = sizeof(name) / sizeof(name[0]); + args.oldval = &old_usespid; args.oldlenp = &old_len; - + old_len = sizeof(old_usespid); - + if (syscall(SYS__sysctl, &args) == -1) { - uInfo("_sysctl(kern_core_uses_pid) get fail: %s", strerror(errno)); + uInfo("_sysctl(kern_core_uses_pid) get fail: %s", strerror(errno)); } - + uInfo("The new core_uses_pid[%" PRIu64 "]: %d", old_len, old_usespid); #endif - } bool taosGetSystemUid(char *uid) { From de09879d50e9be7eb41c59622c3dc494140826a0 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 15 Jan 2021 18:55:29 +0800 Subject: [PATCH 005/147] TD-1207 --- src/dnode/src/dnodeSystem.c | 73 +++++++--------- src/kit/shell/src/shellMain.c | 2 +- src/os/inc/osWindows.h | 12 ++- src/os/src/linux/linuxEnv.c | 15 ++-- src/os/src/windows/wSysinfo.c | 156 +++++++++++++++++++++------------- src/sync/src/syncArbitrator.c | 28 +++--- 6 files changed, 161 insertions(+), 125 deletions(-) diff --git a/src/dnode/src/dnodeSystem.c b/src/dnode/src/dnodeSystem.c index 6c28552e1e..a16ff826bf 100644 --- a/src/dnode/src/dnodeSystem.c +++ b/src/dnode/src/dnodeSystem.c @@ -19,43 +19,9 @@ #include "tconfig.h" #include "dnodeMain.h" +static void signal_handler(int32_t signum, siginfo_t *sigInfo, void *context); static tsem_t exitSem; -#ifdef WINDOWS -static void signal_handler(int32_t signum) { - dInfo("shut down signal is %d", signum); -#else -static void signal_handler(int32_t signum, siginfo_t *sigInfo, void *context) { - if (signum == SIGUSR1) { - taosCfgDynamicOptions("debugFlag 143"); - return; - } - if (signum == SIGUSR2) { - taosCfgDynamicOptions("resetlog"); - return; - } - dInfo("shut down signal is %d, sender PID:%d cmdline:%s", signum, sigInfo->si_pid, taosGetCmdlineByPID(sigInfo->si_pid)); -#endif - - syslog(LOG_INFO, "Shut down signal is %d", signum); - syslog(LOG_INFO, "Shutting down TDengine service..."); - - // protect the application from receive another signal - struct sigaction act = {{0}}; - act.sa_handler = SIG_IGN; - sigaction(SIGTERM, &act, NULL); - sigaction(SIGINT, &act, NULL); - -#ifndef WINDOWS - sigaction(SIGHUP, &act, NULL); - sigaction(SIGUSR1, &act, NULL); - sigaction(SIGUSR2, &act, NULL); -#endif - - // inform main thread to exit - tsem_post(&exitSem); -} - int32_t main(int32_t argc, char *argv[]) { int dump_config = 0; @@ -147,8 +113,6 @@ int32_t main(int32_t argc, char *argv[]) { /* Set termination handler. */ struct sigaction act = {{0}}; - -#ifndef WINDOWS act.sa_flags = SA_SIGINFO; act.sa_sigaction = signal_handler; sigaction(SIGTERM, &act, NULL); @@ -156,11 +120,6 @@ int32_t main(int32_t argc, char *argv[]) { sigaction(SIGINT, &act, NULL); sigaction(SIGUSR1, &act, NULL); sigaction(SIGUSR2, &act, NULL); -#else - act.sa_handler = signal_handler; - sigaction(SIGTERM, &act, NULL); - sigaction(SIGINT, &act, NULL); -#endif // Open /var/log/syslog file to record information. openlog("TDengine:", LOG_PID | LOG_CONS | LOG_NDELAY, LOG_LOCAL1); @@ -187,3 +146,33 @@ int32_t main(int32_t argc, char *argv[]) { closelog(); return EXIT_SUCCESS; } + +static void signal_handler(int32_t signum, siginfo_t *sigInfo, void *context) { + if (signum == SIGUSR1) { + taosCfgDynamicOptions("debugFlag 143"); + return; + } + if (signum == SIGUSR2) { + taosCfgDynamicOptions("resetlog"); + return; + } + + syslog(LOG_INFO, "Shut down signal is %d", signum); + syslog(LOG_INFO, "Shutting down TDengine service..."); + // clean the system. + dInfo("shut down signal is %d, sender PID:%d cmdline:%s", signum, sigInfo->si_pid, taosGetCmdlineByPID(sigInfo->si_pid)); + + // protect the application from receive another signal + struct sigaction act = {{0}}; +#ifndef WINDOWS + act.sa_handler = SIG_IGN; +#endif + sigaction(SIGTERM, &act, NULL); + sigaction(SIGHUP, &act, NULL); + sigaction(SIGINT, &act, NULL); + sigaction(SIGUSR1, &act, NULL); + sigaction(SIGUSR2, &act, NULL); + + // inform main thread to exit + tsem_post(&exitSem); +} \ No newline at end of file diff --git a/src/kit/shell/src/shellMain.c b/src/kit/shell/src/shellMain.c index 496cef41ba..cd801ef07a 100644 --- a/src/kit/shell/src/shellMain.c +++ b/src/kit/shell/src/shellMain.c @@ -21,7 +21,7 @@ pthread_t pid; static tsem_t cancelSem; -void shellQueryInterruptHandler(int32_t signum) { +void shellQueryInterruptHandler(int32_t signum, siginfo_t *sigInfo, void *context) { tsem_post(&cancelSem); } diff --git a/src/os/inc/osWindows.h b/src/os/inc/osWindows.h index 396577a5d7..15b26d208d 100644 --- a/src/os/inc/osWindows.h +++ b/src/os/inc/osWindows.h @@ -193,9 +193,19 @@ int gettimeofday(struct timeval *ptv, void *pTimeZone); //for signal, not dispose #define SIGALRM 1234 +#define SIGHUP 1234 +#define SIGUSR1 1234 +#define SIGUSR2 1234 +#define SA_SIGINFO 1234 + typedef int sigset_t; +typedef struct siginfo_t { + int si_pid; +} siginfo_t; struct sigaction { - void (*sa_handler)(int); + int sa_flags; + void (*sa_handler)(int32_t signum, siginfo_t *sigInfo, void *context); + void (*sa_sigaction)(int32_t signum, siginfo_t *sigInfo, void *context); }; int sigaction(int, struct sigaction *, void *); diff --git a/src/os/src/linux/linuxEnv.c b/src/os/src/linux/linuxEnv.c index 5772885cb4..e3eadbc94b 100644 --- a/src/os/src/linux/linuxEnv.c +++ b/src/os/src/linux/linuxEnv.c @@ -18,7 +18,6 @@ #include "tglobal.h" void osInit() { - #ifdef _TD_POWER_ if (configDir[0] == 0) { strcpy(configDir, "/etc/power"); @@ -43,16 +42,14 @@ void osInit() { char cmdline[1024]; -char *taosGetCmdlineByPID(int pid) -{ - sprintf(cmdline, "/proc/%d/cmdline",pid); - FILE* f = fopen(cmdline,"r"); - if(f){ +char* taosGetCmdlineByPID(int pid) { + sprintf(cmdline, "/proc/%d/cmdline", pid); + FILE* f = fopen(cmdline, "r"); + if (f) { size_t size; size = fread(cmdline, sizeof(char), 1024, f); - if(size>0){ - if('\n'==cmdline[size-1]) - cmdline[size-1]='\0'; + if (size > 0) { + if ('\n' == cmdline[size - 1]) cmdline[size - 1] = '\0'; } fclose(f); } diff --git a/src/os/src/windows/wSysinfo.c b/src/os/src/windows/wSysinfo.c index b3fd4c28b9..0499173a0b 100644 --- a/src/os/src/windows/wSysinfo.c +++ b/src/os/src/windows/wSysinfo.c @@ -32,10 +32,51 @@ #endif #pragma warning(push) -#pragma warning(disable:4091) +#pragma warning(disable : 4091) #include #pragma warning(pop) +static int32_t taosGetTotalMemory() { + MEMORYSTATUSEX memsStat; + memsStat.dwLength = sizeof(memsStat); + if (!GlobalMemoryStatusEx(&memsStat)) { + return 0; + } + + float nMemTotal = memsStat.ullTotalPhys / (1024.0f * 1024.0f); + return (int32_t)nMemTotal; +} + +bool taosGetSysMemory(float *memoryUsedMB) { + MEMORYSTATUSEX memsStat; + memsStat.dwLength = sizeof(memsStat); + if (!GlobalMemoryStatusEx(&memsStat)) { + return false; + } + + float nMemFree = memsStat.ullAvailPhys / (1024.0f * 1024.0f); + float nMemTotal = memsStat.ullTotalPhys / (1024.0f * 1024.0f); + + *memoryUsedMB = nMemTotal - nMemFree; + return true; +} + +bool taosGetProcMemory(float *memoryUsedMB) { + unsigned bytes_used = 0; + +#if defined(_WIN32) && defined(_MSC_VER) + PROCESS_MEMORY_COUNTERS pmc; + HANDLE cur_proc = GetCurrentProcess(); + + if (GetProcessMemoryInfo(cur_proc, &pmc, sizeof(pmc))) { + bytes_used = (unsigned)(pmc.WorkingSetSize + pmc.PagefileUsage); + } +#endif + + *memoryUsedMB = (float)bytes_used / 1024 / 1024; + return true; +} + static void taosGetSystemTimezone() { // get and set default timezone SGlobalCfg *cfg_timezone = taosGetConfigOption("timezone"); @@ -71,16 +112,16 @@ static void taosGetSystemLocale() { } } -void taosPrintOsInfo() {} - -void taosKillSystem() { - uError("function taosKillSystem, exit!"); - exit(0); +static int32_t taosGetCpuCores() { + SYSTEM_INFO info; + GetSystemInfo(&info); + return (int32_t)info.dwNumberOfProcessors; } -void taosGetSystemInfo() { - taosGetSystemTimezone(); - taosGetSystemLocale(); +bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) { + *sysCpuUsage = 0; + *procCpuUsage = 0; + return true; } bool taosGetDisk() { @@ -89,20 +130,35 @@ bool taosGetDisk() { unsigned _int64 i64FreeBytesToCaller; unsigned _int64 i64TotalBytes; unsigned _int64 i64FreeBytes; - char dir[4] = {'C', ':', '\\', '\0'}; - int drive_type; if (tscEmbedded) { - drive_type = GetDriveTypeA(dir); - if (drive_type == DRIVE_FIXED) { - fResult = GetDiskFreeSpaceExA(dir, (PULARGE_INTEGER)&i64FreeBytesToCaller, (PULARGE_INTEGER)&i64TotalBytes, - (PULARGE_INTEGER)&i64FreeBytes); - if (fResult) { - tsTotalDataDirGB = tsTotalLogDirGB = tsTotalTmpDirGB = (float)(i64TotalBytes / unit); - tsAvailDataDirGB = tsAvailLogDirGB = tsAvailTmpDirectorySpace = (float)(i64FreeBytes / unit); - } + fResult = GetDiskFreeSpaceExA(tsDataDir, (PULARGE_INTEGER)&i64FreeBytesToCaller, (PULARGE_INTEGER)&i64TotalBytes, + (PULARGE_INTEGER)&i64FreeBytes); + if (fResult) { + tsTotalDataDirGB = (float)(i64TotalBytes / unit); + tsAvailDataDirGB = (float)(i64FreeBytes / unit); } } + + fResult = GetDiskFreeSpaceExA(tsLogDir, (PULARGE_INTEGER)&i64FreeBytesToCaller, (PULARGE_INTEGER)&i64TotalBytes, + (PULARGE_INTEGER)&i64FreeBytes); + if (fResult) { + tsTotalLogDirGB = (float)(i64TotalBytes / unit); + tsAvailLogDirGB = (float)(i64FreeBytes / unit); + } + + fResult = GetDiskFreeSpaceExA(tsTempDir, (PULARGE_INTEGER)&i64FreeBytesToCaller, (PULARGE_INTEGER)&i64TotalBytes, + (PULARGE_INTEGER)&i64FreeBytes); + if (fResult) { + tsTotalTmpDirGB = (float)(i64TotalBytes / unit); + tsAvailTmpDirectorySpace = (float)(i64FreeBytes / unit); + } + + return true; +} + +bool taosGetBandSpeed(float *bandSpeedKb) { + *bandSpeedKb = 0; return true; } @@ -144,48 +200,30 @@ bool taosGetProcIO(float *readKB, float *writeKB) { return true; } -bool taosGetBandSpeed(float *bandSpeedKb) { - *bandSpeedKb = 0; - return true; +void taosGetSystemInfo() { + tsNumOfCores = taosGetCpuCores(); + tsTotalMemoryMB = taosGetTotalMemory(); + + float tmp1, tmp2; + taosGetDisk(); + taosGetBandSpeed(&tmp1); + taosGetCpuUsage(&tmp1, &tmp2); + taosGetProcIO(&tmp1, &tmp2); + + taosGetSystemTimezone(); + taosGetSystemLocale(); } -bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) { - *sysCpuUsage = 0; - *procCpuUsage = 0; - return true; +void taosPrintOsInfo() { + uInfo(" os numOfCores: %d", tsNumOfCores); + uInfo(" os totalDisk: %f(GB)", tsTotalDataDirGB); + uInfo(" os totalMemory: %d(MB)", tsTotalMemoryMB); + uInfo("=================================="); } -bool taosGetProcMemory(float *memoryUsedMB) { - unsigned bytes_used = 0; -#if 0 -#if defined(_WIN32) && defined(_MSC_VER) - PROCESS_MEMORY_COUNTERS pmc; - HANDLE cur_proc = GetCurrentProcess(); - - if (GetProcessMemoryInfo(cur_proc, &pmc, sizeof(pmc))) { - bytes_used = (unsigned)(pmc.WorkingSetSize + pmc.PagefileUsage); - } -#endif -#endif - - *memoryUsedMB = (float)bytes_used / 1024 / 1024; - - return true; -} - -bool taosGetSysMemory(float *memoryUsedMB) { - MEMORYSTATUSEX memsStat; - float nMemFree; - float nMemTotal; - - memsStat.dwLength = sizeof(memsStat); - if (!GlobalMemoryStatusEx(&memsStat)) { - return false; - } - nMemFree = memsStat.ullAvailPhys / (1024.0f * 1024.0f); - nMemTotal = memsStat.ullTotalPhys / (1024.0f * 1024.0f); - *memoryUsedMB = nMemTotal - nMemFree; - return true; +void taosKillSystem() { + uError("function taosKillSystem, exit!"); + exit(0); } int taosSystem(const char *cmd) { @@ -240,4 +278,6 @@ void taosSetCoreDump() { SetUnhandledExceptionFilter(&FlCrashDump); } bool taosGetSystemUid(char *uid) { sprintf(uid, "uid_not_implemented_yet"); return true; -} \ No newline at end of file +} + +char *taosGetCmdlineByPID(int pid) { return ""; } diff --git a/src/sync/src/syncArbitrator.c b/src/sync/src/syncArbitrator.c index dbcc4b40a6..06b4a61d6f 100644 --- a/src/sync/src/syncArbitrator.c +++ b/src/sync/src/syncArbitrator.c @@ -27,7 +27,7 @@ #include "syncInt.h" #include "syncTcp.h" -static void arbSignalHandler(int32_t signum); +static void arbSignalHandler(int32_t signum, siginfo_t *sigInfo, void *context); static void arbProcessIncommingConnection(SOCKET connFd, uint32_t sourceIp); static void arbProcessBrokenLink(int64_t rid); static int32_t arbProcessPeerMsg(int64_t rid, void *buffer); @@ -35,9 +35,9 @@ static tsem_t tsArbSem; static void * tsArbTcpPool; typedef struct { - char id[TSDB_EP_LEN + 24]; - SOCKET nodeFd; - void * pConn; + char id[TSDB_EP_LEN + 24]; + SOCKET nodeFd; + void * pConn; } SNodeConn; int32_t main(int32_t argc, char *argv[]) { @@ -70,14 +70,13 @@ int32_t main(int32_t argc, char *argv[]) { /* Set termination handler. */ struct sigaction act = {{0}}; - memset(&act, 0, sizeof(struct sigaction)); + act.sa_flags = SA_SIGINFO; + act.sa_sigaction = arbSignalHandler; act.sa_handler = arbSignalHandler; sigaction(SIGTERM, &act, NULL); - sigaction(SIGINT, &act, NULL); -#ifndef WINDOWS sigaction(SIGHUP, &act, NULL); -#endif + sigaction(SIGINT, &act, NULL); tsAsyncLog = 0; strcat(arbLogPath, "/arbitrator.log"); @@ -107,6 +106,7 @@ int32_t main(int32_t argc, char *argv[]) { syncCloseTcpThreadPool(tsArbTcpPool); sInfo("TAOS arbitrator is shut down"); + closelog(); return 0; } @@ -174,16 +174,16 @@ static int32_t arbProcessPeerMsg(int64_t rid, void *buffer) { return 0; } -static void arbSignalHandler(int32_t signum) { +static void arbSignalHandler(int32_t signum, siginfo_t *sigInfo, void *context) { struct sigaction act = {{0}}; - act.sa_handler = SIG_IGN; - sigaction(SIGTERM, &act, NULL); - sigaction(SIGINT, &act, NULL); #ifndef WINDOWS - sigaction(SIGHUP, &act, NULL); + act.sa_handler = SIG_IGN; #endif + sigaction(SIGTERM, &act, NULL); + sigaction(SIGHUP, &act, NULL); + sigaction(SIGINT, &act, NULL); - sInfo("shut down signal is %d", signum); + sInfo("shut down signal is %d, sender PID:%d", signum, sigInfo->si_pid); // inform main thread to exit tsem_post(&tsArbSem); From a649f55195c37c45c7f66a4d42a3268b7603f48b Mon Sep 17 00:00:00 2001 From: freemine Date: Fri, 15 Jan 2021 22:34:48 +0800 Subject: [PATCH 006/147] eok --- src/CMakeLists.txt | 2 +- src/client/CMakeLists.txt | 4 +- src/client/src/tscSQLParser.c | 2 + src/client/src/tscUtil.c | 4 + src/kit/shell/src/shellDarwin.c | 2 + src/os/inc/eok.h | 74 ++++ src/os/inc/osDarwin.h | 11 +- src/os/src/darwin/darwinEnv.c | 3 + src/os/src/darwin/eok.c | 580 ++++++++++++++++++++++++++++++ src/os/src/detail/osSocket.c | 3 +- src/plugins/http/src/httpServer.c | 4 + src/rpc/src/rpcTcp.c | 22 ++ src/sync/src/syncTcp.c | 14 + src/tsdb/inc/tsdbMain.h | 6 +- src/tsdb/src/tsdbCommit.c | 4 + src/tsdb/src/tsdbMain.c | 16 + src/tsdb/src/tsdbMemTable.c | 11 +- src/util/src/tsocket.c | 4 +- tests/examples/c/CMakeLists.txt | 5 + 19 files changed, 763 insertions(+), 8 deletions(-) create mode 100644 src/os/inc/eok.h create mode 100644 src/os/src/darwin/eok.c diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 931a0a132e..f7304ae72f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -19,5 +19,5 @@ ADD_SUBDIRECTORY(wal) ADD_SUBDIRECTORY(cq) ADD_SUBDIRECTORY(dnode) #ADD_SUBDIRECTORY(connector/odbc) -ADD_SUBDIRECTORY(connector/jdbc) +#ADD_SUBDIRECTORY(connector/jdbc) diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index daf7c5e534..3fd29b474e 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -49,12 +49,12 @@ ELSEIF (TD_DARWIN) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/linux) ADD_LIBRARY(taos_static STATIC ${SRC}) - TARGET_LINK_LIBRARIES(taos_static trpc tutil pthread m) + TARGET_LINK_LIBRARIES(taos_static query trpc tutil pthread m) SET_TARGET_PROPERTIES(taos_static PROPERTIES OUTPUT_NAME "taos_static") # generate dynamic library (*.dylib) ADD_LIBRARY(taos SHARED ${SRC}) - TARGET_LINK_LIBRARIES(taos trpc tutil pthread m) + TARGET_LINK_LIBRARIES(taos query trpc tutil pthread m) SET_TARGET_PROPERTIES(taos PROPERTIES CLEAN_DIRECT_OUTPUT 1) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index a08482f570..4a24f2bfdb 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -13,10 +13,12 @@ * along with this program. If not, see . */ +#ifndef __APPLE__ #define _BSD_SOURCE #define _XOPEN_SOURCE 500 #define _DEFAULT_SOURCE #define _GNU_SOURCE +#endif // __APPLE__ #include "os.h" #include "ttype.h" diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index b44ebb3c98..d6151c1a88 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -2486,7 +2486,11 @@ bool tscSetSqlOwner(SSqlObj* pSql) { SSqlRes* pRes = &pSql->res; // set the sql object owner +#ifdef __APPLE__ + pthread_t threadId = (pthread_t)taosGetSelfPthreadId(); +#else uint64_t threadId = taosGetSelfPthreadId(); +#endif if (atomic_val_compare_exchange_64(&pSql->owner, 0, threadId) != 0) { pRes->code = TSDB_CODE_QRY_IN_EXEC; return false; diff --git a/src/kit/shell/src/shellDarwin.c b/src/kit/shell/src/shellDarwin.c index ddf7b21bef..d6aed4401c 100644 --- a/src/kit/shell/src/shellDarwin.c +++ b/src/kit/shell/src/shellDarwin.c @@ -21,6 +21,8 @@ #include "shellCommand.h" #include "tkey.h" +#include "tscLog.h" + #define OPT_ABORT 1 /* �Cabort */ int indicator = 1; diff --git a/src/os/inc/eok.h b/src/os/inc/eok.h new file mode 100644 index 0000000000..8892e50c35 --- /dev/null +++ b/src/os/inc/eok.h @@ -0,0 +1,74 @@ +#ifndef _eok_h_fd274616_996c_400e_9023_ae70be881fa3_ +#define _eok_h_fd274616_996c_400e_9023_ae70be881fa3_ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +enum EPOLL_EVENTS + { + EPOLLIN = 0x001, +#define EPOLLIN EPOLLIN + EPOLLPRI = 0x002, +#define EPOLLPRI EPOLLPRI + EPOLLOUT = 0x004, +#define EPOLLOUT EPOLLOUT + EPOLLRDNORM = 0x040, +#define EPOLLRDNORM EPOLLRDNORM + EPOLLRDBAND = 0x080, +#define EPOLLRDBAND EPOLLRDBAND + EPOLLWRNORM = 0x100, +#define EPOLLWRNORM EPOLLWRNORM + EPOLLWRBAND = 0x200, +#define EPOLLWRBAND EPOLLWRBAND + EPOLLMSG = 0x400, +#define EPOLLMSG EPOLLMSG + EPOLLERR = 0x008, +#define EPOLLERR EPOLLERR + EPOLLHUP = 0x010, +#define EPOLLHUP EPOLLHUP + EPOLLRDHUP = 0x2000, +#define EPOLLRDHUP EPOLLRDHUP + EPOLLEXCLUSIVE = 1u << 28, +#define EPOLLEXCLUSIVE EPOLLEXCLUSIVE + EPOLLWAKEUP = 1u << 29, +#define EPOLLWAKEUP EPOLLWAKEUP + EPOLLONESHOT = 1u << 30, +#define EPOLLONESHOT EPOLLONESHOT + EPOLLET = 1u << 31 +#define EPOLLET EPOLLET + }; + +/* Valid opcodes ( "op" parameter ) to issue to epoll_ctl(). */ +#define EPOLL_CTL_ADD 1 /* Add a file descriptor to the interface. */ +#define EPOLL_CTL_DEL 2 /* Remove a file descriptor from the interface. */ +#define EPOLL_CTL_MOD 3 /* Change file descriptor epoll_event structure. */ + + +typedef union epoll_data +{ + void *ptr; + int fd; + uint32_t u32; + uint64_t u64; +} epoll_data_t; + +struct epoll_event +{ + uint32_t events; /* Epoll events */ + epoll_data_t data; /* User data variable */ +}; + +int epoll_create(int size); +int epoll_ctl(int epfd, int op, int fd, struct epoll_event *event); +int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout); +int epoll_close(int epfd); + +#ifdef __cplusplus +} +#endif + +#endif // _eok_h_fd274616_996c_400e_9023_ae70be881fa3_ + diff --git a/src/os/inc/osDarwin.h b/src/os/inc/osDarwin.h index 1461ec6d3b..52bb661f89 100644 --- a/src/os/inc/osDarwin.h +++ b/src/os/inc/osDarwin.h @@ -91,7 +91,7 @@ extern "C" { typedef int(*__compar_fn_t)(const void *, const void *); // for send function in tsocket.c -#define MSG_NOSIGNAL 0 +// #define MSG_NOSIGNAL 0 #define SO_NO_CHECK 0x1234 #define SOL_TCP 0x1234 #define TCP_KEEPIDLE 0x1234 @@ -100,6 +100,15 @@ typedef int(*__compar_fn_t)(const void *, const void *); #define PTHREAD_MUTEX_RECURSIVE_NP PTHREAD_MUTEX_RECURSIVE #endif +int64_t tsosStr2int64(char *str); + +#include "eok.h" + + + + + + #ifdef __cplusplus } #endif diff --git a/src/os/src/darwin/darwinEnv.c b/src/os/src/darwin/darwinEnv.c index 6adebabec0..28388f24d2 100644 --- a/src/os/src/darwin/darwinEnv.c +++ b/src/os/src/darwin/darwinEnv.c @@ -17,6 +17,8 @@ #include "os.h" #include "tglobal.h" +#include + void osInit() { if (configDir[0] == 0) { strcpy(configDir, "~/TDengine/cfg"); @@ -30,3 +32,4 @@ void osInit() { strcpy(tsScriptDir, "~/TDengine/cfg"); strcpy(tsOsName, "Darwin"); } + diff --git a/src/os/src/darwin/eok.c b/src/os/src/darwin/eok.c new file mode 100644 index 0000000000..ea67223c25 --- /dev/null +++ b/src/os/src/darwin/eok.c @@ -0,0 +1,580 @@ +#include "eok.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define D(fmt, ...) fprintf(stderr, "%s[%d]%s(): " fmt "\n", basename(__FILE__), __LINE__, __func__, ##__VA_ARGS__) +#define A(statement, fmt, ...) do { \ + if (statement) break; \ + fprintf(stderr, "%s[%d]%s(): assert [%s] failed: %d[%s]: " fmt "\n", \ + basename(__FILE__), __LINE__, __func__, \ + #statement, errno, strerror(errno), \ + ##__VA_ARGS__); \ + abort(); \ +} while (0) + +#define E(fmt, ...) do { \ + fprintf(stderr, "%s[%d]%s(): %d[%s]: " fmt "\n", \ + basename(__FILE__), __LINE__, __func__, \ + errno, strerror(errno), \ + ##__VA_ARGS__); \ +} while (0) + +static int eok_dummy = 0; + +typedef struct ep_over_kq_s ep_over_kq_t; +typedef struct eok_event_s eok_event_t; + +struct ep_over_kq_s { + int kq; + int idx; + ep_over_kq_t *next; + + int sv[2]; // 0 for read, 1 for write + + eok_event_t *evs_head; + eok_event_t *evs_tail; + eok_event_t *evs_free; + + struct kevent64_s *kchanges; + int nchanges; + int ichanges; + + struct kevent64_s *kevslist; + int nevslist; + + pthread_mutex_t lock; + + volatile unsigned int lock_valid:1; + volatile unsigned int waiting:1; + volatile unsigned int changed:1; + volatile unsigned int wakenup:1; + volatile unsigned int stopping:1; +}; + +struct eok_event_s { + int fd; + struct epoll_event epev; + volatile unsigned int changed; // 0:registered;1:add;2:mod;3:del + + eok_event_t *next; + eok_event_t *prev; +}; + +typedef struct eoks_s eoks_t; +struct eoks_s { + pthread_mutex_t lock; + ep_over_kq_t *eoks; + int neoks; + ep_over_kq_t *eoks_free; +}; +static eoks_t eoks = { + .lock = PTHREAD_MUTEX_INITIALIZER, + .eoks = NULL, + .neoks = 0, + .eoks_free = NULL, +}; + +static ep_over_kq_t* eoks_alloc(void); +static void eoks_free(ep_over_kq_t *eok); +static ep_over_kq_t* eoks_find(int epfd); + +static eok_event_t* eok_find_ev(ep_over_kq_t *eok, int fd); +static eok_event_t* eok_calloc_ev(ep_over_kq_t *eok); +static void eok_free_ev(ep_over_kq_t *eok, eok_event_t *ev); +static void eok_wakeup(ep_over_kq_t *eok); + +static int eok_chgs_refresh(ep_over_kq_t *eok, eok_event_t *oev, eok_event_t *ev, struct kevent64_s *krev, struct kevent64_s *kwev); + +static struct kevent64_s* eok_alloc_eventslist(ep_over_kq_t *eok, int maxevents); + +int epoll_create(int size) { + (void)size; + int e = 0; + ep_over_kq_t *eok = eoks_alloc(); + if (!eok) { + errno = ENOMEM; + return -1; + } + + A(eok->kq==-1, "internal logic error"); + A(eok->lock_valid, "internal logic error"); + A(eok->idx>=0 && eok->idxnext==NULL, "internal logic error"); + A(eok->sv[0]==-1, "internal logic error"); + A(eok->sv[1]==-1, "internal logic error"); + + eok->kq = kqueue(); + if (eok->kq==-1) { + e = errno; + eoks_free(eok); + errno = e; + return -1; + } + + if (socketpair(AF_LOCAL, SOCK_STREAM, 0, eok->sv)) { + e = errno; + eoks_free(eok); + errno = e; + return -1; + } + + struct epoll_event ev = {0}; + ev.events = EPOLLIN; + ev.data.ptr = &eok_dummy; + if (epoll_ctl(eok->idx, EPOLL_CTL_ADD, eok->sv[0], &ev)) { + A(0, "internal logic error"); + epoll_close(eok->idx); + return -1; + } + + return eok->idx; +} + +int epoll_ctl(int epfd, int op, int fd, struct epoll_event *event) { + int e = 0; + if (epfd<0 || epfd>=eoks.neoks) { + errno = EBADF; + return -1; + } + if (fd==-1) { + errno = EBADF; + return -1; + } + if (event && !(event->events & (EPOLLIN | EPOLLERR | EPOLLHUP | EPOLLRDHUP | EPOLLOUT))) { + e = ENOTSUP; + return -1; + } + + ep_over_kq_t *eok = eoks_find(epfd); + if (!eok) { + errno = EBADF; + return -1; + } + + A(0==pthread_mutex_lock(&eok->lock), ""); + do { + eok_event_t* oev = eok_find_ev(eok, fd); + if (op==EPOLL_CTL_ADD && oev) { + e = EEXIST; + break; + } + if (op!=EPOLL_CTL_ADD && !oev) { + e = ENOENT; + break; + } + + struct kevent64_s krev = {0}; + struct kevent64_s kwev = {0}; + krev.ident = -1; + kwev.ident = -1; + uint16_t flags = 0; + eok_event_t ev = {0}; + ev.fd = fd; + if (event) ev.epev = *event; + struct epoll_event *pev = event; + switch (op) { + case EPOLL_CTL_ADD: { + if (!event) { + e = EINVAL; + break; + } + flags = EV_ADD; + ev.changed = 1; + } break; + case EPOLL_CTL_MOD: { + if (!event) { + e = EINVAL; + break; + } + flags = EV_ADD; + ev.changed = 2; + } break; + case EPOLL_CTL_DEL: { + // event is ignored + pev = &oev->epev; + flags = EV_DELETE; + ev.changed = 3; + } break; + default: { + e = ENOTSUP; + } break; + } + + if (e) break; + + if (pev->events & (EPOLLIN | EPOLLERR | EPOLLHUP | EPOLLRDHUP)) { + flags |= EV_EOF; + EV_SET64(&krev, ev.fd, EVFILT_READ, flags, 0, 0, -1, 0, 0); + D("...."); + } + if (pev->events & EPOLLOUT) { + EV_SET64(&kwev, ev.fd, EVFILT_WRITE, flags, 0, 0, -1, 0, 0); + D("...."); + } + + if (eok_chgs_refresh(eok, oev, &ev, &krev, &kwev)) { + e = errno; + A(e, "internal logic error"); + break; + } + eok->changed = 1; + eok_wakeup(eok); + } while (0); + A(0==pthread_mutex_unlock(&eok->lock), ""); + + if (e) { + errno = e; + return -1; + } + + return 0; +} + +static struct timespec do_timespec_diff(struct timespec *from, struct timespec *to); + +int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) { + int e = 0; + if (epfd<0 || epfd>=eoks.neoks) { + errno = EBADF; + return -1; + } + if (!events) { + errno = EINVAL; + return -1; + } + if (maxevents<=0) { + errno = EINVAL; + return -1; + } + + int r = 0; + + ep_over_kq_t *eok = eoks_find(epfd); + if (!eok) { + errno = EBADF; + return -1; + } + + struct timespec abstime = {0}; + A(TIME_UTC==timespec_get(&abstime, TIME_UTC), "internal logic error"); + + if (timeout!=-1) { + if (timeout<0) timeout = 0; + int64_t t = abstime.tv_nsec + timeout * 1000000; + abstime.tv_sec += t / 1000000000; + abstime.tv_nsec %= 1000000000; + } + + int cnts = 0; + A(0==pthread_mutex_lock(&eok->lock), ""); + do { + cnts = 0; + A(eok->waiting==0, "internal logic error"); + struct kevent64_s *eventslist = eok_alloc_eventslist(eok, maxevents); + if (!eventslist) { + e = ENOMEM; + break; + } + memset(eventslist, 0, maxevents * sizeof(*eventslist)); + + struct timespec now = {0}; + A(TIME_UTC==timespec_get(&now, TIME_UTC), "internal logic error"); + struct timespec to = do_timespec_diff(&now, &abstime); + struct timespec *pto = NULL; + if (timeout!=-1) { + pto = &to; + } + + eok->changed = 0; + eok->wakenup = 0; + eok->waiting = 1; + + struct kevent64_s *kchanges = eok->kchanges; + int nchanges = eok->nchanges; + int ichanges = eok->ichanges; + eok->kchanges = NULL; + eok->nchanges = 0; + eok->ichanges = 0; + + A(0==pthread_mutex_unlock(&eok->lock), ""); + r = kevent64(eok->kq, kchanges, ichanges, eventslist, maxevents, 0, pto); + e = errno; + A(0==pthread_mutex_lock(&eok->lock), ""); + + eok->waiting = 0; + if (kchanges) { + free(kchanges); + kchanges = NULL; + nchanges = 0; + ichanges = 0; + } + + eok->waiting = 0; + if (r<0) break; + if (r==0) { + A(timeout!=-1, "internal logic error"); + } + for (int i=0; iudata && eok->evs_head && eok->evs_tail, "internal logic error"); + eok_event_t *ev = (eok_event_t*)kev->udata; + A(kev->ident == ev->fd, "internal logic error"); + D("..."); + switch (kev->filter) { + case EVFILT_READ: { + A((ev->epev.events & EPOLLIN), "internal logic errro"); + if (ev->epev.data.ptr==&eok_dummy) { + char c = '\0'; + A(1==recv(kev->ident, &c, 1, 0), "internal logic error"); + A(0==memcmp(&c, "1", 1), "internal logic error"); + D("..............."); + } else { + if (ev->changed==3) { + D("already requested to delete"); + // EV_DELETE? + continue; + } + struct epoll_event pev = {0}; + pev.data.ptr = ev->epev.data.ptr; + pev.events = EPOLLIN; + if (kev->flags & EV_EOF) { + pev.events |= (EPOLLHUP | EPOLLERR | EPOLLRDHUP); + } + pev.events &= ev->epev.events; + events[cnts++] = pev; + } + } break; + case EVFILT_WRITE: { + A(0, "not implemented yet"); + } break; + default: { + A(0, "internal logic error"); + } break; + } + } + } while (cnts==0); + A(0==pthread_mutex_unlock(&eok->lock), ""); + + if (e) { + errno = e; + return -1; + } + + return cnts; +} + +static struct timespec do_timespec_diff(struct timespec *from, struct timespec *to) { + struct timespec delta; + delta.tv_sec = to->tv_sec - from->tv_sec; + delta.tv_nsec = to->tv_nsec - from->tv_nsec; + while (delta.tv_nsec<0) { + delta.tv_sec -= 1; + delta.tv_nsec += 1000000000; + } + return delta; +} + +int epoll_close(int epfd) { + if (epfd<0 || epfd>=eoks.neoks) { + errno = EBADF; + return -1; + } + ep_over_kq_t *eok = eoks_find(epfd); + if (!eok) { + errno = EBADF; + return -1; + } + + A(0==pthread_mutex_lock(&eok->lock), ""); + do { + A(eok->stopping==0, "internal logic error"); + eok->stopping = 1; + A(eok->waiting, "internal logic error"); + + if (eok->kq!=-1) { + close(eok->kq); + eok->kq = -1; + } + } while (0); + A(0==pthread_mutex_unlock(&eok->lock), ""); + eoks_free(eok); + + return 0; +} + +static struct kevent64_s* eok_alloc_eventslist(ep_over_kq_t *eok, int maxevents) { + if (maxevents<=eok->nevslist) return eok->kevslist; + struct kevent64_s *p = (struct kevent64_s*)realloc(eok->kevslist, sizeof(*p)*maxevents); + if (!p) return NULL; + eok->kevslist = p; + eok->nevslist = maxevents; + return p; +} + +static eok_event_t* eok_find_ev(ep_over_kq_t *eok, int fd) { + eok_event_t *p = eok->evs_head; + while (p) { + if (p->fd == fd) return p; + p = p->next; + } + errno = ENOENT; + return NULL; +} + +static eok_event_t* eok_calloc_ev(ep_over_kq_t *eok) { + eok_event_t *p = NULL; + if (eok->evs_free) { + p = eok->evs_free; + eok->evs_free = p->next; + p->next = NULL; + } else { + p = (eok_event_t*)calloc(1, sizeof(*p)); + if (!p) return NULL; + } + // dirty link + p->prev = eok->evs_tail; + if (eok->evs_tail) eok->evs_tail->next = p; + else eok->evs_head = p; + eok->evs_tail = p; + + return p; +} + +static void eok_free_ev(ep_over_kq_t *eok, eok_event_t *ev) { + if (ev->prev) ev->prev->next = ev->next; + else eok->evs_head = ev->next; + ev->prev = NULL; + if (ev->next) ev->next->prev = ev->prev; + else eok->evs_tail = ev->prev; + ev->next = eok->evs_free; + eok->evs_free = ev->next; +} + +static void eok_wakeup(ep_over_kq_t *eok) { + if (!eok->waiting) return; + if (eok->wakenup) return; + eok->wakenup = 1; + send(eok->sv[1], "1", 1, 0); +} + +static int eok_chgs_refresh(ep_over_kq_t *eok, eok_event_t *oev, eok_event_t *ev, struct kevent64_s *krev, struct kevent64_s *kwev) { + if (!oev) oev = eok_calloc_ev(eok); + if (!oev) return -1; + int n = 0; + if (krev->ident==ev->fd) ++n; + if (kwev->ident==ev->fd) ++n; + A(n, "internal logic error"); + if (eok->ichanges+n>eok->nchanges) { + struct kevent64_s *p = (struct kevent64_s*)realloc(eok->kchanges, sizeof(*p) * (eok->nchanges + 10)); + if (!p) { + if (ev->changed==1) { + eok_free_ev(eok, oev); + } + errno = ENOMEM; + return -1; + } + eok->kchanges = p; + eok->nchanges += 10; + } + + oev->fd = ev->fd; + oev->epev = ev->epev; + oev->changed = ev->changed; + + if (krev->ident==ev->fd) { + krev->udata = (uint64_t)oev; + eok->kchanges[eok->ichanges++] = *krev; + } + if (kwev->ident==ev->fd) { + kwev->udata = (uint64_t)oev; + eok->kchanges[eok->ichanges++] = *kwev; + } + return 0; +} + +static ep_over_kq_t* eoks_alloc(void) { + ep_over_kq_t *eok = NULL; + + A(0==pthread_mutex_lock(&eoks.lock), ""); + do { + if (eoks.eoks_free) { + eok = eoks.eoks_free; + eoks.eoks_free = eok->next; + eok->next = NULL; + break; + } + ep_over_kq_t *p = (ep_over_kq_t*)realloc(eoks.eoks, sizeof(*p) * (eoks.neoks+1)); + if (!p) break; + eoks.eoks = p; + eok = eoks.eoks + eoks.neoks; + memset(eok, 0, sizeof(*eok)); + eok->idx = eoks.neoks; + eok->kq = -1; + eok->sv[0] = -1; + eok->sv[1] = -1; + eoks.neoks += 1; + } while (0); + A(0==pthread_mutex_unlock(&eoks.lock), ""); + + if (!eok) return NULL; + if (eok->lock_valid) { + return eok; + } + + if (0==pthread_mutex_init(&eok->lock, NULL)) { + eok->lock_valid = 1; + return eok; + } + + eoks_free(eok); + return NULL; +} + +static void eoks_free(ep_over_kq_t *eok) { + A(0==pthread_mutex_lock(&eoks.lock), ""); + do { + A(eok->next==NULL, "internal logic error"); + A(eok->evs_head==NULL, "internal logic error"); + A(eok->waiting==0, "internal logic error"); + if (eok->sv[0]!=-1) { + close(eok->sv[0]); + eok->sv[0] = -1; + } + if (eok->sv[1]!=-1) { + close(eok->sv[1]); + eok->sv[1] = -1; + } + if (eok->kq!=-1) { + close(eok->kq); + eok->kq = -1; + } + eok->next = eoks.eoks_free; + eoks.eoks_free = eok; + } while (0); + A(0==pthread_mutex_unlock(&eoks.lock), ""); +} + +static ep_over_kq_t* eoks_find(int epfd) { + ep_over_kq_t *eok = NULL; + A(0==pthread_mutex_lock(&eoks.lock), ""); + do { + if (epfd<0 || epfd>=eoks.neoks) { + break; + } + A(eoks.eoks, "internal logic error"); + eok = eoks.eoks + epfd; + A(eok->next==NULL, "internal logic error"); + A(eok->lock_valid, "internal logic error"); + } while (0); + A(0==pthread_mutex_unlock(&eoks.lock), ""); + return eok; +} + diff --git a/src/os/src/detail/osSocket.c b/src/os/src/detail/osSocket.c index 729471247f..c6cbbe6b83 100644 --- a/src/os/src/detail/osSocket.c +++ b/src/os/src/detail/osSocket.c @@ -58,6 +58,7 @@ void taosBlockSIGPIPE() { #ifndef TAOS_OS_FUNC_SOCKET_SETSOCKETOPT int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen) { + fprintf(stderr, "==%s[%d]%s()==\n", basename(__FILE__), __LINE__, __func__); return setsockopt(socketfd, level, optname, optval, (socklen_t)optlen); } @@ -73,4 +74,4 @@ const char *taosInetNtoa(struct in_addr ipInt) { return inet_ntoa(ipInt); } -#endif \ No newline at end of file +#endif diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index 4896d50c6c..1cc73aef06 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -52,7 +52,11 @@ static void httpStopThread(HttpThread* pThread) { close(fd); } +#ifdef __APPLE__ + epoll_close(pThread->pollFd); +#else close(pThread->pollFd); +#endif pthread_mutex_destroy(&(pThread->threadMutex)); } diff --git a/src/rpc/src/rpcTcp.c b/src/rpc/src/rpcTcp.c index 178b96c423..ff2a5882f8 100644 --- a/src/rpc/src/rpcTcp.c +++ b/src/rpc/src/rpcTcp.c @@ -133,6 +133,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread } pThreadObj->pollFd = (int64_t)epoll_create(10); // size does not matter + fprintf(stderr, "==%s[%d]%s()==\n", basename(__FILE__), __LINE__, __func__); if (pThreadObj->pollFd < 0) { tError("%s failed to create TCP epoll", label); code = -1; @@ -293,6 +294,7 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int num, void * } pThreadObj->pollFd = (SOCKET)epoll_create(10); // size does not matter + fprintf(stderr, "==%s[%d]%s()==\n", basename(__FILE__), __LINE__, __func__); if (pThreadObj->pollFd < 0) { tError("%s failed to create TCP client epoll", label); free(pThreadObj); @@ -307,7 +309,11 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int num, void * int code = pthread_create(&(pThreadObj->thread), &thattr, taosProcessTcpData, (void *)(pThreadObj)); pthread_attr_destroy(&thattr); if (code != 0) { +#ifdef __APPLE__ + epoll_close(pThreadObj->pollFd); +#else taosCloseSocket(pThreadObj->pollFd); +#endif free(pThreadObj); terrno = TAOS_SYSTEM_ERROR(errno); tError("%s failed to create TCP read data thread(%s)", label, strerror(errno)); @@ -337,6 +343,10 @@ void taosCleanUpTcpClient(void *chandle) { void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port) { SThreadObj * pThreadObj = shandle; + fprintf(stderr, "==%s[%d]%s()==\n", basename(__FILE__), __LINE__, __func__); + fprintf(stderr, "pThreadObj->ip:%d\n", pThreadObj->ip); + fprintf(stderr, "PF_INET/AF_INET:%d/%d\n", PF_INET, AF_INET); + fprintf(stderr, "ip/port:%x/%d\n", ip, port); SOCKET fd = taosOpenTcpClientSocket(ip, port, pThreadObj->ip); if (fd < 0) return NULL; @@ -348,6 +358,7 @@ void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uin localPort = (uint16_t)ntohs(sin.sin_port); } + fprintf(stderr, "==%s[%d]%s()==\n", basename(__FILE__), __LINE__, __func__); SFdObj *pFdObj = taosMallocFdObj(pThreadObj, fd); if (pFdObj) { @@ -358,6 +369,7 @@ void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uin pThreadObj->label, thandle, ip, port, localPort, pFdObj, pThreadObj->numOfFds); } else { tError("%s failed to malloc client FdObj(%s)", pThreadObj->label, strerror(errno)); + fprintf(stderr, "==%s[%d]%s()==\n", basename(__FILE__), __LINE__, __func__); taosCloseSocket(fd); } @@ -480,27 +492,32 @@ static void *taosProcessTcpData(void *param) { if (fdNum < 0) continue; for (int i = 0; i < fdNum; ++i) { + fprintf(stderr, "==%s[%d]%s()==\n", basename(__FILE__), __LINE__, __func__); pFdObj = events[i].data.ptr; if (events[i].events & EPOLLERR) { tDebug("%s %p FD:%p epoll errors", pThreadObj->label, pFdObj->thandle, pFdObj); + fprintf(stderr, "==%s[%d]%s()==\n", basename(__FILE__), __LINE__, __func__); taosReportBrokenLink(pFdObj); continue; } if (events[i].events & EPOLLRDHUP) { tDebug("%s %p FD:%p RD hang up", pThreadObj->label, pFdObj->thandle, pFdObj); + fprintf(stderr, "==%s[%d]%s()==\n", basename(__FILE__), __LINE__, __func__); taosReportBrokenLink(pFdObj); continue; } if (events[i].events & EPOLLHUP) { tDebug("%s %p FD:%p hang up", pThreadObj->label, pFdObj->thandle, pFdObj); + fprintf(stderr, "==%s[%d]%s()==\n", basename(__FILE__), __LINE__, __func__); taosReportBrokenLink(pFdObj); continue; } if (taosReadTcpData(pFdObj, &recvInfo) < 0) { + fprintf(stderr, "==%s[%d]%s()==\n", basename(__FILE__), __LINE__, __func__); shutdown(pFdObj->fd, SHUT_WR); continue; } @@ -512,7 +529,11 @@ static void *taosProcessTcpData(void *param) { if (pThreadObj->stop) break; } +#ifdef __APPLE__ + if (pThreadObj->pollFd >=0) epoll_close(pThreadObj->pollFd); +#else if (pThreadObj->pollFd >=0) taosCloseSocket(pThreadObj->pollFd); +#endif while (pThreadObj->pHead) { SFdObj *pFdObj = pThreadObj->pHead; @@ -542,6 +563,7 @@ static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, SOCKET fd) { event.events = EPOLLIN | EPOLLRDHUP; event.data.ptr = pFdObj; + fprintf(stderr, "==%s[%d]%s()==\n", basename(__FILE__), __LINE__, __func__); if (epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) { tfree(pFdObj); terrno = TAOS_SYSTEM_ERROR(errno); diff --git a/src/sync/src/syncTcp.c b/src/sync/src/syncTcp.c index 4744666737..be85853819 100644 --- a/src/sync/src/syncTcp.c +++ b/src/sync/src/syncTcp.c @@ -136,12 +136,15 @@ void *syncAllocateTcpConn(void *param, int64_t rid, int32_t connFd) { event.events = EPOLLIN | EPOLLRDHUP; event.data.ptr = pConn; + fprintf(stderr, ">>>>>>>>>>>>>>>>>\n"); if (epoll_ctl(pThread->pollFd, EPOLL_CTL_ADD, connFd, &event) < 0) { + fprintf(stderr, "<<<<<<<<<<<<<<<<<\n"); sError("failed to add fd:%d since %s", connFd, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); tfree(pConn); pConn = NULL; } else { + fprintf(stderr, "<<<<<<<<<<<<<<<<<\n"); pThread->numOfFds++; sDebug("%p fd:%d is added to epoll thread, num:%d", pThread, connFd, pThread->numOfFds); } @@ -167,7 +170,9 @@ static void taosProcessBrokenLink(SConnObj *pConn) { (*pInfo->processBrokenLink)(pConn->handleId); pThread->numOfFds--; + fprintf(stderr, "<<<<<<<<<<<<<<<<<\n"); epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, pConn->fd, NULL); + fprintf(stderr, "<<<<<<<<<<<<<<<<<\n"); sDebug("%p fd:%d is removed from epoll thread, num:%d", pThread, pConn->fd, pThread->numOfFds); taosClose(pConn->fd); tfree(pConn); @@ -233,7 +238,11 @@ static void *syncProcessTcpData(void *param) { sDebug("%p TCP epoll thread exits", pThread); +#ifdef __APPLE__ + epoll_close(pThread->pollFd); +#else close(pThread->pollFd); +#endif tfree(pThread); tfree(buffer); return NULL; @@ -278,6 +287,7 @@ static SThreadObj *syncGetTcpThread(SPoolObj *pPool) { pThread->pPool = pPool; pThread->pollFd = epoll_create(10); // size does not matter + fprintf(stderr, "...............\n"); if (pThread->pollFd < 0) { tfree(pThread); return NULL; @@ -290,7 +300,11 @@ static SThreadObj *syncGetTcpThread(SPoolObj *pPool) { pthread_attr_destroy(&thattr); if (ret != 0) { +#ifdef __APPLE__ + epoll_close(pThread->pollFd); +#else close(pThread->pollFd); +#endif tfree(pThread); return NULL; } diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 5067974903..984839162e 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -233,7 +233,11 @@ typedef struct { SMemTable* mem; SMemTable* imem; STsdbFileH* tsdbFileH; +#ifdef __APPLE__ + sem_t *readyToCommit; +#else sem_t readyToCommit; +#endif pthread_mutex_t mutex; bool repoLocked; int32_t code; // Commit code @@ -616,4 +620,4 @@ int tsdbScheduleCommit(STsdbRepo *pRepo); } #endif -#endif \ No newline at end of file +#endif diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c index 696270d670..9551d1b3a5 100644 --- a/src/tsdb/src/tsdbCommit.c +++ b/src/tsdb/src/tsdbCommit.c @@ -166,7 +166,11 @@ static void tsdbEndCommit(STsdbRepo *pRepo, int eno) { pRepo->imem = NULL; tsdbUnlockRepo(pRepo); tsdbUnRefMemTable(pRepo, pIMem); +#ifdef __APPLE__ + sem_post(pRepo->readyToCommit); +#else sem_post(&(pRepo->readyToCommit)); +#endif } static int tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey) { diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index b34b2fa9e6..602aea70e3 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -146,7 +146,11 @@ int tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit) { if (toCommit) { tsdbAsyncCommit(pRepo); +#ifdef __APPLE__ + sem_wait(pRepo->readyToCommit); +#else sem_wait(&(pRepo->readyToCommit)); +#endif terrno = pRepo->code; } tsdbUnRefMemTable(pRepo, pRepo->mem); @@ -643,11 +647,19 @@ static STsdbRepo *tsdbNewRepo(char *rootDir, STsdbAppH *pAppH, STsdbCfg *pCfg) { goto _err; } +#ifdef __APPLE__ + pRepo->readyToCommit = sem_open(NULL, O_CREAT, 0644, 1); + if (!pRepo->readyToCommit) { + terrno = TAOS_SYSTEM_ERROR(code); + goto _err; + } +#else code = sem_init(&(pRepo->readyToCommit), 0, 1); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(code); goto _err; } +#endif pRepo->repoLocked = false; @@ -693,7 +705,11 @@ static void tsdbFreeRepo(STsdbRepo *pRepo) { // tsdbFreeMemTable(pRepo->mem); // tsdbFreeMemTable(pRepo->imem); tfree(pRepo->rootDir); +#ifdef __APPLE__ + sem_close(pRepo->readyToCommit); +#else sem_destroy(&(pRepo->readyToCommit)); +#endif pthread_mutex_destroy(&pRepo->mutex); free(pRepo); } diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 07f001f68a..202405df00 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -207,7 +207,11 @@ void *tsdbAllocBytes(STsdbRepo *pRepo, int bytes) { int tsdbAsyncCommit(STsdbRepo *pRepo) { if (pRepo->mem == NULL) return 0; +#ifdef __APPLE__ + sem_wait(pRepo->readyToCommit); +#else sem_wait(&(pRepo->readyToCommit)); +#endif ASSERT(pRepo->imem == NULL); @@ -229,8 +233,13 @@ int tsdbSyncCommit(TSDB_REPO_T *repo) { STsdbRepo *pRepo = (STsdbRepo *)repo; tsdbAsyncCommit(pRepo); +#ifdef __APPLE__ + sem_wait(pRepo->readyToCommit); + sem_post(pRepo->readyToCommit); +#else sem_wait(&(pRepo->readyToCommit)); sem_post(&(pRepo->readyToCommit)); +#endif if (pRepo->code != TSDB_CODE_SUCCESS) { terrno = pRepo->code; @@ -927,4 +936,4 @@ static int tsdbUpdateTableLatestInfo(STsdbRepo *pRepo, STable *pTable, SDataRow } return 0; -} \ No newline at end of file +} diff --git a/src/util/src/tsocket.c b/src/util/src/tsocket.c index 1a5c3bd64d..6b15d3677c 100644 --- a/src/util/src/tsocket.c +++ b/src/util/src/tsocket.c @@ -35,7 +35,7 @@ int32_t taosGetFqdn(char *fqdn) { hints.ai_flags = AI_CANONNAME; int32_t ret = getaddrinfo(hostname, NULL, &hints, &result); if (!result) { - uError("failed to get fqdn, code:%d, reason:%s", ret, gai_strerror(ret)); + uError("failed to get fqdn for hostname <%s>, code:%d, reason:%s", hostname, ret, gai_strerror(ret)); return -1; } @@ -341,6 +341,7 @@ int32_t taosKeepTcpAlive(SOCKET sockFd) { return -1; } +#ifndef __APPLE__ int32_t probes = 3; if (taosSetSockOpt(sockFd, SOL_TCP, TCP_KEEPCNT, (void *)&probes, sizeof(probes)) < 0) { uError("fd:%d setsockopt SO_KEEPCNT failed: %d (%s)", sockFd, errno, strerror(errno)); @@ -361,6 +362,7 @@ int32_t taosKeepTcpAlive(SOCKET sockFd) { taosCloseSocket(sockFd); return -1; } +#endif int32_t nodelay = 1; if (taosSetSockOpt(sockFd, IPPROTO_TCP, TCP_NODELAY, (void *)&nodelay, sizeof(nodelay)) < 0) { diff --git a/tests/examples/c/CMakeLists.txt b/tests/examples/c/CMakeLists.txt index 59bcb6eaff..4689ed8868 100644 --- a/tests/examples/c/CMakeLists.txt +++ b/tests/examples/c/CMakeLists.txt @@ -6,3 +6,8 @@ IF (TD_LINUX) ADD_EXECUTABLE(demo demo.c) TARGET_LINK_LIBRARIES(demo taos_static trpc tutil pthread ) ENDIF () +IF (TD_DARWIN) + INCLUDE_DIRECTORIES(. ${TD_COMMUNITY_DIR}/src/inc ${TD_COMMUNITY_DIR}/src/client/inc ${TD_COMMUNITY_DIR}/inc) + ADD_EXECUTABLE(epoll epoll.c) + TARGET_LINK_LIBRARIES(epoll taos_static trpc tutil pthread ) +ENDIF () From c46f6ac022309951f2af325018dacf04d5ccb064 Mon Sep 17 00:00:00 2001 From: freemine Date: Fri, 15 Jan 2021 22:39:56 +0800 Subject: [PATCH 007/147] add epoll.c tests --- tests/examples/c/epoll.c | 183 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 183 insertions(+) create mode 100644 tests/examples/c/epoll.c diff --git a/tests/examples/c/epoll.c b/tests/examples/c/epoll.c new file mode 100644 index 0000000000..e0d63bae32 --- /dev/null +++ b/tests/examples/c/epoll.c @@ -0,0 +1,183 @@ +#ifdef __APPLE__ +#include "eok.h" +#else +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define D(fmt, ...) fprintf(stderr, "%s[%d]%s(): " fmt "\n", basename(__FILE__), __LINE__, __func__, ##__VA_ARGS__) +#define A(statement, fmt, ...) do { \ + if (statement) break; \ + fprintf(stderr, "%s[%d]%s(): assert [%s] failed: %d[%s]: " fmt "\n", \ + basename(__FILE__), __LINE__, __func__, \ + #statement, errno, strerror(errno), \ + ##__VA_ARGS__); \ + abort(); \ +} while (0) + +#define E(fmt, ...) do { \ + fprintf(stderr, "%s[%d]%s(): %d[%s]: " fmt "\n", \ + basename(__FILE__), __LINE__, __func__, \ + errno, strerror(errno), \ + ##__VA_ARGS__); \ +} while (0) + +typedef struct ep_s ep_t; +struct ep_s { + int ep; + + pthread_mutex_t lock; + int sv[2]; // 0 for read, 1 for write; + pthread_t thread; + + volatile unsigned int stopping:1; + volatile unsigned int waiting:1; + volatile unsigned int wakenup:1; +}; + +static int ep_dummy = 0; + +static ep_t* ep_create(void); +static void ep_destroy(ep_t *ep); +static void* routine(void* arg); +static int open_connect(unsigned short port); + +int main(int argc, char *argv[]) { + ep_t* ep = ep_create(); + A(ep, "failed"); + int skt = open_connect(6789); + if (skt!=-1) { + struct epoll_event ev = {0}; + ev.events = EPOLLIN | EPOLLERR | EPOLLHUP | EPOLLRDHUP; + ev.data.ptr = &skt; + A(0==epoll_ctl(ep->ep, EPOLL_CTL_ADD, skt, &ev), ""); + } + getchar(); + ep_destroy(ep); + D(""); + return 0; +} + +ep_t* ep_create(void) { + ep_t *ep = (ep_t*)calloc(1, sizeof(*ep)); + A(ep, "out of memory"); + A(-1!=(ep->ep = epoll_create(1)), ""); + ep->sv[0] = -1; + ep->sv[1] = -1; + A(0==socketpair(AF_LOCAL, SOCK_STREAM, 0, ep->sv), ""); + A(0==pthread_mutex_init(&ep->lock, NULL), ""); + A(0==pthread_mutex_lock(&ep->lock), ""); + struct epoll_event ev = {0}; + ev.events = EPOLLIN; + ev.data.ptr = &ep_dummy; + A(0==epoll_ctl(ep->ep, EPOLL_CTL_ADD, ep->sv[0], &ev), ""); + A(0==pthread_create(&ep->thread, NULL, routine, ep), ""); + A(0==pthread_mutex_unlock(&ep->lock), ""); + return ep; +} + +static void ep_destroy(ep_t *ep) { + A(ep, "invalid argument"); + ep->stopping = 1; + A(1==send(ep->sv[1], "1", 1, 0), ""); + A(0==pthread_join(ep->thread, NULL), ""); + A(0==pthread_mutex_destroy(&ep->lock), ""); + A(0==close(ep->sv[0]), ""); + A(0==close(ep->sv[1]), ""); + A(0==close(ep->ep), ""); + free(ep); +} + +static void* routine(void* arg) { + A(arg, "invalid argument"); + ep_t *ep = (ep_t*)arg; + + while (!ep->stopping) { + struct epoll_event evs[10] = {0}; + + A(0==pthread_mutex_lock(&ep->lock), ""); + A(ep->waiting==0, "internal logic error"); + ep->waiting = 1; + A(0==pthread_mutex_unlock(&ep->lock), ""); + + int r = epoll_wait(ep->ep, evs, sizeof(evs)/sizeof(evs[0]), -1); + A(r>0, "indefinite epoll_wait shall not timeout"); + + A(0==pthread_mutex_lock(&ep->lock), ""); + A(ep->waiting==1, "internal logic error"); + ep->waiting = 0; + A(0==pthread_mutex_unlock(&ep->lock), ""); + + for (int i=0; idata.ptr == &ep_dummy) { + char c = '\0'; + A(1==recv(ep->sv[0], &c, 1, 0), "internal logic error"); + A(0==pthread_mutex_lock(&ep->lock), ""); + ep->wakenup = 0; + A(0==pthread_mutex_unlock(&ep->lock), ""); + D("........"); + continue; + } + A(ev->data.ptr, "internal logic error"); + int skt = *(int*)ev->data.ptr; + if (ev->events & EPOLLIN) { + char buf[4]; + int n = recv(skt, buf, sizeof(buf)-1, 0); + A(n>=0 && nevents, buf); + } + if (ev->events & EPOLLRDHUP) { + A(0==epoll_ctl(ep->ep, EPOLL_CTL_DEL, skt, NULL), ""); + } + continue; + } + } + return NULL; +} + +static int open_connect(unsigned short port) { + int r = 0; + int skt = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + if (skt==-1) { + E("socket() failed"); + return -1; + } + do { + struct sockaddr_in si = {0}; + si.sin_family = AF_INET; + si.sin_addr.s_addr = inet_addr("127.0.0.1"); + si.sin_port = htons(port); + r = connect(skt, (struct sockaddr*)&si, sizeof(si)); + if (r) { + E("connect(%u) failed", port); + break; + } + memset(&si, 0, sizeof(si)); + socklen_t len = sizeof(si); + r = getsockname(skt, (struct sockaddr *)&si, &len); + if (r) { + E("getsockname() failed"); + } + A(len==sizeof(si), "internal logic error"); + D("connected: %d", ntohs(si.sin_port)); + return skt; + } while (0); + close(skt); + return -1; +} + From a47e778b5cc30388110abc7e6e227bf2ea65dc74 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 15 Jan 2021 22:45:31 +0800 Subject: [PATCH 008/147] TD-1207 --- src/os/src/darwin/darwinFile.c | 38 ++++++++-- src/os/src/detail/osFile.c | 4 -- src/os/src/windows/wFile.c | 125 +++++++++++++++++++++++++++------ src/os/src/windows/wSysinfo.c | 17 ++++- 4 files changed, 153 insertions(+), 31 deletions(-) diff --git a/src/os/src/darwin/darwinFile.c b/src/os/src/darwin/darwinFile.c index dacf4db741..1e77cd68d8 100644 --- a/src/os/src/darwin/darwinFile.c +++ b/src/os/src/darwin/darwinFile.c @@ -51,7 +51,37 @@ int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t co return writeLen; } -int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t* offset, int64_t size) { - uError("taosSendFile not implemented yet"); - return -1; -} \ No newline at end of file +int64_t taosSendFile(SOCKET dfd, int32_t sfd, int64_t* offset, int64_t count) { + lseek(sfd, (int32_t)(*offset), 0); + int64_t writeLen = 0; + uint8_t buffer[_SEND_FILE_STEP_] = { 0 }; + + for (int64_t len = 0; len < (count - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) { + int32_t rlen = (int32_t)read(sfd, buffer, _SEND_FILE_STEP_); + if (rlen <= 0) { + return writeLen; + } + else if (rlen < _SEND_FILE_STEP_) { + taosWriteSocket(dfd, buffer, rlen); + return (int64_t)(writeLen + rlen); + } + else { + taosWriteSocket(dfd, buffer, _SEND_FILE_STEP_); + writeLen += _SEND_FILE_STEP_; + } + } + + int64_t remain = count - writeLen; + if (remain > 0) { + int32_t rlen = read(sfd, buffer, (int32_t)remain); + if (rlen <= 0) { + return writeLen; + } + else { + taosWriteSocket(sfd, buffer, (int32_t)remain); + writeLen += remain; + } + } + + return writeLen; +} diff --git a/src/os/src/detail/osFile.c b/src/os/src/detail/osFile.c index 6ba38635f4..bb68622731 100644 --- a/src/os/src/detail/osFile.c +++ b/src/os/src/detail/osFile.c @@ -126,10 +126,6 @@ int64_t taosSendFile(SOCKET dfd, int32_t sfd, int64_t *offset, int64_t size) { int64_t sentbytes; while (leftbytes > 0) { - /* - * TODO : Think to check if file is larger than 1GB - */ - // if (leftbytes > 1000000000) leftbytes = 1000000000; sentbytes = sendfile(dfd, sfd, offset, leftbytes); if (sentbytes == -1) { if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) { diff --git a/src/os/src/windows/wFile.c b/src/os/src/windows/wFile.c index a127c17ce7..e61218cac3 100644 --- a/src/os/src/windows/wFile.c +++ b/src/os/src/windows/wFile.c @@ -15,18 +15,19 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "tulog.h" +#include "osSocket.h" #include "tglobal.h" +#include "tulog.h" void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath) { - const char* tdengineTmpFileNamePrefix = "tdengine-"; - char tmpPath[PATH_MAX]; + const char *tdengineTmpFileNamePrefix = "tdengine-"; + char tmpPath[PATH_MAX]; int32_t len = (int32_t)strlen(tsTempDir); memcpy(tmpPath, tsTempDir, len); if (tmpPath[len - 1] != '/' && tmpPath[len - 1] != '\\') { - tmpPath[len++] = '\\'; + tmpPath[len++] = '\\'; } strcpy(tmpPath + len, tdengineTmpFileNamePrefix); @@ -35,7 +36,7 @@ void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath) { strcat(tmpPath, fileNamePrefix); strcat(tmpPath, "-%d-%s"); } - + char rand[8] = {0}; taosRandStr(rand, tListLen(rand) - 1); snprintf(dstPath, PATH_MAX, tmpPath, getpid(), rand); @@ -46,18 +47,16 @@ void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath) { int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t count) { fseek(in_file, (int32_t)(*offset), 0); int64_t writeLen = 0; - uint8_t buffer[_SEND_FILE_STEP_] = { 0 }; - + uint8_t buffer[_SEND_FILE_STEP_] = {0}; + for (int64_t len = 0; len < (count - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) { size_t rlen = fread(buffer, 1, _SEND_FILE_STEP_, in_file); if (rlen <= 0) { return writeLen; - } - else if (rlen < _SEND_FILE_STEP_) { + } else if (rlen < _SEND_FILE_STEP_) { fwrite(buffer, 1, rlen, out_file); return (int64_t)(writeLen + rlen); - } - else { + } else { fwrite(buffer, 1, _SEND_FILE_STEP_, in_file); writeLen += _SEND_FILE_STEP_; } @@ -65,12 +64,11 @@ int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t co int64_t remain = count - writeLen; if (remain > 0) { - size_t rlen = fread(buffer, 1, (size_t) remain, in_file); + size_t rlen = fread(buffer, 1, (size_t)remain, in_file); if (rlen <= 0) { return writeLen; - } - else { - fwrite(buffer, 1, (size_t) remain, out_file); + } else { + fwrite(buffer, 1, (size_t)remain, out_file); writeLen += remain; } } @@ -78,12 +76,99 @@ int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t co return writeLen; } -int64_t taosSendFile(SOCKET dfd, int32_t sfd, int64_t* offset, int64_t size) { - uError("taosSendFile no implemented yet"); +int64_t taosSendFile(SOCKET dfd, int32_t sfd, int64_t *offset, int64_t count) { + lseek(sfd, (int32_t)(*offset), 0); + int64_t writeLen = 0; + uint8_t buffer[_SEND_FILE_STEP_] = {0}; + + for (int64_t len = 0; len < (count - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) { + int32_t rlen = (int32_t)read(sfd, buffer, _SEND_FILE_STEP_); + if (rlen <= 0) { + return writeLen; + } else if (rlen < _SEND_FILE_STEP_) { + taosWriteSocket(dfd, buffer, rlen); + return (int64_t)(writeLen + rlen); + } else { + taosWriteSocket(dfd, buffer, _SEND_FILE_STEP_); + writeLen += _SEND_FILE_STEP_; + } + } + + int64_t remain = count - writeLen; + if (remain > 0) { + int32_t rlen = read(sfd, buffer, (int32_t)remain); + if (rlen <= 0) { + return writeLen; + } else { + taosWriteSocket(sfd, buffer, (int32_t)remain); + writeLen += remain; + } + } + + return writeLen; +} + +int32_t taosFtruncate(int32_t fd, int64_t l_size) { + if (fd < 0) { + errno = EBADF; + uError("%s\n", "fd arg was negative"); + return -1; + } + + HANDLE h = (HANDLE)_get_osfhandle(fd); + + LARGE_INTEGER li_0; + li_0.QuadPart = (int64_t)0; + BOOL cur = SetFilePointerEx(h, li_0, NULL, FILE_CURRENT); + if (!cur) { + uError("SetFilePointerEx Error getting current position in file.\n"); + return -1; + } + + LARGE_INTEGER li_size; + li_size.QuadPart = l_size; + BOOL cur2 = SetFilePointerEx(h, li_size, NULL, FILE_BEGIN); + if (cur2 == 0) { + int error = GetLastError(); + uError("SetFilePointerEx GetLastError is: %d\n", error); + switch (error) { + case ERROR_INVALID_HANDLE: + errno = EBADF; + break; + default: + errno = EIO; + break; + } + return -1; + } + + if (!SetEndOfFile(h)) { + int error = GetLastError(); + uError("SetEndOfFile GetLastError is:%d", error); + switch (error) { + case ERROR_INVALID_HANDLE: + errno = EBADF; + break; + default: + errno = EIO; + break; + } + return -1; + } + return 0; } -int32_t taosFtruncate(int32_t fd, int64_t length) { - uError("taosFtruncate no implemented yet"); + +int fsync(int filedes) { + if (filedes < 0) { + errno = EBADF; + uError("%s\n", "fd arg was negative"); + return -1; + } + + HANDLE h = (HANDLE)_get_osfhandle(filedes); + FlushFileBuffers(h); + return 0; -} \ No newline at end of file + } diff --git a/src/os/src/windows/wSysinfo.c b/src/os/src/windows/wSysinfo.c index 0499173a0b..c024521219 100644 --- a/src/os/src/windows/wSysinfo.c +++ b/src/os/src/windows/wSysinfo.c @@ -31,6 +31,8 @@ #pragma comment(lib, "Mswsock.lib ") #endif +#include + #pragma warning(push) #pragma warning(disable : 4091) #include @@ -233,8 +235,6 @@ int taosSystem(const char *cmd) { int flock(int fd, int option) { return 0; } -int fsync(int filedes) { return 0; } - int sigaction(int sig, struct sigaction *d, void *p) { return 0; } LONG WINAPI FlCrashDump(PEXCEPTION_POINTERS ep) { @@ -276,7 +276,18 @@ LONG WINAPI FlCrashDump(PEXCEPTION_POINTERS ep) { void taosSetCoreDump() { SetUnhandledExceptionFilter(&FlCrashDump); } bool taosGetSystemUid(char *uid) { - sprintf(uid, "uid_not_implemented_yet"); + GUID guid; + CoCreateGuid(&guid); + + sprintf( + uid, + "%08X-%04X-%04X-%02X%02X-%02X%02X%02X%02X%02X%02X", + guid.Data1, guid.Data2, guid.Data3, + guid.Data4[0], guid.Data4[1], + guid.Data4[2], guid.Data4[3], + guid.Data4[4], guid.Data4[5], + guid.Data4[6], guid.Data4[7]); + return true; } From 499644d5a7afc6c12d72d682b31915c9fc284b07 Mon Sep 17 00:00:00 2001 From: freemine Date: Fri, 15 Jan 2021 23:11:24 +0800 Subject: [PATCH 009/147] epoll.c test in CMakeLists.txt --- tests/examples/c/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/examples/c/CMakeLists.txt b/tests/examples/c/CMakeLists.txt index 4689ed8868..7cacefda57 100644 --- a/tests/examples/c/CMakeLists.txt +++ b/tests/examples/c/CMakeLists.txt @@ -5,6 +5,8 @@ IF (TD_LINUX) AUX_SOURCE_DIRECTORY(. SRC) ADD_EXECUTABLE(demo demo.c) TARGET_LINK_LIBRARIES(demo taos_static trpc tutil pthread ) + ADD_EXECUTABLE(epoll epoll.c) + TARGET_LINK_LIBRARIES(epoll taos_static trpc tutil pthread ) ENDIF () IF (TD_DARWIN) INCLUDE_DIRECTORIES(. ${TD_COMMUNITY_DIR}/src/inc ${TD_COMMUNITY_DIR}/src/client/inc ${TD_COMMUNITY_DIR}/inc) From 65a28796cdc267c12ee8706adc288a6ec5f3adcf Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 16 Jan 2021 12:59:40 +0800 Subject: [PATCH 010/147] TD-1207 --- src/dnode/src/dnodeSystem.c | 69 ++++++++++++++++------------------- src/kit/shell/src/shellMain.c | 12 +++--- src/os/inc/os.h | 1 + src/os/inc/osSignal.h | 56 ++++++++++++++++++++++++++++ src/os/inc/osWindows.h | 18 --------- src/os/src/detail/osSignal.c | 38 +++++++++++++++++++ src/os/src/linux/linuxEnv.c | 4 ++ src/os/src/windows/wSysinfo.c | 2 - src/sync/src/syncArbitrator.c | 29 ++++++--------- tests/tsim/src/simExe.c | 4 +- tests/tsim/src/simMain.c | 2 +- 11 files changed, 148 insertions(+), 87 deletions(-) create mode 100644 src/os/inc/osSignal.h create mode 100644 src/os/src/detail/osSignal.c diff --git a/src/dnode/src/dnodeSystem.c b/src/dnode/src/dnodeSystem.c index a16ff826bf..bdaecf2e0f 100644 --- a/src/dnode/src/dnodeSystem.c +++ b/src/dnode/src/dnodeSystem.c @@ -19,8 +19,10 @@ #include "tconfig.h" #include "dnodeMain.h" -static void signal_handler(int32_t signum, siginfo_t *sigInfo, void *context); static tsem_t exitSem; +static void siguser1Handler(int32_t signum); +static void siguser2Handler(int32_t signum); +static void sigintHandler(int32_t signum); int32_t main(int32_t argc, char *argv[]) { int dump_config = 0; @@ -28,7 +30,7 @@ int32_t main(int32_t argc, char *argv[]) { // Set global configuration file for (int32_t i = 1; i < argc; ++i) { if (strcmp(argv[i], "-c") == 0) { - if (i < argc - 1) { + if (i < argc - 1) { if (strlen(argv[++i]) >= TSDB_FILENAME_LEN) { printf("config file path overflow"); exit(EXIT_FAILURE); @@ -80,8 +82,8 @@ int32_t main(int32_t argc, char *argv[]) { taosSetRandomFileFailOutput(NULL); } } else if (strcmp(argv[i], "--random-file-fail-factor") == 0) { - if ( (i+1) < argc ) { - int factor = atoi(argv[i+1]); + if ((i + 1) < argc) { + int factor = atoi(argv[i + 1]); printf("The factor of random failure is %d\n", factor); taosSetRandomFileFailFactor(factor); } else { @@ -93,7 +95,7 @@ int32_t main(int32_t argc, char *argv[]) { } if (0 != dump_config) { - tscEmbedded = 1; + tscEmbedded = 1; taosInitGlobalCfg(); taosReadGlobalLogCfg(); @@ -112,14 +114,12 @@ int32_t main(int32_t argc, char *argv[]) { } /* Set termination handler. */ - struct sigaction act = {{0}}; - act.sa_flags = SA_SIGINFO; - act.sa_sigaction = signal_handler; - sigaction(SIGTERM, &act, NULL); - sigaction(SIGHUP, &act, NULL); - sigaction(SIGINT, &act, NULL); - sigaction(SIGUSR1, &act, NULL); - sigaction(SIGUSR2, &act, NULL); + taosSetSignal(SIGUSR1, siguser1Handler); + taosSetSignal(SIGUSR2, siguser2Handler); + taosSetSignal(SIGTERM, sigintHandler); + taosSetSignal(SIGHUP, sigintHandler); + taosSetSignal(SIGINT, sigintHandler); + taosSetSignal(SIGABRT, sigintHandler); // Open /var/log/syslog file to record information. openlog("TDengine:", LOG_PID | LOG_CONS | LOG_NDELAY, LOG_LOCAL1); @@ -147,32 +147,25 @@ int32_t main(int32_t argc, char *argv[]) { return EXIT_SUCCESS; } -static void signal_handler(int32_t signum, siginfo_t *sigInfo, void *context) { - if (signum == SIGUSR1) { - taosCfgDynamicOptions("debugFlag 143"); - return; - } - if (signum == SIGUSR2) { - taosCfgDynamicOptions("resetlog"); - return; - } +static void siguser1Handler(int32_t signum) { taosCfgDynamicOptions("debugFlag 143"); } - syslog(LOG_INFO, "Shut down signal is %d", signum); - syslog(LOG_INFO, "Shutting down TDengine service..."); - // clean the system. - dInfo("shut down signal is %d, sender PID:%d cmdline:%s", signum, sigInfo->si_pid, taosGetCmdlineByPID(sigInfo->si_pid)); +static void siguser2Handler(int32_t signum) { taosCfgDynamicOptions("resetlog"); } - // protect the application from receive another signal - struct sigaction act = {{0}}; -#ifndef WINDOWS - act.sa_handler = SIG_IGN; -#endif - sigaction(SIGTERM, &act, NULL); - sigaction(SIGHUP, &act, NULL); - sigaction(SIGINT, &act, NULL); - sigaction(SIGUSR1, &act, NULL); - sigaction(SIGUSR2, &act, NULL); +static void sigintHandler(int32_t signum) { + // clean the system. + dInfo("shut down signal is %d", signum); - // inform main thread to exit - tsem_post(&exitSem); + syslog(LOG_INFO, "Shut down signal is %d", signum); + syslog(LOG_INFO, "Shutting down TDengine service..."); + + // protect the application from receive another signal + taosIgnSignal(SIGUSR1); + taosIgnSignal(SIGUSR2); + taosIgnSignal(SIGTERM); + taosIgnSignal(SIGHUP); + taosIgnSignal(SIGINT); + taosIgnSignal(SIGABRT); + + // inform main thread to exit + tsem_post(&exitSem); } \ No newline at end of file diff --git a/src/kit/shell/src/shellMain.c b/src/kit/shell/src/shellMain.c index cd801ef07a..c7a0dab2dd 100644 --- a/src/kit/shell/src/shellMain.c +++ b/src/kit/shell/src/shellMain.c @@ -21,7 +21,7 @@ pthread_t pid; static tsem_t cancelSem; -void shellQueryInterruptHandler(int32_t signum, siginfo_t *sigInfo, void *context) { +void shellQueryInterruptHandler(int32_t signum) { tsem_post(&cancelSem); } @@ -130,12 +130,10 @@ int main(int argc, char* argv[]) { pthread_create(&spid, NULL, cancelHandler, NULL); /* Interrupt handler. */ - struct sigaction act; - memset(&act, 0, sizeof(struct sigaction)); - - act.sa_handler = shellQueryInterruptHandler; - sigaction(SIGTERM, &act, NULL); - sigaction(SIGINT, &act, NULL); + taosSetSignal(SIGTERM, shellQueryInterruptHandler); + taosSetSignal(SIGINT, shellQueryInterruptHandler); + taosSetSignal(SIGHUP, shellQueryInterruptHandler); + taosSetSignal(SIGABRT, shellQueryInterruptHandler); /* Get grant information */ shellGetGrantInfo(con); diff --git a/src/os/inc/os.h b/src/os/inc/os.h index 9383ae48dc..8312b74a50 100644 --- a/src/os/inc/os.h +++ b/src/os/inc/os.h @@ -62,6 +62,7 @@ extern "C" { #include "osMemory.h" #include "osRand.h" #include "osSemphone.h" +#include "osSignal.h" #include "osSocket.h" #include "osString.h" #include "osSysinfo.h" diff --git a/src/os/inc/osSignal.h b/src/os/inc/osSignal.h new file mode 100644 index 0000000000..8b047b9e25 --- /dev/null +++ b/src/os/inc/osSignal.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_OS_SIGNAL_H +#define TDENGINE_OS_SIGNAL_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "os.h" +#include "taosdef.h" +#include + +#ifndef SIGALRM + #define SIGALRM 1234 +#endif + +#ifndef SIGHUP + #define SIGHUP 1234 +#endif + +#ifndef SIGCHLD + #define SIGCHLD 1234 +#endif + +#ifndef SIGUSR1 + #define SIGUSR1 1234 +#endif + +#ifndef SIGUSR2 + #define SIGUSR2 1234 +#endif + +typedef void (*FSignalHandler)(int32_t signum); +void taosSetSignal(int32_t signum, FSignalHandler sigfp); +void taosIgnSignal(int32_t signum); +void taosDflSignal(int32_t signum); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_TTIME_H diff --git a/src/os/inc/osWindows.h b/src/os/inc/osWindows.h index 15b26d208d..1b28cfa90f 100644 --- a/src/os/inc/osWindows.h +++ b/src/os/inc/osWindows.h @@ -191,24 +191,6 @@ int gettimeofday(struct timeval *ptv, void *pTimeZone); #define PATH_MAX 256 #endif -//for signal, not dispose -#define SIGALRM 1234 -#define SIGHUP 1234 -#define SIGUSR1 1234 -#define SIGUSR2 1234 -#define SA_SIGINFO 1234 - -typedef int sigset_t; -typedef struct siginfo_t { - int si_pid; -} siginfo_t; -struct sigaction { - int sa_flags; - void (*sa_handler)(int32_t signum, siginfo_t *sigInfo, void *context); - void (*sa_sigaction)(int32_t signum, siginfo_t *sigInfo, void *context); -}; -int sigaction(int, struct sigaction *, void *); - typedef struct { int we_wordc; char **we_wordv; diff --git a/src/os/src/detail/osSignal.c b/src/os/src/detail/osSignal.c new file mode 100644 index 0000000000..c97a3343de --- /dev/null +++ b/src/os/src/detail/osSignal.c @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "tconfig.h" +#include "tglobal.h" +#include "tulog.h" + +#ifndef TAOS_OS_FUNC_SIGNAL + +void taosSetSignal(int32_t signum, FSignalHandler sigfp) { + struct sigaction act = {{0}}; + act.sa_handler = sigfp; + sigaction(signum, &act, NULL); +} + +void taosIgnSignal(int32_t signum) { + signal(signum, SIG_IGN); +} + +void taosDflSignal(int32_t signum) { + signal(signum, SIG_DFL); +} + +#endif diff --git a/src/os/src/linux/linuxEnv.c b/src/os/src/linux/linuxEnv.c index e3eadbc94b..abcdabcfd5 100644 --- a/src/os/src/linux/linuxEnv.c +++ b/src/os/src/linux/linuxEnv.c @@ -43,6 +43,7 @@ void osInit() { char cmdline[1024]; char* taosGetCmdlineByPID(int pid) { +#if 0 sprintf(cmdline, "/proc/%d/cmdline", pid); FILE* f = fopen(cmdline, "r"); if (f) { @@ -54,4 +55,7 @@ char* taosGetCmdlineByPID(int pid) { fclose(f); } return cmdline; +#else + return ""; +#endif } diff --git a/src/os/src/windows/wSysinfo.c b/src/os/src/windows/wSysinfo.c index c024521219..9a79f5a6bf 100644 --- a/src/os/src/windows/wSysinfo.c +++ b/src/os/src/windows/wSysinfo.c @@ -235,8 +235,6 @@ int taosSystem(const char *cmd) { int flock(int fd, int option) { return 0; } -int sigaction(int sig, struct sigaction *d, void *p) { return 0; } - LONG WINAPI FlCrashDump(PEXCEPTION_POINTERS ep) { typedef BOOL(WINAPI * FxMiniDumpWriteDump)(IN HANDLE hProcess, IN DWORD ProcessId, IN HANDLE hFile, IN MINIDUMP_TYPE DumpType, diff --git a/src/sync/src/syncArbitrator.c b/src/sync/src/syncArbitrator.c index 06b4a61d6f..4fc5e2196c 100644 --- a/src/sync/src/syncArbitrator.c +++ b/src/sync/src/syncArbitrator.c @@ -27,7 +27,7 @@ #include "syncInt.h" #include "syncTcp.h" -static void arbSignalHandler(int32_t signum, siginfo_t *sigInfo, void *context); +static void arbSignalHandler(int32_t signum); static void arbProcessIncommingConnection(SOCKET connFd, uint32_t sourceIp); static void arbProcessBrokenLink(int64_t rid); static int32_t arbProcessPeerMsg(int64_t rid, void *buffer); @@ -69,14 +69,10 @@ int32_t main(int32_t argc, char *argv[]) { } /* Set termination handler. */ - struct sigaction act = {{0}}; - act.sa_flags = SA_SIGINFO; - act.sa_sigaction = arbSignalHandler; - - act.sa_handler = arbSignalHandler; - sigaction(SIGTERM, &act, NULL); - sigaction(SIGHUP, &act, NULL); - sigaction(SIGINT, &act, NULL); + taosSetSignal(SIGTERM, arbSignalHandler); + taosSetSignal(SIGINT, arbSignalHandler); + taosSetSignal(SIGHUP, arbSignalHandler); + taosSetSignal(SIGABRT, arbSignalHandler); tsAsyncLog = 0; strcat(arbLogPath, "/arbitrator.log"); @@ -174,16 +170,13 @@ static int32_t arbProcessPeerMsg(int64_t rid, void *buffer) { return 0; } -static void arbSignalHandler(int32_t signum, siginfo_t *sigInfo, void *context) { - struct sigaction act = {{0}}; -#ifndef WINDOWS - act.sa_handler = SIG_IGN; -#endif - sigaction(SIGTERM, &act, NULL); - sigaction(SIGHUP, &act, NULL); - sigaction(SIGINT, &act, NULL); +static void arbSignalHandler(int32_t signum) { + taosIgnSignal(SIGTERM); + taosIgnSignal(SIGINT); + taosIgnSignal(SIGABRT); + taosIgnSignal(SIGHUP); - sInfo("shut down signal is %d, sender PID:%d", signum, sigInfo->si_pid); + sInfo("shut down signal is %d", signum); // inform main thread to exit tsem_post(&tsArbSem); diff --git a/tests/tsim/src/simExe.c b/tests/tsim/src/simExe.c index a23dff13be..10e87eefe6 100644 --- a/tests/tsim/src/simExe.c +++ b/tests/tsim/src/simExe.c @@ -314,9 +314,7 @@ bool simExecuteSystemCmd(SScript *script, char *option) { simError("script:%s, failed to execute %s , code %d, errno:%d %s, repeatTimes:%d", script->fileName, buf, code, errno, strerror(errno), repeatTimes); taosMsleep(1000); -#ifdef LINUX - signal(SIGCHLD, SIG_DFL); -#endif + taosDflSignal(SIGCHLD); if (repeatTimes++ >= 10) { exit(0); } diff --git a/tests/tsim/src/simMain.c b/tests/tsim/src/simMain.c index 8f13254f68..de560901aa 100644 --- a/tests/tsim/src/simMain.c +++ b/tests/tsim/src/simMain.c @@ -51,7 +51,7 @@ int32_t main(int32_t argc, char *argv[]) { } simInfo("simulator is running ..."); - signal(SIGINT, simHandleSignal); + taosSetSignal(SIGINT, simHandleSignal); SScript *script = simParseScript(scriptFile); if (script == NULL) { From 1e7037e8e17210a0e64bf9f130e11b4734e2675f Mon Sep 17 00:00:00 2001 From: freemine Date: Sat, 16 Jan 2021 13:11:33 +0800 Subject: [PATCH 011/147] epoll and kqueue --- src/os/src/darwin/eok.c | 199 +++++++++++++++++++++++++++++++++++++-- tests/examples/c/epoll.c | 155 ++++++++++++++++++++++++++---- 2 files changed, 329 insertions(+), 25 deletions(-) diff --git a/src/os/src/darwin/eok.c b/src/os/src/darwin/eok.c index ea67223c25..f83846b734 100644 --- a/src/os/src/darwin/eok.c +++ b/src/os/src/darwin/eok.c @@ -39,6 +39,7 @@ struct ep_over_kq_s { int sv[2]; // 0 for read, 1 for write + int evs_count; eok_event_t *evs_head; eok_event_t *evs_tail; eok_event_t *evs_free; @@ -82,6 +83,139 @@ static eoks_t eoks = { .eoks_free = NULL, }; +static const char* op_str(int op) { + switch (op) { + case EPOLL_CTL_ADD: return "EPOLL_CTL_ADD"; + case EPOLL_CTL_MOD: return "EPOLL_CTL_MOD"; + case EPOLL_CTL_DEL: return "EPOLL_CTL_DEL"; + default: return "UNKNOWN"; + } +} + +static __thread char buf_slots[10][1024] = {0}; +static __thread int buf_slots_linelen = sizeof(buf_slots[0])/sizeof(buf_slots[0][0]); +static __thread int buf_slots_count = sizeof(buf_slots)/(sizeof(buf_slots[0])/sizeof(buf_slots[0][0])); +static const char* events_str(uint32_t events, int slots) { + A(slots>=0 && slots0 && (events & (ev))==(ev)) { \ + n = snprintf(p, len, "%s%s", p!=buf ? "|" : "", #ev); \ + p += n; \ + len -= n; \ + } + CHK_EV(EPOLLIN); + CHK_EV(EPOLLPRI); + CHK_EV(EPOLLOUT); + CHK_EV(EPOLLRDNORM); + CHK_EV(EPOLLRDBAND); + CHK_EV(EPOLLWRNORM); + CHK_EV(EPOLLWRBAND); + CHK_EV(EPOLLMSG); + CHK_EV(EPOLLERR); + CHK_EV(EPOLLHUP); + CHK_EV(EPOLLRDHUP); + CHK_EV(EPOLLEXCLUSIVE); + CHK_EV(EPOLLWAKEUP); + CHK_EV(EPOLLONESHOT); + CHK_EV(EPOLLET); +#undef CHK_EV + return buf; +} + +static const char* kev_flags_str(uint16_t flags, int slots) { + A(slots>=0 && slots0 && (flags & (ev))==(ev)) { \ + n = snprintf(p, len, "%s%s", p!=buf ? "|" : "", #ev); \ + p += n; \ + len -= n; \ + } + CHK_EV(EV_ADD); + CHK_EV(EV_DELETE); + CHK_EV(EV_ENABLE); + CHK_EV(EV_DISABLE); + CHK_EV(EV_ONESHOT); + CHK_EV(EV_CLEAR); + CHK_EV(EV_RECEIPT); + CHK_EV(EV_DISPATCH); + CHK_EV(EV_UDATA_SPECIFIC); + CHK_EV(EV_DISPATCH2); + CHK_EV(EV_VANISHED); + CHK_EV(EV_SYSFLAGS); + CHK_EV(EV_FLAG0); + CHK_EV(EV_FLAG1); + CHK_EV(EV_EOF); + CHK_EV(EV_ERROR); +#undef CHK_EV + return buf; +} + static ep_over_kq_t* eoks_alloc(void); static void eoks_free(ep_over_kq_t *eok); static ep_over_kq_t* eoks_find(int epfd); @@ -135,10 +269,15 @@ int epoll_create(int size) { return -1; } + D("epoll_create epfd:[%d]", eok->idx); return eok->idx; } int epoll_ctl(int epfd, int op, int fd, struct epoll_event *event) { + D("epoll_ctling epfd:[%d], op:[%s], fd:[%d], events:[%04x:%s]", + epfd, op_str(op), fd, + event ? event->events : 0, + event ? events_str(event->events, 0) : "NULL"); int e = 0; if (epfd<0 || epfd>=eoks.neoks) { errno = EBADF; @@ -198,6 +337,7 @@ int epoll_ctl(int epfd, int op, int fd, struct epoll_event *event) { ev.changed = 2; } break; case EPOLL_CTL_DEL: { + D("epoll_ctl adding..."); // event is ignored pev = &oev->epev; flags = EV_DELETE; @@ -213,11 +353,9 @@ int epoll_ctl(int epfd, int op, int fd, struct epoll_event *event) { if (pev->events & (EPOLLIN | EPOLLERR | EPOLLHUP | EPOLLRDHUP)) { flags |= EV_EOF; EV_SET64(&krev, ev.fd, EVFILT_READ, flags, 0, 0, -1, 0, 0); - D("...."); } if (pev->events & EPOLLOUT) { EV_SET64(&kwev, ev.fd, EVFILT_WRITE, flags, 0, 0, -1, 0, 0); - D("...."); } if (eok_chgs_refresh(eok, oev, &ev, &krev, &kwev)) { @@ -244,14 +382,17 @@ int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) int e = 0; if (epfd<0 || epfd>=eoks.neoks) { errno = EBADF; + E("epoll_waiting epfd:[%d], maxevents:[%d], timeout:[%d] failed", epfd, maxevents, timeout); return -1; } if (!events) { errno = EINVAL; + E("epoll_waiting epfd:[%d], maxevents:[%d], timeout:[%d] failed", epfd, maxevents, timeout); return -1; } if (maxevents<=0) { errno = EINVAL; + E("epoll_waiting epfd:[%d], maxevents:[%d], timeout:[%d] failed", epfd, maxevents, timeout); return -1; } @@ -260,6 +401,7 @@ int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) ep_over_kq_t *eok = eoks_find(epfd); if (!eok) { errno = EBADF; + E("epoll_waiting epfd:[%d], maxevents:[%d], timeout:[%d] failed", epfd, maxevents, timeout); return -1; } @@ -281,6 +423,7 @@ int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) struct kevent64_s *eventslist = eok_alloc_eventslist(eok, maxevents); if (!eventslist) { e = ENOMEM; + E("epoll_waiting epfd:[%d], maxevents:[%d], timeout:[%d] failed", epfd, maxevents, timeout); break; } memset(eventslist, 0, maxevents * sizeof(*eventslist)); @@ -305,8 +448,15 @@ int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) eok->ichanges = 0; A(0==pthread_mutex_unlock(&eok->lock), ""); + if (ichanges>0) { + D("kevent64 changing [%d] changes and waiting...", ichanges); + } + errno = 0; r = kevent64(eok->kq, kchanges, ichanges, eventslist, maxevents, 0, pto); e = errno; + if (e) { + E("kevent64 waiting done, with r[%d]", r); + } A(0==pthread_mutex_lock(&eok->lock), ""); eok->waiting = 0; @@ -318,7 +468,6 @@ int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) } eok->waiting = 0; - if (r<0) break; if (r==0) { A(timeout!=-1, "internal logic error"); } @@ -327,7 +476,10 @@ int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) A(kev->udata && eok->evs_head && eok->evs_tail, "internal logic error"); eok_event_t *ev = (eok_event_t*)kev->udata; A(kev->ident == ev->fd, "internal logic error"); - D("..."); + if (kev->flags & EV_ERROR) { + D("error when processing change list for fd[%d], error[%s], kev_flags:[%04x:%s]", + ev->fd, strerror(kev->data), kev->flags, kev_flags_str(kev->flags, 0)); + } switch (kev->filter) { case EVFILT_READ: { A((ev->epev.events & EPOLLIN), "internal logic errro"); @@ -335,10 +487,10 @@ int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) char c = '\0'; A(1==recv(kev->ident, &c, 1, 0), "internal logic error"); A(0==memcmp(&c, "1", 1), "internal logic error"); - D("..............."); + D("wokenup"); } else { if (ev->changed==3) { - D("already requested to delete"); + D("already requested to delete for fd[%d]", ev->fd); // EV_DELETE? continue; } @@ -347,8 +499,13 @@ int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) pev.events = EPOLLIN; if (kev->flags & EV_EOF) { pev.events |= (EPOLLHUP | EPOLLERR | EPOLLRDHUP); + D(".........."); } - pev.events &= ev->epev.events; + pev.events = pev.events & ev->epev.events; + D("events found for fd[%d]: [%04x:%s], which was registered: [%04x:%s], kev_flags: [%04x:%s]", + ev->fd, pev.events, events_str(pev.events, 0), + ev->epev.events, events_str(ev->epev.events, 1), + kev->flags, kev_flags_str(kev->flags, 2)); events[cnts++] = pev; } } break; @@ -360,11 +517,26 @@ int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) } break; } } + if (r>=0) { + eok_event_t *p = eok->evs_head; + while (p) { + eok_event_t *next = p->next; + if (p->changed==3) { + D("removing registered event for fd[%d]: [%04x:%s]", p->fd, p->epev.events, events_str(p->epev.events, 0)); + eok_free_ev(eok, p); + } + p = next; + } + } } while (cnts==0); + if (cnts>0) { + D("kevent64 waiting done with [%d] events", cnts); + } A(0==pthread_mutex_unlock(&eok->lock), ""); if (e) { errno = e; + E("epoll_wait failed"); return -1; } @@ -383,6 +555,7 @@ static struct timespec do_timespec_diff(struct timespec *from, struct timespec * } int epoll_close(int epfd) { + D("epoll_closing epfd: [%d]", epfd); if (epfd<0 || epfd>=eoks.neoks) { errno = EBADF; return -1; @@ -445,6 +618,8 @@ static eok_event_t* eok_calloc_ev(ep_over_kq_t *eok) { else eok->evs_head = p; eok->evs_tail = p; + eok->evs_count += 1; + return p; } @@ -456,6 +631,8 @@ static void eok_free_ev(ep_over_kq_t *eok, eok_event_t *ev) { else eok->evs_tail = ev->prev; ev->next = eok->evs_free; eok->evs_free = ev->next; + + eok->evs_count -= 1; } static void eok_wakeup(ep_over_kq_t *eok) { @@ -486,17 +663,23 @@ static int eok_chgs_refresh(ep_over_kq_t *eok, eok_event_t *oev, eok_event_t *ev } oev->fd = ev->fd; - oev->epev = ev->epev; + if (ev->changed!=3) { + oev->epev = ev->epev; + } oev->changed = ev->changed; + n = 0; if (krev->ident==ev->fd) { krev->udata = (uint64_t)oev; eok->kchanges[eok->ichanges++] = *krev; + ++n; } if (kwev->ident==ev->fd) { kwev->udata = (uint64_t)oev; eok->kchanges[eok->ichanges++] = *kwev; + ++n; } + D("add changes[%d] for fd[%d], and changes/registers [%d/%d]", n, ev->fd, eok->ichanges, eok->evs_count); return 0; } diff --git a/tests/examples/c/epoll.c b/tests/examples/c/epoll.c index e0d63bae32..6047c9e4ea 100644 --- a/tests/examples/c/epoll.c +++ b/tests/examples/c/epoll.c @@ -54,18 +54,74 @@ static ep_t* ep_create(void); static void ep_destroy(ep_t *ep); static void* routine(void* arg); static int open_connect(unsigned short port); +static int open_listen(unsigned short port); + +typedef struct client_s client_t; +struct client_s { + int skt; + void (*on_event)(ep_t *ep, struct epoll_event *events, client_t *client); + volatile unsigned int state; // 1: listenning; 2: connected +}; + +static void echo_event(ep_t *ep, struct epoll_event *ev, client_t *client); int main(int argc, char *argv[]) { ep_t* ep = ep_create(); A(ep, "failed"); int skt = open_connect(6789); if (skt!=-1) { - struct epoll_event ev = {0}; - ev.events = EPOLLIN | EPOLLERR | EPOLLHUP | EPOLLRDHUP; - ev.data.ptr = &skt; - A(0==epoll_ctl(ep->ep, EPOLL_CTL_ADD, skt, &ev), ""); + client_t *client = (client_t*)calloc(1, sizeof(*client)); + if (client) { + client->skt = skt; + client->on_event = echo_event; + client->state = 2; + struct epoll_event ev = {0}; + ev.events = EPOLLIN | EPOLLERR | EPOLLHUP | EPOLLRDHUP; + ev.data.ptr = client; + A(0==epoll_ctl(ep->ep, EPOLL_CTL_ADD, skt, &ev), ""); + } + } + skt = open_listen(0); + if (skt!=-1) { + client_t *client = (client_t*)calloc(1, sizeof(*client)); + if (client) { + client->skt = skt; + client->on_event = echo_event; + client->state = 1; + struct epoll_event ev = {0}; + ev.events = EPOLLIN | EPOLLERR | EPOLLHUP | EPOLLRDHUP; + ev.data.ptr = client; + A(0==epoll_ctl(ep->ep, EPOLL_CTL_ADD, skt, &ev), ""); + } + } + // char c = '\0'; + // while ((c=getchar())!=EOF) { + // switch (c) { + // case 'q': break; + // default: continue; + // } + // } + // getchar(); + char *line = NULL; + size_t linecap = 0; + ssize_t linelen; + while ((linelen = getline(&line, &linecap, stdin)) > 0) { + line[strlen(line)-1] = '\0'; + if (0==strcmp(line, "exit")) break; + if (0==strcmp(line, "quit")) break; + if (line==strstr(line, "close")) { + int fd = 0; + sscanf(line, "close %d", &fd); + if (fd<=2) { + fprintf(stderr, "fd [%d] invalid\n", fd); + continue; + } + A(0==epoll_ctl(ep->ep, EPOLL_CTL_DEL, fd, NULL), ""); + continue; + } + if (strlen(line)==0) continue; + fprintf(stderr, "unknown cmd:[%s]\n", line); } - getchar(); ep_destroy(ep); D(""); return 0; @@ -114,7 +170,7 @@ static void* routine(void* arg) { A(0==pthread_mutex_unlock(&ep->lock), ""); int r = epoll_wait(ep->ep, evs, sizeof(evs)/sizeof(evs[0]), -1); - A(r>0, "indefinite epoll_wait shall not timeout"); + A(r>0, "indefinite epoll_wait shall not timeout:[%d]", r); A(0==pthread_mutex_lock(&ep->lock), ""); A(ep->waiting==1, "internal logic error"); @@ -133,23 +189,50 @@ static void* routine(void* arg) { continue; } A(ev->data.ptr, "internal logic error"); - int skt = *(int*)ev->data.ptr; - if (ev->events & EPOLLIN) { - char buf[4]; - int n = recv(skt, buf, sizeof(buf)-1, 0); - A(n>=0 && nevents, buf); - } - if (ev->events & EPOLLRDHUP) { - A(0==epoll_ctl(ep->ep, EPOLL_CTL_DEL, skt, NULL), ""); - } + client_t *client = (client_t*)ev->data.ptr; + client->on_event(ep, ev, client); continue; } } return NULL; } +static int open_listen(unsigned short port) { + int r = 0; + int skt = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + if (skt==-1) { + E("socket() failed"); + return -1; + } + do { + struct sockaddr_in si = {0}; + si.sin_family = AF_INET; + si.sin_addr.s_addr = inet_addr("127.0.0.1"); + si.sin_port = htons(port); + r = bind(skt, (struct sockaddr*)&si, sizeof(si)); + if (r) { + E("bind(%u) failed", port); + break; + } + r = listen(skt, 100); + if (r) { + E("listen() failed"); + break; + } + memset(&si, 0, sizeof(si)); + socklen_t len = sizeof(si); + r = getsockname(skt, (struct sockaddr *)&si, &len); + if (r) { + E("getsockname() failed"); + } + A(len==sizeof(si), "internal logic error"); + D("listenning at: %d", ntohs(si.sin_port)); + return skt; + } while (0); + close(skt); + return -1; +} + static int open_connect(unsigned short port) { int r = 0; int skt = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); @@ -181,3 +264,41 @@ static int open_connect(unsigned short port) { return -1; } +static void echo_event(ep_t *ep, struct epoll_event *ev, client_t *client) { + if (ev->events & EPOLLIN) { + if (client->state==1) { + struct sockaddr_in si = {0}; + socklen_t silen = sizeof(si); + int skt = accept(client->skt, (struct sockaddr*)&si, &silen); + if (skt!=-1) { + client_t *server = (client_t*)calloc(1, sizeof(*server)); + if (server) { + server->skt = skt; + server->on_event = echo_event; + server->state = 2; + struct epoll_event ev = {0}; + ev.events = EPOLLIN | EPOLLERR | EPOLLHUP | EPOLLRDHUP; + ev.data.ptr = server; + A(0==epoll_ctl(ep->ep, EPOLL_CTL_ADD, skt, &ev), ""); + } + } + } + if (client->state==2) { + char buf[4]; + int n = recv(client->skt, buf, sizeof(buf)-1, 0); + A(n>=0 && nevents, buf); + } + } + if (ev->events & (EPOLLERR | EPOLLHUP | EPOLLRDHUP)) { + A(0==pthread_mutex_lock(&ep->lock), ""); + A(0==epoll_ctl(ep->ep, EPOLL_CTL_DEL, client->skt, NULL), ""); + A(0==pthread_mutex_unlock(&ep->lock), ""); + close(client->skt); + client->skt = -1; + client->on_event = NULL; + free(client); + } +} + From 3998f0895e2a1a384bd93b4c1fd100ed4ee4f643 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 16 Jan 2021 23:16:17 +0800 Subject: [PATCH 012/147] TD-1207 --- src/os/inc/osWindows.h | 2 ++ src/os/src/windows/wFile.c | 10 +++++----- src/os/src/windows/wSignal.c | 31 +++++++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 5 deletions(-) create mode 100644 src/os/src/windows/wSignal.c diff --git a/src/os/inc/osWindows.h b/src/os/inc/osWindows.h index 1b28cfa90f..a984a5ba88 100644 --- a/src/os/inc/osWindows.h +++ b/src/os/inc/osWindows.h @@ -191,6 +191,8 @@ int gettimeofday(struct timeval *ptv, void *pTimeZone); #define PATH_MAX 256 #endif +#define TAOS_OS_FUNC_SIGNAL + typedef struct { int we_wordc; char **we_wordv; diff --git a/src/os/src/windows/wFile.c b/src/os/src/windows/wFile.c index e61218cac3..4ad195dc6f 100644 --- a/src/os/src/windows/wFile.c +++ b/src/os/src/windows/wFile.c @@ -77,7 +77,8 @@ int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t co } int64_t taosSendFile(SOCKET dfd, int32_t sfd, int64_t *offset, int64_t count) { - lseek(sfd, (int32_t)(*offset), 0); + if (offset != NULL) lseek(sfd, (int32_t)(*offset), 0); + int64_t writeLen = 0; uint8_t buffer[_SEND_FILE_STEP_] = {0}; @@ -168,7 +169,6 @@ int fsync(int filedes) { } HANDLE h = (HANDLE)_get_osfhandle(filedes); - FlushFileBuffers(h); - - return 0; - } + + return FlushFileBuffers(h); +} diff --git a/src/os/src/windows/wSignal.c b/src/os/src/windows/wSignal.c new file mode 100644 index 0000000000..6767076fc4 --- /dev/null +++ b/src/os/src/windows/wSignal.c @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include + +void taosSetSignal(int32_t signum, FSignalHandler sigfp) { + if (signum == SIGUSR1) return; + signal(signum, sigfp); +} + +void taosIgnSignal(int32_t signum) { + signal(signum, SIG_IGN); +} + +void taosDflSignal(int32_t signum) { + signal(signum, SIG_DFL); +} From 9fbe6e471eb9b9e4f35d883915702ed9c67fe59c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 16 Jan 2021 23:58:45 +0800 Subject: [PATCH 013/147] TD-1207 --- deps/zlib-1.2.11/src/gzlib.c | 4 +- src/client/src/tscParseInsert.c | 2 +- src/client/src/tscSub.c | 4 +- src/client/src/tscSubquery.c | 6 +- src/os/src/detail/osDir.c | 5 +- src/os/src/windows/wSignal.c | 2 + src/query/src/qTsbuf.c | 4 +- src/sync/src/syncRestore.c | 2 +- src/sync/src/syncRetrieve.c | 6 +- src/tsdb/src/tsdbCommit.c | 4 +- src/tsdb/src/tsdbFile.c | 4 +- src/tsdb/src/tsdbMain.c | 4 +- src/util/src/tfile.c | 4 +- src/util/src/tkvstore.c | 12 +-- tests/script/sh/deploy.bat | 155 ++++++++++++++++++++++++++++++++ tests/script/sh/exec.bat | 115 ++++++++++++++++++++++++ tests/script/sh/stop_dnodes.bat | 24 +++++ tests/script/wtest.bat | 37 ++++---- 18 files changed, 344 insertions(+), 50 deletions(-) create mode 100644 tests/script/sh/deploy.bat create mode 100644 tests/script/sh/exec.bat create mode 100644 tests/script/sh/stop_dnodes.bat diff --git a/deps/zlib-1.2.11/src/gzlib.c b/deps/zlib-1.2.11/src/gzlib.c index 381cdc2c7d..f3bd685be0 100644 --- a/deps/zlib-1.2.11/src/gzlib.c +++ b/deps/zlib-1.2.11/src/gzlib.c @@ -240,9 +240,9 @@ local gzFile gz_open(path, fd, mode) /* open the file with the appropriate flags (or just use fd) */ state->fd = fd > -1 ? fd : ( #ifdef WIDECHAR - fd == -2 ? _wopen(path, oflag, 0666) : + fd == -2 ? _wopen(path, oflag | O_BINARY, 0666) : #endif - open((const char *)path, oflag, 0666)); + open((const char *)path, oflag | O_BINARY, 0666)); if (state->fd == -1) { free(state->path); free(state); diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 7151c33393..27abb0d2da 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -1545,7 +1545,7 @@ void tscProcessMultiVnodesImportFromFile(SSqlObj *pSql) { SSqlObj *pNew = createSubqueryObj(pSql, 0, parseFileSendDataBlock, pSupporter, TSDB_SQL_INSERT, NULL); pCmd->count = 1; - FILE *fp = fopen(pCmd->payload, "r"); + FILE *fp = fopen(pCmd->payload, "rb"); if (fp == NULL) { pSql->res.code = TAOS_SYSTEM_ERROR(errno); tscError("%p failed to open file %s to load data from file, code:%s", pSql, pCmd->payload, tstrerror(pSql->res.code)); diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c index 7f0b174ad3..527531b31a 100644 --- a/src/client/src/tscSub.c +++ b/src/client/src/tscSub.c @@ -313,7 +313,7 @@ static int tscLoadSubscriptionProgress(SSub* pSub) { char buf[TSDB_MAX_SQL_LEN]; sprintf(buf, "%s/subscribe/%s", tsDataDir, pSub->topic); - FILE* fp = fopen(buf, "r"); + FILE* fp = fopen(buf, "rb"); if (fp == NULL) { tscDebug("subscription progress file does not exist: %s", pSub->topic); return 1; @@ -368,7 +368,7 @@ void tscSaveSubscriptionProgress(void* sub) { } sprintf(path, "%s/subscribe/%s", tsDataDir, pSub->topic); - FILE* fp = fopen(path, "w+"); + FILE* fp = fopen(path, "wb+"); if (fp == NULL) { tscError("failed to create progress file for subscription: %s", pSub->topic); return; diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 2622246111..c207e1a6cf 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -930,7 +930,7 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow if (numOfRows > 0) { // write the compressed timestamp to disk file if(pSupporter->f == NULL) { - pSupporter->f = fopen(pSupporter->path, "w"); + pSupporter->f = fopen(pSupporter->path, "wb"); if (pSupporter->f == NULL) { tscError("%p failed to create tmp file:%s, reason:%s", pSql, pSupporter->path, strerror(errno)); @@ -974,7 +974,7 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow // continue to retrieve ts-comp data from vnode if (!pRes->completed) { taosGetTmpfilePath("ts-join", pSupporter->path); - pSupporter->f = fopen(pSupporter->path, "w"); + pSupporter->f = fopen(pSupporter->path, "wb"); pRes->row = pRes->numOfRows; taos_fetch_rows_a(tres, tsCompRetrieveCallback, param); @@ -1000,7 +1000,7 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow taosGetTmpfilePath("ts-join", pSupporter->path); // TODO check for failure - pSupporter->f = fopen(pSupporter->path, "w"); + pSupporter->f = fopen(pSupporter->path, "wb"); pRes->row = pRes->numOfRows; // set the callback function diff --git a/src/os/src/detail/osDir.c b/src/os/src/detail/osDir.c index d3f0fda1a5..4f2985548c 100644 --- a/src/os/src/detail/osDir.c +++ b/src/os/src/detail/osDir.c @@ -54,6 +54,9 @@ int taosMkDir(const char *path, mode_t mode) { void taosRename(char* oldName, char *newName) { // if newName in not empty, rename return fail. // the newName must be empty or does not exist +#ifdef WINDOWS + remove(newName); +#endif if (rename(oldName, newName)) { uError("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno)); } else { @@ -117,7 +120,7 @@ int32_t taosCompressFile(char *srcFileName, char *destFileName) { goto cmp_end; } - int32_t fd = open(destFileName, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO); + int32_t fd = open(destFileName, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, S_IRWXU | S_IRWXG | S_IRWXO); if (fd < 0) { ret = -2; goto cmp_end; diff --git a/src/os/src/windows/wSignal.c b/src/os/src/windows/wSignal.c index 6767076fc4..8a3c2cfbd4 100644 --- a/src/os/src/windows/wSignal.c +++ b/src/os/src/windows/wSignal.c @@ -23,9 +23,11 @@ void taosSetSignal(int32_t signum, FSignalHandler sigfp) { } void taosIgnSignal(int32_t signum) { + if (signum == SIGUSR1) return; signal(signum, SIG_IGN); } void taosDflSignal(int32_t signum) { + if (signum == SIGUSR1) return; signal(signum, SIG_DFL); } diff --git a/src/query/src/qTsbuf.c b/src/query/src/qTsbuf.c index a53f8935ee..1f43c5b33c 100644 --- a/src/query/src/qTsbuf.c +++ b/src/query/src/qTsbuf.c @@ -23,7 +23,7 @@ STSBuf* tsBufCreate(bool autoDelete, int32_t order) { pTSBuf->autoDelete = autoDelete; taosGetTmpfilePath("join", pTSBuf->path); - pTSBuf->f = fopen(pTSBuf->path, "w+"); + pTSBuf->f = fopen(pTSBuf->path, "wb+"); if (pTSBuf->f == NULL) { free(pTSBuf); return NULL; @@ -59,7 +59,7 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { tstrncpy(pTSBuf->path, path, sizeof(pTSBuf->path)); - pTSBuf->f = fopen(pTSBuf->path, "r+"); + pTSBuf->f = fopen(pTSBuf->path, "rb+"); if (pTSBuf->f == NULL) { free(pTSBuf); return NULL; diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c index bd4f9bd3e1..99f4ce1c17 100644 --- a/src/sync/src/syncRestore.c +++ b/src/sync/src/syncRestore.c @@ -128,7 +128,7 @@ static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { minfo.name[sizeof(minfo.name) - 1] = 0; snprintf(name, sizeof(name), "%s/%s", pNode->path, minfo.name); - int32_t dfd = open(name, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO); + int32_t dfd = open(name, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, S_IRWXU | S_IRWXG | S_IRWXO); if (dfd < 0) { sError("%s, failed to open file:%s while restore file since %s", pPeer->id, minfo.name, strerror(errno)); break; diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c index c7017c1af8..e748898e6e 100644 --- a/src/sync/src/syncRetrieve.c +++ b/src/sync/src/syncRetrieve.c @@ -152,7 +152,7 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) { snprintf(name, sizeof(name), "%s/%s", pNode->path, fileInfo.name); // send the file to peer - int32_t sfd = open(name, O_RDONLY); + int32_t sfd = open(name, O_RDONLY | O_BINARY); if (sfd < 0) { code = -1; sError("%s, failed to open file:%s while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno)); @@ -221,7 +221,7 @@ static int32_t syncReadOneWalRecord(int32_t sfd, SWalHead *pHead) { } static int32_t syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion, int64_t offset) { - int32_t sfd = open(name, O_RDONLY); + int32_t sfd = open(name, O_RDONLY | O_BINARY); if (sfd < 0) { sError("%s, failed to open wal:%s for retrieve since:%s", pPeer->id, name, tstrerror(errno)); return -1; @@ -376,7 +376,7 @@ static int32_t syncRetrieveWal(SSyncPeer *pPeer) { size = fstat.st_size; sDebug("%s, retrieve wal:%s size:%d", pPeer->id, fname, size); - int32_t sfd = open(fname, O_RDONLY); + int32_t sfd = open(fname, O_RDONLY | O_BINARY); if (sfd < 0) { code = -1; sError("%s, failed to open wal:%s for retrieve since %s, code:0x%x", pPeer->id, fname, strerror(errno), code); diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c index 696270d670..9e7b0cf108 100644 --- a/src/tsdb/src/tsdbCommit.c +++ b/src/tsdb/src/tsdbCommit.c @@ -269,11 +269,11 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe pthread_rwlock_wrlock(&(pFileH->fhlock)); - (void)rename(helperNewHeadF(pHelper)->fname, helperHeadF(pHelper)->fname); + (void)taosRename(helperNewHeadF(pHelper)->fname, helperHeadF(pHelper)->fname); pGroup->files[TSDB_FILE_TYPE_HEAD].info = helperNewHeadF(pHelper)->info; if (newLast) { - (void)rename(helperNewLastF(pHelper)->fname, helperLastF(pHelper)->fname); + (void)taosRename(helperNewLastF(pHelper)->fname, helperLastF(pHelper)->fname); pGroup->files[TSDB_FILE_TYPE_LAST].info = helperNewLastF(pHelper)->info; } else { pGroup->files[TSDB_FILE_TYPE_LAST].info = helperLastF(pHelper)->info; diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 5d8933d141..7a8622b110 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -175,7 +175,7 @@ int tsdbOpenFileH(STsdbRepo *pRepo) { } sprintf(fname2, "%s/%s_back_%" PRId64, tDataDir, dp->d_name, taosGetTimestamp(TSDB_TIME_PRECISION_MILLI)); - (void)rename(fname1, fname2); + (void)taosRename(fname1, fname2); tsdbDebug("vgId:%d file %s exists, backup it as %s", REPO_ID(pRepo), fname1, fname2); @@ -339,7 +339,7 @@ SFileGroup *tsdbGetFileGroupNext(SFileGroupIter *pIter) { int tsdbOpenFile(SFile *pFile, int oflag) { ASSERT(!TSDB_IS_FILE_OPENED(pFile)); - pFile->fd = open(pFile->fname, oflag, 0755); + pFile->fd = open(pFile->fname, oflag | O_BINARY, 0755); if (pFile->fd < 0) { tsdbError("failed to open file %s since %s", pFile->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index b34b2fa9e6..3903f1dedc 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -537,7 +537,7 @@ static int32_t tsdbSaveConfig(char *rootDir, STsdbCfg *pCfg) { goto _err; } - fd = open(fname, O_WRONLY | O_CREAT, 0755); + fd = open(fname, O_WRONLY | O_CREAT | O_BINARY, 0755); if (fd < 0) { tsdbError("vgId:%d failed to open file %s since %s", pCfg->tsdbId, fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); @@ -583,7 +583,7 @@ static int tsdbLoadConfig(char *rootDir, STsdbCfg *pCfg) { goto _err; } - fd = open(fname, O_RDONLY); + fd = open(fname, O_RDONLY | O_BINARY); if (fd < 0) { tsdbError("failed to open file %s since %s", fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); diff --git a/src/util/src/tfile.c b/src/util/src/tfile.c index 64fea2843b..dd621b3199 100644 --- a/src/util/src/tfile.c +++ b/src/util/src/tfile.c @@ -54,12 +54,12 @@ static int64_t tfOpenImp(int32_t fd) { } int64_t tfOpen(const char *pathname, int32_t flags) { - int32_t fd = open(pathname, flags); + int32_t fd = open(pathname, flags | O_BINARY); return tfOpenImp(fd); } int64_t tfOpenM(const char *pathname, int32_t flags, mode_t mode) { - int32_t fd = open(pathname, flags, mode); + int32_t fd = open(pathname, flags | O_BINARY, mode); return tfOpenImp(fd); } diff --git a/src/util/src/tkvstore.c b/src/util/src/tkvstore.c index 2b1d13c78b..0abba410b0 100644 --- a/src/util/src/tkvstore.c +++ b/src/util/src/tkvstore.c @@ -50,7 +50,7 @@ static void * tdDecodeKVRecord(void *buf, SKVRecord *pRecord); static int tdRestoreKVStore(SKVStore *pStore); int tdCreateKVStore(char *fname) { - int fd = open(fname, O_RDWR | O_CREAT, 0755); + int fd = open(fname, O_RDWR | O_CREAT | O_BINARY, 0755); if (fd < 0) { uError("failed to open file %s since %s", fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); @@ -96,14 +96,14 @@ SKVStore *tdOpenKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void *appH SKVStore *pStore = tdNewKVStore(fname, iFunc, aFunc, appH); if (pStore == NULL) return NULL; - pStore->fd = open(pStore->fname, O_RDWR); + pStore->fd = open(pStore->fname, O_RDWR | O_BINARY); if (pStore->fd < 0) { uError("failed to open file %s since %s", pStore->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); goto _err; } - pStore->sfd = open(pStore->fsnap, O_RDONLY); + pStore->sfd = open(pStore->fsnap, O_RDONLY | O_BINARY); if (pStore->sfd < 0) { if (errno != ENOENT) { uError("failed to open file %s since %s", pStore->fsnap, strerror(errno)); @@ -172,14 +172,14 @@ void tdCloseKVStore(SKVStore *pStore) { tdFreeKVStore(pStore); } int tdKVStoreStartCommit(SKVStore *pStore) { ASSERT(pStore->fd < 0); - pStore->fd = open(pStore->fname, O_RDWR); + pStore->fd = open(pStore->fname, O_RDWR | O_BINARY); if (pStore->fd < 0) { uError("failed to open file %s since %s", pStore->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); goto _err; } - pStore->sfd = open(pStore->fsnap, O_WRONLY | O_CREAT, 0755); + pStore->sfd = open(pStore->fsnap, O_WRONLY | O_CREAT | O_BINARY, 0755); if (pStore->sfd < 0) { uError("failed to open file %s since %s", pStore->fsnap, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); @@ -336,7 +336,7 @@ void tsdbGetStoreInfo(char *fname, uint32_t *magic, int64_t *size) { char buf[TD_KVSTORE_HEADER_SIZE] = "\0"; SStoreInfo info = {0}; - int fd = open(fname, O_RDONLY); + int fd = open(fname, O_RDONLY | O_BINARY); if (fd < 0) goto _err; if (taosRead(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) goto _err; diff --git a/tests/script/sh/deploy.bat b/tests/script/sh/deploy.bat new file mode 100644 index 0000000000..9b61a33d45 --- /dev/null +++ b/tests/script/sh/deploy.bat @@ -0,0 +1,155 @@ +#!/bin/bash + +echo "Executing deploy.sh" + +if [ $# != 4 ]; then + echo "argument list need input : " + echo " -n nodeName" + echo " -i nodePort" + exit 1 +fi + +NODE_NAME= +NODE= +while getopts "n:i:" arg +do + case $arg in + n) + NODE_NAME=$OPTARG + ;; + i) + NODE=$OPTARG + ;; + ?) + echo "unkonw argument" + ;; + esac +done + +SCRIPT_DIR=`dirname $0` +cd $SCRIPT_DIR/../ +SCRIPT_DIR=`pwd` +echo "SCRIPT_DIR: $SCRIPT_DIR" + +IN_TDINTERNAL="community" +if [[ "$SCRIPT_DIR" == *"$IN_TDINTERNAL"* ]]; then + cd ../../.. +else + cd ../../ +fi + +TAOS_DIR=`pwd` +TAOSD_DIR=`find . -name "taosd"|grep bin|head -n1` + +if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then + BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2,3` +else + BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2` +fi + +BUILD_DIR=$TAOS_DIR/$BIN_DIR/build + +SIM_DIR=$TAOS_DIR/sim + +NODE_DIR=$SIM_DIR/$NODE_NAME +EXE_DIR=$BUILD_DIR/bin +CFG_DIR=$NODE_DIR/cfg +LOG_DIR=$NODE_DIR/log +DATA_DIR=$NODE_DIR/data + +rm -rf $NODE_DIR + +mkdir -p $SIM_DIR +mkdir -p $NODE_DIR +mkdir -p $LOG_DIR +mkdir -p $DATA_DIR + +#cp -rf $TAOS_DIR/cfg $NODE_DIR/ +mkdir -p $CFG_DIR + +#allow normal user to read/write log +chmod -R 777 $NODE_DIR + +TAOS_CFG=$NODE_DIR/cfg/taos.cfg +touch -f $TAOS_CFG + +TAOS_FLAG=$SIM_DIR/tsim/flag +if [ -f "$TAOS_FLAG" ] ; then + TAOS_CFG=/etc/taos/taos.cfg + DATA_DIR=/var/lib/taos + LOG_DIR=/var/log/taos + sudo rm -f /etc/taos/*.cfg + sudo cp -rf $TAOS_DIR/cfg/*.cfg /etc/taos + sudo rm -rf $DATA_DIR + sudo rm -rf $LOG_DIR +fi + +HOSTNAME=`hostname -f` + +if [ $NODE -eq 1 ]; then + NODE=7100 +elif [ $NODE -eq 2 ]; then + NODE=7200 +elif [ $NODE -eq 3 ]; then + NODE=7300 +elif [ $NODE -eq 4 ]; then + NODE=7400 +elif [ $NODE -eq 5 ]; then + NODE=7500 +elif [ $NODE -eq 6 ]; then + NODE=7600 +elif [ $NODE -eq 7 ]; then + NODE=7700 +elif [ $NODE -eq 8 ]; then + NODE=7800 +fi + +echo " " >> $TAOS_CFG +echo "firstEp ${HOSTNAME}:7100" >> $TAOS_CFG +echo "secondEp ${HOSTNAME}:7200" >> $TAOS_CFG +echo "serverPort ${NODE}" >> $TAOS_CFG +echo "dataDir $DATA_DIR" >> $TAOS_CFG +echo "logDir $LOG_DIR" >> $TAOS_CFG +echo "debugFlag 0" >> $TAOS_CFG +echo "mDebugFlag 143" >> $TAOS_CFG +echo "sdbDebugFlag 143" >> $TAOS_CFG +echo "dDebugFlag 143" >> $TAOS_CFG +echo "vDebugFlag 143" >> $TAOS_CFG +echo "tsdbDebugFlag 143" >> $TAOS_CFG +echo "cDebugFlag 143" >> $TAOS_CFG +echo "jnidebugFlag 143" >> $TAOS_CFG +echo "odbcdebugFlag 143" >> $TAOS_CFG +echo "httpDebugFlag 143" >> $TAOS_CFG +echo "monDebugFlag 143" >> $TAOS_CFG +echo "mqttDebugFlag 143" >> $TAOS_CFG +echo "qdebugFlag 143" >> $TAOS_CFG +echo "rpcDebugFlag 143" >> $TAOS_CFG +echo "tmrDebugFlag 131" >> $TAOS_CFG +echo "udebugFlag 143" >> $TAOS_CFG +echo "sdebugFlag 143" >> $TAOS_CFG +echo "wdebugFlag 143" >> $TAOS_CFG +echo "cqdebugFlag 143" >> $TAOS_CFG +echo "monitor 0" >> $TAOS_CFG +echo "monitorInterval 1" >> $TAOS_CFG +echo "http 0" >> $TAOS_CFG +echo "slaveQuery 0" >> $TAOS_CFG +echo "numOfThreadsPerCore 2.0" >> $TAOS_CFG +echo "defaultPass taosdata" >> $TAOS_CFG +echo "numOfLogLines 20000000" >> $TAOS_CFG +echo "mnodeEqualVnodeNum 0" >> $TAOS_CFG +echo "balanceInterval 1" >> $TAOS_CFG +echo "clog 2" >> $TAOS_CFG +#echo "cache 1" >> $TAOS_CFG +echo "days 10" >> $TAOS_CFG +echo "statusInterval 1" >> $TAOS_CFG +echo "maxVgroupsPerDb 4" >> $TAOS_CFG +echo "minTablesPerVnode 4" >> $TAOS_CFG +echo "maxTablesPerVnode 1000" >> $TAOS_CFG +echo "tableIncStepPerVnode 10000" >> $TAOS_CFG +echo "asyncLog 0" >> $TAOS_CFG +echo "numOfMnodes 1" >> $TAOS_CFG +echo "locale en_US.UTF-8" >> $TAOS_CFG +echo "fsync 0" >> $TAOS_CFG +echo "telemetryReporting 0" >> $TAOS_CFG +echo " " >> $TAOS_CFG + diff --git a/tests/script/sh/exec.bat b/tests/script/sh/exec.bat new file mode 100644 index 0000000000..1c84a6b10e --- /dev/null +++ b/tests/script/sh/exec.bat @@ -0,0 +1,115 @@ +#!/bin/bash + +# if [ $# != 4 || $# != 5 ]; then + # echo "argument list need input : " + # echo " -n nodeName" + # echo " -s start/stop" + # echo " -c clear" + # exit 1 +# fi + +NODE_NAME= +EXEC_OPTON= +CLEAR_OPTION="false" +while getopts "n:s:u:x:ct" arg +do + case $arg in + n) + NODE_NAME=$OPTARG + ;; + s) + EXEC_OPTON=$OPTARG + ;; + c) + CLEAR_OPTION="clear" + ;; + t) + SHELL_OPTION="true" + ;; + u) + USERS=$OPTARG + ;; + x) + SIGNAL=$OPTARG + ;; + ?) + echo "unkown argument" + ;; + esac +done + +SCRIPT_DIR=`dirname $0` +cd $SCRIPT_DIR/../ +SCRIPT_DIR=`pwd` + +IN_TDINTERNAL="community" +if [[ "$SCRIPT_DIR" == *"$IN_TDINTERNAL"* ]]; then + cd ../../.. +else + cd ../../ +fi + +TAOS_DIR=`pwd` +TAOSD_DIR=`find . -name "taosd"|grep bin|head -n1` + +if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then + BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2,3` +else + BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2` +fi + +BUILD_DIR=$TAOS_DIR/$BIN_DIR/build + +SIM_DIR=$TAOS_DIR/sim +NODE_DIR=$SIM_DIR/$NODE_NAME +EXE_DIR=$BUILD_DIR/bin +CFG_DIR=$NODE_DIR/cfg +LOG_DIR=$NODE_DIR/log +DATA_DIR=$NODE_DIR/data +MGMT_DIR=$NODE_DIR/data/mgmt +TSDB_DIR=$NODE_DIR/data/tsdb + +TAOS_CFG=$NODE_DIR/cfg/taos.cfg + +echo ------------ $EXEC_OPTON $NODE_NAME + +TAOS_FLAG=$SIM_DIR/tsim/flag +if [ -f "$TAOS_FLAG" ]; then + EXE_DIR=/usr/local/bin/taos +fi + +if [ "$CLEAR_OPTION" = "clear" ]; then + echo rm -rf $MGMT_DIR $TSDB_DIR + rm -rf $TSDB_DIR + rm -rf $MGMT_DIR +fi + +if [ "$EXEC_OPTON" = "start" ]; then + echo "ExcuteCmd:" $EXE_DIR/taosd -c $CFG_DIR + + if [ "$SHELL_OPTION" = "true" ]; then + TT=`date +%s` + mkdir ${LOG_DIR}/${TT} + nohup valgrind --log-file=${LOG_DIR}/${TT}/valgrind.log --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 & + else + nohup $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 & + fi + +else + #relative path + RCFG_DIR=sim/$NODE_NAME/cfg + PID=`ps -ef|grep taosd | grep $RCFG_DIR | grep -v grep | awk '{print $2}'` + while [ -n "$PID" ] + do + if [ "$SIGNAL" = "SIGKILL" ]; then + echo try to kill by signal SIGKILL + kill -9 $PID + else + echo try to kill by signal SIGINT + kill -SIGINT $PID + fi + sleep 1 + PID=`ps -ef|grep taosd | grep $RCFG_DIR | grep -v grep | awk '{print $2}'` + done +fi + diff --git a/tests/script/sh/stop_dnodes.bat b/tests/script/sh/stop_dnodes.bat new file mode 100644 index 0000000000..1a6e7153c3 --- /dev/null +++ b/tests/script/sh/stop_dnodes.bat @@ -0,0 +1,24 @@ +#!/bin/sh + +PID=`ps -ef|grep /usr/bin/taosd | grep -v grep | awk '{print $2}'` +if [ -n "$PID" ]; then + echo systemctl stop taosd + systemctl stop taosd +fi + +PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` +while [ -n "$PID" ]; do + echo kill -9 $PID + pkill -9 taosd + fuser -k -n tcp 6030 + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` +done + +PID=`ps -ef|grep -w tarbitrator | grep -v grep | awk '{print $2}'` +while [ -n "$PID" ]; do + echo kill -9 $PID + pkill -9 tarbitrator + fuser -k -n tcp 6040 + PID=`ps -ef|grep -w tarbitrator | grep -v grep | awk '{print $2}'` +done + diff --git a/tests/script/wtest.bat b/tests/script/wtest.bat index 6cdd63b42d..8ca1f22518 100644 --- a/tests/script/wtest.bat +++ b/tests/script/wtest.bat @@ -6,8 +6,8 @@ echo Start TDengine Testing Case ... set "SCRIPT_DIR=%~dp0" echo SCRIPT_DIR: %SCRIPT_DIR% -set "BUILD_DIR=%~dp0..\..\debug\32\build\bin" -set "TSIM=%~dp0..\..\debug\32\build\bin\tsim" +set "BUILD_DIR=%~dp0..\..\debug\build\bin" +set "TSIM=%~dp0..\..\debug\build\bin\tsim" echo BUILD_DIR: %BUILD_DIR% set "SIM_DIR=%~dp0..\..\sim" @@ -32,29 +32,24 @@ if exist %LOG_DIR% rmdir /s/q %LOG_DIR% if not exist %CFG_DIR% mkdir %CFG_DIR% if not exist %LOG_DIR% mkdir %LOG_DIR% -echo firstEp %FIRSTEP% > %TAOS_CFG% -echo serverPort 6030 >> %TAOS_CFG% -echo wal 2 >> %TAOS_CFG% -echo asyncLog 0 >> %TAOS_CFG% -echo locale en_US.UTF-8 >> %TAOS_CFG% -echo logDir %LOG_DIR% >> %TAOS_CFG% -echo scriptDir %SCRIPT_DIR% >> %TAOS_CFG% -echo numOfLogLines 100000000 >> %TAOS_CFG% -echo tmrDebugFlag 131 >> %TAOS_CFG% -echo rpcDebugFlag 143 >> %TAOS_CFG% -echo cDebugFlag 143 >> %TAOS_CFG% -echo qdebugFlag 143 >> %TAOS_CFG% -echo udebugFlag 143 >> %TAOS_CFG% +echo firstEp localhost > %TAOS_CFG% +echo serverPort 7100 >> %TAOS_CFG% +echo logDir %LOG_DIR% >> %TAOS_CFG% +echo scriptDir %SCRIPT_DIR% >> %TAOS_CFG% +echo numOfLogLines 100000000 >> %TAOS_CFG% +echo rpcDebugFlag 143 >> %TAOS_CFG% +echo tmrDebugFlag 131 >> %TAOS_CFG% +echo cDebugFlag 143 >> %TAOS_CFG% +echo udebugFlag 143 >> %TAOS_CFG% +echo wal 0 >> %TAOS_CFG% +echo asyncLog 0 >> %TAOS_CFG% +echo locale en_US.UTF-8 >> %TAOS_CFG% +echo enableCoreFile 1 >> %TAOS_CFG% -set "FILE_NAME=windows\testSuite.sim" -set "FIRSTEP=192.168.1.182" +set "FILE_NAME=testSuite.sim" if "%1" == "-f" set "FILE_NAME=%2" -if "%1" == "-h" set "FIRSTEP=%2" -if "%3" == "-f" set "FILE_NAME=%4" -if "%3" == "-h" set "FIRSTEP=%4" echo FILE_NAME: %FILE_NAME% -echo FIRSTEP: %FIRSTEP% echo ExcuteCmd: %tsim% -c %CFG_DIR% -f %FILE_NAME% %tsim% -c %CFG_DIR% -f %FILE_NAME% \ No newline at end of file From 985de782080ae45639aaf701df760c2a5ffcea1b Mon Sep 17 00:00:00 2001 From: freemine Date: Sun, 17 Jan 2021 00:47:16 +0800 Subject: [PATCH 014/147] with epoll over kqueue, taos can be built on MacOSX --- src/kit/shell/src/shellDarwin.c | 3 + src/os/inc/eok.h | 2 +- src/os/src/darwin/darwinEnv.c | 2 - src/os/src/darwin/eok.c | 176 +++++++++++++++++++++----------- src/os/src/detail/osSocket.c | 1 - src/rpc/src/rpcTcp.c | 14 --- src/sync/src/syncTcp.c | 6 -- src/tsdb/src/tsdbMain.c | 4 +- src/util/src/tsocket.c | 3 +- 9 files changed, 127 insertions(+), 84 deletions(-) diff --git a/src/kit/shell/src/shellDarwin.c b/src/kit/shell/src/shellDarwin.c index d6aed4401c..dbec3fdb05 100644 --- a/src/kit/shell/src/shellDarwin.c +++ b/src/kit/shell/src/shellDarwin.c @@ -350,6 +350,9 @@ void *shellLoopQuery(void *arg) { reset_terminal_mode(); } while (shellRunCommand(con, command) == 0); + tfree(command); + exitShell(); + pthread_cleanup_pop(1); return NULL; diff --git a/src/os/inc/eok.h b/src/os/inc/eok.h index 8892e50c35..a0fd5b5d8e 100644 --- a/src/os/inc/eok.h +++ b/src/os/inc/eok.h @@ -70,5 +70,5 @@ int epoll_close(int epfd); } #endif -#endif // _eok_h_fd274616_996c_400e_9023_ae70be881fa3_ +#endif // _eok_h_fd274616_996c_400e_9023_ae70be881fa3_ diff --git a/src/os/src/darwin/darwinEnv.c b/src/os/src/darwin/darwinEnv.c index 28388f24d2..da4b32139e 100644 --- a/src/os/src/darwin/darwinEnv.c +++ b/src/os/src/darwin/darwinEnv.c @@ -17,8 +17,6 @@ #include "os.h" #include "tglobal.h" -#include - void osInit() { if (configDir[0] == 0) { strcpy(configDir, "~/TDengine/cfg"); diff --git a/src/os/src/darwin/eok.c b/src/os/src/darwin/eok.c index f83846b734..3f7137f6a1 100644 --- a/src/os/src/darwin/eok.c +++ b/src/os/src/darwin/eok.c @@ -34,20 +34,31 @@ typedef struct eok_event_s eok_event_t; struct ep_over_kq_s { int kq; + + // !!! + // idx in the eoks list + // used as pseudo-file-desciptor + // must be 'closed' with epoll_close int idx; + ep_over_kq_t *next; int sv[2]; // 0 for read, 1 for write + // all registered 'epoll events, key by fd' int evs_count; eok_event_t *evs_head; eok_event_t *evs_tail; eok_event_t *evs_free; + // all kev changes list pending to be processed by kevent64 + // key by tuple (ident,filter), ident === fd in this case struct kevent64_s *kchanges; int nchanges; int ichanges; + // kev eventslist for kevent64 to store active events + // they remain alive among kevent64 calls struct kevent64_s *kevslist; int nevslist; @@ -76,6 +87,7 @@ struct eoks_s { int neoks; ep_over_kq_t *eoks_free; }; + static eoks_t eoks = { .lock = PTHREAD_MUTEX_INITIALIZER, .eoks = NULL, @@ -93,8 +105,9 @@ static const char* op_str(int op) { } static __thread char buf_slots[10][1024] = {0}; -static __thread int buf_slots_linelen = sizeof(buf_slots[0])/sizeof(buf_slots[0][0]); -static __thread int buf_slots_count = sizeof(buf_slots)/(sizeof(buf_slots[0])/sizeof(buf_slots[0][0])); +static __thread int buf_slots_linelen = sizeof(buf_slots[0])/sizeof(buf_slots[0][0]); +static __thread int buf_slots_count = sizeof(buf_slots)/(sizeof(buf_slots[0])/sizeof(buf_slots[0][0])); + static const char* events_str(uint32_t events, int slots) { A(slots>=0 && slots on linux // EPOLLIN = 0x001, // #define EPOLLIN EPOLLIN // EPOLLPRI = 0x002, @@ -164,6 +178,7 @@ static const char* kev_flags_str(uint16_t flags, int slots) { size_t len = buf_slots_linelen; int n = 0; buf[0] = '\0'; + // copied to // #define EV_ADD 0x0001 /* add event to kq (implies enable) */ // #define EV_DELETE 0x0002 /* delete event from kq */ // #define EV_ENABLE 0x0004 /* enable event */ @@ -233,10 +248,7 @@ int epoll_create(int size) { (void)size; int e = 0; ep_over_kq_t *eok = eoks_alloc(); - if (!eok) { - errno = ENOMEM; - return -1; - } + if (!eok) return -1; A(eok->kq==-1, "internal logic error"); A(eok->lock_valid, "internal logic error"); @@ -263,13 +275,14 @@ int epoll_create(int size) { struct epoll_event ev = {0}; ev.events = EPOLLIN; ev.data.ptr = &eok_dummy; + D("epoll_create epfd:[%d]", eok->idx); if (epoll_ctl(eok->idx, EPOLL_CTL_ADD, eok->sv[0], &ev)) { - A(0, "internal logic error"); + e = errno; epoll_close(eok->idx); + errno = e; return -1; } - D("epoll_create epfd:[%d]", eok->idx); return eok->idx; } @@ -309,36 +322,34 @@ int epoll_ctl(int epfd, int op, int fd, struct epoll_event *event) { e = ENOENT; break; } + if (op!=EPOLL_CTL_DEL && !event) { + e = EINVAL; + break; + } + // prepare krev/kwev struct kevent64_s krev = {0}; struct kevent64_s kwev = {0}; krev.ident = -1; kwev.ident = -1; uint16_t flags = 0; + // prepare internal eok event eok_event_t ev = {0}; ev.fd = fd; if (event) ev.epev = *event; struct epoll_event *pev = event; switch (op) { case EPOLL_CTL_ADD: { - if (!event) { - e = EINVAL; - break; - } flags = EV_ADD; ev.changed = 1; } break; case EPOLL_CTL_MOD: { - if (!event) { - e = EINVAL; - break; - } flags = EV_ADD; ev.changed = 2; } break; case EPOLL_CTL_DEL: { - D("epoll_ctl adding..."); // event is ignored + // pev points to registered epoll_event pev = &oev->epev; flags = EV_DELETE; ev.changed = 3; @@ -350,6 +361,7 @@ int epoll_ctl(int epfd, int op, int fd, struct epoll_event *event) { if (e) break; + // udata will be delayed to be set if (pev->events & (EPOLLIN | EPOLLERR | EPOLLHUP | EPOLLRDHUP)) { flags |= EV_EOF; EV_SET64(&krev, ev.fd, EVFILT_READ, flags, 0, 0, -1, 0, 0); @@ -358,6 +370,7 @@ int epoll_ctl(int epfd, int op, int fd, struct epoll_event *event) { EV_SET64(&kwev, ev.fd, EVFILT_WRITE, flags, 0, 0, -1, 0, 0); } + // refresh registered evlist and changelist in a transaction way if (eok_chgs_refresh(eok, oev, &ev, &krev, &kwev)) { e = errno; A(e, "internal logic error"); @@ -380,11 +393,6 @@ static struct timespec do_timespec_diff(struct timespec *from, struct timespec * int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) { int e = 0; - if (epfd<0 || epfd>=eoks.neoks) { - errno = EBADF; - E("epoll_waiting epfd:[%d], maxevents:[%d], timeout:[%d] failed", epfd, maxevents, timeout); - return -1; - } if (!events) { errno = EINVAL; E("epoll_waiting epfd:[%d], maxevents:[%d], timeout:[%d] failed", epfd, maxevents, timeout); @@ -396,15 +404,6 @@ int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) return -1; } - int r = 0; - - ep_over_kq_t *eok = eoks_find(epfd); - if (!eok) { - errno = EBADF; - E("epoll_waiting epfd:[%d], maxevents:[%d], timeout:[%d] failed", epfd, maxevents, timeout); - return -1; - } - struct timespec abstime = {0}; A(TIME_UTC==timespec_get(&abstime, TIME_UTC), "internal logic error"); @@ -412,7 +411,17 @@ int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) if (timeout<0) timeout = 0; int64_t t = abstime.tv_nsec + timeout * 1000000; abstime.tv_sec += t / 1000000000; - abstime.tv_nsec %= 1000000000; + abstime.tv_nsec = t % 1000000000; + } + + int r = 0; + + ep_over_kq_t *eok = eoks_find(epfd); + if (!eok) { + errno = EBADF; + E("epoll_waiting epfd:[%d], maxevents:[%d], timeout:[%d] failed", epfd, maxevents, timeout); + errno = EBADF; + return -1; } int cnts = 0; @@ -431,43 +440,52 @@ int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) struct timespec now = {0}; A(TIME_UTC==timespec_get(&now, TIME_UTC), "internal logic error"); struct timespec to = do_timespec_diff(&now, &abstime); - struct timespec *pto = NULL; - if (timeout!=-1) { - pto = &to; + struct timespec *pto = &to; + if (timeout==-1) { + pto = NULL; } - eok->changed = 0; - eok->wakenup = 0; - eok->waiting = 1; - + // taking the changelist struct kevent64_s *kchanges = eok->kchanges; int nchanges = eok->nchanges; int ichanges = eok->ichanges; + // let outside world to add changes eok->kchanges = NULL; eok->nchanges = 0; eok->ichanges = 0; + eok->changed = 0; + eok->wakenup = 0; + eok->waiting = 1; + A(0==pthread_mutex_unlock(&eok->lock), ""); if (ichanges>0) { - D("kevent64 changing [%d] changes and waiting...", ichanges); + D("kevent64 epfd[%d] changing [%d] changes and waiting...", eok->idx, ichanges); } errno = 0; r = kevent64(eok->kq, kchanges, ichanges, eventslist, maxevents, 0, pto); e = errno; if (e) { - E("kevent64 waiting done, with r[%d]", r); + E("kevent64 epfd[%d] waiting done, with r[%d]", eok->idx, r); } A(0==pthread_mutex_lock(&eok->lock), ""); eok->waiting = 0; + if (kchanges) { - free(kchanges); - kchanges = NULL; + if (eok->kchanges==NULL) { + // reuse + A(eok->nchanges==0 && eok->ichanges==0, "internal logic error"); + eok->kchanges = kchanges; + eok->nchanges = nchanges; + } else { + free(kchanges); + kchanges = NULL; + } nchanges = 0; ichanges = 0; } - eok->waiting = 0; if (r==0) { A(timeout!=-1, "internal logic error"); } @@ -484,28 +502,37 @@ int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) case EVFILT_READ: { A((ev->epev.events & EPOLLIN), "internal logic errro"); if (ev->epev.data.ptr==&eok_dummy) { + // it's coming from wakeup socket pair char c = '\0'; A(1==recv(kev->ident, &c, 1, 0), "internal logic error"); A(0==memcmp(&c, "1", 1), "internal logic error"); D("wokenup"); + continue; } else { if (ev->changed==3) { D("already requested to delete for fd[%d]", ev->fd); + // TODO: write a unit test for this case // EV_DELETE? continue; } + // converting to epoll_event + // we shall collect all kevents for the uniq fd into one epoll_evnt + // but currently, taos never use EPOLLOUT + // just let it this way for the moment struct epoll_event pev = {0}; pev.data.ptr = ev->epev.data.ptr; pev.events = EPOLLIN; if (kev->flags & EV_EOF) { + // take all these as EOF for the moment pev.events |= (EPOLLHUP | EPOLLERR | EPOLLRDHUP); - D(".........."); } + // rounded to what user care pev.events = pev.events & ev->epev.events; D("events found for fd[%d]: [%04x:%s], which was registered: [%04x:%s], kev_flags: [%04x:%s]", ev->fd, pev.events, events_str(pev.events, 0), ev->epev.events, events_str(ev->epev.events, 1), kev->flags, kev_flags_str(kev->flags, 2)); + // now we get ev and store it events[cnts++] = pev; } } break; @@ -518,6 +545,8 @@ int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) } } if (r>=0) { + // we can safely rule out delete-requested-events from the regitered evlists + // if only changelist are correctly registered eok_event_t *p = eok->evs_head; while (p) { eok_event_t *next = p->next; @@ -528,6 +557,14 @@ int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) p = next; } } + if (cnts==0) { + // if no user-cared-events is up + // we check to see if time is up + A(TIME_UTC==timespec_get(&now, TIME_UTC), "internal logic error"); + to = do_timespec_diff(&now, &abstime); + if (to.tv_sec==0 && to.tv_nsec==0) break; + // time is not up yet, continue loop + } } while (cnts==0); if (cnts>0) { D("kevent64 waiting done with [%d] events", cnts); @@ -540,6 +577,7 @@ int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) return -1; } + // tell user how many events are valid return cnts; } @@ -547,19 +585,20 @@ static struct timespec do_timespec_diff(struct timespec *from, struct timespec * struct timespec delta; delta.tv_sec = to->tv_sec - from->tv_sec; delta.tv_nsec = to->tv_nsec - from->tv_nsec; + // norm and round up while (delta.tv_nsec<0) { delta.tv_sec -= 1; delta.tv_nsec += 1000000000; } + if (delta.tv_sec < 0) { + delta.tv_sec = 0; + delta.tv_nsec = 0; + } return delta; } int epoll_close(int epfd) { D("epoll_closing epfd: [%d]", epfd); - if (epfd<0 || epfd>=eoks.neoks) { - errno = EBADF; - return -1; - } ep_over_kq_t *eok = eoks_find(epfd); if (!eok) { errno = EBADF; @@ -568,9 +607,11 @@ int epoll_close(int epfd) { A(0==pthread_mutex_lock(&eok->lock), ""); do { + // panic if it would be double-closed A(eok->stopping==0, "internal logic error"); eok->stopping = 1; - A(eok->waiting, "internal logic error"); + // panic if epoll_wait is pending + A(eok->waiting==0, "internal logic error"); if (eok->kq!=-1) { close(eok->kq); @@ -608,9 +649,12 @@ static eok_event_t* eok_calloc_ev(ep_over_kq_t *eok) { p = eok->evs_free; eok->evs_free = p->next; p->next = NULL; + A(p->prev==NULL, "internal logic error"); } else { p = (eok_event_t*)calloc(1, sizeof(*p)); if (!p) return NULL; + A(p->next==NULL, "internal logic error"); + A(p->prev==NULL, "internal logic error"); } // dirty link p->prev = eok->evs_tail; @@ -626,9 +670,9 @@ static eok_event_t* eok_calloc_ev(ep_over_kq_t *eok) { static void eok_free_ev(ep_over_kq_t *eok, eok_event_t *ev) { if (ev->prev) ev->prev->next = ev->next; else eok->evs_head = ev->next; - ev->prev = NULL; if (ev->next) ev->next->prev = ev->prev; else eok->evs_tail = ev->prev; + ev->prev = NULL; ev->next = eok->evs_free; eok->evs_free = ev->next; @@ -639,7 +683,7 @@ static void eok_wakeup(ep_over_kq_t *eok) { if (!eok->waiting) return; if (eok->wakenup) return; eok->wakenup = 1; - send(eok->sv[1], "1", 1, 0); + A(1==send(eok->sv[1], "1", 1, 0), ""); } static int eok_chgs_refresh(ep_over_kq_t *eok, eok_event_t *oev, eok_event_t *ev, struct kevent64_s *krev, struct kevent64_s *kwev) { @@ -653,6 +697,8 @@ static int eok_chgs_refresh(ep_over_kq_t *eok, eok_event_t *oev, eok_event_t *ev struct kevent64_s *p = (struct kevent64_s*)realloc(eok->kchanges, sizeof(*p) * (eok->nchanges + 10)); if (!p) { if (ev->changed==1) { + // roll back + A(oev, "internal logic error"); eok_free_ev(eok, oev); } errno = ENOMEM; @@ -662,12 +708,15 @@ static int eok_chgs_refresh(ep_over_kq_t *eok, eok_event_t *oev, eok_event_t *ev eok->nchanges += 10; } + // copy to registered event slot oev->fd = ev->fd; if (ev->changed!=3) { + // if add/mod, copy epoll_event oev->epev = ev->epev; } oev->changed = ev->changed; + // copy to changes list n = 0; if (krev->ident==ev->fd) { krev->udata = (uint64_t)oev; @@ -679,7 +728,7 @@ static int eok_chgs_refresh(ep_over_kq_t *eok, eok_event_t *oev, eok_event_t *ev eok->kchanges[eok->ichanges++] = *kwev; ++n; } - D("add changes[%d] for fd[%d], and changes/registers [%d/%d]", n, ev->fd, eok->ichanges, eok->evs_count); + D("add #changes[%d] for fd[%d], and now #changes/registers [%d/%d]", n, ev->fd, eok->ichanges, eok->evs_count); return 0; } @@ -707,7 +756,10 @@ static ep_over_kq_t* eoks_alloc(void) { } while (0); A(0==pthread_mutex_unlock(&eoks.lock), ""); - if (!eok) return NULL; + if (!eok) { + errno = ENOMEM; + return NULL; + } if (eok->lock_valid) { return eok; } @@ -718,6 +770,7 @@ static ep_over_kq_t* eoks_alloc(void) { } eoks_free(eok); + errno = ENOMEM; return NULL; } @@ -725,16 +778,23 @@ static void eoks_free(ep_over_kq_t *eok) { A(0==pthread_mutex_lock(&eoks.lock), ""); do { A(eok->next==NULL, "internal logic error"); - A(eok->evs_head==NULL, "internal logic error"); + + // leave eok->kchanges as is + A(eok->ichanges==0, "internal logic error"); + A(eok->waiting==0, "internal logic error"); - if (eok->sv[0]!=-1) { + if (eok->evs_count==1) { + A(eok->evs_head && eok->evs_tail && eok->evs_head==eok->evs_tail, "internal logic error"); + A(eok->evs_head->fd==eok->sv[0] && eok->sv[0]!=-1 && eok->sv[1]!=-1, "internal logic error"); + // fd is critical system resource close(eok->sv[0]); eok->sv[0] = -1; - } - if (eok->sv[1]!=-1) { close(eok->sv[1]); eok->sv[1] = -1; + eok_free_ev(eok, eok->evs_head); } + A(eok->evs_head==NULL && eok->evs_tail==NULL && eok->evs_count==0, "internal logic error"); + A(eok->sv[0]==-1 && eok->sv[1]==-1, "internal logic error"); if (eok->kq!=-1) { close(eok->kq); eok->kq = -1; diff --git a/src/os/src/detail/osSocket.c b/src/os/src/detail/osSocket.c index c6cbbe6b83..659c8510dc 100644 --- a/src/os/src/detail/osSocket.c +++ b/src/os/src/detail/osSocket.c @@ -58,7 +58,6 @@ void taosBlockSIGPIPE() { #ifndef TAOS_OS_FUNC_SOCKET_SETSOCKETOPT int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen) { - fprintf(stderr, "==%s[%d]%s()==\n", basename(__FILE__), __LINE__, __func__); return setsockopt(socketfd, level, optname, optval, (socklen_t)optlen); } diff --git a/src/rpc/src/rpcTcp.c b/src/rpc/src/rpcTcp.c index ff2a5882f8..c65c0db66b 100644 --- a/src/rpc/src/rpcTcp.c +++ b/src/rpc/src/rpcTcp.c @@ -133,7 +133,6 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread } pThreadObj->pollFd = (int64_t)epoll_create(10); // size does not matter - fprintf(stderr, "==%s[%d]%s()==\n", basename(__FILE__), __LINE__, __func__); if (pThreadObj->pollFd < 0) { tError("%s failed to create TCP epoll", label); code = -1; @@ -294,7 +293,6 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int num, void * } pThreadObj->pollFd = (SOCKET)epoll_create(10); // size does not matter - fprintf(stderr, "==%s[%d]%s()==\n", basename(__FILE__), __LINE__, __func__); if (pThreadObj->pollFd < 0) { tError("%s failed to create TCP client epoll", label); free(pThreadObj); @@ -343,10 +341,6 @@ void taosCleanUpTcpClient(void *chandle) { void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port) { SThreadObj * pThreadObj = shandle; - fprintf(stderr, "==%s[%d]%s()==\n", basename(__FILE__), __LINE__, __func__); - fprintf(stderr, "pThreadObj->ip:%d\n", pThreadObj->ip); - fprintf(stderr, "PF_INET/AF_INET:%d/%d\n", PF_INET, AF_INET); - fprintf(stderr, "ip/port:%x/%d\n", ip, port); SOCKET fd = taosOpenTcpClientSocket(ip, port, pThreadObj->ip); if (fd < 0) return NULL; @@ -358,7 +352,6 @@ void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uin localPort = (uint16_t)ntohs(sin.sin_port); } - fprintf(stderr, "==%s[%d]%s()==\n", basename(__FILE__), __LINE__, __func__); SFdObj *pFdObj = taosMallocFdObj(pThreadObj, fd); if (pFdObj) { @@ -369,7 +362,6 @@ void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uin pThreadObj->label, thandle, ip, port, localPort, pFdObj, pThreadObj->numOfFds); } else { tError("%s failed to malloc client FdObj(%s)", pThreadObj->label, strerror(errno)); - fprintf(stderr, "==%s[%d]%s()==\n", basename(__FILE__), __LINE__, __func__); taosCloseSocket(fd); } @@ -492,32 +484,27 @@ static void *taosProcessTcpData(void *param) { if (fdNum < 0) continue; for (int i = 0; i < fdNum; ++i) { - fprintf(stderr, "==%s[%d]%s()==\n", basename(__FILE__), __LINE__, __func__); pFdObj = events[i].data.ptr; if (events[i].events & EPOLLERR) { tDebug("%s %p FD:%p epoll errors", pThreadObj->label, pFdObj->thandle, pFdObj); - fprintf(stderr, "==%s[%d]%s()==\n", basename(__FILE__), __LINE__, __func__); taosReportBrokenLink(pFdObj); continue; } if (events[i].events & EPOLLRDHUP) { tDebug("%s %p FD:%p RD hang up", pThreadObj->label, pFdObj->thandle, pFdObj); - fprintf(stderr, "==%s[%d]%s()==\n", basename(__FILE__), __LINE__, __func__); taosReportBrokenLink(pFdObj); continue; } if (events[i].events & EPOLLHUP) { tDebug("%s %p FD:%p hang up", pThreadObj->label, pFdObj->thandle, pFdObj); - fprintf(stderr, "==%s[%d]%s()==\n", basename(__FILE__), __LINE__, __func__); taosReportBrokenLink(pFdObj); continue; } if (taosReadTcpData(pFdObj, &recvInfo) < 0) { - fprintf(stderr, "==%s[%d]%s()==\n", basename(__FILE__), __LINE__, __func__); shutdown(pFdObj->fd, SHUT_WR); continue; } @@ -563,7 +550,6 @@ static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, SOCKET fd) { event.events = EPOLLIN | EPOLLRDHUP; event.data.ptr = pFdObj; - fprintf(stderr, "==%s[%d]%s()==\n", basename(__FILE__), __LINE__, __func__); if (epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) { tfree(pFdObj); terrno = TAOS_SYSTEM_ERROR(errno); diff --git a/src/sync/src/syncTcp.c b/src/sync/src/syncTcp.c index be85853819..1e5761316a 100644 --- a/src/sync/src/syncTcp.c +++ b/src/sync/src/syncTcp.c @@ -136,15 +136,12 @@ void *syncAllocateTcpConn(void *param, int64_t rid, int32_t connFd) { event.events = EPOLLIN | EPOLLRDHUP; event.data.ptr = pConn; - fprintf(stderr, ">>>>>>>>>>>>>>>>>\n"); if (epoll_ctl(pThread->pollFd, EPOLL_CTL_ADD, connFd, &event) < 0) { - fprintf(stderr, "<<<<<<<<<<<<<<<<<\n"); sError("failed to add fd:%d since %s", connFd, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); tfree(pConn); pConn = NULL; } else { - fprintf(stderr, "<<<<<<<<<<<<<<<<<\n"); pThread->numOfFds++; sDebug("%p fd:%d is added to epoll thread, num:%d", pThread, connFd, pThread->numOfFds); } @@ -170,9 +167,7 @@ static void taosProcessBrokenLink(SConnObj *pConn) { (*pInfo->processBrokenLink)(pConn->handleId); pThread->numOfFds--; - fprintf(stderr, "<<<<<<<<<<<<<<<<<\n"); epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, pConn->fd, NULL); - fprintf(stderr, "<<<<<<<<<<<<<<<<<\n"); sDebug("%p fd:%d is removed from epoll thread, num:%d", pThread, pConn->fd, pThread->numOfFds); taosClose(pConn->fd); tfree(pConn); @@ -287,7 +282,6 @@ static SThreadObj *syncGetTcpThread(SPoolObj *pPool) { pThread->pPool = pPool; pThread->pollFd = epoll_create(10); // size does not matter - fprintf(stderr, "...............\n"); if (pThread->pollFd < 0) { tfree(pThread); return NULL; diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 602aea70e3..5493188c09 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -649,13 +649,15 @@ static STsdbRepo *tsdbNewRepo(char *rootDir, STsdbAppH *pAppH, STsdbCfg *pCfg) { #ifdef __APPLE__ pRepo->readyToCommit = sem_open(NULL, O_CREAT, 0644, 1); - if (!pRepo->readyToCommit) { + if (pRepo->readyToCommit==SEM_FAILED) { + code = errno; terrno = TAOS_SYSTEM_ERROR(code); goto _err; } #else code = sem_init(&(pRepo->readyToCommit), 0, 1); if (code != 0) { + code = errno; terrno = TAOS_SYSTEM_ERROR(code); goto _err; } diff --git a/src/util/src/tsocket.c b/src/util/src/tsocket.c index 6b15d3677c..9fc10576ac 100644 --- a/src/util/src/tsocket.c +++ b/src/util/src/tsocket.c @@ -35,7 +35,7 @@ int32_t taosGetFqdn(char *fqdn) { hints.ai_flags = AI_CANONNAME; int32_t ret = getaddrinfo(hostname, NULL, &hints, &result); if (!result) { - uError("failed to get fqdn for hostname <%s>, code:%d, reason:%s", hostname, ret, gai_strerror(ret)); + uError("failed to get fqdn, code:%d, reason:%s", ret, gai_strerror(ret)); return -1; } @@ -342,6 +342,7 @@ int32_t taosKeepTcpAlive(SOCKET sockFd) { } #ifndef __APPLE__ + // all fails on macosx int32_t probes = 3; if (taosSetSockOpt(sockFd, SOL_TCP, TCP_KEEPCNT, (void *)&probes, sizeof(probes)) < 0) { uError("fd:%d setsockopt SO_KEEPCNT failed: %d (%s)", sockFd, errno, strerror(errno)); From 0c19d10929de5f598bdccb35c4295c72fcbdeecb Mon Sep 17 00:00:00 2001 From: freemine Date: Sun, 17 Jan 2021 01:04:09 +0800 Subject: [PATCH 015/147] add ENABLE_LOG macro to swith on/off D/E stuffs --- src/os/src/darwin/eok.c | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/os/src/darwin/eok.c b/src/os/src/darwin/eok.c index 3f7137f6a1..322aa7cf86 100644 --- a/src/os/src/darwin/eok.c +++ b/src/os/src/darwin/eok.c @@ -10,7 +10,19 @@ #include #include +#ifdef ENABLE_LOG #define D(fmt, ...) fprintf(stderr, "%s[%d]%s(): " fmt "\n", basename(__FILE__), __LINE__, __func__, ##__VA_ARGS__) +#define E(fmt, ...) do { \ + fprintf(stderr, "%s[%d]%s(): %d[%s]: " fmt "\n", \ + basename(__FILE__), __LINE__, __func__, \ + errno, strerror(errno), \ + ##__VA_ARGS__); \ +} while (0) +#else // !ENABLE_LOG +#define D(fmt, ...) (void)fmt +#define E(fmt, ...) (void)fmt +#endif // ENABLE_LOG + #define A(statement, fmt, ...) do { \ if (statement) break; \ fprintf(stderr, "%s[%d]%s(): assert [%s] failed: %d[%s]: " fmt "\n", \ @@ -20,13 +32,6 @@ abort(); \ } while (0) -#define E(fmt, ...) do { \ - fprintf(stderr, "%s[%d]%s(): %d[%s]: " fmt "\n", \ - basename(__FILE__), __LINE__, __func__, \ - errno, strerror(errno), \ - ##__VA_ARGS__); \ -} while (0) - static int eok_dummy = 0; typedef struct ep_over_kq_s ep_over_kq_t; @@ -95,6 +100,7 @@ static eoks_t eoks = { .eoks_free = NULL, }; +#ifdef ENABLE_LOG static const char* op_str(int op) { switch (op) { case EPOLL_CTL_ADD: return "EPOLL_CTL_ADD"; @@ -230,6 +236,7 @@ static const char* kev_flags_str(uint16_t flags, int slots) { #undef CHK_EV return buf; } +#endif // ENABLE_LOG static ep_over_kq_t* eoks_alloc(void); static void eoks_free(ep_over_kq_t *eok); From 54af96b6f06e04725eec3ea48ff876d10b76e06f Mon Sep 17 00:00:00 2001 From: freemine Date: Sun, 17 Jan 2021 01:37:58 +0800 Subject: [PATCH 016/147] adding TAOS license header --- src/CMakeLists.txt | 2 +- src/os/inc/eok.h | 15 +++++++++++++++ src/os/src/darwin/eok.c | 15 +++++++++++++++ tests/examples/c/epoll.c | 15 +++++++++++++++ 4 files changed, 46 insertions(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 71ed16afa1..04bc61ed9e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -19,5 +19,5 @@ ADD_SUBDIRECTORY(wal) ADD_SUBDIRECTORY(cq) ADD_SUBDIRECTORY(dnode) #ADD_SUBDIRECTORY(connector/odbc) -#ADD_SUBDIRECTORY(connector/jdbc) +ADD_SUBDIRECTORY(connector/jdbc) diff --git a/src/os/inc/eok.h b/src/os/inc/eok.h index a0fd5b5d8e..4d38e8e632 100644 --- a/src/os/inc/eok.h +++ b/src/os/inc/eok.h @@ -1,3 +1,18 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + #ifndef _eok_h_fd274616_996c_400e_9023_ae70be881fa3_ #define _eok_h_fd274616_996c_400e_9023_ae70be881fa3_ diff --git a/src/os/src/darwin/eok.c b/src/os/src/darwin/eok.c index 322aa7cf86..f9465d29dd 100644 --- a/src/os/src/darwin/eok.c +++ b/src/os/src/darwin/eok.c @@ -1,3 +1,18 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + #include "eok.h" #include diff --git a/tests/examples/c/epoll.c b/tests/examples/c/epoll.c index 6047c9e4ea..1ef9b3eddf 100644 --- a/tests/examples/c/epoll.c +++ b/tests/examples/c/epoll.c @@ -1,3 +1,18 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + #ifdef __APPLE__ #include "eok.h" #else From 60a9725d031d36ff0f0903863080d434182555e4 Mon Sep 17 00:00:00 2001 From: freemine Date: Sun, 17 Jan 2021 02:07:25 +0800 Subject: [PATCH 017/147] replace array initializer with memset --- tests/examples/c/epoll.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/examples/c/epoll.c b/tests/examples/c/epoll.c index 1ef9b3eddf..eb66156c14 100644 --- a/tests/examples/c/epoll.c +++ b/tests/examples/c/epoll.c @@ -177,7 +177,8 @@ static void* routine(void* arg) { ep_t *ep = (ep_t*)arg; while (!ep->stopping) { - struct epoll_event evs[10] = {0}; + struct epoll_event evs[10]; + memset(evs, 0, sizeof(evs)); A(0==pthread_mutex_lock(&ep->lock), ""); A(ep->waiting==0, "internal logic error"); From 6530365e580e034615b4c550a9d0514a2b5f4be8 Mon Sep 17 00:00:00 2001 From: freemine Date: Sun, 17 Jan 2021 08:13:51 +0800 Subject: [PATCH 018/147] bugFix: check timeout as well --- src/os/src/darwin/eok.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/os/src/darwin/eok.c b/src/os/src/darwin/eok.c index f9465d29dd..e2f7c55284 100644 --- a/src/os/src/darwin/eok.c +++ b/src/os/src/darwin/eok.c @@ -25,6 +25,8 @@ #include #include +#define ENABLE_LOG + #ifdef ENABLE_LOG #define D(fmt, ...) fprintf(stderr, "%s[%d]%s(): " fmt "\n", basename(__FILE__), __LINE__, __func__, ##__VA_ARGS__) #define E(fmt, ...) do { \ @@ -582,9 +584,11 @@ int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) if (cnts==0) { // if no user-cared-events is up // we check to see if time is up - A(TIME_UTC==timespec_get(&now, TIME_UTC), "internal logic error"); - to = do_timespec_diff(&now, &abstime); - if (to.tv_sec==0 && to.tv_nsec==0) break; + if (timeout!=-1) { + A(TIME_UTC==timespec_get(&now, TIME_UTC), "internal logic error"); + to = do_timespec_diff(&now, &abstime); + if (to.tv_sec==0 && to.tv_nsec==0) break; + } // time is not up yet, continue loop } } while (cnts==0); From c5dfa590ec3bc5546e23d15357e92dda2156452c Mon Sep 17 00:00:00 2001 From: freemine Date: Sun, 17 Jan 2021 08:15:31 +0800 Subject: [PATCH 019/147] typo: by default, ENABLE_LOG is not defined --- src/os/src/darwin/eok.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/os/src/darwin/eok.c b/src/os/src/darwin/eok.c index e2f7c55284..a6dceb3df3 100644 --- a/src/os/src/darwin/eok.c +++ b/src/os/src/darwin/eok.c @@ -25,8 +25,6 @@ #include #include -#define ENABLE_LOG - #ifdef ENABLE_LOG #define D(fmt, ...) fprintf(stderr, "%s[%d]%s(): " fmt "\n", basename(__FILE__), __LINE__, __func__, ##__VA_ARGS__) #define E(fmt, ...) do { \ From f8ee5fc4133ddbe413c02d52abcb3b3148acd455 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 17 Jan 2021 13:07:37 +0800 Subject: [PATCH 020/147] TD-1207 --- src/plugins/http/inc/httpInt.h | 2 ++ src/plugins/http/src/httpServer.c | 5 +++++ src/plugins/http/src/httpSystem.c | 5 +++++ src/rpc/src/rpcTcp.c | 16 +++++++++++++++- src/sync/src/syncTcp.c | 13 +++++++++++++ 5 files changed, 40 insertions(+), 1 deletion(-) diff --git a/src/plugins/http/inc/httpInt.h b/src/plugins/http/inc/httpInt.h index 31b06714d6..00a7bcf6c5 100644 --- a/src/plugins/http/inc/httpInt.h +++ b/src/plugins/http/inc/httpInt.h @@ -177,6 +177,8 @@ typedef struct HttpServer { char label[HTTP_LABEL_SIZE]; uint32_t serverIp; uint16_t serverPort; + int8_t stop; + int8_t reserve; SOCKET fd; int32_t numOfThreads; int32_t methodScannerLen; diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index ee6addd6fa..5a04a021cd 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -171,6 +171,11 @@ static void *httpAcceptHttpConnection(void *arg) { while (1) { socklen_t addrlen = sizeof(clientAddr); connFd = (int32_t)accept(pServer->fd, (struct sockaddr *)&clientAddr, &addrlen); + if (pServer->stop) { + httpDebug("http server:%s socket stop, exiting...", pServer->label); + break; + } + if (connFd == -1) { if (errno == EINVAL) { httpDebug("http server:%s socket was shutdown, exiting...", pServer->label); diff --git a/src/plugins/http/src/httpSystem.c b/src/plugins/http/src/httpSystem.c index 34a70a658b..df39c080a5 100644 --- a/src/plugins/http/src/httpSystem.c +++ b/src/plugins/http/src/httpSystem.c @@ -89,7 +89,12 @@ int32_t httpStartSystem() { void httpStopSystem() { tsHttpServer.status = HTTP_SERVER_CLOSING; + tsHttpServer.stop = 1; +#ifdef WINDOWS + closesocket(tsHttpServer.fd); +#else shutdown(tsHttpServer.fd, SHUT_RD); +#endif tgCleanupHandle(); } diff --git a/src/rpc/src/rpcTcp.c b/src/rpc/src/rpcTcp.c index 08ae4b5b8b..4cc7530784 100644 --- a/src/rpc/src/rpcTcp.c +++ b/src/rpc/src/rpcTcp.c @@ -59,6 +59,8 @@ typedef struct { SOCKET fd; uint32_t ip; uint16_t port; + int8_t stop; + int8_t reserve; char label[TSDB_LABEL_LEN]; int numOfThreads; void * shandle; @@ -188,8 +190,15 @@ void taosStopTcpServer(void *handle) { SServerObj *pServerObj = handle; if (pServerObj == NULL) return; - if(pServerObj->fd >=0) shutdown(pServerObj->fd, SHUT_RD); + pServerObj->stop = 1; + if (pServerObj->fd >= 0) { +#ifdef WINDOWS + closesocket(pServerObj->fd); +#else + shutdown(pServerObj->fd, SHUT_RD); +#endif + } if (taosCheckPthreadValid(pServerObj->thread)) { if (taosComparePthread(pServerObj->thread, pthread_self())) { pthread_detach(pthread_self()); @@ -230,6 +239,11 @@ static void *taosAcceptTcpConnection(void *arg) { while (1) { socklen_t addrlen = sizeof(caddr); connFd = accept(pServerObj->fd, (struct sockaddr *)&caddr, &addrlen); + if (pServerObj->stop) { + tDebug("%s TCP server stop accepting new connections", pServerObj->label); + break; + } + if (connFd == -1) { if (errno == EINVAL) { tDebug("%s TCP server stop accepting new connections, exiting", pServerObj->label); diff --git a/src/sync/src/syncTcp.c b/src/sync/src/syncTcp.c index 829c9ceec6..72ba8e26b2 100644 --- a/src/sync/src/syncTcp.c +++ b/src/sync/src/syncTcp.c @@ -46,6 +46,7 @@ typedef struct SPoolObj { pthread_t thread; int32_t nextId; SOCKET acceptFd; // FD for accept new connection + int8_t stop; } SPoolObj; typedef struct { @@ -106,7 +107,14 @@ void syncCloseTcpThreadPool(void *param) { SPoolObj * pPool = param; SThreadObj *pThread; + pPool->stop = 1; + +#ifdef WINDOWS + closesocket(pPool->acceptFd); +#else shutdown(pPool->acceptFd, SHUT_RD); +#endif + pthread_join(pPool->thread, NULL); for (int32_t i = 0; i < pPool->info.numOfThreads; ++i) { @@ -257,6 +265,11 @@ static void *syncAcceptPeerTcpConnection(void *argv) { struct sockaddr_in clientAddr; socklen_t addrlen = sizeof(clientAddr); SOCKET connFd = accept(pPool->acceptFd, (struct sockaddr *)&clientAddr, &addrlen); + if (pPool->stop) { + sDebug("%p TCP server accept is stopped", pPool); + break; + } + if (connFd < 0) { if (errno == EINVAL) { sDebug("%p TCP server accept is exiting...", pPool); From 5d9ebb25b0f84c4543bb2cbb82cb2ff3b227fcd5 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 17 Jan 2021 20:47:53 +0800 Subject: [PATCH 021/147] TD-1207 --- src/dnode/src/dnodeSystem.c | 14 ++++++++------ src/os/inc/osSignal.h | 4 ++++ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/dnode/src/dnodeSystem.c b/src/dnode/src/dnodeSystem.c index bdaecf2e0f..3a50eca9e6 100644 --- a/src/dnode/src/dnodeSystem.c +++ b/src/dnode/src/dnodeSystem.c @@ -120,6 +120,7 @@ int32_t main(int32_t argc, char *argv[]) { taosSetSignal(SIGHUP, sigintHandler); taosSetSignal(SIGINT, sigintHandler); taosSetSignal(SIGABRT, sigintHandler); + taosSetSignal(SIGBREAK, sigintHandler); // Open /var/log/syslog file to record information. openlog("TDengine:", LOG_PID | LOG_CONS | LOG_NDELAY, LOG_LOCAL1); @@ -152,12 +153,6 @@ static void siguser1Handler(int32_t signum) { taosCfgDynamicOptions("debugFlag 1 static void siguser2Handler(int32_t signum) { taosCfgDynamicOptions("resetlog"); } static void sigintHandler(int32_t signum) { - // clean the system. - dInfo("shut down signal is %d", signum); - - syslog(LOG_INFO, "Shut down signal is %d", signum); - syslog(LOG_INFO, "Shutting down TDengine service..."); - // protect the application from receive another signal taosIgnSignal(SIGUSR1); taosIgnSignal(SIGUSR2); @@ -165,6 +160,13 @@ static void sigintHandler(int32_t signum) { taosIgnSignal(SIGHUP); taosIgnSignal(SIGINT); taosIgnSignal(SIGABRT); + taosIgnSignal(SIGBREAK); + + // clean the system. + dInfo("shut down signal is %d", signum); + + syslog(LOG_INFO, "Shut down signal is %d", signum); + syslog(LOG_INFO, "Shutting down TDengine service..."); // inform main thread to exit tsem_post(&exitSem); diff --git a/src/os/inc/osSignal.h b/src/os/inc/osSignal.h index 8b047b9e25..57582a8a28 100644 --- a/src/os/inc/osSignal.h +++ b/src/os/inc/osSignal.h @@ -44,6 +44,10 @@ extern "C" { #define SIGUSR2 1234 #endif +#ifndef SIGBREAK + #define SIGBREAK 1234 +#endif + typedef void (*FSignalHandler)(int32_t signum); void taosSetSignal(int32_t signum, FSignalHandler sigfp); void taosIgnSignal(int32_t signum); From 0c96f7653ec4d5c697ff6d504afc59bc40002fb2 Mon Sep 17 00:00:00 2001 From: freemine Date: Sun, 17 Jan 2021 21:40:39 +0800 Subject: [PATCH 022/147] 1. bugfix: eok.c: array of pointers rather than POD buffer 2. initial port of taosd for MacOSX --- cmake/define.inc | 2 + deps/CMakeLists.txt | 6 +- packaging/tools/install_client.sh | 2 +- packaging/tools/make_install.sh | 14 ++- src/balance/CMakeLists.txt | 4 + src/client/CMakeLists.txt | 22 +++++ src/cq/CMakeLists.txt | 12 ++- src/dnode/CMakeLists.txt | 41 +++++++- src/dnode/src/dnodeSystem.c | 4 + src/dnode/src/dnodeTelemetry.c | 7 ++ src/kit/shell/CMakeLists.txt | 13 ++- src/kit/taosdemo/CMakeLists.txt | 11 ++- src/kit/taosdemox/CMakeLists.txt | 34 +++++-- src/kit/taosdump/CMakeLists.txt | 12 ++- src/mnode/CMakeLists.txt | 13 ++- src/mnode/src/mnodeCluster.c | 7 ++ src/os/src/darwin/eok.c | 99 ++++++++++++++------ src/os/src/detail/CMakeLists.txt | 1 + src/plugins/http/CMakeLists.txt | 19 +++- src/plugins/http/src/httpServer.c | 46 +++++++++ src/plugins/monitor/CMakeLists.txt | 14 ++- src/plugins/mqtt/CMakeLists.txt | 12 +++ src/query/CMakeLists.txt | 5 + src/rpc/src/rpcTcp.c | 10 +- src/rpc/test/CMakeLists.txt | 14 +++ src/sync/CMakeLists.txt | 15 ++- src/sync/src/syncRetrieve.c | 2 +- src/sync/test/CMakeLists.txt | 1 - src/util/CMakeLists.txt | 3 +- src/vnode/CMakeLists.txt | 6 ++ src/wal/CMakeLists.txt | 8 +- src/wal/test/CMakeLists.txt | 8 ++ tests/comparisonTest/tdengine/CMakeLists.txt | 5 + tests/examples/c/CMakeLists.txt | 3 + tests/examples/c/demo.c | 2 +- 35 files changed, 413 insertions(+), 64 deletions(-) diff --git a/cmake/define.inc b/cmake/define.inc index 5d4d94ff42..ba12df6f0c 100755 --- a/cmake/define.inc +++ b/cmake/define.inc @@ -128,6 +128,8 @@ IF (TD_DARWIN_64) SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -Wno-missing-braces -fPIC -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") SET(DEBUG_FLAGS "-O0 -g3 -DDEBUG") SET(RELEASE_FLAGS "-Og") + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/lz4/inc) ENDIF () IF (TD_WINDOWS) diff --git a/deps/CMakeLists.txt b/deps/CMakeLists.txt index 1d725add21..1e59396c70 100644 --- a/deps/CMakeLists.txt +++ b/deps/CMakeLists.txt @@ -12,4 +12,8 @@ ADD_SUBDIRECTORY(MsvcLibX) IF (TD_LINUX AND TD_MQTT) ADD_SUBDIRECTORY(MQTT-C) -ENDIF () \ No newline at end of file +ENDIF () + +IF (TD_DARWIN AND TD_MQTT) + ADD_SUBDIRECTORY(MQTT-C) +ENDIF () diff --git a/packaging/tools/install_client.sh b/packaging/tools/install_client.sh index dd116e9bfb..84ca3b5131 100755 --- a/packaging/tools/install_client.sh +++ b/packaging/tools/install_client.sh @@ -21,7 +21,7 @@ else cd ${script_dir} script_dir="$(pwd)" data_dir="/var/lib/taos" - log_dir="~/TDengineLog" + log_dir=~/TDengineLog fi log_link_dir="/usr/local/taos/log" diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh index 831012851a..0727fe2a1e 100755 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -24,7 +24,7 @@ data_dir="/var/lib/taos" if [ "$osType" != "Darwin" ]; then log_dir="/var/log/taos" else - log_dir="~/TDengineLog" + log_dir=~/TDengineLog fi data_link_dir="/usr/local/taos/data" @@ -178,7 +178,9 @@ function install_bin() { function install_lib() { # Remove links ${csudo} rm -f ${lib_link_dir}/libtaos.* || : - ${csudo} rm -f ${lib64_link_dir}/libtaos.* || : + if [ "$osType" != "Darwin" ]; then + ${csudo} rm -f ${lib64_link_dir}/libtaos.* || : + fi if [ "$osType" != "Darwin" ]; then ${csudo} cp ${binary_dir}/build/lib/libtaos.so.${verNumber} ${install_main_dir}/driver && ${csudo} chmod 777 ${install_main_dir}/driver/* @@ -190,12 +192,14 @@ function install_lib() { ${csudo} ln -sf ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so fi else - ${csudo} cp ${binary_dir}/build/lib/libtaos.* ${install_main_dir}/driver && ${csudo} chmod 777 ${install_main_dir}/driver/* - ${csudo} ln -sf ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.1.dylib + ${csudo} cp -Rf ${binary_dir}/build/lib/libtaos.* ${install_main_dir}/driver && ${csudo} chmod 777 ${install_main_dir}/driver/* + ${csudo} ln -sf ${install_main_dir}/driver/libtaos.1.dylib ${lib_link_dir}/libtaos.1.dylib ${csudo} ln -sf ${lib_link_dir}/libtaos.1.dylib ${lib_link_dir}/libtaos.dylib fi - ${csudo} ldconfig + if [ "$osType" != "Darwin" ]; then + ${csudo} ldconfig + fi } function install_header() { diff --git a/src/balance/CMakeLists.txt b/src/balance/CMakeLists.txt index fdc43ea32f..a3ae74d825 100644 --- a/src/balance/CMakeLists.txt +++ b/src/balance/CMakeLists.txt @@ -11,3 +11,7 @@ AUX_SOURCE_DIRECTORY(src SRC) IF (TD_LINUX) ADD_LIBRARY(balance ${SRC}) ENDIF () + +IF (TD_DARWIN) + ADD_LIBRARY(balance ${SRC}) +ENDIF () diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index 3f8f9d2934..660ad564a5 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -28,6 +28,28 @@ IF (TD_LINUX) ADD_SUBDIRECTORY(tests) +ELSEIF (TD_DARWIN) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/linux) + + # set the static lib name + ADD_LIBRARY(taos_static STATIC ${SRC}) + TARGET_LINK_LIBRARIES(taos_static common query trpc tutil pthread m) + SET_TARGET_PROPERTIES(taos_static PROPERTIES OUTPUT_NAME "taos_static") + SET_TARGET_PROPERTIES(taos_static PROPERTIES CLEAN_DIRECT_OUTPUT 1) + + # generate dynamic library (*.dylib) + ADD_LIBRARY(taos SHARED ${SRC}) + TARGET_LINK_LIBRARIES(taos common query trpc tutil pthread m) + SET_TARGET_PROPERTIES(taos PROPERTIES CLEAN_DIRECT_OUTPUT 1) + + #set version of .dylib + #VERSION dylib version + #SOVERSION dylib version + #MESSAGE(STATUS "build version ${TD_VER_NUMBER}") + SET_TARGET_PROPERTIES(taos PROPERTIES VERSION ${TD_VER_NUMBER} SOVERSION 1) + + ADD_SUBDIRECTORY(tests) + ELSEIF (TD_WINDOWS) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/windows) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/windows/win32) diff --git a/src/cq/CMakeLists.txt b/src/cq/CMakeLists.txt index 9da831c9c1..34146241a5 100644 --- a/src/cq/CMakeLists.txt +++ b/src/cq/CMakeLists.txt @@ -10,7 +10,17 @@ IF (TD_LINUX) ADD_LIBRARY(tcq ${SRC}) IF (TD_SOMODE_STATIC) TARGET_LINK_LIBRARIES(tcq tutil common taos_static) - ELSE () + ELSE () + TARGET_LINK_LIBRARIES(tcq tutil common taos) + ENDIF () + ADD_SUBDIRECTORY(test) +ENDIF () + +IF (TD_DARWIN) + ADD_LIBRARY(tcq ${SRC}) + IF (TD_SOMODE_STATIC) + TARGET_LINK_LIBRARIES(tcq tutil common taos_static) + ELSE () TARGET_LINK_LIBRARIES(tcq tutil common taos) ENDIF () ADD_SUBDIRECTORY(test) diff --git a/src/dnode/CMakeLists.txt b/src/dnode/CMakeLists.txt index 699ca00a25..905ad9d920 100644 --- a/src/dnode/CMakeLists.txt +++ b/src/dnode/CMakeLists.txt @@ -23,7 +23,7 @@ IF (TD_LINUX) IF (TD_ACCOUNT) TARGET_LINK_LIBRARIES(taosd account) ENDIF () - + IF (TD_GRANT) TARGET_LINK_LIBRARIES(taosd grant) ENDIF () @@ -47,3 +47,42 @@ IF (TD_LINUX) COMMENT "prepare taosd environment") ADD_CUSTOM_TARGET(${PREPARE_ENV_TARGET} ALL WORKING_DIRECTORY ${TD_EXECUTABLE_OUTPUT_PATH} DEPENDS ${PREPARE_ENV_CMD}) ENDIF () + +IF (TD_DARWIN) + ADD_EXECUTABLE(taosd ${SRC}) + TARGET_LINK_LIBRARIES(taosd mnode monitor http tsdb twal vnode cJson lz4 balance sync) + + IF (TD_SOMODE_STATIC) + TARGET_LINK_LIBRARIES(taosd taos_static) + ELSE () + TARGET_LINK_LIBRARIES(taosd taos) + ENDIF () + + IF (TD_ACCOUNT) + TARGET_LINK_LIBRARIES(taosd account) + ENDIF () + + IF (TD_GRANT) + TARGET_LINK_LIBRARIES(taosd grant) + ENDIF () + + IF (TD_MQTT) + TARGET_LINK_LIBRARIES(taosd mqtt) + ENDIF () + + # SET(PREPARE_ENV_CMD "prepare_env_cmd") + # SET(PREPARE_ENV_TARGET "prepare_env_target") + # ADD_CUSTOM_COMMAND(OUTPUT ${PREPARE_ENV_CMD} + # POST_BUILD + # COMMAND echo "make test directory" + # DEPENDS taosd + # COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/cfg/ + # COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/log/ + # COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/data/ + # COMMAND ${CMAKE_COMMAND} -E echo dataDir ${TD_TESTS_OUTPUT_DIR}/data > ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg + # COMMAND ${CMAKE_COMMAND} -E echo logDir ${TD_TESTS_OUTPUT_DIR}/log >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg + # COMMAND ${CMAKE_COMMAND} -E echo charset UTF-8 >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg + # COMMENT "prepare taosd environment") + # ADD_CUSTOM_TARGET(${PREPARE_ENV_TARGET} ALL WORKING_DIRECTORY ${TD_EXECUTABLE_OUTPUT_PATH} DEPENDS ${PREPARE_ENV_CMD}) +ENDIF () + diff --git a/src/dnode/src/dnodeSystem.c b/src/dnode/src/dnodeSystem.c index a4d7e791e6..893049df2e 100644 --- a/src/dnode/src/dnodeSystem.c +++ b/src/dnode/src/dnodeSystem.c @@ -160,7 +160,11 @@ static void signal_handler(int32_t signum, siginfo_t *sigInfo, void *context) { syslog(LOG_INFO, "Shut down signal is %d", signum); syslog(LOG_INFO, "Shutting down TDengine service..."); // clean the system. +#ifdef __APPLE__ + dInfo("shut down signal is %d, sender PID:%d", signum, sigInfo->si_pid); +#else // __APPLE__ dInfo("shut down signal is %d, sender PID:%d cmdline:%s", signum, sigInfo->si_pid, taosGetCmdlineByPID(sigInfo->si_pid)); +#endif // protect the application from receive another signal struct sigaction act = {{0}}; diff --git a/src/dnode/src/dnodeTelemetry.c b/src/dnode/src/dnodeTelemetry.c index ff9598ecc5..1f7084c3b5 100644 --- a/src/dnode/src/dnodeTelemetry.c +++ b/src/dnode/src/dnodeTelemetry.c @@ -236,6 +236,13 @@ static void sendTelemetryReport() { taosCloseSocket(fd); } +#ifdef __APPLE__ +static int sem_timedwait(tsem_t *sem, struct timespec *to) { + fprintf(stderr, "%s[%d]%s(): not implemented yet!\n", basename(__FILE__), __LINE__, __func__); + abort(); +} +#endif + static void* telemetryThread(void* param) { struct timespec end = {0}; clock_gettime(CLOCK_REALTIME, &end); diff --git a/src/kit/shell/CMakeLists.txt b/src/kit/shell/CMakeLists.txt index 45da99e572..c4f3cc5696 100644 --- a/src/kit/shell/CMakeLists.txt +++ b/src/kit/shell/CMakeLists.txt @@ -9,14 +9,14 @@ IF (TD_LINUX) AUX_SOURCE_DIRECTORY(./src SRC) LIST(REMOVE_ITEM SRC ./src/shellWindows.c) LIST(REMOVE_ITEM SRC ./src/shellDarwin.c) - ADD_EXECUTABLE(shell ${SRC}) - + ADD_EXECUTABLE(shell ${SRC}) + IF (TD_SOMODE_STATIC) TARGET_LINK_LIBRARIES(shell taos_static) ELSE () TARGET_LINK_LIBRARIES(shell taos) ENDIF () - + SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME taos) ELSEIF (TD_WINDOWS) LIST(APPEND SRC ./src/shellEngine.c) @@ -27,7 +27,7 @@ ELSEIF (TD_WINDOWS) IF (TD_POWER) SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME power) - ELSE () + ELSE () SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME taos) ENDIF () ELSEIF (TD_DARWIN) @@ -37,7 +37,10 @@ ELSEIF (TD_DARWIN) LIST(APPEND SRC ./src/shellCommand.c) LIST(APPEND SRC ./src/shellImport.c) ADD_EXECUTABLE(shell ${SRC}) - TARGET_LINK_LIBRARIES(shell taos_static) + # linking with dylib + TARGET_LINK_LIBRARIES(shell taos) + # linking taos statically + # TARGET_LINK_LIBRARIES(shell taos_static) SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME taos) ENDIF () diff --git a/src/kit/taosdemo/CMakeLists.txt b/src/kit/taosdemo/CMakeLists.txt index 91c743939c..f74dbc2de4 100644 --- a/src/kit/taosdemo/CMakeLists.txt +++ b/src/kit/taosdemo/CMakeLists.txt @@ -7,7 +7,7 @@ INCLUDE_DIRECTORIES(inc) IF (TD_LINUX) AUX_SOURCE_DIRECTORY(. SRC) ADD_EXECUTABLE(taosdemo ${SRC}) - + IF (TD_SOMODE_STATIC) TARGET_LINK_LIBRARIES(taosdemo taos_static) ELSE () @@ -17,4 +17,13 @@ ELSEIF (TD_WINDOWS) AUX_SOURCE_DIRECTORY(. SRC) ADD_EXECUTABLE(taosdemo ${SRC}) TARGET_LINK_LIBRARIES(taosdemo taos_static) +ELSEIF (TD_DARWIN) + AUX_SOURCE_DIRECTORY(. SRC) + ADD_EXECUTABLE(taosdemo ${SRC}) + + IF (TD_SOMODE_STATIC) + TARGET_LINK_LIBRARIES(taosdemo taos_static) + ELSE () + TARGET_LINK_LIBRARIES(taosdemo taos) + ENDIF () ENDIF () diff --git a/src/kit/taosdemox/CMakeLists.txt b/src/kit/taosdemox/CMakeLists.txt index 7db6c04b28..3993cb0feb 100644 --- a/src/kit/taosdemox/CMakeLists.txt +++ b/src/kit/taosdemox/CMakeLists.txt @@ -4,22 +4,44 @@ PROJECT(TDengine) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/libcurl/include) -IF (TD_LINUX) +IF (TD_LINUX) AUX_SOURCE_DIRECTORY(. SRC) ADD_EXECUTABLE(taosdemox ${SRC}) - - #find_program(HAVE_CURL NAMES curl) - IF ((NOT TD_ARM_64) AND (NOT TD_ARM_32)) + + #find_program(HAVE_CURL NAMES curl) + IF ((NOT TD_ARM_64) AND (NOT TD_ARM_32)) ADD_DEFINITIONS(-DTD_LOWA_CURL) LINK_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/libcurl/lib) ADD_LIBRARY(curl STATIC IMPORTED) SET_PROPERTY(TARGET curl PROPERTY IMPORTED_LOCATION ${TD_COMMUNITY_DIR}/deps/libcurl/lib/libcurl.a) - TARGET_LINK_LIBRARIES(taosdemox curl) + TARGET_LINK_LIBRARIES(taosdemox curl) ENDIF () - + IF (TD_SOMODE_STATIC) TARGET_LINK_LIBRARIES(taosdemox taos_static cJson) ELSE () TARGET_LINK_LIBRARIES(taosdemox taos cJson) ENDIF () ENDIF () + +IF (TD_DARWIN) + # missing a few dependencies, such as + # AUX_SOURCE_DIRECTORY(. SRC) + # ADD_EXECUTABLE(taosdemox ${SRC}) + # + # #find_program(HAVE_CURL NAMES curl) + # IF ((NOT TD_ARM_64) AND (NOT TD_ARM_32)) + # ADD_DEFINITIONS(-DTD_LOWA_CURL) + # LINK_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/libcurl/lib) + # ADD_LIBRARY(curl STATIC IMPORTED) + # SET_PROPERTY(TARGET curl PROPERTY IMPORTED_LOCATION ${TD_COMMUNITY_DIR}/deps/libcurl/lib/libcurl.a) + # TARGET_LINK_LIBRARIES(taosdemox curl) + # ENDIF () + # + # IF (TD_SOMODE_STATIC) + # TARGET_LINK_LIBRARIES(taosdemox taos_static cJson) + # ELSE () + # TARGET_LINK_LIBRARIES(taosdemox taos cJson) + # ENDIF () +ENDIF () + diff --git a/src/kit/taosdump/CMakeLists.txt b/src/kit/taosdump/CMakeLists.txt index dc9ac6d4a7..b50ad85c08 100644 --- a/src/kit/taosdump/CMakeLists.txt +++ b/src/kit/taosdump/CMakeLists.txt @@ -12,5 +12,15 @@ IF (TD_LINUX) TARGET_LINK_LIBRARIES(taosdump taos_static) ELSE () TARGET_LINK_LIBRARIES(taosdump taos) - ENDIF () + ENDIF () +ENDIF () + +IF (TD_DARWIN) + # missing for macosx + # ADD_EXECUTABLE(taosdump ${SRC}) + # IF (TD_SOMODE_STATIC) + # TARGET_LINK_LIBRARIES(taosdump taos_static) + # ELSE () + # TARGET_LINK_LIBRARIES(taosdump taos) + # ENDIF () ENDIF () diff --git a/src/mnode/CMakeLists.txt b/src/mnode/CMakeLists.txt index 4123098694..ae4a37320c 100644 --- a/src/mnode/CMakeLists.txt +++ b/src/mnode/CMakeLists.txt @@ -9,4 +9,15 @@ IF (TD_LINUX) AUX_SOURCE_DIRECTORY(src SRC) ADD_LIBRARY(mnode ${SRC}) -ENDIF () \ No newline at end of file +ENDIF () + +IF (TD_DARWIN) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/dnode/inc) + + INCLUDE_DIRECTORIES(inc) + AUX_SOURCE_DIRECTORY(src SRC) + + ADD_LIBRARY(mnode ${SRC}) +ENDIF () + diff --git a/src/mnode/src/mnodeCluster.c b/src/mnode/src/mnodeCluster.c index a35e304810..08bf20e1a8 100644 --- a/src/mnode/src/mnodeCluster.c +++ b/src/mnode/src/mnodeCluster.c @@ -138,6 +138,13 @@ void mnodeDecClusterRef(SClusterObj *pCluster) { sdbDecRef(tsClusterSdb, pCluster); } +#ifdef __APPLE__ +bool taosGetSystemUid(char *uid) { + fprintf(stderr, "%s[%d]%s(): not implemented yet!\n", basename(__FILE__), __LINE__, __func__); + abort(); + return false; +} +#endif // __APPLE__ static int32_t mnodeCreateCluster() { int32_t numOfClusters = sdbGetNumOfRows(tsClusterSdb); if (numOfClusters != 0) return TSDB_CODE_SUCCESS; diff --git a/src/os/src/darwin/eok.c b/src/os/src/darwin/eok.c index a6dceb3df3..1be5ec4ac1 100644 --- a/src/os/src/darwin/eok.c +++ b/src/os/src/darwin/eok.c @@ -103,9 +103,9 @@ struct eok_event_s { typedef struct eoks_s eoks_t; struct eoks_s { pthread_mutex_t lock; - ep_over_kq_t *eoks; - int neoks; - ep_over_kq_t *eoks_free; + ep_over_kq_t **eoks; // note: this memory leaks when process terminates + int neoks; // we can add an extra api to let user clean + ep_over_kq_t *eoks_free; // currently, we just keep it simple stupid }; static eoks_t eoks = { @@ -297,7 +297,7 @@ int epoll_create(int size) { struct epoll_event ev = {0}; ev.events = EPOLLIN; ev.data.ptr = &eok_dummy; - D("epoll_create epfd:[%d]", eok->idx); + D("epoll_create epfd:[%d] and sv0[%d]", eok->idx, eok->sv[0]); if (epoll_ctl(eok->idx, EPOLL_CTL_ADD, eok->sv[0], &ev)) { e = errno; epoll_close(eok->idx); @@ -517,8 +517,8 @@ int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) eok_event_t *ev = (eok_event_t*)kev->udata; A(kev->ident == ev->fd, "internal logic error"); if (kev->flags & EV_ERROR) { - D("error when processing change list for fd[%d], error[%s], kev_flags:[%04x:%s]", - ev->fd, strerror(kev->data), kev->flags, kev_flags_str(kev->flags, 0)); + D("epfd[%d] error when processing change list for fd[%d], error[%s], kev_flags:[%04x:%s]", + epfd, ev->fd, strerror(kev->data), kev->flags, kev_flags_str(kev->flags, 0)); } switch (kev->filter) { case EVFILT_READ: { @@ -528,11 +528,11 @@ int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) char c = '\0'; A(1==recv(kev->ident, &c, 1, 0), "internal logic error"); A(0==memcmp(&c, "1", 1), "internal logic error"); - D("wokenup"); + D("epfd[%d] wokenup", epfd); continue; } else { if (ev->changed==3) { - D("already requested to delete for fd[%d]", ev->fd); + D("epfd[%d] already requested to delete for fd[%d]", epfd, ev->fd); // TODO: write a unit test for this case // EV_DELETE? continue; @@ -550,7 +550,8 @@ int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) } // rounded to what user care pev.events = pev.events & ev->epev.events; - D("events found for fd[%d]: [%04x:%s], which was registered: [%04x:%s], kev_flags: [%04x:%s]", + D("epfd[%d] events found for fd[%d]: [%04x:%s], which was registered: [%04x:%s], kev_flags: [%04x:%s]", + epfd, ev->fd, pev.events, events_str(pev.events, 0), ev->epev.events, events_str(ev->epev.events, 1), kev->flags, kev_flags_str(kev->flags, 2)); @@ -573,7 +574,7 @@ int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) while (p) { eok_event_t *next = p->next; if (p->changed==3) { - D("removing registered event for fd[%d]: [%04x:%s]", p->fd, p->epev.events, events_str(p->epev.events, 0)); + D("epfd[%d] removing registered event for fd[%d]: [%04x:%s]", epfd, p->fd, p->epev.events, events_str(p->epev.events, 0)); eok_free_ev(eok, p); } p = next; @@ -591,13 +592,13 @@ int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) } } while (cnts==0); if (cnts>0) { - D("kevent64 waiting done with [%d] events", cnts); + D("kevent64 epfd[%d] waiting done with [%d] events", epfd, cnts); } A(0==pthread_mutex_unlock(&eok->lock), ""); if (e) { errno = e; - E("epoll_wait failed"); + E("epfd[%d] epoll_wait failed", epfd); return -1; } @@ -752,7 +753,7 @@ static int eok_chgs_refresh(ep_over_kq_t *eok, eok_event_t *oev, eok_event_t *ev eok->kchanges[eok->ichanges++] = *kwev; ++n; } - D("add #changes[%d] for fd[%d], and now #changes/registers [%d/%d]", n, ev->fd, eok->ichanges, eok->evs_count); + D("epfd[%d]: add #changes[%d] for fd[%d], and now #changes/registers [%d/%d]", eok->idx, n, ev->fd, eok->ichanges, eok->evs_count); return 0; } @@ -764,14 +765,21 @@ static ep_over_kq_t* eoks_alloc(void) { if (eoks.eoks_free) { eok = eoks.eoks_free; eoks.eoks_free = eok->next; + A(eoks.eoks, "internal logic error"); + A(eok->idx>=0 && eok->idxidx)==NULL, "internal logic error"); + *(eoks.eoks + eok->idx) = eok; eok->next = NULL; + eok->stopping = 0; break; } - ep_over_kq_t *p = (ep_over_kq_t*)realloc(eoks.eoks, sizeof(*p) * (eoks.neoks+1)); - if (!p) break; - eoks.eoks = p; - eok = eoks.eoks + eoks.neoks; - memset(eok, 0, sizeof(*eok)); + eok = (ep_over_kq_t*)calloc(1, sizeof(*eok)); + if (!eok) break; + eok->idx = -1; + ep_over_kq_t **ar = (ep_over_kq_t**)realloc(eoks.eoks, sizeof(**ar) * (eoks.neoks+1)); + if (!ar) break; + eoks.eoks = ar; + *(eoks.eoks + eoks.neoks) = eok; eok->idx = eoks.neoks; eok->kq = -1; eok->sv[0] = -1; @@ -784,6 +792,11 @@ static ep_over_kq_t* eoks_alloc(void) { errno = ENOMEM; return NULL; } + if (eok->idx==-1) { + free(eok); + errno = ENOMEM; + return NULL; + } if (eok->lock_valid) { return eok; } @@ -801,23 +814,50 @@ static ep_over_kq_t* eoks_alloc(void) { static void eoks_free(ep_over_kq_t *eok) { A(0==pthread_mutex_lock(&eoks.lock), ""); do { + A(eok->idx>=0 && eok->idxnext==NULL, "internal logic error"); // leave eok->kchanges as is A(eok->ichanges==0, "internal logic error"); A(eok->waiting==0, "internal logic error"); - if (eok->evs_count==1) { - A(eok->evs_head && eok->evs_tail && eok->evs_head==eok->evs_tail, "internal logic error"); - A(eok->evs_head->fd==eok->sv[0] && eok->sv[0]!=-1 && eok->sv[1]!=-1, "internal logic error"); - // fd is critical system resource - close(eok->sv[0]); - eok->sv[0] = -1; - close(eok->sv[1]); - eok->sv[1] = -1; - eok_free_ev(eok, eok->evs_head); + eok_event_t *ev = eok->evs_head; + while (ev) { + eok_event_t *next = ev->next; + if (ev->fd==eok->sv[0]) { + // fd is critical system resource + close(eok->sv[0]); + eok->sv[0] = -1; + close(eok->sv[1]); + eok->sv[1] = -1; + eok_free_ev(eok, ev); + } else { + // user forget calling epoll_ctl(EPOLL_CTL_DEL) before calling epoll_close/close? + // calling close(ev->fd) here smells really bad +#ifdef LET_IT_BE + // we just let it be and reclaim ev + eok_free_ev(eok, ev); +#else + // panic otherwise, if LET_IT_BE not defined + A(eok->evs_head==NULL && eok->evs_tail==NULL && eok->evs_count==0, + "epfd[%d] fd[%d]: internal logic error: have you epoll_ctl(EPOLL_CTL_DEL) everything before calling epoll_close?", + eok->idx, ev->fd); +#endif + } + ev = next; } - A(eok->evs_head==NULL && eok->evs_tail==NULL && eok->evs_count==0, "internal logic error"); + // if (eok->evs_count==1) { + // A(eok->evs_head && eok->evs_tail && eok->evs_head==eok->evs_tail, "internal logic error"); + // A(eok->evs_head->fd==eok->sv[0] && eok->sv[0]!=-1 && eok->sv[1]!=-1, "internal logic error"); + // // fd is critical system resource + // close(eok->sv[0]); + // eok->sv[0] = -1; + // close(eok->sv[1]); + // eok->sv[1] = -1; + // eok_free_ev(eok, eok->evs_head); + // } + A(eok->evs_head==NULL && eok->evs_tail==NULL && eok->evs_count==0, + "internal logic error: have you epoll_ctl(EPOLL_CTL_DEL) everything before calling epoll_close?"); A(eok->sv[0]==-1 && eok->sv[1]==-1, "internal logic error"); if (eok->kq!=-1) { close(eok->kq); @@ -825,6 +865,7 @@ static void eoks_free(ep_over_kq_t *eok) { } eok->next = eoks.eoks_free; eoks.eoks_free = eok; + *(eoks.eoks + eok->idx) = NULL; } while (0); A(0==pthread_mutex_unlock(&eoks.lock), ""); } @@ -837,7 +878,7 @@ static ep_over_kq_t* eoks_find(int epfd) { break; } A(eoks.eoks, "internal logic error"); - eok = eoks.eoks + epfd; + eok = *(eoks.eoks + epfd); A(eok->next==NULL, "internal logic error"); A(eok->lock_valid, "internal logic error"); } while (0); diff --git a/src/os/src/detail/CMakeLists.txt b/src/os/src/detail/CMakeLists.txt index e4052a4fb3..1c5e55a522 100644 --- a/src/os/src/detail/CMakeLists.txt +++ b/src/os/src/detail/CMakeLists.txt @@ -13,3 +13,4 @@ TARGET_LINK_LIBRARIES(osdetail os) IF (TD_ARM_32 OR TD_LINUX_32) TARGET_LINK_LIBRARIES(osdetail atomic) ENDIF () + diff --git a/src/plugins/http/CMakeLists.txt b/src/plugins/http/CMakeLists.txt index 56c3c25d7c..274f50d24a 100644 --- a/src/plugins/http/CMakeLists.txt +++ b/src/plugins/http/CMakeLists.txt @@ -8,14 +8,29 @@ INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) INCLUDE_DIRECTORIES(inc) AUX_SOURCE_DIRECTORY(src SRC) - + IF (TD_LINUX) ADD_LIBRARY(http ${SRC}) TARGET_LINK_LIBRARIES(http z) IF (TD_SOMODE_STATIC) TARGET_LINK_LIBRARIES(http taos_static) - ELSE () + ELSE () + TARGET_LINK_LIBRARIES(http taos) + ENDIF () + + IF (TD_ADMIN) + TARGET_LINK_LIBRARIES(http admin) + ENDIF () +ENDIF () + +IF (TD_DARWIN) + ADD_LIBRARY(http ${SRC}) + TARGET_LINK_LIBRARIES(http z) + + IF (TD_SOMODE_STATIC) + TARGET_LINK_LIBRARIES(http taos_static) + ELSE () TARGET_LINK_LIBRARIES(http taos) ENDIF () diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index 1cc73aef06..97a5b4bc23 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -31,11 +31,36 @@ static bool httpReadData(HttpContext *pContext); +#ifdef __APPLE__ +static int sv_dummy = 0; +#endif + static void httpStopThread(HttpThread* pThread) { pThread->stop = true; // signal the thread to stop, try graceful method first, // and use pthread_cancel when failed +#ifdef __APPLE__ + int sv[2]; + sv[0] = sv[1] = -1; + int r = socketpair(PF_LOCAL, SOCK_STREAM, 0, sv); + do { + if (r) break; + struct epoll_event ev = {0}; + ev.events = EPOLLIN; + ev.data.ptr = &sv_dummy; + pThread->stop = true; + r = epoll_ctl(pThread->pollFd, EPOLL_CTL_ADD, sv[0], &ev); + if (r) break; + if (1!=send(sv[1], "1", 1, 0)) { + r = -1; + break; + } + } while (0); + if (r) { + pthread_cancel(pThread->thread); + } +#else // __APPLE__ struct epoll_event event = { .events = EPOLLIN }; eventfd_t fd = eventfd(1, 0); if (fd == -1) { @@ -46,11 +71,23 @@ static void httpStopThread(HttpThread* pThread) { httpError("%s, failed to call epoll_ctl, will call pthread_cancel instead, which may result in data corruption: %s", pThread->label, strerror(errno)); pthread_cancel(pThread->thread); } +#endif // __APPLE__ pthread_join(pThread->thread, NULL); +#ifdef __APPLE__ + if (sv[0]!=-1) { + close(sv[0]); + sv[0] = -1; + } + if (sv[1]!=-1) { + close(sv[1]); + sv[1] = -1; + } +#else // __APPLE__ if (fd != -1) { close(fd); } +#endif // __APPLE__ #ifdef __APPLE__ epoll_close(pThread->pollFd); @@ -97,6 +134,15 @@ static void httpProcessHttpData(void *param) { if (fdNum <= 0) continue; for (int32_t i = 0; i < fdNum; ++i) { +#ifdef __APPLE__ + if (events[i].data.ptr == &sv_dummy) { + // no need to drain the recv buffer of sv[0] + // since there's only one time to send at most 1 byte to sv[0] + // btw, pThread->stop shall be already set, thus never reached here + httpDebug("if you see this line, there's internal logic error"); + continue; + } +#endif // __APPLE__ pContext = httpGetContext(events[i].data.ptr); if (pContext == NULL) { httpError("context:%p, is already released, close connect", events[i].data.ptr); diff --git a/src/plugins/monitor/CMakeLists.txt b/src/plugins/monitor/CMakeLists.txt index 90189c9d75..6b26c0447a 100644 --- a/src/plugins/monitor/CMakeLists.txt +++ b/src/plugins/monitor/CMakeLists.txt @@ -8,10 +8,20 @@ AUX_SOURCE_DIRECTORY(./src SRC) IF (TD_LINUX) ADD_LIBRARY(monitor ${SRC}) - + IF (TD_SOMODE_STATIC) TARGET_LINK_LIBRARIES(monitor taos_static) ELSE () TARGET_LINK_LIBRARIES(monitor taos) ENDIF () -ENDIF () +ENDIF () + +IF (TD_DARWIN) + ADD_LIBRARY(monitor ${SRC}) + + IF (TD_SOMODE_STATIC) + TARGET_LINK_LIBRARIES(monitor taos_static) + ELSE () + TARGET_LINK_LIBRARIES(monitor taos) + ENDIF () +ENDIF () diff --git a/src/plugins/mqtt/CMakeLists.txt b/src/plugins/mqtt/CMakeLists.txt index 3761f70134..b6de421517 100644 --- a/src/plugins/mqtt/CMakeLists.txt +++ b/src/plugins/mqtt/CMakeLists.txt @@ -18,3 +18,15 @@ IF (TD_LINUX) TARGET_LINK_LIBRARIES(mqtt taos) ENDIF () ENDIF () + +IF (TD_DARWIN) + ADD_LIBRARY(mqtt ${SRC}) + TARGET_LINK_LIBRARIES(mqtt cJson mqttc) + + IF (TD_SOMODE_STATIC) + TARGET_LINK_LIBRARIES(mqtt taos_static) + ELSE () + TARGET_LINK_LIBRARIES(mqtt taos) + ENDIF () +ENDIF () + diff --git a/src/query/CMakeLists.txt b/src/query/CMakeLists.txt index e403251858..967e86de3c 100644 --- a/src/query/CMakeLists.txt +++ b/src/query/CMakeLists.txt @@ -14,3 +14,8 @@ IF (TD_LINUX) TARGET_LINK_LIBRARIES(query m rt) ADD_SUBDIRECTORY(tests) ENDIF () + +IF (TD_DARWIN) + TARGET_LINK_LIBRARIES(query m) + ADD_SUBDIRECTORY(tests) +ENDIF () diff --git a/src/rpc/src/rpcTcp.c b/src/rpc/src/rpcTcp.c index 2bdc58390d..ba90f93073 100644 --- a/src/rpc/src/rpcTcp.c +++ b/src/rpc/src/rpcTcp.c @@ -308,7 +308,10 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int num, void * pthread_attr_destroy(&thattr); if (code != 0) { #ifdef __APPLE__ - epoll_close(pThreadObj->pollFd); + if (pThreadObj->pollFd!=-1) { + epoll_close(pThreadObj->pollFd); + pThreadObj->pollFd = -1; + } #else taosCloseSocket(pThreadObj->pollFd); #endif @@ -517,7 +520,10 @@ static void *taosProcessTcpData(void *param) { } #ifdef __APPLE__ - if (pThreadObj->pollFd >=0) epoll_close(pThreadObj->pollFd); + if (pThreadObj->pollFd >=0) { + epoll_close(pThreadObj->pollFd); + pThreadObj->pollFd = -1; + } #else if (pThreadObj->pollFd >=0) taosCloseSocket(pThreadObj->pollFd); #endif diff --git a/src/rpc/test/CMakeLists.txt b/src/rpc/test/CMakeLists.txt index 9a4bcc353d..e923105860 100644 --- a/src/rpc/test/CMakeLists.txt +++ b/src/rpc/test/CMakeLists.txt @@ -16,3 +16,17 @@ IF (TD_LINUX) ADD_EXECUTABLE(rserver ${SERVER_SRC}) TARGET_LINK_LIBRARIES(rserver trpc) ENDIF () + +IF (TD_DARWIN) + LIST(APPEND CLIENT_SRC ./rclient.c) + ADD_EXECUTABLE(rclient ${CLIENT_SRC}) + TARGET_LINK_LIBRARIES(rclient trpc) + + LIST(APPEND SCLIENT_SRC ./rsclient.c) + ADD_EXECUTABLE(rsclient ${SCLIENT_SRC}) + TARGET_LINK_LIBRARIES(rsclient trpc) + + LIST(APPEND SERVER_SRC ./rserver.c) + ADD_EXECUTABLE(rserver ${SERVER_SRC}) + TARGET_LINK_LIBRARIES(rserver trpc) +ENDIF () diff --git a/src/sync/CMakeLists.txt b/src/sync/CMakeLists.txt index 6a53380841..3721163f79 100644 --- a/src/sync/CMakeLists.txt +++ b/src/sync/CMakeLists.txt @@ -3,7 +3,7 @@ PROJECT(TDengine) INCLUDE_DIRECTORIES(inc) AUX_SOURCE_DIRECTORY(src SRC) - + IF (TD_LINUX) LIST(REMOVE_ITEM SRC src/syncArbitrator.c) ADD_LIBRARY(sync ${SRC}) @@ -16,3 +16,16 @@ IF (TD_LINUX) #ADD_SUBDIRECTORY(test) ENDIF () + +IF (TD_DARWIN) + LIST(REMOVE_ITEM SRC src/syncArbitrator.c) + ADD_LIBRARY(sync ${SRC}) + TARGET_LINK_LIBRARIES(sync tutil pthread common) + + LIST(APPEND BIN_SRC src/syncArbitrator.c) + LIST(APPEND BIN_SRC src/syncTcp.c) + ADD_EXECUTABLE(tarbitrator ${BIN_SRC}) + TARGET_LINK_LIBRARIES(tarbitrator sync common osdetail tutil) + + #ADD_SUBDIRECTORY(test) +ENDIF () diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c index cb2379583f..7678a7d284 100644 --- a/src/sync/src/syncRetrieve.c +++ b/src/sync/src/syncRetrieve.c @@ -14,7 +14,7 @@ */ #define _DEFAULT_SOURCE -#include +// #include #include "os.h" #include "taoserror.h" #include "tlog.h" diff --git a/src/sync/test/CMakeLists.txt b/src/sync/test/CMakeLists.txt index 256e87580d..ab2e6c307b 100644 --- a/src/sync/test/CMakeLists.txt +++ b/src/sync/test/CMakeLists.txt @@ -13,4 +13,3 @@ IF (TD_LINUX) TARGET_LINK_LIBRARIES(syncServer sync trpc common) ENDIF () - diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index 48b4d76561..3606aea76b 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -5,7 +5,7 @@ INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/rpc/inc) AUX_SOURCE_DIRECTORY(src SRC) ADD_LIBRARY(tutil ${SRC}) TARGET_LINK_LIBRARIES(tutil pthread osdetail lz4 z) - + IF (TD_LINUX) TARGET_LINK_LIBRARIES(tutil m rt) # ADD_SUBDIRECTORY(tests) @@ -28,5 +28,6 @@ IF (TD_LINUX) ELSEIF (TD_WINDOWS) TARGET_LINK_LIBRARIES(tutil iconv regex winmm IPHLPAPI ws2_32 wepoll) ELSEIF(TD_DARWIN) + TARGET_LINK_LIBRARIES(tutil m) TARGET_LINK_LIBRARIES(tutil iconv) ENDIF() diff --git a/src/vnode/CMakeLists.txt b/src/vnode/CMakeLists.txt index c953883361..3a26172411 100644 --- a/src/vnode/CMakeLists.txt +++ b/src/vnode/CMakeLists.txt @@ -13,3 +13,9 @@ IF (TD_LINUX) ADD_LIBRARY(vnode ${SRC}) TARGET_LINK_LIBRARIES(vnode tsdb tcq) ENDIF () + +IF (TD_DARWIN) + ADD_LIBRARY(vnode ${SRC}) + TARGET_LINK_LIBRARIES(vnode tsdb tcq) +ENDIF () + diff --git a/src/wal/CMakeLists.txt b/src/wal/CMakeLists.txt index 681bed5425..d3cd86df50 100644 --- a/src/wal/CMakeLists.txt +++ b/src/wal/CMakeLists.txt @@ -8,4 +8,10 @@ IF (TD_LINUX) ADD_LIBRARY(twal ${SRC}) TARGET_LINK_LIBRARIES(twal tutil common) ADD_SUBDIRECTORY(test) -ENDIF () +ENDIF () + +IF (TD_DARWIN) + ADD_LIBRARY(twal ${SRC}) + TARGET_LINK_LIBRARIES(twal tutil common) + ADD_SUBDIRECTORY(test) +ENDIF () diff --git a/src/wal/test/CMakeLists.txt b/src/wal/test/CMakeLists.txt index b8338b1738..aec0602ac0 100644 --- a/src/wal/test/CMakeLists.txt +++ b/src/wal/test/CMakeLists.txt @@ -10,4 +10,12 @@ IF (TD_LINUX) ENDIF () +IF (TD_DARWIN) + INCLUDE_DIRECTORIES(../inc) + + LIST(APPEND WALTEST_SRC ./waltest.c) + ADD_EXECUTABLE(waltest ${WALTEST_SRC}) + TARGET_LINK_LIBRARIES(waltest twal osdetail tutil) + +ENDIF () diff --git a/tests/comparisonTest/tdengine/CMakeLists.txt b/tests/comparisonTest/tdengine/CMakeLists.txt index aaa18592ed..a12e36ab6b 100644 --- a/tests/comparisonTest/tdengine/CMakeLists.txt +++ b/tests/comparisonTest/tdengine/CMakeLists.txt @@ -5,3 +5,8 @@ IF (TD_LINUX) add_executable(tdengineTest tdengineTest.c) target_link_libraries(tdengineTest taos_static tutil common pthread) ENDIF() + +IF (TD_DARWIN) + add_executable(tdengineTest tdengineTest.c) + target_link_libraries(tdengineTest taos_static tutil common pthread) +ENDIF() diff --git a/tests/examples/c/CMakeLists.txt b/tests/examples/c/CMakeLists.txt index 7cacefda57..5af7935c6b 100644 --- a/tests/examples/c/CMakeLists.txt +++ b/tests/examples/c/CMakeLists.txt @@ -10,6 +10,9 @@ IF (TD_LINUX) ENDIF () IF (TD_DARWIN) INCLUDE_DIRECTORIES(. ${TD_COMMUNITY_DIR}/src/inc ${TD_COMMUNITY_DIR}/src/client/inc ${TD_COMMUNITY_DIR}/inc) + AUX_SOURCE_DIRECTORY(. SRC) + ADD_EXECUTABLE(demo demo.c) + TARGET_LINK_LIBRARIES(demo taos_static trpc tutil pthread ) ADD_EXECUTABLE(epoll epoll.c) TARGET_LINK_LIBRARIES(epoll taos_static trpc tutil pthread ) ENDIF () diff --git a/tests/examples/c/demo.c b/tests/examples/c/demo.c index 54e81d33b9..f5ad37acef 100644 --- a/tests/examples/c/demo.c +++ b/tests/examples/c/demo.c @@ -86,7 +86,7 @@ void Test(TAOS *taos, char *qstr, int index) { int i = 0; for (i = 0; i < 10; ++i) { - sprintf(qstr, "insert into m1 values (%" PRId64 ", %d, %d, %d, %d, %f, %lf, '%s')", 1546300800000 + i * 1000, i, i, i, i*10000000, i*1.0, i*2.0, "hello"); + sprintf(qstr, "insert into m1 values (%" PRId64 ", %d, %d, %d, %d, %f, %lf, '%s')", 1546300800000LL + i * 1000, i, i, i, i*10000000, i*1.0, i*2.0, "hello"); printf("qstr: %s\n", qstr); // note: how do you wanna do if taos_query returns non-NULL From 7a78290715a86124ae7e2fc59191f8a544e7e144 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 17 Jan 2021 21:45:40 +0800 Subject: [PATCH 023/147] TD-1207 change socket to int32 --- src/dnode/src/dnodeCheck.c | 8 ++++---- src/dnode/src/dnodeTelemetry.c | 2 +- src/os/inc/osFile.h | 2 +- src/os/inc/osSocket.h | 5 ++--- src/os/src/darwin/darwinFile.c | 2 +- src/os/src/darwin/darwinSocket.c | 2 +- src/os/src/detail/osFile.c | 2 +- src/os/src/detail/osSocket.c | 4 ++-- src/os/src/windows/wFile.c | 2 +- src/os/src/windows/wSocket.c | 4 ++-- src/plugins/http/inc/httpInt.h | 4 ++-- src/plugins/http/src/httpServer.c | 2 +- src/rpc/src/rpcTcp.c | 20 ++++++++++---------- src/rpc/src/rpcUdp.c | 2 +- src/sync/inc/syncInt.h | 4 ++-- src/sync/inc/syncTcp.h | 4 ++-- src/sync/src/syncArbitrator.c | 10 +++++----- src/sync/src/syncMain.c | 8 ++++---- src/sync/src/syncTcp.c | 14 +++++++------- src/util/inc/tsocket.h | 20 ++++++++++---------- src/util/src/tnettest.c | 24 ++++++++++++------------ src/util/src/tsocket.c | 28 ++++++++++++++-------------- 22 files changed, 86 insertions(+), 87 deletions(-) diff --git a/src/dnode/src/dnodeCheck.c b/src/dnode/src/dnodeCheck.c index 8955fb5643..2c8dcfdcc9 100644 --- a/src/dnode/src/dnodeCheck.c +++ b/src/dnode/src/dnodeCheck.c @@ -30,10 +30,10 @@ static SCheckItem tsCheckItem[TSDB_CHECK_ITEM_MAX] = {{0}}; int64_t tsMinFreeMemSizeForStart = 0; static int32_t bindTcpPort(int32_t port) { - SOCKET serverSocket; + int32_t serverSocket; struct sockaddr_in server_addr; - if ((serverSocket = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0) { + if ((serverSocket = ( int32_t)socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0) { dError("socket() fail: %s", strerror(errno)); return -1; } @@ -60,10 +60,10 @@ static int32_t bindTcpPort(int32_t port) { } static int32_t bindUdpPort(int32_t port) { - SOCKET serverSocket; + int32_t serverSocket; struct sockaddr_in server_addr; - if ((serverSocket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0) { + if ((serverSocket = (int32_t)socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0) { dError("socket() fail: %s", strerror(errno)); return -1; } diff --git a/src/dnode/src/dnodeTelemetry.c b/src/dnode/src/dnodeTelemetry.c index 9291831884..2834ea3391 100644 --- a/src/dnode/src/dnodeTelemetry.c +++ b/src/dnode/src/dnodeTelemetry.c @@ -200,7 +200,7 @@ static void sendTelemetryReport() { dTrace("failed to get IP address of " TELEMETRY_SERVER ", reason:%s", strerror(errno)); return; } - SOCKET fd = taosOpenTcpClientSocket(ip, TELEMETRY_PORT, 0); + int32_t fd = taosOpenTcpClientSocket(ip, TELEMETRY_PORT, 0); if (fd < 0) { dTrace("failed to create socket for telemetry, reason:%s", strerror(errno)); return; diff --git a/src/os/inc/osFile.h b/src/os/inc/osFile.h index 19cc78472c..d72f9cefb9 100644 --- a/src/os/inc/osFile.h +++ b/src/os/inc/osFile.h @@ -39,7 +39,7 @@ int32_t taosRenameFile(char *fullPath, char *suffix, char delimiter, char **dstP } // TAOS_OS_FUNC_FILE_SENDIFLE -int64_t taosSendFile(SOCKET dfd, int32_t sfd, int64_t *offset, int64_t size); +int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t *offset, int64_t size); int64_t taosFSendFile(FILE *outfile, FILE *infile, int64_t *offset, int64_t size); #ifdef TAOS_RANDOM_FILE_FAIL diff --git a/src/os/inc/osSocket.h b/src/os/inc/osSocket.h index 4e2671ce2b..9683adf45a 100644 --- a/src/os/inc/osSocket.h +++ b/src/os/inc/osSocket.h @@ -33,7 +33,6 @@ extern "C" { x = FD_INITIALIZER; \ } \ } - typedef int32_t SOCKET; #endif #ifndef TAOS_OS_DEF_EPOLL @@ -58,13 +57,13 @@ extern "C" { #endif // TAOS_OS_FUNC_SOCKET -int32_t taosSetNonblocking(SOCKET sock, int32_t on); +int32_t taosSetNonblocking(int32_t sock, int32_t on); void taosIgnSIGPIPE(); void taosBlockSIGPIPE(); void taosSetMaskSIGPIPE(); // TAOS_OS_FUNC_SOCKET_SETSOCKETOPT -int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen); +int32_t taosSetSockOpt(int32_t socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen); // TAOS_OS_FUNC_SOCKET_INET uint32_t taosInetAddr(char *ipAddr); diff --git a/src/os/src/darwin/darwinFile.c b/src/os/src/darwin/darwinFile.c index 1e77cd68d8..00c1d8532f 100644 --- a/src/os/src/darwin/darwinFile.c +++ b/src/os/src/darwin/darwinFile.c @@ -51,7 +51,7 @@ int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t co return writeLen; } -int64_t taosSendFile(SOCKET dfd, int32_t sfd, int64_t* offset, int64_t count) { +int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t* offset, int64_t count) { lseek(sfd, (int32_t)(*offset), 0); int64_t writeLen = 0; uint8_t buffer[_SEND_FILE_STEP_] = { 0 }; diff --git a/src/os/src/darwin/darwinSocket.c b/src/os/src/darwin/darwinSocket.c index 69a4666d34..9137d4b125 100644 --- a/src/os/src/darwin/darwinSocket.c +++ b/src/os/src/darwin/darwinSocket.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "os.h" -int taosSetSockOpt(SOCKET socketfd, int level, int optname, void *optval, int optlen) { +int taosSetSockOpt(int32_t socketfd, int level, int optname, void *optval, int optlen) { if (level == SOL_SOCKET && optname == SO_SNDBUF) { return 0; } diff --git a/src/os/src/detail/osFile.c b/src/os/src/detail/osFile.c index bb68622731..9560ce59bb 100644 --- a/src/os/src/detail/osFile.c +++ b/src/os/src/detail/osFile.c @@ -121,7 +121,7 @@ int64_t taosLSeekImp(int32_t fd, int64_t offset, int32_t whence) { #ifndef TAOS_OS_FUNC_FILE_SENDIFLE -int64_t taosSendFile(SOCKET dfd, int32_t sfd, int64_t *offset, int64_t size) { +int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t *offset, int64_t size) { int64_t leftbytes = size; int64_t sentbytes; diff --git a/src/os/src/detail/osSocket.c b/src/os/src/detail/osSocket.c index d03e8086bf..f929b3e717 100644 --- a/src/os/src/detail/osSocket.c +++ b/src/os/src/detail/osSocket.c @@ -19,7 +19,7 @@ #ifndef TAOS_OS_FUNC_SOCKET -int32_t taosSetNonblocking(SOCKET sock, int32_t on) { +int32_t taosSetNonblocking(int32_t sock, int32_t on) { int32_t flags = 0; if ((flags = fcntl(sock, F_GETFL, 0)) < 0) { uError("fcntl(F_GETFL) error: %d (%s)\n", errno, strerror(errno)); @@ -67,7 +67,7 @@ void taosSetMaskSIGPIPE() { #ifndef TAOS_OS_FUNC_SOCKET_SETSOCKETOPT -int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen) { +int32_t taosSetSockOpt(int32_t socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen) { return setsockopt(socketfd, level, optname, optval, (socklen_t)optlen); } diff --git a/src/os/src/windows/wFile.c b/src/os/src/windows/wFile.c index 4ad195dc6f..e388e5fc67 100644 --- a/src/os/src/windows/wFile.c +++ b/src/os/src/windows/wFile.c @@ -76,7 +76,7 @@ int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t co return writeLen; } -int64_t taosSendFile(SOCKET dfd, int32_t sfd, int64_t *offset, int64_t count) { +int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t *offset, int64_t count) { if (offset != NULL) lseek(sfd, (int32_t)(*offset), 0); int64_t writeLen = 0; diff --git a/src/os/src/windows/wSocket.c b/src/os/src/windows/wSocket.c index 4e6ee15e77..679842a7e9 100644 --- a/src/os/src/windows/wSocket.c +++ b/src/os/src/windows/wSocket.c @@ -34,7 +34,7 @@ void taosWinSocketInit() { } } -int32_t taosSetNonblocking(SOCKET sock, int32_t on) { +int32_t taosSetNonblocking(int32_t sock, int32_t on) { u_long mode; if (on) { mode = 1; @@ -50,7 +50,7 @@ void taosIgnSIGPIPE() {} void taosBlockSIGPIPE() {} void taosSetMaskSIGPIPE() {} -int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen) { +int32_t taosSetSockOpt(int32_t socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen) { if (level == SOL_SOCKET && optname == TCP_KEEPCNT) { return 0; } diff --git a/src/plugins/http/inc/httpInt.h b/src/plugins/http/inc/httpInt.h index 00a7bcf6c5..c23cfe0637 100644 --- a/src/plugins/http/inc/httpInt.h +++ b/src/plugins/http/inc/httpInt.h @@ -166,7 +166,7 @@ typedef struct HttpThread { HttpContext * pHead; pthread_mutex_t threadMutex; bool stop; - SOCKET pollFd; + int32_t pollFd; int32_t numOfContexts; int32_t threadId; char label[HTTP_LABEL_SIZE]; @@ -179,7 +179,7 @@ typedef struct HttpServer { uint16_t serverPort; int8_t stop; int8_t reserve; - SOCKET fd; + int32_t fd; int32_t numOfThreads; int32_t methodScannerLen; int32_t requestNum; diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index 5a04a021cd..72e0c7129e 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -260,7 +260,7 @@ bool httpInitConnect() { return false; } - pThread->pollFd = (SOCKET)epoll_create(HTTP_MAX_EVENTS); // size does not matter + pThread->pollFd = (int32_t)epoll_create(HTTP_MAX_EVENTS); // size does not matter if (pThread->pollFd < 0) { httpError("http thread:%s, failed to create HTTP epoll", pThread->label); pthread_mutex_destroy(&(pThread->threadMutex)); diff --git a/src/rpc/src/rpcTcp.c b/src/rpc/src/rpcTcp.c index 4cc7530784..6b3682a441 100644 --- a/src/rpc/src/rpcTcp.c +++ b/src/rpc/src/rpcTcp.c @@ -31,7 +31,7 @@ typedef struct SFdObj { void *signature; - SOCKET fd; // TCP socket FD + int32_t fd; // TCP socket FD int closedByApp; // 1: already closed by App void *thandle; // handle from upper layer, like TAOS uint32_t ip; @@ -47,7 +47,7 @@ typedef struct SThreadObj { pthread_mutex_t mutex; uint32_t ip; bool stop; - SOCKET pollFd; + int32_t pollFd; int numOfFds; int threadId; char label[TSDB_LABEL_LEN]; @@ -56,7 +56,7 @@ typedef struct SThreadObj { } SThreadObj; typedef struct { - SOCKET fd; + int32_t fd; uint32_t ip; uint16_t port; int8_t stop; @@ -69,7 +69,7 @@ typedef struct { } SServerObj; static void *taosProcessTcpData(void *param); -static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, SOCKET fd); +static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, int32_t fd); static void taosFreeFdObj(SFdObj *pFdObj); static void taosReportBrokenLink(SFdObj *pFdObj); static void *taosAcceptTcpConnection(void *arg); @@ -134,7 +134,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread break; } - pThreadObj->pollFd = (SOCKET)epoll_create(10); // size does not matter + pThreadObj->pollFd = (int32_t)epoll_create(10); // size does not matter if (pThreadObj->pollFd < 0) { tError("%s failed to create TCP epoll", label); code = -1; @@ -227,7 +227,7 @@ void taosCleanUpTcpServer(void *handle) { } static void *taosAcceptTcpConnection(void *arg) { - SOCKET connFd = -1; + int32_t connFd = -1; struct sockaddr_in caddr; int threadId = 0; SThreadObj *pThreadObj; @@ -238,7 +238,7 @@ static void *taosAcceptTcpConnection(void *arg) { while (1) { socklen_t addrlen = sizeof(caddr); - connFd = accept(pServerObj->fd, (struct sockaddr *)&caddr, &addrlen); + connFd = (int32_t)accept(pServerObj->fd, (struct sockaddr *)&caddr, &addrlen); if (pServerObj->stop) { tDebug("%s TCP server stop accepting new connections", pServerObj->label); break; @@ -306,7 +306,7 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int num, void * return NULL; } - pThreadObj->pollFd = (SOCKET)epoll_create(10); // size does not matter + pThreadObj->pollFd = (int32_t)epoll_create(10); // size does not matter if (pThreadObj->pollFd < 0) { tError("%s failed to create TCP client epoll", label); free(pThreadObj); @@ -351,7 +351,7 @@ void taosCleanUpTcpClient(void *chandle) { void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port) { SThreadObj * pThreadObj = shandle; - SOCKET fd = taosOpenTcpClientSocket(ip, port, pThreadObj->ip); + int32_t fd = taosOpenTcpClientSocket(ip, port, pThreadObj->ip); if (fd < 0) return NULL; struct sockaddr_in sin; @@ -541,7 +541,7 @@ static void *taosProcessTcpData(void *param) { return NULL; } -static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, SOCKET fd) { +static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, int32_t fd) { struct epoll_event event; SFdObj *pFdObj = (SFdObj *)calloc(sizeof(SFdObj), 1); diff --git a/src/rpc/src/rpcUdp.c b/src/rpc/src/rpcUdp.c index 22301fcecc..30a2fb2899 100644 --- a/src/rpc/src/rpcUdp.c +++ b/src/rpc/src/rpcUdp.c @@ -31,7 +31,7 @@ typedef struct { int index; - SOCKET fd; + int32_t fd; uint16_t port; // peer port uint16_t localPort; // local port char label[TSDB_LABEL_LEN]; // copy from udpConnSet; diff --git a/src/sync/inc/syncInt.h b/src/sync/inc/syncInt.h index e43140d4e6..eef687d647 100644 --- a/src/sync/inc/syncInt.h +++ b/src/sync/inc/syncInt.h @@ -82,8 +82,8 @@ typedef struct SsyncPeer { uint64_t sversion; // track the peer version in retrieve process uint64_t lastFileVer; // track the file version while retrieve uint64_t lastWalVer; // track the wal version while retrieve - SOCKET syncFd; - SOCKET peerFd; // forward FD + int32_t syncFd; + int32_t peerFd; // forward FD int32_t numOfRetrieves; // number of retrieves tried int32_t fileChanged; // a flag to indicate file is changed during retrieving process int32_t refCount; diff --git a/src/sync/inc/syncTcp.h b/src/sync/inc/syncTcp.h index b322c3440c..d4674fee6b 100644 --- a/src/sync/inc/syncTcp.h +++ b/src/sync/inc/syncTcp.h @@ -27,12 +27,12 @@ typedef struct { int32_t bufferSize; void (*processBrokenLink)(int64_t handleId); int32_t (*processIncomingMsg)(int64_t handleId, void *buffer); - void (*processIncomingConn)(SOCKET fd, uint32_t ip); + void (*processIncomingConn)(int32_t fd, uint32_t ip); } SPoolInfo; void *syncOpenTcpThreadPool(SPoolInfo *pInfo); void syncCloseTcpThreadPool(void *); -void *syncAllocateTcpConn(void *, int64_t rid, SOCKET connFd); +void *syncAllocateTcpConn(void *, int64_t rid, int32_t connFd); void syncFreeTcpConn(void *); #ifdef __cplusplus diff --git a/src/sync/src/syncArbitrator.c b/src/sync/src/syncArbitrator.c index 4fc5e2196c..d65a7b3043 100644 --- a/src/sync/src/syncArbitrator.c +++ b/src/sync/src/syncArbitrator.c @@ -28,16 +28,16 @@ #include "syncTcp.h" static void arbSignalHandler(int32_t signum); -static void arbProcessIncommingConnection(SOCKET connFd, uint32_t sourceIp); +static void arbProcessIncommingConnection(int32_t connFd, uint32_t sourceIp); static void arbProcessBrokenLink(int64_t rid); static int32_t arbProcessPeerMsg(int64_t rid, void *buffer); static tsem_t tsArbSem; static void * tsArbTcpPool; typedef struct { - char id[TSDB_EP_LEN + 24]; - SOCKET nodeFd; - void * pConn; + char id[TSDB_EP_LEN + 24]; + int32_t nodeFd; + void * pConn; } SNodeConn; int32_t main(int32_t argc, char *argv[]) { @@ -106,7 +106,7 @@ int32_t main(int32_t argc, char *argv[]) { return 0; } -static void arbProcessIncommingConnection(SOCKET connFd, uint32_t sourceIp) { +static void arbProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { char ipstr[24]; tinet_ntoa(ipstr, sourceIp); sDebug("peer TCP connection from ip:%s", ipstr); diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index b43f5824be..2cfd393aad 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -45,7 +45,7 @@ static void syncCheckPeerConnection(void *param, void *tmrId); static int32_t syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack, int8_t type, uint16_t tranId); static void syncProcessBrokenLink(int64_t rid); static int32_t syncProcessPeerMsg(int64_t rid, void *buffer); -static void syncProcessIncommingConnection(SOCKET connFd, uint32_t sourceIp); +static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp); static void syncRemovePeer(SSyncPeer *pPeer); static void syncAddArbitrator(SSyncNode *pNode); static void syncFreeNode(void *); @@ -1114,8 +1114,8 @@ static void syncSetupPeerConnection(SSyncPeer *pPeer) { return; } - SOCKET connFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0); - if (connFd < 0) { + int32_t connFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0); + if ((int32_t)connFd < 0) { sDebug("%s, failed to open tcp socket since %s", pPeer->id, strerror(errno)); taosTmrReset(syncCheckPeerConnection, SYNC_CHECK_INTERVAL, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer); return; @@ -1179,7 +1179,7 @@ static void syncCreateRestoreDataThread(SSyncPeer *pPeer) { } } -static void syncProcessIncommingConnection(SOCKET connFd, uint32_t sourceIp) { +static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { char ipstr[24]; int32_t i; diff --git a/src/sync/src/syncTcp.c b/src/sync/src/syncTcp.c index 72ba8e26b2..1210caa84c 100644 --- a/src/sync/src/syncTcp.c +++ b/src/sync/src/syncTcp.c @@ -35,7 +35,7 @@ typedef struct SThreadObj { pthread_t thread; bool stop; - SOCKET pollFd; + int32_t pollFd; int32_t numOfFds; struct SPoolObj *pPool; } SThreadObj; @@ -45,14 +45,14 @@ typedef struct SPoolObj { SThreadObj **pThread; pthread_t thread; int32_t nextId; - SOCKET acceptFd; // FD for accept new connection + int32_t acceptFd; // FD for accept new connection int8_t stop; } SPoolObj; typedef struct { SThreadObj *pThread; int64_t handleId; - SOCKET fd; + int32_t fd; int32_t closedByApp; } SConnObj; @@ -128,7 +128,7 @@ void syncCloseTcpThreadPool(void *param) { tfree(pPool); } -void *syncAllocateTcpConn(void *param, int64_t rid, SOCKET connFd) { +void *syncAllocateTcpConn(void *param, int64_t rid, int32_t connFd) { struct epoll_event event; SPoolObj *pPool = param; @@ -264,13 +264,13 @@ static void *syncAcceptPeerTcpConnection(void *argv) { while (1) { struct sockaddr_in clientAddr; socklen_t addrlen = sizeof(clientAddr); - SOCKET connFd = accept(pPool->acceptFd, (struct sockaddr *)&clientAddr, &addrlen); + int32_t connFd = (int32_t)accept(pPool->acceptFd, (struct sockaddr *)&clientAddr, &addrlen); if (pPool->stop) { sDebug("%p TCP server accept is stopped", pPool); break; } - if (connFd < 0) { + if ((int32_t)connFd < 0) { if (errno == EINVAL) { sDebug("%p TCP server accept is exiting...", pPool); break; @@ -298,7 +298,7 @@ static SThreadObj *syncGetTcpThread(SPoolObj *pPool) { if (pThread == NULL) return NULL; pThread->pPool = pPool; - pThread->pollFd = (SOCKET)epoll_create(10); // size does not matter + pThread->pollFd = (int32_t)epoll_create(10); // size does not matter if (pThread->pollFd < 0) { tfree(pThread); return NULL; diff --git a/src/util/inc/tsocket.h b/src/util/inc/tsocket.h index 6b449d6bc9..fc0d71c4d8 100644 --- a/src/util/inc/tsocket.h +++ b/src/util/inc/tsocket.h @@ -28,17 +28,17 @@ extern "C" { #define EPOLLWAKEUP (1u << 29) #endif -int32_t taosReadn(SOCKET sock, char *buffer, int32_t len); -int32_t taosWriteMsg(SOCKET fd, void *ptr, int32_t nbytes); -int32_t taosReadMsg(SOCKET fd, void *ptr, int32_t nbytes); -int32_t taosNonblockwrite(SOCKET fd, char *ptr, int32_t nbytes); -int32_t taosCopyFds(SOCKET sfd, SOCKET dfd, int64_t len); -int32_t taosSetNonblocking(SOCKET sock, int32_t on); +int32_t taosReadn(int32_t sock, char *buffer, int32_t len); +int32_t taosWriteMsg(int32_t fd, void *ptr, int32_t nbytes); +int32_t taosReadMsg(int32_t fd, void *ptr, int32_t nbytes); +int32_t taosNonblockwrite(int32_t fd, char *ptr, int32_t nbytes); +int32_t taosCopyFds(int32_t sfd, int32_t dfd, int64_t len); +int32_t taosSetNonblocking(int32_t sock, int32_t on); -SOCKET taosOpenUdpSocket(uint32_t localIp, uint16_t localPort); -SOCKET taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp); -SOCKET taosOpenTcpServerSocket(uint32_t ip, uint16_t port); -int32_t taosKeepTcpAlive(SOCKET sockFd); +int32_t taosOpenUdpSocket(uint32_t localIp, uint16_t localPort); +int32_t taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp); +int32_t taosOpenTcpServerSocket(uint32_t ip, uint16_t port); +int32_t taosKeepTcpAlive(int32_t sockFd); int32_t taosGetFqdn(char *); uint32_t taosGetIpv4FromFqdn(const char *); diff --git a/src/util/src/tnettest.c b/src/util/src/tnettest.c index e1b834b949..70c38d36dc 100644 --- a/src/util/src/tnettest.c +++ b/src/util/src/tnettest.c @@ -39,7 +39,7 @@ typedef struct { static void *taosNetBindUdpPort(void *sarg) { STestInfo *pinfo = (STestInfo *)sarg; int32_t port = pinfo->port; - SOCKET serverSocket; + int32_t serverSocket; char buffer[BUFFER_SIZE]; int32_t iDataNum; socklen_t sin_size; @@ -48,7 +48,7 @@ static void *taosNetBindUdpPort(void *sarg) { struct sockaddr_in server_addr; struct sockaddr_in clientAddr; - if ((serverSocket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0) { + if ((serverSocket = (int32_t)socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0) { uError("failed to create UDP socket since %s", strerror(errno)); return NULL; } @@ -104,14 +104,14 @@ static void *taosNetBindTcpPort(void *sarg) { struct sockaddr_in server_addr; struct sockaddr_in clientAddr; - STestInfo *pinfo = sarg; + STestInfo *pinfo = sarg; int32_t port = pinfo->port; - SOCKET serverSocket; + int32_t serverSocket; int32_t addr_len = sizeof(clientAddr); - SOCKET client; + int32_t client; char buffer[BUFFER_SIZE]; - if ((serverSocket = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0) { + if ((serverSocket = (int32_t)socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0) { uError("failed to create TCP socket since %s", strerror(errno)); return NULL; } @@ -148,7 +148,7 @@ static void *taosNetBindTcpPort(void *sarg) { uInfo("TCP server at port:%d is listening", port); while (1) { - client = accept(serverSocket, (struct sockaddr *)&clientAddr, (socklen_t *)&addr_len); + client = (int32_t)accept(serverSocket, (struct sockaddr *)&clientAddr, (socklen_t *)&addr_len); if (client < 0) { uDebug("TCP: failed to accept at port:%d since %s", port, strerror(errno)); continue; @@ -178,10 +178,10 @@ static void *taosNetBindTcpPort(void *sarg) { } static int32_t taosNetCheckTcpPort(STestInfo *info) { - SOCKET clientSocket; - char buffer[BUFFER_SIZE] = {0}; + int32_t clientSocket; + char buffer[BUFFER_SIZE] = {0}; - if ((clientSocket = socket(AF_INET, SOCK_STREAM, 0)) < 0) { + if ((clientSocket = (int32_t)socket(AF_INET, SOCK_STREAM, 0)) < 0) { uError("failed to create TCP client socket since %s", strerror(errno)); return -1; } @@ -226,14 +226,14 @@ static int32_t taosNetCheckTcpPort(STestInfo *info) { } static int32_t taosNetCheckUdpPort(STestInfo *info) { - SOCKET clientSocket; + int32_t clientSocket; char buffer[BUFFER_SIZE] = {0}; int32_t iDataNum = 0; int32_t bufSize = 1024000; struct sockaddr_in serverAddr; - if ((clientSocket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0) { + if ((clientSocket = (int32_t)socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0) { uError("failed to create udp client socket since %s", strerror(errno)); return -1; } diff --git a/src/util/src/tsocket.c b/src/util/src/tsocket.c index 49b69ea0a1..d41c1518b8 100644 --- a/src/util/src/tsocket.c +++ b/src/util/src/tsocket.c @@ -102,7 +102,7 @@ uint32_t ip2uint(const char *const ip_addr) { return *((uint32_t *)ip); } -int32_t taosWriteMsg(SOCKET fd, void *buf, int32_t nbytes) { +int32_t taosWriteMsg(int32_t fd, void *buf, int32_t nbytes) { int32_t nleft, nwritten; char * ptr = (char *)buf; @@ -128,7 +128,7 @@ int32_t taosWriteMsg(SOCKET fd, void *buf, int32_t nbytes) { return (nbytes - nleft); } -int32_t taosReadMsg(SOCKET fd, void *buf, int32_t nbytes) { +int32_t taosReadMsg(int32_t fd, void *buf, int32_t nbytes) { int32_t nleft, nread; char * ptr = (char *)buf; @@ -159,7 +159,7 @@ int32_t taosReadMsg(SOCKET fd, void *buf, int32_t nbytes) { return (nbytes - nleft); } -int32_t taosNonblockwrite(SOCKET fd, char *ptr, int32_t nbytes) { +int32_t taosNonblockwrite(int32_t fd, char *ptr, int32_t nbytes) { taosSetNonblocking(fd, 1); int32_t nleft, nwritten, nready; @@ -201,7 +201,7 @@ int32_t taosNonblockwrite(SOCKET fd, char *ptr, int32_t nbytes) { return (nbytes - nleft); } -int32_t taosReadn(SOCKET fd, char *ptr, int32_t nbytes) { +int32_t taosReadn(int32_t fd, char *ptr, int32_t nbytes) { int32_t nread, nready, nleft = nbytes; fd_set fset; @@ -239,9 +239,9 @@ int32_t taosReadn(SOCKET fd, char *ptr, int32_t nbytes) { return (nbytes - nleft); } -SOCKET taosOpenUdpSocket(uint32_t ip, uint16_t port) { +int32_t taosOpenUdpSocket(uint32_t ip, uint16_t port) { struct sockaddr_in localAddr; - SOCKET sockFd; + int32_t sockFd; int32_t bufSize = 1024000; uDebug("open udp socket:0x%x:%hu", ip, port); @@ -279,14 +279,14 @@ SOCKET taosOpenUdpSocket(uint32_t ip, uint16_t port) { return sockFd; } -SOCKET taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clientIp) { - SOCKET sockFd = 0; +int32_t taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clientIp) { + int32_t sockFd = 0; int32_t ret; struct sockaddr_in serverAddr, clientAddr; int32_t bufSize = 1024 * 1024; - sockFd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP); - + sockFd = (int32_t)socket(PF_INET, SOCK_STREAM, IPPROTO_TCP); + if (sockFd <= 2) { uError("failed to open the socket: %d (%s)", errno, strerror(errno)); taosCloseSocketNoCheck(sockFd); @@ -346,7 +346,7 @@ SOCKET taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clie return sockFd; } -int32_t taosKeepTcpAlive(SOCKET sockFd) { +int32_t taosKeepTcpAlive(int32_t sockFd) { int32_t alive = 1; if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_KEEPALIVE, (void *)&alive, sizeof(alive)) < 0) { uError("fd:%d setsockopt SO_KEEPALIVE failed: %d (%s)", sockFd, errno, strerror(errno)); @@ -394,9 +394,9 @@ int32_t taosKeepTcpAlive(SOCKET sockFd) { return 0; } -SOCKET taosOpenTcpServerSocket(uint32_t ip, uint16_t port) { +int32_t taosOpenTcpServerSocket(uint32_t ip, uint16_t port) { struct sockaddr_in serverAdd; - SOCKET sockFd; + int32_t sockFd; int32_t reuse; uDebug("open tcp server socket:0x%x:%hu", ip, port); @@ -449,7 +449,7 @@ void tinet_ntoa(char *ipstr, uint32_t ip) { #define COPY_SIZE 32768 // sendfile shall be used -int32_t taosCopyFds(SOCKET sfd, SOCKET dfd, int64_t len) { +int32_t taosCopyFds(int32_t sfd, int32_t dfd, int64_t len) { int64_t leftLen; int32_t readLen, writeLen; char temp[COPY_SIZE]; From 0ef43ca365355fdc40c7c333a26bebc254593b63 Mon Sep 17 00:00:00 2001 From: freemine Date: Sun, 17 Jan 2021 21:53:02 +0800 Subject: [PATCH 024/147] literal big integer --- tests/examples/c/demo.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/examples/c/demo.c b/tests/examples/c/demo.c index f5ad37acef..9ad3e6f902 100644 --- a/tests/examples/c/demo.c +++ b/tests/examples/c/demo.c @@ -86,7 +86,7 @@ void Test(TAOS *taos, char *qstr, int index) { int i = 0; for (i = 0; i < 10; ++i) { - sprintf(qstr, "insert into m1 values (%" PRId64 ", %d, %d, %d, %d, %f, %lf, '%s')", 1546300800000LL + i * 1000, i, i, i, i*10000000, i*1.0, i*2.0, "hello"); + sprintf(qstr, "insert into m1 values (%" PRId64 ", %d, %d, %d, %d, %f, %lf, '%s')", (uint64_t)(1546300800000 + i * 1000), i, i, i, i*10000000, i*1.0, i*2.0, "hello"); printf("qstr: %s\n", qstr); // note: how do you wanna do if taos_query returns non-NULL From f6e18e01d1a567f2d57e267eed2bea101bc77e8e Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 18 Jan 2021 09:41:35 +0800 Subject: [PATCH 025/147] TD-1207 --- deps/zlib-1.2.11/src/gzlib.c | 4 ++++ src/os/inc/osDef.h | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/deps/zlib-1.2.11/src/gzlib.c b/deps/zlib-1.2.11/src/gzlib.c index f3bd685be0..4c135aabff 100644 --- a/deps/zlib-1.2.11/src/gzlib.c +++ b/deps/zlib-1.2.11/src/gzlib.c @@ -5,6 +5,10 @@ #include #include "gzguts.h" +#ifndef O_BINARY + #define O_BINARY 0 +#endif + #if defined(_WIN32) && !defined(__BORLANDC__) && !defined(__MINGW32__) # define LSEEK _lseeki64 #else diff --git a/src/os/inc/osDef.h b/src/os/inc/osDef.h index d718bef6da..bdbe95a75d 100644 --- a/src/os/inc/osDef.h +++ b/src/os/inc/osDef.h @@ -20,6 +20,10 @@ extern "C" { #endif +#ifndef O_BINARY + #define O_BINARY 0 +#endif + #ifndef STDERR_FILENO #define STDERR_FILENO (2) #endif From 1eace47c8570775e08b2131ce498aa7bdfdcdebe Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 18 Jan 2021 19:52:42 +0800 Subject: [PATCH 026/147] TD-1207 --- src/os/inc/osDef.h | 6 +- tests/script/jenkins/wbasic.txt | 393 ++++++++++++++++++++++++++++++++ tests/script/sh/cfg.bat | 42 ++++ tests/script/sh/deploy.bat | 231 ++++++++----------- tests/script/sh/exec.bat | 134 +++-------- tests/script/sh/stop_dnodes.bat | 25 +- tests/script/wtest.bat | 43 ++-- tests/tsim/src/simExe.c | 26 +++ 8 files changed, 614 insertions(+), 286 deletions(-) create mode 100644 tests/script/jenkins/wbasic.txt create mode 100644 tests/script/sh/cfg.bat diff --git a/src/os/inc/osDef.h b/src/os/inc/osDef.h index bdbe95a75d..e5b5e0f895 100644 --- a/src/os/inc/osDef.h +++ b/src/os/inc/osDef.h @@ -20,8 +20,10 @@ extern "C" { #endif -#ifndef O_BINARY - #define O_BINARY 0 +#ifndef WINDOWS + #ifndef O_BINARY + #define O_BINARY 0 + #endif #endif #ifndef STDERR_FILENO diff --git a/tests/script/jenkins/wbasic.txt b/tests/script/jenkins/wbasic.txt new file mode 100644 index 0000000000..984b15fc5a --- /dev/null +++ b/tests/script/jenkins/wbasic.txt @@ -0,0 +1,393 @@ +wtest.bat -f issue/TD-2677.sim +wtest.bat -f issue/TD-2680.sim +wtest.bat -f issue/TD-2713.sim + +wtest.bat -f general/alter/cached_schema_after_alter.sim +wtest.bat -f general/alter/count.sim +wtest.bat -f general/alter/dnode.sim +wtest.bat -f general/alter/import.sim +wtest.bat -f general/alter/insert1.sim +wtest.bat -f general/alter/insert2.sim +wtest.bat -f general/alter/metrics.sim +wtest.bat -f general/alter/table.sim + +wtest.bat -f general/cache/new_metrics.sim +wtest.bat -f general/cache/restart_metrics.sim +wtest.bat -f general/cache/restart_table.sim + +wtest.bat -f general/connection/connection.sim + +wtest.bat -f general/column/commit.sim +wtest.bat -f general/column/metrics.sim +wtest.bat -f general/column/table.sim + +wtest.bat -f general/compress/commitlog.sim +wtest.bat -f general/compress/compress.sim +wtest.bat -f general/compress/compress2.sim +wtest.bat -f general/compress/uncompress.sim + +wtest.bat -f general/compute/avg.sim +wtest.bat -f general/compute/bottom.sim +wtest.bat -f general/compute/count.sim +wtest.bat -f general/compute/diff.sim +wtest.bat -f general/compute/diff2.sim +wtest.bat -f general/compute/first.sim +wtest.bat -f general/compute/interval.sim +wtest.bat -f general/compute/last.sim +wtest.bat -f general/compute/leastsquare.sim +wtest.bat -f general/compute/max.sim +wtest.bat -f general/compute/min.sim +wtest.bat -f general/compute/null.sim +wtest.bat -f general/compute/percentile.sim +wtest.bat -f general/compute/stddev.sim +wtest.bat -f general/compute/sum.sim +wtest.bat -f general/compute/top.sim + +wtest.bat -f general/db/alter_option.sim +wtest.bat -f general/db/alter_tables_d2.sim +wtest.bat -f general/db/alter_tables_v1.sim +wtest.bat -f general/db/alter_tables_v4.sim +wtest.bat -f general/db/alter_vgroups.sim +wtest.bat -f general/db/basic.sim +wtest.bat -f general/db/basic1.sim +wtest.bat -f general/db/basic2.sim +wtest.bat -f general/db/basic3.sim +wtest.bat -f general/db/basic4.sim +wtest.bat -f general/db/basic5.sim +wtest.bat -f general/db/delete_reuse1.sim +wtest.bat -f general/db/delete_reuse2.sim +wtest.bat -f general/db/delete_reusevnode.sim +wtest.bat -f general/db/delete_reusevnode2.sim +wtest.bat -f general/db/delete_writing1.sim +wtest.bat -f general/db/delete_writing2.sim +wtest.bat -f general/db/delete.sim +wtest.bat -f general/db/len.sim +wtest.bat -f general/db/repeat.sim +wtest.bat -f general/db/tables.sim +wtest.bat -f general/db/vnodes.sim +wtest.bat -f general/db/nosuchfile.sim + +wtest.bat -f general/field/2.sim +wtest.bat -f general/field/3.sim +wtest.bat -f general/field/4.sim +wtest.bat -f general/field/5.sim +wtest.bat -f general/field/6.sim +wtest.bat -f general/field/bigint.sim +wtest.bat -f general/field/binary.sim +wtest.bat -f general/field/bool.sim +wtest.bat -f general/field/single.sim +wtest.bat -f general/field/smallint.sim +wtest.bat -f general/field/tinyint.sim + +wtest.bat -f general/http/autocreate.sim +wtest.bat -f general/http/chunked.sim +wtest.bat -f general/http/gzip.sim +wtest.bat -f general/http/restful.sim +wtest.bat -f general/http/restful_insert.sim +wtest.bat -f general/http/restful_limit.sim +wtest.bat -f general/http/restful_full.sim +wtest.bat -f general/http/prepare.sim +wtest.bat -f general/http/telegraf.sim +wtest.bat -f general/http/grafana_bug.sim +wtest.bat -f general/http/grafana.sim + +wtest.bat -f general/import/basic.sim +wtest.bat -f general/import/commit.sim +wtest.bat -f general/import/large.sim +wtest.bat -f general/import/replica1.sim + +wtest.bat -f general/insert/basic.sim +wtest.bat -f general/insert/insert_drop.sim +wtest.bat -f general/insert/query_block1_memory.sim +wtest.bat -f general/insert/query_block2_memory.sim +wtest.bat -f general/insert/query_block1_file.sim +wtest.bat -f general/insert/query_block2_file.sim +wtest.bat -f general/insert/query_file_memory.sim +wtest.bat -f general/insert/query_multi_file.sim +wtest.bat -f general/insert/tcp.sim + +wtest.bat -f general/parser/alter.sim +wtest.bat -f general/parser/alter1.sim +wtest.bat -f general/parser/alter_stable.sim +wtest.bat -f general/parser/auto_create_tb.sim +wtest.bat -f general/parser/auto_create_tb_drop_tb.sim +wtest.bat -f general/parser/col_arithmetic_operation.sim +wtest.bat -f general/parser/columnValue.sim +wtest.bat -f general/parser/commit.sim +wtest.bat -f general/parser/create_db.sim +wtest.bat -f general/parser/create_mt.sim +wtest.bat -f general/parser/create_tb.sim +wtest.bat -f general/parser/dbtbnameValidate.sim +wtest.bat -f general/parser/import_commit1.sim +wtest.bat -f general/parser/import_commit2.sim +wtest.bat -f general/parser/import_commit3.sim +wtest.bat -f general/parser/insert_tb.sim +wtest.bat -f general/parser/first_last.sim +wtest.bat -f general/parser/lastrow.sim +wtest.bat -f general/parser/nchar.sim +wtest.bat -f general/parser/null_char.sim +wtest.bat -f general/parser/single_row_in_tb.sim +wtest.bat -f general/parser/select_from_cache_disk.sim +wtest.bat -f general/parser/mixed_blocks.sim +wtest.bat -f general/parser/selectResNum.sim +wtest.bat -f general/parser/limit.sim +wtest.bat -f general/parser/limit1.sim +wtest.bat -f general/parser/limit1_tblocks100.sim +wtest.bat -f general/parser/select_across_vnodes.sim +wtest.bat -f general/parser/slimit1.sim +wtest.bat -f general/parser/tbnameIn.sim +wtest.bat -f general/parser/projection_limit_offset.sim +wtest.bat -f general/parser/limit2.sim +wtest.bat -f general/parser/fill.sim +wtest.bat -f general/parser/fill_stb.sim +wtest.bat -f general/parser/where.sim +wtest.bat -f general/parser/slimit.sim +wtest.bat -f general/parser/select_with_tags.sim +wtest.bat -f general/parser/interp.sim +wtest.bat -f general/parser/tags_dynamically_specifiy.sim +wtest.bat -f general/parser/groupby.sim +wtest.bat -f general/parser/set_tag_vals.sim +wtest.bat -f general/parser/tags_filter.sim +wtest.bat -f general/parser/slimit_alter_tags.sim +wtest.bat -f general/parser/join.sim +wtest.bat -f general/parser/join_multivnode.sim +wtest.bat -f general/parser/binary_escapeCharacter.sim +wtest.bat -f general/parser/repeatAlter.sim +wtest.bat -f general/parser/union.sim +wtest.bat -f general/parser/topbot.sim +wtest.bat -f general/parser/function.sim + +wtest.bat -f general/stable/disk.sim +wtest.bat -f general/stable/dnode3.sim +wtest.bat -f general/stable/metrics.sim +wtest.bat -f general/stable/refcount.sim +wtest.bat -f general/stable/show.sim +wtest.bat -f general/stable/values.sim +wtest.bat -f general/stable/vnode3.sim + +wtest.bat -f general/table/autocreate.sim +wtest.bat -f general/table/basic1.sim +wtest.bat -f general/table/basic2.sim +wtest.bat -f general/table/basic3.sim +wtest.bat -f general/table/bigint.sim +wtest.bat -f general/table/binary.sim +wtest.bat -f general/table/bool.sim +wtest.bat -f general/table/column_name.sim +wtest.bat -f general/table/column_num.sim +wtest.bat -f general/table/column_value.sim +wtest.bat -f general/table/column2.sim +wtest.bat -f general/table/date.sim +wtest.bat -f general/table/db.table.sim +wtest.bat -f general/table/delete_reuse1.sim +wtest.bat -f general/table/delete_reuse2.sim +wtest.bat -f general/table/delete_writing.sim +wtest.bat -f general/table/describe.sim +wtest.bat -f general/table/double.sim +wtest.bat -f general/table/fill.sim +wtest.bat -f general/table/float.sim +wtest.bat -f general/table/int.sim +wtest.bat -f general/table/limit.sim +wtest.bat -f general/table/smallint.sim +wtest.bat -f general/table/table_len.sim +wtest.bat -f general/table/table.sim +wtest.bat -f general/table/tinyint.sim +wtest.bat -f general/table/vgroup.sim + +wtest.bat -f general/tag/3.sim +wtest.bat -f general/tag/4.sim +wtest.bat -f general/tag/5.sim +wtest.bat -f general/tag/6.sim +wtest.bat -f general/tag/add.sim +wtest.bat -f general/tag/bigint.sim +wtest.bat -f general/tag/binary_binary.sim +wtest.bat -f general/tag/binary.sim +wtest.bat -f general/tag/bool_binary.sim +wtest.bat -f general/tag/bool_int.sim +wtest.bat -f general/tag/bool.sim +wtest.bat -f general/tag/change.sim +wtest.bat -f general/tag/column.sim +wtest.bat -f general/tag/commit.sim +wtest.bat -f general/tag/create.sim +wtest.bat -f general/tag/delete.sim +wtest.bat -f general/tag/double.sim +wtest.bat -f general/tag/filter.sim +wtest.bat -f general/tag/float.sim +wtest.bat -f general/tag/int_binary.sim +wtest.bat -f general/tag/int_float.sim +wtest.bat -f general/tag/int.sim +wtest.bat -f general/tag/set.sim +wtest.bat -f general/tag/smallint.sim +wtest.bat -f general/tag/tinyint.sim + +wtest.bat -f general/user/authority.sim +wtest.bat -f general/user/monitor.sim +wtest.bat -f general/user/pass_alter.sim +wtest.bat -f general/user/pass_len.sim +wtest.bat -f general/user/user_create.sim +wtest.bat -f general/user/user_len.sim + +wtest.bat -f general/vector/metrics_field.sim +wtest.bat -f general/vector/metrics_mix.sim +wtest.bat -f general/vector/metrics_query.sim +wtest.bat -f general/vector/metrics_tag.sim +wtest.bat -f general/vector/metrics_time.sim +wtest.bat -f general/vector/multi.sim +wtest.bat -f general/vector/single.sim +wtest.bat -f general/vector/table_field.sim +wtest.bat -f general/vector/table_mix.sim +wtest.bat -f general/vector/table_query.sim +wtest.bat -f general/vector/table_time.sim + +wtest.bat -f general/wal/sync.sim +wtest.bat -f general/wal/kill.sim +wtest.bat -f general/wal/maxtables.sim + +wtest.bat -f unique/account/account_create.sim +wtest.bat -f unique/account/account_delete.sim +wtest.bat -f unique/account/account_len.sim +wtest.bat -f unique/account/authority.sim +wtest.bat -f unique/account/basic.sim +wtest.bat -f unique/account/paras.sim +wtest.bat -f unique/account/pass_alter.sim +wtest.bat -f unique/account/pass_len.sim +wtest.bat -f unique/account/usage.sim +wtest.bat -f unique/account/user_create.sim +wtest.bat -f unique/account/user_len.sim + +wtest.bat -f unique/big/balance.sim +wtest.bat -f unique/big/maxvnodes.sim +wtest.bat -f unique/big/tcp.sim + +wtest.bat -f unique/cluster/alter.sim +wtest.bat -f unique/cluster/balance1.sim +wtest.bat -f unique/cluster/balance2.sim +wtest.bat -f unique/cluster/balance3.sim +wtest.bat -f unique/cluster/cache.sim +wtest.bat -f unique/cluster/vgroup100.sim + +wtest.bat -f unique/column/replica3.sim + +wtest.bat -f unique/db/commit.sim +wtest.bat -f unique/db/delete.sim +wtest.bat -f unique/db/delete_part.sim +wtest.bat -f unique/db/replica_add12.sim +wtest.bat -f unique/db/replica_add13.sim +wtest.bat -f unique/db/replica_add23.sim +wtest.bat -f unique/db/replica_reduce21.sim +wtest.bat -f unique/db/replica_reduce32.sim +wtest.bat -f unique/db/replica_reduce31.sim +wtest.bat -f unique/db/replica_part.sim + +wtest.bat -f unique/dnode/alternativeRole.sim +wtest.bat -f unique/dnode/monitor.sim +wtest.bat -f unique/dnode/monitor_bug.sim +wtest.bat -f unique/dnode/simple.sim +wtest.bat -f unique/dnode/balance1.sim +wtest.bat -f unique/dnode/balance2.sim +wtest.bat -f unique/dnode/balance3.sim +wtest.bat -f unique/dnode/balancex.sim +wtest.bat -f unique/dnode/data1.sim +wtest.bat -f unique/dnode/m2.sim +wtest.bat -f unique/dnode/m3.sim +wtest.bat -f unique/dnode/lossdata.sim +wtest.bat -f unique/dnode/offline1.sim +wtest.bat -f unique/dnode/offline2.sim +wtest.bat -f unique/dnode/offline3.sim +wtest.bat -f unique/dnode/reason.sim +wtest.bat -f unique/dnode/remove1.sim +wtest.bat -f unique/dnode/remove2.sim +wtest.bat -f unique/dnode/vnode_clean.sim + +wtest.bat -f unique/http/admin.sim +wtest.bat -f unique/http/opentsdb.sim + +wtest.bat -f unique/import/replica2.sim +wtest.bat -f unique/import/replica3.sim + +wtest.bat -f unique/stable/balance_replica1.sim +wtest.bat -f unique/stable/dnode2_stop.sim +wtest.bat -f unique/stable/dnode2.sim +wtest.bat -f unique/stable/dnode3.sim +wtest.bat -f unique/stable/replica2_dnode4.sim +wtest.bat -f unique/stable/replica2_vnode3.sim +wtest.bat -f unique/stable/replica3_dnode6.sim +wtest.bat -f unique/stable/replica3_vnode3.sim + +wtest.bat -f unique/mnode/mgmt20.sim +wtest.bat -f unique/mnode/mgmt21.sim +wtest.bat -f unique/mnode/mgmt22.sim +wtest.bat -f unique/mnode/mgmt23.sim +wtest.bat -f unique/mnode/mgmt24.sim +wtest.bat -f unique/mnode/mgmt25.sim +wtest.bat -f unique/mnode/mgmt26.sim +wtest.bat -f unique/mnode/mgmt30.sim +wtest.bat -f unique/mnode/mgmt33.sim +wtest.bat -f unique/mnode/mgmt34.sim +wtest.bat -f unique/mnode/mgmtr2.sim + +wtest.bat -f unique/vnode/many.sim +wtest.bat -f unique/vnode/replica2_basic2.sim +wtest.bat -f unique/vnode/replica2_repeat.sim +wtest.bat -f unique/vnode/replica3_basic.sim +wtest.bat -f unique/vnode/replica3_repeat.sim +wtest.bat -f unique/vnode/replica3_vgroup.sim + +wtest.bat -f general/stream/metrics_del.sim +wtest.bat -f general/stream/metrics_replica1_vnoden.sim +wtest.bat -f general/stream/restart_stream.sim +wtest.bat -f general/stream/stream_3.sim +wtest.bat -f general/stream/stream_restart.sim +wtest.bat -f general/stream/table_del.sim +wtest.bat -f general/stream/table_replica1_vnoden.sim + +wtest.bat -f unique/arbitrator/check_cluster_cfg_para.sim +#wtest.bat -f unique/arbitrator/dn2_mn1_cache_file_sync.sim +wtest.bat -f unique/arbitrator/dn3_mn1_full_createTableFail.sim +wtest.bat -f unique/arbitrator/dn3_mn1_multiCreateDropTable.sim +#wtest.bat -f unique/arbitrator/dn3_mn1_nw_disable_timeout_autoDropDnode.sim +#wtest.bat -f unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim +wtest.bat -f unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim +wtest.bat -f unique/arbitrator/dn3_mn1_replica_change.sim +#wtest.bat -f unique/arbitrator/dn3_mn1_stopDnode_timeout.sim +# lower the priority while file corruption +#wtest.bat -f unique/arbitrator/dn3_mn1_vnode_change.sim +#wtest.bat -f unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim +#wtest.bat -f unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim +#wtest.bat -f unique/arbitrator/dn3_mn1_vnode_createErrData_online.sim +wtest.bat -f unique/arbitrator/dn3_mn1_vnode_noCorruptFile_offline.sim +wtest.bat -f unique/arbitrator/dn3_mn1_vnode_delDir.sim +wtest.bat -f unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim +wtest.bat -f unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim +wtest.bat -f unique/arbitrator/dn3_mn1_vnode_nomaster.sim +wtest.bat -f unique/arbitrator/dn3_mn2_killDnode.sim +wtest.bat -f unique/arbitrator/insert_duplicationTs.sim +wtest.bat -f unique/arbitrator/offline_replica2_alterTable_online.sim +wtest.bat -f unique/arbitrator/offline_replica2_alterTag_online.sim +wtest.bat -f unique/arbitrator/offline_replica2_createTable_online.sim +wtest.bat -f unique/arbitrator/offline_replica2_dropDb_online.sim +wtest.bat -f unique/arbitrator/offline_replica2_dropTable_online.sim +wtest.bat -f unique/arbitrator/offline_replica3_alterTable_online.sim +wtest.bat -f unique/arbitrator/offline_replica3_alterTag_online.sim +wtest.bat -f unique/arbitrator/offline_replica3_createTable_online.sim +wtest.bat -f unique/arbitrator/offline_replica3_dropDb_online.sim +wtest.bat -f unique/arbitrator/offline_replica3_dropTable_online.sim +wtest.bat -f unique/arbitrator/replica_changeWithArbitrator.sim +wtest.bat -f unique/arbitrator/sync_replica2_alterTable_add.sim +wtest.bat -f unique/arbitrator/sync_replica2_alterTable_drop.sim + +wtest.bat -f unique/arbitrator/sync_replica2_dropDb.sim +wtest.bat -f unique/arbitrator/sync_replica2_dropTable.sim +wtest.bat -f unique/arbitrator/sync_replica3_alterTable_add.sim +wtest.bat -f unique/arbitrator/sync_replica3_alterTable_drop.sim +wtest.bat -f unique/arbitrator/sync_replica3_dropDb.sim +wtest.bat -f unique/arbitrator/sync_replica3_dropTable.sim + +wtest.bat -f unique/migrate/mn2_vn2_repl2_rmMnodeDir.sim +wtest.bat -f unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim +wtest.bat -f unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir_stopAll_starAll.sim +wtest.bat -f unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim + +wtest.bat -f general/connection/test_old_data.sim +wtest.bat -f unique/dnode/datatrans_3node.sim +wtest.bat -f unique/dnode/datatrans_3node_2.sim + diff --git a/tests/script/sh/cfg.bat b/tests/script/sh/cfg.bat new file mode 100644 index 0000000000..49ab34383a --- /dev/null +++ b/tests/script/sh/cfg.bat @@ -0,0 +1,42 @@ +@echo off + +if %1 == -n set NODE_NAME=%2 +if %1 == -c set CONFIG_NAME=%2 +if %1 == -v set CONFIG_VALUE=%2 +if %3 == -n set NODE_NAME=%4 +if %3 == -c set CONFIG_NAME=%4 +if %3 == -v set CONFIG_VALUE=%4 +if %5 == -n set NODE_NAME=%6 +if %5 == -c set CONFIG_NAME=%6 +if %5 == -v set CONFIG_VALUE=%6 + +rem echo NODE_NAME: %NODE_NAME% +rem echo NODE: %NODE% + +set SCRIPT_DIR=%~dp0..\ +rem echo SCRIPT_DIR: %SCRIPT_DIR% + +set BUILD_DIR=%SCRIPT_DIR%..\..\..\debug\build\bin\ +set TSIM=%BUILD_DIR%tsim +rem echo BUILD_DIR: %BUILD_DIR% +rem echo TSIM: %TSIM% + +set SIM_DIR=%SCRIPT_DIR%..\..\..\sim\ +rem echo SIM_DIR: %SIM_DIR% + +set NODE_DIR=%SIM_DIR%%NODE_NAME%\ +rem echo NODE_DIR: %NODE_DIR% + +set CFG_DIR=%NODE_DIR%cfg\ +rem echo CFG_DIR: %CFG_DIR% + +set LOG_DIR=%NODE_DIR%log\ +rem echo LOG_DIR: %LOG_DIR% + +set DATA_DIR=%NODE_DIR%data\ +rem echo DATA_DIR: %DATA_DIR% + +set TAOS_CFG=%CFG_DIR%taos.cfg +rem echo TAOS_CFG: %TAOS_CFG% + +echo %CONFIG_NAME% %CONFIG_VALUE% >> %TAOS_CFG% diff --git a/tests/script/sh/deploy.bat b/tests/script/sh/deploy.bat index 9b61a33d45..0b419b1e9b 100644 --- a/tests/script/sh/deploy.bat +++ b/tests/script/sh/deploy.bat @@ -1,155 +1,106 @@ -#!/bin/bash +@echo off -echo "Executing deploy.sh" +rem echo Executing deploy.sh -if [ $# != 4 ]; then - echo "argument list need input : " - echo " -n nodeName" - echo " -i nodePort" - exit 1 -fi +if %1 == -n set NODE_NAME=%2 +if %1 == -i set NODE=%2 +if %3 == -n set NODE_NAME=%4 +if %3 == -i set NODE=%4 -NODE_NAME= -NODE= -while getopts "n:i:" arg -do - case $arg in - n) - NODE_NAME=$OPTARG - ;; - i) - NODE=$OPTARG - ;; - ?) - echo "unkonw argument" - ;; - esac -done +rem echo NODE_NAME: %NODE_NAME% +rem echo NODE: %NODE% -SCRIPT_DIR=`dirname $0` -cd $SCRIPT_DIR/../ -SCRIPT_DIR=`pwd` -echo "SCRIPT_DIR: $SCRIPT_DIR" +set SCRIPT_DIR=%~dp0..\ +rem echo SCRIPT_DIR: %SCRIPT_DIR% -IN_TDINTERNAL="community" -if [[ "$SCRIPT_DIR" == *"$IN_TDINTERNAL"* ]]; then - cd ../../.. -else - cd ../../ -fi +set BUILD_DIR=%SCRIPT_DIR%..\..\..\debug\build\bin\ +set TSIM=%BUILD_DIR%tsim +rem echo BUILD_DIR: %BUILD_DIR% +rem echo TSIM: %TSIM% -TAOS_DIR=`pwd` -TAOSD_DIR=`find . -name "taosd"|grep bin|head -n1` +set SIM_DIR=%SCRIPT_DIR%..\..\..\sim\ +rem echo SIM_DIR: %SIM_DIR% -if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then - BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2,3` -else - BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2` -fi +set NODE_DIR=%SIM_DIR%%NODE_NAME%\ +rem echo NODE_DIR: %NODE_DIR% -BUILD_DIR=$TAOS_DIR/$BIN_DIR/build +set CFG_DIR=%NODE_DIR%cfg\ +rem echo CFG_DIR: %CFG_DIR% -SIM_DIR=$TAOS_DIR/sim +set LOG_DIR=%NODE_DIR%log\ +rem echo LOG_DIR: %LOG_DIR% -NODE_DIR=$SIM_DIR/$NODE_NAME -EXE_DIR=$BUILD_DIR/bin -CFG_DIR=$NODE_DIR/cfg -LOG_DIR=$NODE_DIR/log -DATA_DIR=$NODE_DIR/data +set DATA_DIR=%NODE_DIR%data\ +rem echo DATA_DIR: %DATA_DIR% -rm -rf $NODE_DIR +set TAOS_CFG=%CFG_DIR%taos.cfg +rem echo TAOS_CFG: %TAOS_CFG% -mkdir -p $SIM_DIR -mkdir -p $NODE_DIR -mkdir -p $LOG_DIR -mkdir -p $DATA_DIR +if not exist %SIM_DIR% mkdir %SIM_DIR% +if not exist %NODE_DIR% mkdir %NODE_DIR% +if exist %CFG_DIR% rmdir /s/q %CFG_DIR% +if exist %LOG_DIR% rmdir /s/q %LOG_DIR% +if exist %DATA_DIR% rmdir /s/q %DATA_DIR% +if not exist %CFG_DIR% mkdir %CFG_DIR% +if not exist %LOG_DIR% mkdir %LOG_DIR% +if not exist %DATA_DIR% mkdir %DATA_DIR% -#cp -rf $TAOS_DIR/cfg $NODE_DIR/ -mkdir -p $CFG_DIR +if %NODE% == 1 set NODE=7100 +if %NODE% == 2 set NODE=7200 +if %NODE% == 3 set NODE=7300 +if %NODE% == 4 set NODE=7400 +if %NODE% == 5 set NODE=7500 +if %NODE% == 6 set NODE=7600 +if %NODE% == 7 set NODE=7700 +if %NODE% == 8 set NODE=7800 -#allow normal user to read/write log -chmod -R 777 $NODE_DIR - -TAOS_CFG=$NODE_DIR/cfg/taos.cfg -touch -f $TAOS_CFG - -TAOS_FLAG=$SIM_DIR/tsim/flag -if [ -f "$TAOS_FLAG" ] ; then - TAOS_CFG=/etc/taos/taos.cfg - DATA_DIR=/var/lib/taos - LOG_DIR=/var/log/taos - sudo rm -f /etc/taos/*.cfg - sudo cp -rf $TAOS_DIR/cfg/*.cfg /etc/taos - sudo rm -rf $DATA_DIR - sudo rm -rf $LOG_DIR -fi - -HOSTNAME=`hostname -f` - -if [ $NODE -eq 1 ]; then - NODE=7100 -elif [ $NODE -eq 2 ]; then - NODE=7200 -elif [ $NODE -eq 3 ]; then - NODE=7300 -elif [ $NODE -eq 4 ]; then - NODE=7400 -elif [ $NODE -eq 5 ]; then - NODE=7500 -elif [ $NODE -eq 6 ]; then - NODE=7600 -elif [ $NODE -eq 7 ]; then - NODE=7700 -elif [ $NODE -eq 8 ]; then - NODE=7800 -fi - -echo " " >> $TAOS_CFG -echo "firstEp ${HOSTNAME}:7100" >> $TAOS_CFG -echo "secondEp ${HOSTNAME}:7200" >> $TAOS_CFG -echo "serverPort ${NODE}" >> $TAOS_CFG -echo "dataDir $DATA_DIR" >> $TAOS_CFG -echo "logDir $LOG_DIR" >> $TAOS_CFG -echo "debugFlag 0" >> $TAOS_CFG -echo "mDebugFlag 143" >> $TAOS_CFG -echo "sdbDebugFlag 143" >> $TAOS_CFG -echo "dDebugFlag 143" >> $TAOS_CFG -echo "vDebugFlag 143" >> $TAOS_CFG -echo "tsdbDebugFlag 143" >> $TAOS_CFG -echo "cDebugFlag 143" >> $TAOS_CFG -echo "jnidebugFlag 143" >> $TAOS_CFG -echo "odbcdebugFlag 143" >> $TAOS_CFG -echo "httpDebugFlag 143" >> $TAOS_CFG -echo "monDebugFlag 143" >> $TAOS_CFG -echo "mqttDebugFlag 143" >> $TAOS_CFG -echo "qdebugFlag 143" >> $TAOS_CFG -echo "rpcDebugFlag 143" >> $TAOS_CFG -echo "tmrDebugFlag 131" >> $TAOS_CFG -echo "udebugFlag 143" >> $TAOS_CFG -echo "sdebugFlag 143" >> $TAOS_CFG -echo "wdebugFlag 143" >> $TAOS_CFG -echo "cqdebugFlag 143" >> $TAOS_CFG -echo "monitor 0" >> $TAOS_CFG -echo "monitorInterval 1" >> $TAOS_CFG -echo "http 0" >> $TAOS_CFG -echo "slaveQuery 0" >> $TAOS_CFG -echo "numOfThreadsPerCore 2.0" >> $TAOS_CFG -echo "defaultPass taosdata" >> $TAOS_CFG -echo "numOfLogLines 20000000" >> $TAOS_CFG -echo "mnodeEqualVnodeNum 0" >> $TAOS_CFG -echo "balanceInterval 1" >> $TAOS_CFG -echo "clog 2" >> $TAOS_CFG -#echo "cache 1" >> $TAOS_CFG -echo "days 10" >> $TAOS_CFG -echo "statusInterval 1" >> $TAOS_CFG -echo "maxVgroupsPerDb 4" >> $TAOS_CFG -echo "minTablesPerVnode 4" >> $TAOS_CFG -echo "maxTablesPerVnode 1000" >> $TAOS_CFG -echo "tableIncStepPerVnode 10000" >> $TAOS_CFG -echo "asyncLog 0" >> $TAOS_CFG -echo "numOfMnodes 1" >> $TAOS_CFG -echo "locale en_US.UTF-8" >> $TAOS_CFG -echo "fsync 0" >> $TAOS_CFG -echo "telemetryReporting 0" >> $TAOS_CFG -echo " " >> $TAOS_CFG +set "fqdn=" +for /f "skip=1" %%A in ( + 'wmic computersystem get caption' +) do if not defined fqdn set "fqdn=%%A" +echo firstEp %fqdn% > %TAOS_CFG% +echo fqdn %fqdn% >> %TAOS_CFG% +echo serverPort %NODE% >> %TAOS_CFG% +echo dataDir %DATA_DIR% >> %TAOS_CFG% +echo logDir %LOG_DIR% >> %TAOS_CFG% +echo debugFlag 0 >> %TAOS_CFG% +echo mDebugFlag 143 >> %TAOS_CFG% +echo sdbDebugFlag 143 >> %TAOS_CFG% +echo dDebugFlag 143 >> %TAOS_CFG% +echo vDebugFlag 143 >> %TAOS_CFG% +echo tsdbDebugFlag 143 >> %TAOS_CFG% +echo cDebugFlag 143 >> %TAOS_CFG% +echo jnidebugFlag 143 >> %TAOS_CFG% +echo odbcdebugFlag 143 >> %TAOS_CFG% +echo httpDebugFlag 143 >> %TAOS_CFG% +echo monDebugFlag 143 >> %TAOS_CFG% +echo mqttDebugFlag 143 >> %TAOS_CFG% +echo qdebugFlag 143 >> %TAOS_CFG% +echo rpcDebugFlag 143 >> %TAOS_CFG% +echo tmrDebugFlag 131 >> %TAOS_CFG% +echo udebugFlag 143 >> %TAOS_CFG% +echo sdebugFlag 143 >> %TAOS_CFG% +echo wdebugFlag 143 >> %TAOS_CFG% +echo cqdebugFlag 143 >> %TAOS_CFG% +echo monitor 0 >> %TAOS_CFG% +echo monitorInterval 1 >> %TAOS_CFG% +echo http 0 >> %TAOS_CFG% +echo slaveQuery 0 >> %TAOS_CFG% +echo numOfThreadsPerCore 2.0 >> %TAOS_CFG% +echo defaultPass taosdata >> %TAOS_CFG% +echo numOfLogLines 20000000 >> %TAOS_CFG% +echo mnodeEqualVnodeNum 0 >> %TAOS_CFG% +echo balanceInterval 1 >> %TAOS_CFG% +echo clog 2 >> %TAOS_CFG% +echo days 10 >> %TAOS_CFG% +echo statusInterval 1 >> %TAOS_CFG% +echo maxVgroupsPerDb 4 >> %TAOS_CFG% +echo minTablesPerVnode 4 >> %TAOS_CFG% +echo maxTablesPerVnode 1000 >> %TAOS_CFG% +echo tableIncStepPerVnode 10000 >> %TAOS_CFG% +echo asyncLog 0 >> %TAOS_CFG% +echo numOfMnodes 1 >> %TAOS_CFG% +echo locale en_US.UTF-8 >> %TAOS_CFG% +echo fsync 0 >> %TAOS_CFG% +echo telemetryReporting 0 >> %TAOS_CFG% diff --git a/tests/script/sh/exec.bat b/tests/script/sh/exec.bat index 1c84a6b10e..47b7910210 100644 --- a/tests/script/sh/exec.bat +++ b/tests/script/sh/exec.bat @@ -1,115 +1,41 @@ -#!/bin/bash +@echo off -# if [ $# != 4 || $# != 5 ]; then - # echo "argument list need input : " - # echo " -n nodeName" - # echo " -s start/stop" - # echo " -c clear" - # exit 1 -# fi +rem echo Executing exec.sh -NODE_NAME= -EXEC_OPTON= -CLEAR_OPTION="false" -while getopts "n:s:u:x:ct" arg -do - case $arg in - n) - NODE_NAME=$OPTARG - ;; - s) - EXEC_OPTON=$OPTARG - ;; - c) - CLEAR_OPTION="clear" - ;; - t) - SHELL_OPTION="true" - ;; - u) - USERS=$OPTARG - ;; - x) - SIGNAL=$OPTARG - ;; - ?) - echo "unkown argument" - ;; - esac -done +if %1 == -n set NODE_NAME=%2 +if %1 == -s set EXEC_OPTON=%2 +if %3 == -n set NODE_NAME=%4 +if %3 == -s set EXEC_OPTON=%4 -SCRIPT_DIR=`dirname $0` -cd $SCRIPT_DIR/../ -SCRIPT_DIR=`pwd` +rem echo NODE_NAME: %NODE_NAME% +rem echo NODE: %EXEC_OPTON% -IN_TDINTERNAL="community" -if [[ "$SCRIPT_DIR" == *"$IN_TDINTERNAL"* ]]; then - cd ../../.. -else - cd ../../ -fi +set SCRIPT_DIR=%~dp0..\ +rem echo SCRIPT_DIR: %SCRIPT_DIR% -TAOS_DIR=`pwd` -TAOSD_DIR=`find . -name "taosd"|grep bin|head -n1` +set BUILD_DIR=%SCRIPT_DIR%..\..\..\debug\build\bin\ +set TAOSD=%BUILD_DIR%taosd +rem echo BUILD_DIR: %BUILD_DIR% +rem echo TAOSD: %TAOSD% -if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then - BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2,3` -else - BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2` -fi +set SIM_DIR=%SCRIPT_DIR%..\..\..\sim\ +rem echo SIM_DIR: %SIM_DIR% -BUILD_DIR=$TAOS_DIR/$BIN_DIR/build +set NODE_DIR=%SIM_DIR%%NODE_NAME%\ +rem echo NODE_DIR: %NODE_DIR% -SIM_DIR=$TAOS_DIR/sim -NODE_DIR=$SIM_DIR/$NODE_NAME -EXE_DIR=$BUILD_DIR/bin -CFG_DIR=$NODE_DIR/cfg -LOG_DIR=$NODE_DIR/log -DATA_DIR=$NODE_DIR/data -MGMT_DIR=$NODE_DIR/data/mgmt -TSDB_DIR=$NODE_DIR/data/tsdb +set CFG_DIR=%NODE_DIR%cfg\ +rem echo CFG_DIR: %CFG_DIR% -TAOS_CFG=$NODE_DIR/cfg/taos.cfg +set TAOS_CFG=%CFG_DIR%taos.cfg +rem echo TAOS_CFG: %TAOS_CFG% -echo ------------ $EXEC_OPTON $NODE_NAME - -TAOS_FLAG=$SIM_DIR/tsim/flag -if [ -f "$TAOS_FLAG" ]; then - EXE_DIR=/usr/local/bin/taos -fi - -if [ "$CLEAR_OPTION" = "clear" ]; then - echo rm -rf $MGMT_DIR $TSDB_DIR - rm -rf $TSDB_DIR - rm -rf $MGMT_DIR -fi - -if [ "$EXEC_OPTON" = "start" ]; then - echo "ExcuteCmd:" $EXE_DIR/taosd -c $CFG_DIR - - if [ "$SHELL_OPTION" = "true" ]; then - TT=`date +%s` - mkdir ${LOG_DIR}/${TT} - nohup valgrind --log-file=${LOG_DIR}/${TT}/valgrind.log --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 & - else - nohup $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 & - fi - -else - #relative path - RCFG_DIR=sim/$NODE_NAME/cfg - PID=`ps -ef|grep taosd | grep $RCFG_DIR | grep -v grep | awk '{print $2}'` - while [ -n "$PID" ] - do - if [ "$SIGNAL" = "SIGKILL" ]; then - echo try to kill by signal SIGKILL - kill -9 $PID - else - echo try to kill by signal SIGINT - kill -SIGINT $PID - fi - sleep 1 - PID=`ps -ef|grep taosd | grep $RCFG_DIR | grep -v grep | awk '{print $2}'` - done -fi +if %EXEC_OPTON% == start ( + echo start %TAOSD% -c %CFG_DIR% + start %TAOSD% -c %CFG_DIR% +) +if %EXEC_OPTON% == stop ( + rem echo wmic process where "name='taosd.exe' and CommandLine like '%%%NODE_NAME%%%'" list INSTANCE + wmic process where "name='taosd.exe' and CommandLine like '%%%NODE_NAME%%%'" call terminate > NUL 2>&1 +) diff --git a/tests/script/sh/stop_dnodes.bat b/tests/script/sh/stop_dnodes.bat index 1a6e7153c3..e44b7b4e29 100644 --- a/tests/script/sh/stop_dnodes.bat +++ b/tests/script/sh/stop_dnodes.bat @@ -1,24 +1,5 @@ -#!/bin/sh +@echo off -PID=`ps -ef|grep /usr/bin/taosd | grep -v grep | awk '{print $2}'` -if [ -n "$PID" ]; then - echo systemctl stop taosd - systemctl stop taosd -fi - -PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` -while [ -n "$PID" ]; do - echo kill -9 $PID - pkill -9 taosd - fuser -k -n tcp 6030 - PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` -done - -PID=`ps -ef|grep -w tarbitrator | grep -v grep | awk '{print $2}'` -while [ -n "$PID" ]; do - echo kill -9 $PID - pkill -9 tarbitrator - fuser -k -n tcp 6040 - PID=`ps -ef|grep -w tarbitrator | grep -v grep | awk '{print $2}'` -done +rem echo taskkill /F /IM taosd.exe +taskkill /F /IM taosd.exe > NUL 2>&1 \ No newline at end of file diff --git a/tests/script/wtest.bat b/tests/script/wtest.bat index 8ca1f22518..a89c1df67a 100644 --- a/tests/script/wtest.bat +++ b/tests/script/wtest.bat @@ -1,29 +1,30 @@ @echo off echo TDengine in windows -echo Start TDengine Testing Case ... +rem echo Start TDengine Testing Case ... set "SCRIPT_DIR=%~dp0" -echo SCRIPT_DIR: %SCRIPT_DIR% +rem echo SCRIPT_DIR: %SCRIPT_DIR% -set "BUILD_DIR=%~dp0..\..\debug\build\bin" -set "TSIM=%~dp0..\..\debug\build\bin\tsim" -echo BUILD_DIR: %BUILD_DIR% +set "BUILD_DIR=%SCRIPT_DIR%..\..\..\debug\build\bin\" +set "TSIM=%BUILD_DIR%tsim" +rem echo BUILD_DIR: %BUILD_DIR% +rem echo TSIM: %TSIM% -set "SIM_DIR=%~dp0..\..\sim" -echo SIM_DIR: %SIM_DIR% +set "SIM_DIR=%SCRIPT_DIR%..\..\..\sim\" +rem echo SIM_DIR: %SIM_DIR% -set "TSIM_DIR=%~dp0..\..\sim\tsim" -echo TSIM_DIR: %TSIM_DIR% +set "TSIM_DIR=%SIM_DIR%tsim\" +rem echo TSIM_DIR: %TSIM_DIR% -set "CFG_DIR=%~dp0..\..\sim\tsim\cfg" -echo CFG_DIR: %CFG_DIR% +set "CFG_DIR=%TSIM_DIR%cfg\" +rem echo CFG_DIR: %CFG_DIR% -set "LOG_DIR=%~dp0..\..\sim\tsim\log" -echo LOG_DIR: %LOG_DIR% +set "LOG_DIR=%TSIM_DIR%log\" +rem echo LOG_DIR: %LOG_DIR% -set "TAOS_CFG=%~dp0..\..\sim\tsim\cfg\taos.cfg" -echo TAOS_CFG: %TAOS_CFG% +set "TAOS_CFG=%CFG_DIR%taos.cfg" +rem echo TAOS_CFG: %TAOS_CFG% if not exist %SIM_DIR% mkdir %SIM_DIR% if not exist %TSIM_DIR% mkdir %TSIM_DIR% @@ -32,7 +33,13 @@ if exist %LOG_DIR% rmdir /s/q %LOG_DIR% if not exist %CFG_DIR% mkdir %CFG_DIR% if not exist %LOG_DIR% mkdir %LOG_DIR% -echo firstEp localhost > %TAOS_CFG% +set "fqdn=" +for /f "skip=1" %%A in ( + 'wmic computersystem get caption' +) do if not defined fqdn set "fqdn=%%A" + +echo firstEp %fqdn% > %TAOS_CFG% +echo fqdn %fqdn% >> %TAOS_CFG% echo serverPort 7100 >> %TAOS_CFG% echo logDir %LOG_DIR% >> %TAOS_CFG% echo scriptDir %SCRIPT_DIR% >> %TAOS_CFG% @@ -49,7 +56,7 @@ echo enableCoreFile 1 >> %TAOS_CFG% set "FILE_NAME=testSuite.sim" if "%1" == "-f" set "FILE_NAME=%2" -echo FILE_NAME: %FILE_NAME% +rem echo FILE_NAME: %FILE_NAME% echo ExcuteCmd: %tsim% -c %CFG_DIR% -f %FILE_NAME% -%tsim% -c %CFG_DIR% -f %FILE_NAME% \ No newline at end of file +%TSIM% -c %CFG_DIR% -f %FILE_NAME% \ No newline at end of file diff --git a/tests/tsim/src/simExe.c b/tests/tsim/src/simExe.c index 10e87eefe6..83ca46599c 100644 --- a/tests/tsim/src/simExe.c +++ b/tests/tsim/src/simExe.c @@ -301,11 +301,37 @@ bool simExecuteRunBackCmd(SScript *script, char *option) { return true; } +void simReplaceShToBat(char *dst) { + char* sh = strstr(dst, ".sh"); + if (sh != NULL) { + int32_t dstLen = (int32_t)strlen(dst); + char *end = dst + dstLen; + *(end + 1) = 0; + + for (char *p = end; p >= sh; p--) { + *(p + 1) = *p; + } + + sh[0] = '.'; + sh[1] = 'b'; + sh[2] = 'a'; + sh[3] = 't'; + sh[4] = ' '; + } + + simDebug("system cmd is %s", dst); +} + bool simExecuteSystemCmd(SScript *script, char *option) { char buf[4096] = {0}; +#ifndef WINDOWS sprintf(buf, "cd %s; ", tsScriptDir); simVisuallizeOption(script, option, buf + strlen(buf)); +#else + sprintf(buf, "%s%s", tsScriptDir, option); + simReplaceShToBat(buf); +#endif simLogSql(buf, true); int32_t code = system(buf); From d0534df8c01ef7958337a57644d9d3fe59702dd4 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 19 Jan 2021 15:26:44 +0800 Subject: [PATCH 027/147] TD-1207 --- src/dnode/src/dnodeSystem.c | 3 +++ src/os/inc/osSignal.h | 2 +- src/os/src/windows/wSignal.c | 13 ++++++++++--- tests/script/sh/deploy.bat | 2 +- tests/script/sh/exec.bat | 9 ++++++++- tests/script/wtest.bat | 2 +- 6 files changed, 24 insertions(+), 7 deletions(-) diff --git a/src/dnode/src/dnodeSystem.c b/src/dnode/src/dnodeSystem.c index 3a50eca9e6..dcbf249a51 100644 --- a/src/dnode/src/dnodeSystem.c +++ b/src/dnode/src/dnodeSystem.c @@ -145,6 +145,8 @@ int32_t main(int32_t argc, char *argv[]) { syslog(LOG_INFO, "Shut down TDengine service successfully"); dInfo("TDengine is shut down!"); closelog(); + + tsem_post(&exitSem); return EXIT_SUCCESS; } @@ -170,4 +172,5 @@ static void sigintHandler(int32_t signum) { // inform main thread to exit tsem_post(&exitSem); + tsem_wait(&exitSem); } \ No newline at end of file diff --git a/src/os/inc/osSignal.h b/src/os/inc/osSignal.h index 57582a8a28..8016c49ba8 100644 --- a/src/os/inc/osSignal.h +++ b/src/os/inc/osSignal.h @@ -29,7 +29,7 @@ extern "C" { #endif #ifndef SIGHUP - #define SIGHUP 1234 + #define SIGHUP 1230 #endif #ifndef SIGCHLD diff --git a/src/os/src/windows/wSignal.c b/src/os/src/windows/wSignal.c index 8a3c2cfbd4..3988f0c6e1 100644 --- a/src/os/src/windows/wSignal.c +++ b/src/os/src/windows/wSignal.c @@ -16,18 +16,25 @@ #define _DEFAULT_SOURCE #include "os.h" #include +#include void taosSetSignal(int32_t signum, FSignalHandler sigfp) { if (signum == SIGUSR1) return; - signal(signum, sigfp); + + // SIGHUP doesn't exist in windows, we handle it in the way of ctrlhandler + if (signum == SIGHUP) { + SetConsoleCtrlHandler((PHANDLER_ROUTINE)sigfp, TRUE); + } else { + signal(signum, sigfp); + } } void taosIgnSignal(int32_t signum) { - if (signum == SIGUSR1) return; + if (signum == SIGUSR1 || signum == SIGHUP) return; signal(signum, SIG_IGN); } void taosDflSignal(int32_t signum) { - if (signum == SIGUSR1) return; + if (signum == SIGUSR1 || signum == SIGHUP) return; signal(signum, SIG_DFL); } diff --git a/tests/script/sh/deploy.bat b/tests/script/sh/deploy.bat index 0b419b1e9b..7a81761e5b 100644 --- a/tests/script/sh/deploy.bat +++ b/tests/script/sh/deploy.bat @@ -54,7 +54,7 @@ if %NODE% == 6 set NODE=7600 if %NODE% == 7 set NODE=7700 if %NODE% == 8 set NODE=7800 -set "fqdn=" +rem set "fqdn=" for /f "skip=1" %%A in ( 'wmic computersystem get caption' ) do if not defined fqdn set "fqdn=%%A" diff --git a/tests/script/sh/exec.bat b/tests/script/sh/exec.bat index 47b7910210..9ad6667644 100644 --- a/tests/script/sh/exec.bat +++ b/tests/script/sh/exec.bat @@ -37,5 +37,12 @@ if %EXEC_OPTON% == start ( if %EXEC_OPTON% == stop ( rem echo wmic process where "name='taosd.exe' and CommandLine like '%%%NODE_NAME%%%'" list INSTANCE - wmic process where "name='taosd.exe' and CommandLine like '%%%NODE_NAME%%%'" call terminate > NUL 2>&1 + rem wmic process where "name='taosd.exe' and CommandLine like '%%%NODE_NAME%%%'" call terminate > NUL 2>&1 + + for /f "tokens=1 skip=1" %%A in ( + 'wmic process where "name='taosd.exe'" get processId ' + ) do ( + rem echo taskkill /IM %%A + taskkill /IM %%A > NUL 2>&1 + ) ) diff --git a/tests/script/wtest.bat b/tests/script/wtest.bat index a89c1df67a..0b5cdda527 100644 --- a/tests/script/wtest.bat +++ b/tests/script/wtest.bat @@ -33,7 +33,7 @@ if exist %LOG_DIR% rmdir /s/q %LOG_DIR% if not exist %CFG_DIR% mkdir %CFG_DIR% if not exist %LOG_DIR% mkdir %LOG_DIR% -set "fqdn=" +rem set "fqdn=" for /f "skip=1" %%A in ( 'wmic computersystem get caption' ) do if not defined fqdn set "fqdn=%%A" From fffe865871c100e2a1a257fba55ac77b9d8d9da4 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 20 Jan 2021 09:40:47 +0800 Subject: [PATCH 028/147] TD-1207 --- src/dnode/src/dnodeSystem.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/dnode/src/dnodeSystem.c b/src/dnode/src/dnodeSystem.c index dcbf249a51..b7ca6f053c 100644 --- a/src/dnode/src/dnodeSystem.c +++ b/src/dnode/src/dnodeSystem.c @@ -146,7 +146,9 @@ int32_t main(int32_t argc, char *argv[]) { dInfo("TDengine is shut down!"); closelog(); +#ifdef WINDOWS tsem_post(&exitSem); +#endif return EXIT_SUCCESS; } @@ -172,5 +174,7 @@ static void sigintHandler(int32_t signum) { // inform main thread to exit tsem_post(&exitSem); +#ifdef WINDOWS tsem_wait(&exitSem); +#endif } \ No newline at end of file From d73cdd656dd6705d84efb275aae17e9010ff52ea Mon Sep 17 00:00:00 2001 From: freemine Date: Wed, 20 Jan 2021 10:05:53 +0800 Subject: [PATCH 029/147] 1. bugFix: typo in eok.c which results in memory leakage 2. for the sake of SIGALRM used in the base code, add taos_block_sigalrm and so forth 3. TODO: getaddrinfo would block a few unnecessary seconds when host ~ *.local better give warning/suggestion message to user, or just bypass getaddrinfo and fail the function --- src/os/inc/eok.h | 4 ++++ src/os/inc/osDarwin.h | 2 ++ src/os/src/darwin/darwinTimer.c | 13 +++++++++++++ src/os/src/darwin/eok.c | 16 +++++++--------- src/os/src/detail/osTimer.c | 6 +++++- 5 files changed, 31 insertions(+), 10 deletions(-) diff --git a/src/os/inc/eok.h b/src/os/inc/eok.h index 4d38e8e632..0874ca975b 100644 --- a/src/os/inc/eok.h +++ b/src/os/inc/eok.h @@ -22,6 +22,8 @@ extern "C" { #endif +#ifdef __APPLE__ + enum EPOLL_EVENTS { EPOLLIN = 0x001, @@ -81,6 +83,8 @@ int epoll_ctl(int epfd, int op, int fd, struct epoll_event *event); int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout); int epoll_close(int epfd); +#endif // __APPLE__ + #ifdef __cplusplus } #endif diff --git a/src/os/inc/osDarwin.h b/src/os/inc/osDarwin.h index 52bb661f89..935708eb1b 100644 --- a/src/os/inc/osDarwin.h +++ b/src/os/inc/osDarwin.h @@ -104,6 +104,8 @@ int64_t tsosStr2int64(char *str); #include "eok.h" +void taos_block_sigalrm(void); + diff --git a/src/os/src/darwin/darwinTimer.c b/src/os/src/darwin/darwinTimer.c index 5fe65fb99e..121e2e9427 100644 --- a/src/os/src/darwin/darwinTimer.c +++ b/src/os/src/darwin/darwinTimer.c @@ -34,3 +34,16 @@ void taosUninitTimer() { setitimer(ITIMER_REAL, &tv, NULL); } +void taos_block_sigalrm(void) { + // since SIGALRM has been used + // consideration: any better solution? + static __thread int already_set = 0; + if (!already_set) { + sigset_t set; + sigemptyset(&set); + sigaddset(&set, SIGALRM); + pthread_sigmask(SIG_BLOCK, &set, NULL); + already_set = 1; + } +} + diff --git a/src/os/src/darwin/eok.c b/src/os/src/darwin/eok.c index 1be5ec4ac1..9c6a263a40 100644 --- a/src/os/src/darwin/eok.c +++ b/src/os/src/darwin/eok.c @@ -15,15 +15,11 @@ #include "eok.h" -#include -#include -#include -#include -#include -#include +#include "os.h" + #include -#include -#include + +#define LET_IT_BE #ifdef ENABLE_LOG #define D(fmt, ...) fprintf(stderr, "%s[%d]%s(): " fmt "\n", basename(__FILE__), __LINE__, __func__, ##__VA_ARGS__) @@ -414,6 +410,8 @@ int epoll_ctl(int epfd, int op, int fd, struct epoll_event *event) { static struct timespec do_timespec_diff(struct timespec *from, struct timespec *to); int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) { + taos_block_sigalrm(); + int e = 0; if (!events) { errno = EINVAL; @@ -699,7 +697,7 @@ static void eok_free_ev(ep_over_kq_t *eok, eok_event_t *ev) { else eok->evs_tail = ev->prev; ev->prev = NULL; ev->next = eok->evs_free; - eok->evs_free = ev->next; + eok->evs_free = ev; eok->evs_count -= 1; } diff --git a/src/os/src/detail/osTimer.c b/src/os/src/detail/osTimer.c index 1d3ba30def..c1135ce292 100644 --- a/src/os/src/detail/osTimer.c +++ b/src/os/src/detail/osTimer.c @@ -116,6 +116,9 @@ void taosUninitTimer() { pthread_sigmask(SIG_BLOCK, &set, NULL); */ void taosMsleep(int mseconds) { +#ifdef __APPLE__ + taos_block_sigalrm(); +#endif // __APPLE__ #if 1 usleep(mseconds * 1000); #else @@ -136,6 +139,7 @@ void taosMsleep(int mseconds) { /* pthread_sigmask(SIG_UNBLOCK, &set, NULL); */ #endif + } -#endif \ No newline at end of file +#endif From b825035d75a584573013979238d0adf3f996d77b Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Wed, 20 Jan 2021 15:18:28 +0800 Subject: [PATCH 030/147] [TD-2671] : Super Table will use keyword "STABLE" since now. --- .../webdocs/markdowndocs/TAOS SQL-ch.md | 31 ++++---- .../webdocs/markdowndocs/administrator-ch.md | 78 +++++++++---------- 2 files changed, 56 insertions(+), 53 deletions(-) diff --git a/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md b/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md index 4563b5b5fc..560ee87dcc 100644 --- a/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md +++ b/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md @@ -213,16 +213,19 @@ TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMic 如果表是通过[超级表](../super-table/)创建，更改表结构的操作只能对超级表进行。同时针对超级表的结构更改对所有通过该结构创建的表生效。对于不是通过超级表创建的表，可以直接修改表结构 ## 超级表STable管理 + +注意：在 2.0.15 以前的版本中，并不支持 STABLE 保留字，而是写作 TABLE。也即，在本节后文的指令说明中，CREATE、DROP、ALTER 三个指令在老版本中保留字需写作 TABLE 而不是 STABLE。 + - **创建超级表** ```mysql - CREATE TABLE [IF NOT EXISTS] stb_name (timestamp_field_name TIMESTAMP, field1_name data_type1 [, field2_name data_type2 ...]) TAGS (tag1_name tag_type1, tag2_name tag_type2 [, tag3_name tag_type3]); + CREATE STABLE [IF NOT EXISTS] stb_name (timestamp_field_name TIMESTAMP, field1_name data_type1 [, field2_name data_type2 ...]) TAGS (tag1_name tag_type1, tag2_name tag_type2 [, tag3_name tag_type3]); ``` - 创建STable, 与创建表的SQL语法相似，但需指定TAGS字段的名称和类型 + 创建 STable，与创建表的 SQL 语法相似，但需指定 TAGS 字段的名称和类型 说明： - 1) TAGS 列的数据类型不能是timestamp类型； + 1) TAGS 列的数据类型不能是 timestamp 类型； 2) TAGS 列名不能与其他列名相同； @@ -233,16 +236,16 @@ TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMic - **删除超级表** ```mysql - DROP TABLE [IF EXISTS] stb_name; + DROP STABLE [IF EXISTS] stb_name; ``` - 删除STable会自动删除通过STable创建的子表。 + 删除 STable 会自动删除通过 STable 创建的子表。 - **显示当前数据库下的所有超级表信息** ```mysql SHOW STABLES [LIKE tb_name_wildcar]; ``` - 查看数据库内全部STable，及其相关信息，包括STable的名称、创建时间、列数量、标签（TAG）数量、通过该STable建表的数量。 + 查看数据库内全部 STable，及其相关信息，包括 STable 的名称、创建时间、列数量、标签（TAG）数量、通过该 STable 建表的数量。 - **获取超级表的结构信息** @@ -253,43 +256,43 @@ TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMic - **超级表增加列** ```mysql - ALTER TABLE stb_name ADD COLUMN field_name data_type; + ALTER STABLE stb_name ADD COLUMN field_name data_type; ``` - **超级表删除列** ```mysql - ALTER TABLE stb_name DROP COLUMN field_name; + ALTER STABLE stb_name DROP COLUMN field_name; ``` ## 超级表 STable 中 TAG 管理 - **添加标签** ```mysql - ALTER TABLE stb_name ADD TAG new_tag_name tag_type; + ALTER STABLE stb_name ADD TAG new_tag_name tag_type; ``` - 为STable增加一个新的标签，并指定新标签的类型。标签总数不能超过128个，总长度不超过16k个字符。 + 为 STable 增加一个新的标签，并指定新标签的类型。标签总数不能超过 128 个，总长度不超过 16k 个字符。 - **删除标签** ```mysql - ALTER TABLE stb_name DROP TAG tag_name; + ALTER STABLE stb_name DROP TAG tag_name; ``` 删除超级表的一个标签，从超级表删除某个标签后，该超级表下的所有子表也会自动删除该标签。 - **修改标签名** ```mysql - ALTER TABLE stb_name CHANGE TAG old_tag_name new_tag_name; + ALTER STABLE stb_name CHANGE TAG old_tag_name new_tag_name; ``` 修改超级表的标签名，从超级表修改某个标签名后，该超级表下的所有子表也会自动更新该标签名。 - **修改子表标签值** ```mysql - ALTER TABLE tb_name SET TAG tag_name=new_tag_value; + ALTER STABLE tb_name SET TAG tag_name=new_tag_value; ``` - 说明：除了更新标签的值的操作是针对子表进行，其他所有的标签操作（添加标签、删除标签等）均只能作用于STable，不能对单个子表操作。对STable添加标签以后，依托于该STable建立的所有表将自动增加了一个标签，所有新增标签的默认值都是NULL。 + 说明：除了更新标签的值的操作是针对子表进行，其他所有的标签操作（添加标签、删除标签等）均只能作用于 STable，不能对单个子表操作。对 STable 添加标签以后，依托于该 STable 建立的所有表将自动增加了一个标签，所有新增标签的默认值都是 NULL。 ## 数据写入 diff --git a/documentation20/webdocs/markdowndocs/administrator-ch.md b/documentation20/webdocs/markdowndocs/administrator-ch.md index 9eee5b2b69..7f477f6757 100644 --- a/documentation20/webdocs/markdowndocs/administrator-ch.md +++ b/documentation20/webdocs/markdowndocs/administrator-ch.md @@ -12,7 +12,7 @@ Memory Size = maxVgroupsPerDb * (blocks * cache + 10Mb) + numOfTables * (tagSizePerTable + 0.5Kb) ``` -示例：假设是4核机器，cache是缺省大小16M, blocks是缺省值6，假设有10万张表，标签总长度是256字节，则总的内存需求为：4\*(16\*6+10) + 100000*(0.25+0.5)/1000 = 499M。 +示例：假设是4核机器，cache是缺省大小16M, blocks是缺省值6，假设有10万张表，标签总长度是256字节，则总的内存需求为：4\*(16\*6+10) + 100000\*(0.25+0.5)/1000 = 499M。 实际运行的系统往往会根据数据特点的不同，将数据存放在不同的DB里。因此做规划时，也需要考虑。 @@ -35,7 +35,7 @@ TDengine相对于通用数据库，有超高的压缩比，在绝大多数场景 Raw DataSize = numOfTables * rowSizePerTable * rowsPerTable ``` -示例：1000万台智能电表，每台电表每15分钟采集一次数据，每次采集的数据128字节，那么一年的原始数据量是：10000000\*128\*24\*60/15*365 = 44.8512T。TDengine大概需要消耗44.851/5=8.97024T空间。 +示例：1000万台智能电表，每台电表每15分钟采集一次数据，每次采集的数据128字节，那么一年的原始数据量是：10000000\*128\*24\*60/15\*365 = 44.8512T。TDengine大概需要消耗44.851/5=8.97024T空间。 用户可以通过参数keep，设置数据在磁盘中的最大保存时长。为进一步减少存储成本，TDengine还提供多级存储，最冷的数据可以存放在最廉价的存储介质上，应用的访问不用做任何调整，只是读取速度降低了。 @@ -181,7 +181,7 @@ taos -C 或 taos --dump-config 客户端的输入的字符均采用操作系统当前默认的编码格式，在Linux系统上多为UTF-8，部分中文系统编码则可能是GB18030或GBK等。在docker环境中默认的编码是POSIX。在中文版Windows系统中，编码则是CP936。客户端需要确保正确设置自己所使用的字符集，即客户端运行的操作系统当前编码字符集，才能保证nchar中的数据正确转换为UCS4-LE编码格式。 - 在 Linux 中 locale 的命名规则为: <语言>_<地区>.<字符集编码> 如：zh_CN.UTF-8，zh代表中文，CN代表大陆地区，UTF-8表示字符集。字符集编码为客户端正确解析本地字符串提供编码转换的说明。Linux系统与 Mac OSX 系统可以通过设置locale来确定系统的字符编码，由于Windows使用的locale中不是POSIX标准的locale格式，因此在Windows下需要采用另一个配置参数charset来指定字符编码。在Linux 系统中也可以使用charset来指定字符编码。 + 在 Linux 中 locale 的命名规则为: <语言>\_<地区>.<字符集编码> 如：zh_CN.UTF-8，zh代表中文，CN代表大陆地区，UTF-8表示字符集。字符集编码为客户端正确解析本地字符串提供编码转换的说明。Linux系统与 Mac OSX 系统可以通过设置locale来确定系统的字符编码，由于Windows使用的locale中不是POSIX标准的locale格式，因此在Windows下需要采用另一个配置参数charset来指定字符编码。在Linux 系统中也可以使用charset来指定字符编码。 - charset @@ -452,39 +452,39 @@ TDengine的所有可执行文件默认存放在 _/usr/local/taos/bin_ 目录下 | 关键字列表 | | | | | | ---------- | ----------- | ------------ | ---------- | --------- | -| ABLOCKS | CONNECTION | GT | MINUS | SHOW | -| ABORT | CONNECTIONS | ID | MNODES | SLASH | -| ACCOUNT | COPY | IF | MODULES | SLIDING | -| ACCOUNTS | COUNT | IGNORE | NCHAR | SMALLINT | -| ADD | CREATE | IMMEDIATE | NE | SPREAD | -| AFTER | CTIME | IMPORT | NONE | STAR | -| ALL | DATABASE | IN | NOT | STATEMENT | -| ALTER | DATABASES | INITIALLY | NOTNULL | STDDEV | -| AND | DAYS | INSERT | NOW | STREAM | -| AS | DEFERRED | INSTEAD | OF | STREAMS | -| ASC | DELIMITERS | INTEGER | OFFSET | STRING | -| ATTACH | DESC | INTERVAL | OR | SUM | -| AVG | DESCRIBE | INTO | ORDER | TABLE | -| BEFORE | DETACH | IP | PASS | TABLES | -| BEGIN | DIFF | IS | PERCENTILE | TAG | -| BETWEEN | DIVIDE | ISNULL | PLUS | TAGS | -| BIGINT | DNODE | JOIN | PRAGMA | TBLOCKS | -| BINARY | DNODES | KEEP | PREV | TBNAME | -| BITAND | DOT | KEY | PRIVILEGE | TIMES | -| BITNOT | DOUBLE | KILL | QUERIES | TIMESTAMP | -| BITOR | DROP | LAST | QUERY | TINYINT | -| BOOL | EACH | LE | RAISE | TOP | -| BOTTOM | END | LEASTSQUARES | REM | TRIGGER | -| BY | EQ | LIKE | REPLACE | UMINUS | -| CACHE | EXISTS | LIMIT | REPLICA | UPLUS | -| CASCADE | EXPLAIN | LINEAR | RESET | USE | -| CHANGE | FAIL | LOCAL | RESTRICT | USER | -| CLOG | FILL | LP | ROW | USERS | -| CLUSTER | FIRST | LSHIFT | ROWS | USING | -| COLON | FLOAT | LT | RP | VALUES | -| COLUMN | FOR | MATCH | RSHIFT | VARIABLE | -| COMMA | FROM | MAX | SCORES | VGROUPS | -| COMP | GE | METRIC | SELECT | VIEW | -| CONCAT | GLOB | METRICS | SEMI | WAVG | -| CONFIGS | GRANTS | MIN | SET | WHERE | -| CONFLICT | GROUP | | | | \ No newline at end of file +| ABLOCKS | CONNECTION | GT | MNODES | SLIDING | +| ABORT | CONNECTIONS | ID | MODULES | SMALLINT | +| ACCOUNT | COPY | IF | NCHAR | SPREAD | +| ACCOUNTS | COUNT | IGNORE | NE | STABLE | +| ADD | CREATE | IMMEDIATE | NONE | STABLES | +| AFTER | CTIME | IMPORT | NOT | STAR | +| ALL | DATABASE | IN | NOTNULL | STATEMENT | +| ALTER | DATABASES | INITIALLY | NOW | STDDEV | +| AND | DAYS | INSERT | OF | STREAM | +| AS | DEFERRED | INSTEAD | OFFSET | STREAMS | +| ASC | DELIMITERS | INTEGER | OR | STRING | +| ATTACH | DESC | INTERVAL | ORDER | SUM | +| AVG | DESCRIBE | INTO | PASS | TABLE | +| BEFORE | DETACH | IP | PERCENTILE | TABLES | +| BEGIN | DIFF | IS | PLUS | TAG | +| BETWEEN | DIVIDE | ISNULL | PRAGMA | TAGS | +| BIGINT | DNODE | JOIN | PREV | TBLOCKS | +| BINARY | DNODES | KEEP | PRIVILEGE | TBNAME | +| BITAND | DOT | KEY | QUERIES | TIMES | +| BITNOT | DOUBLE | KILL | QUERY | TIMESTAMP | +| BITOR | DROP | LAST | RAISE | TINYINT | +| BOOL | EACH | LE | REM | TOP | +| BOTTOM | END | LEASTSQUARES | REPLACE | TRIGGER | +| BY | EQ | LIKE | REPLICA | UMINUS | +| CACHE | EXISTS | LIMIT | RESET | UPLUS | +| CASCADE | EXPLAIN | LINEAR | RESTRICT | USE | +| CHANGE | FAIL | LOCAL | ROW | USER | +| CLOG | FILL | LP | ROWS | USERS | +| CLUSTER | FIRST | LSHIFT | RP | USING | +| COLON | FLOAT | LT | RSHIFT | VALUES | +| COLUMN | FOR | MATCH | SCORES | VARIABLE | +| COMMA | FROM | MAX | SELECT | VGROUPS | +| COMP | GE | METRIC | SEMI | VIEW | +| CONCAT | GLOB | METRICS | SET | WAVG | +| CONFIGS | GRANTS | MIN | SHOW | WHERE | +| CONFLICT | GROUP | MINUS | SLASH | | From 05ce993084f68699cc8cd3bd95218756f7db0973 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 21 Jan 2021 15:05:09 +0800 Subject: [PATCH 031/147] TD-1207 --- tests/script/sh/deploy.bat | 2 +- tests/script/sh/exec.bat | 4 ++-- tests/script/sh/stop_dnodes.bat | 1 + tests/tsim/src/simSystem.c | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/script/sh/deploy.bat b/tests/script/sh/deploy.bat index 7a81761e5b..04c7b8a660 100644 --- a/tests/script/sh/deploy.bat +++ b/tests/script/sh/deploy.bat @@ -59,7 +59,7 @@ for /f "skip=1" %%A in ( 'wmic computersystem get caption' ) do if not defined fqdn set "fqdn=%%A" -echo firstEp %fqdn% > %TAOS_CFG% +echo firstEp %fqdn%:7100 > %TAOS_CFG% echo fqdn %fqdn% >> %TAOS_CFG% echo serverPort %NODE% >> %TAOS_CFG% echo dataDir %DATA_DIR% >> %TAOS_CFG% diff --git a/tests/script/sh/exec.bat b/tests/script/sh/exec.bat index 9ad6667644..6651c7aa8f 100644 --- a/tests/script/sh/exec.bat +++ b/tests/script/sh/exec.bat @@ -40,9 +40,9 @@ if %EXEC_OPTON% == stop ( rem wmic process where "name='taosd.exe' and CommandLine like '%%%NODE_NAME%%%'" call terminate > NUL 2>&1 for /f "tokens=1 skip=1" %%A in ( - 'wmic process where "name='taosd.exe'" get processId ' + 'wmic process where "name='taosd.exe' and CommandLine like '%%%NODE_NAME%%%'" get processId ' ) do ( rem echo taskkill /IM %%A taskkill /IM %%A > NUL 2>&1 - ) + ) ) diff --git a/tests/script/sh/stop_dnodes.bat b/tests/script/sh/stop_dnodes.bat index e44b7b4e29..ab7af2ca92 100644 --- a/tests/script/sh/stop_dnodes.bat +++ b/tests/script/sh/stop_dnodes.bat @@ -2,4 +2,5 @@ rem echo taskkill /F /IM taosd.exe +wmic process where "name='taosd.exe'" call terminate > NUL 2>&1 taskkill /F /IM taosd.exe > NUL 2>&1 \ No newline at end of file diff --git a/tests/tsim/src/simSystem.c b/tests/tsim/src/simSystem.c index bf47c56718..65086eace5 100644 --- a/tests/tsim/src/simSystem.c +++ b/tests/tsim/src/simSystem.c @@ -78,8 +78,8 @@ char *simParseHostName(char *varName) { } bool simSystemInit() { - taosGetFqdn(simHostName); taos_init(); + taosGetFqdn(simHostName); simInitsimCmdList(); memset(simScriptList, 0, sizeof(SScript *) * MAX_MAIN_SCRIPT_NUM); return true; From cd79df1de2050f311f926607c3b2c5619ee266ab Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 21 Jan 2021 19:32:01 +0800 Subject: [PATCH 032/147] TD-1207 --- src/rpc/src/rpcUdp.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/rpc/src/rpcUdp.c b/src/rpc/src/rpcUdp.c index 30a2fb2899..7281647136 100644 --- a/src/rpc/src/rpcUdp.c +++ b/src/rpc/src/rpcUdp.c @@ -140,6 +140,7 @@ void taosStopUdpConnection(void *handle) { pConn = pSet->udpConn + i; if (pConn->fd >=0) shutdown(pConn->fd, SHUT_RDWR); if (pConn->fd >=0) taosCloseSocket(pConn->fd); + pConn->fd = -1; } for (int i = 0; i < pSet->threads; ++i) { @@ -197,9 +198,16 @@ static void *taosRecvUdpData(void *param) { while (1) { dataLen = recvfrom(pConn->fd, pConn->buffer, RPC_MAX_UDP_SIZE, 0, (struct sockaddr *)&sourceAdd, &addLen); - if(dataLen <= 0) { - tDebug("%s UDP socket was closed, exiting(%s)", pConn->label, strerror(errno)); - break; + if (dataLen <= 0) { + tDebug("%s UDP socket was closed, exiting(%s), dataLen:%d fd:%d", pConn->label, strerror(errno), (int32_t)dataLen, + pConn->fd); + + //for windows usage, remote shutdown also returns - 1 in windows client + if (pConn->fd == -1) { + break; + } else { + continue; + } } port = ntohs(sourceAdd.sin_port); From 51dc09dbf2218ab4e6b858ba44926e24e17e3bf0 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 21 Jan 2021 16:35:48 +0000 Subject: [PATCH 033/147] TD-2453 --- src/client/src/tscParseInsert.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 0b419a3305..a5906f5539 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -905,6 +905,13 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { return tscInvalidSQLErrMsg(pCmd->payload, "keyword TAGS expected", sToken.z); } + index = 0; + sToken = tStrGetToken(sql, &index, false, 0, NULL); + sql += index; + if (sToken.type != TK_LP) { + return tscInvalidSQLErrMsg(pCmd->payload, NULL, sToken.z); + } + SKVRowBuilder kvRowBuilder = {0}; if (tdInitKVRowBuilder(&kvRowBuilder) < 0) { return TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -1554,6 +1561,7 @@ void tscImportDataFromFile(SSqlObj *pSql) { tscError("%p failed to open file %s to load data from file, code:%s", pSql, pCmd->payload, tstrerror(pSql->res.code)); tfree(pSupporter); + taos_free_result(pNew); tscAsyncResultOnError(pSql); return; } From fa7e9bd6d262afbf6547dcd49e4db4170c2e2ed7 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 22 Jan 2021 11:59:14 +0800 Subject: [PATCH 034/147] TD-1207 --- src/plugins/http/src/httpAuth.c | 8 +-- src/plugins/http/src/httpContext.c | 11 ++- src/plugins/http/src/httpGcHandle.c | 2 +- src/plugins/http/src/httpGcJson.c | 27 ++++---- src/plugins/http/src/httpGzip.c | 92 +++++++++++++------------ src/plugins/http/src/httpHandle.c | 2 +- src/plugins/http/src/httpJson.c | 47 ++++++------- src/plugins/http/src/httpParser.c | 96 +++++++++++++++------------ src/plugins/http/src/httpQueue.c | 14 ++-- src/plugins/http/src/httpResp.c | 5 +- src/plugins/http/src/httpRestHandle.c | 1 - src/plugins/http/src/httpRestJson.c | 23 ++++--- src/plugins/http/src/httpServer.c | 34 +++++----- src/plugins/http/src/httpSession.c | 5 +- src/plugins/http/src/httpSql.c | 10 +-- src/plugins/http/src/httpSystem.c | 6 +- src/plugins/http/src/httpTgHandle.c | 45 +++++++------ src/plugins/http/src/httpUtil.c | 55 +++++++-------- tests/script/general/http/restful.sim | 2 - 19 files changed, 247 insertions(+), 238 deletions(-) diff --git a/src/plugins/http/src/httpAuth.c b/src/plugins/http/src/httpAuth.c index 973f69f24e..2ef5406823 100644 --- a/src/plugins/http/src/httpAuth.c +++ b/src/plugins/http/src/httpAuth.c @@ -26,7 +26,7 @@ int32_t httpParseBasicAuthToken(HttpContext *pContext, char *token, int32_t len) { token[len] = '\0'; int32_t outlen = 0; - char *base64 = (char *)base64_decode(token, len, &outlen); + char * base64 = (char *)base64_decode(token, len, &outlen); if (base64 == NULL || outlen == 0) { httpError("context:%p, fd:%d, basic token:%s parsed error", pContext, pContext->fd, token); free(base64); @@ -49,7 +49,7 @@ int32_t httpParseBasicAuthToken(HttpContext *pContext, char *token, int32_t len) strncpy(pContext->user, base64, (size_t)user_len); pContext->user[user_len] = 0; - char *password = user + 1; + char * password = user + 1; int32_t pass_len = (int32_t)((base64 + outlen) - password); if (pass_len < 1 || pass_len >= HTTP_PASSWORD_LEN) { httpError("context:%p, fd:%d, basic token:%s parse password error", pContext, pContext->fd, token); @@ -66,7 +66,7 @@ int32_t httpParseBasicAuthToken(HttpContext *pContext, char *token, int32_t len) int32_t httpParseTaosdAuthToken(HttpContext *pContext, char *token, int32_t len) { token[len] = '\0'; - int32_t outlen = 0; + int32_t outlen = 0; unsigned char *base64 = base64_decode(token, len, &outlen); if (base64 == NULL || outlen == 0) { httpError("context:%p, fd:%d, taosd token:%s parsed error", pContext, pContext->fd, token); @@ -97,7 +97,7 @@ int32_t httpParseTaosdAuthToken(HttpContext *pContext, char *token, int32_t len) } int32_t httpGenTaosdAuthToken(HttpContext *pContext, char *token, int32_t maxLen) { - char buffer[sizeof(pContext->user) + sizeof(pContext->pass)] = {0}; + char buffer[sizeof(pContext->user) + sizeof(pContext->pass)] = {0}; size_t size = sizeof(pContext->user); tstrncpy(buffer, pContext->user, size); size = sizeof(pContext->pass); diff --git a/src/plugins/http/src/httpContext.c b/src/plugins/http/src/httpContext.c index 266228e7ac..db77c479c0 100644 --- a/src/plugins/http/src/httpContext.c +++ b/src/plugins/http/src/httpContext.c @@ -48,7 +48,7 @@ static void httpDestroyContext(void *data) { httpRemoveContextFromEpoll(pContext); httpReleaseSession(pContext); atomic_sub_fetch_32(&pThread->numOfContexts, 1); - + httpDebug("context:%p, is destroyed, refCount:%d data:%p thread:%s numOfContexts:%d", pContext, pContext->refCount, data, pContext->pThread->label, pContext->pThread->numOfContexts); pContext->pThread = 0; @@ -100,9 +100,7 @@ const char *httpContextStateStr(HttpContextState state) { } } -void httpNotifyContextClose(HttpContext *pContext) { - shutdown(pContext->fd, SHUT_WR); -} +void httpNotifyContextClose(HttpContext *pContext) { shutdown(pContext->fd, SHUT_WR); } bool httpAlterContextState(HttpContext *pContext, HttpContextState srcState, HttpContextState destState) { return (atomic_val_compare_exchange_32(&pContext->state, srcState, destState) == srcState); @@ -123,8 +121,8 @@ HttpContext *httpCreateContext(int32_t fd) { pContext->ppContext = ppContext; httpDebug("context:%p, fd:%d, is created, data:%p", pContext, fd, ppContext); - // set the ref to 0 - taosCacheRelease(tsHttpServer.contextCache, (void**)&ppContext, false); + // set the ref to 0 + taosCacheRelease(tsHttpServer.contextCache, (void **)&ppContext, false); return pContext; } @@ -174,7 +172,6 @@ bool httpInitContext(HttpContext *pContext) { pContext->encodeMethod = NULL; memset(&pContext->singleCmd, 0, sizeof(HttpSqlCmd)); - httpTrace("context:%p, fd:%d, parsed:%d", pContext, pContext->fd, pContext->parsed); return true; } diff --git a/src/plugins/http/src/httpGcHandle.c b/src/plugins/http/src/httpGcHandle.c index 4c10249672..925c74e7cd 100644 --- a/src/plugins/http/src/httpGcHandle.c +++ b/src/plugins/http/src/httpGcHandle.c @@ -192,7 +192,7 @@ bool gcProcessQueryRequest(HttpContext* pContext) { break; } - cJSON* alias = cJSON_GetObjectItem(query, "alias"); + cJSON* alias = cJSON_GetObjectItem(query, "alias"); int32_t aliasBuffer = -1; if (!(alias == NULL || alias->valuestring == NULL || strlen(alias->valuestring) == 0)) { aliasBuffer = httpAddToSqlCmdBuffer(pContext, alias->valuestring); diff --git a/src/plugins/http/src/httpGcJson.c b/src/plugins/http/src/httpGcJson.c index 3a053afd87..8c223a1500 100644 --- a/src/plugins/http/src/httpGcJson.c +++ b/src/plugins/http/src/httpGcJson.c @@ -86,7 +86,7 @@ bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, JsonBuf *jsonBuf = httpMallocJsonBuf(pContext); if (jsonBuf == NULL) return false; - int32_t num_fields = taos_num_fields(result); + int32_t num_fields = taos_num_fields(result); TAOS_FIELD *fields = taos_fetch_fields(result); if (num_fields == 0) { return false; @@ -101,7 +101,7 @@ bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, // such as select count(*) count(*) from sys.cpu group by ipaddr interval(1d) int32_t dataFields = -1; int32_t groupFields = -1; - bool hasTimestamp = fields[0].type == TSDB_DATA_TYPE_TIMESTAMP; + bool hasTimestamp = fields[0].type == TSDB_DATA_TYPE_TIMESTAMP; if (hasTimestamp) { dataFields = 1; if (num_fields > 2) groupFields = num_fields - 1; @@ -125,15 +125,15 @@ bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, cmd->numOfRows--; continue; } - int32_t* length = taos_fetch_lengths(result); + int32_t *length = taos_fetch_lengths(result); // for group by if (groupFields != -1) { - char target[HTTP_GC_TARGET_SIZE] = {0}; + char target[HTTP_GC_TARGET_SIZE] = {0}; int32_t len; - len = snprintf(target,HTTP_GC_TARGET_SIZE,"%s{",aliasBuffer); - for (int32_t i = dataFields + 1; istate) { + switch (gzip->state) { case EHTTP_GZIP_READY: { inflateEnd(gzip->gzip); } break; - default: break; + default: + break; } if (gzip->gzip) { free(gzip->gzip); @@ -60,43 +59,43 @@ static void ehttp_gzip_cleanup(ehttp_gzip_t *gzip) { gzip->state = EHTTP_GZIP_CLOSED; } -ehttp_gzip_t* ehttp_gzip_create_decompressor(ehttp_gzip_conf_t conf, ehttp_gzip_callbacks_t callbacks, void *arg) { - ehttp_gzip_t *gzip = (ehttp_gzip_t*)calloc(1, sizeof(*gzip)); +ehttp_gzip_t *ehttp_gzip_create_decompressor(ehttp_gzip_conf_t conf, ehttp_gzip_callbacks_t callbacks, void *arg) { + ehttp_gzip_t *gzip = (ehttp_gzip_t *)calloc(1, sizeof(*gzip)); if (!gzip) return NULL; do { - gzip->conf = conf; - gzip->callbacks = callbacks; - gzip->arg = arg; + gzip->conf = conf; + gzip->callbacks = callbacks; + gzip->arg = arg; if (gzip->callbacks.on_data == NULL) gzip->callbacks.on_data = dummy_on_data; - gzip->gzip = (z_stream*)calloc(1, sizeof(*gzip->gzip)); + gzip->gzip = (z_stream *)calloc(1, sizeof(*gzip->gzip)); if (gzip->conf.get_header) { - gzip->header = (gz_header*)calloc(1, sizeof(*gzip->header)); + gzip->header = (gz_header *)calloc(1, sizeof(*gzip->header)); } - if (gzip->conf.chunk_size<=0) gzip->conf.chunk_size = EHTTP_GZIP_CHUNK_SIZE_DEFAULT; - gzip->chunk = (char*)malloc(gzip->conf.chunk_size); + if (gzip->conf.chunk_size <= 0) gzip->conf.chunk_size = EHTTP_GZIP_CHUNK_SIZE_DEFAULT; + gzip->chunk = (char *)malloc(gzip->conf.chunk_size); if (!gzip->gzip || (gzip->conf.get_header && !gzip->header) || !gzip->chunk) break; - gzip->gzip->zalloc = Z_NULL; - gzip->gzip->zfree = Z_NULL; - gzip->gzip->opaque = Z_NULL; + gzip->gzip->zalloc = Z_NULL; + gzip->gzip->zfree = Z_NULL; + gzip->gzip->opaque = Z_NULL; - // 863 windowBits can also be greater than 15 for optional gzip decoding. Add - // 864 32 to windowBits to enable zlib and gzip decoding with automatic header - // 865 detection, or add 16 to decode only the gzip format (the zlib format will - // 866 return a Z_DATA_ERROR). If a gzip stream is being decoded, strm->adler is a - // 867 CRC-32 instead of an Adler-32. Unlike the gunzip utility and gzread() (see - // 868 below), inflate() will not automatically decode concatenated gzip streams. - // 869 inflate() will return Z_STREAM_END at the end of the gzip stream. The state - // 870 would need to be reset to continue decoding a subsequent gzip stream. - int32_t ret = inflateInit2(gzip->gzip, 32); // 32/16? 32/16 + MAX_WBITS + // 863 windowBits can also be greater than 15 for optional gzip decoding. Add + // 864 32 to windowBits to enable zlib and gzip decoding with automatic header + // 865 detection, or add 16 to decode only the gzip format (the zlib format will + // 866 return a Z_DATA_ERROR). If a gzip stream is being decoded, strm->adler is a + // 867 CRC-32 instead of an Adler-32. Unlike the gunzip utility and gzread() (see + // 868 below), inflate() will not automatically decode concatenated gzip streams. + // 869 inflate() will return Z_STREAM_END at the end of the gzip stream. The state + // 870 would need to be reset to continue decoding a subsequent gzip stream. + int32_t ret = inflateInit2(gzip->gzip, 32); // 32/16? 32/16 + MAX_WBITS if (ret != Z_OK) break; if (gzip->header) { ret = inflateGetHeader(gzip->gzip, gzip->header); } if (ret != Z_OK) break; - gzip->gzip->next_out = (z_const Bytef*)gzip->chunk; - gzip->gzip->avail_out = gzip->conf.chunk_size; + gzip->gzip->next_out = (z_const Bytef *)gzip->chunk; + gzip->gzip->avail_out = gzip->conf.chunk_size; gzip->state = EHTTP_GZIP_READY; return gzip; } while (0); @@ -105,7 +104,7 @@ ehttp_gzip_t* ehttp_gzip_create_decompressor(ehttp_gzip_conf_t conf, ehttp_gzip_ return NULL; } -ehttp_gzip_t* ehttp_gzip_create_compressor(ehttp_gzip_conf_t conf, ehttp_gzip_callbacks_t callbacks, void *arg); +ehttp_gzip_t *ehttp_gzip_create_compressor(ehttp_gzip_conf_t conf, ehttp_gzip_callbacks_t callbacks, void *arg); void ehttp_gzip_destroy(ehttp_gzip_t *gzip) { ehttp_gzip_cleanup(gzip); @@ -129,16 +128,16 @@ int32_t ehttp_gzip_write(ehttp_gzip_t *gzip, const char *buf, int32_t len) { } if (ret != Z_OK && ret != Z_STREAM_END) return -1; - if (gzip->gzip->avail_out>0) { - if (ret!=Z_STREAM_END) continue; + if (gzip->gzip->avail_out > 0) { + if (ret != Z_STREAM_END) continue; } - int32_t len = (int32_t)(gzip->gzip->next_out - (z_const Bytef*)gzip->chunk); + int32_t len = (int32_t)(gzip->gzip->next_out - (z_const Bytef *)gzip->chunk); gzip->gzip->next_out[0] = '\0'; gzip->callbacks.on_data(gzip, gzip->arg, gzip->chunk, len); - gzip->gzip->next_out = (z_const Bytef*)gzip->chunk; - gzip->gzip->avail_out = gzip->conf.chunk_size; + gzip->gzip->next_out = (z_const Bytef *)gzip->chunk; + gzip->gzip->avail_out = gzip->conf.chunk_size; } return 0; @@ -147,21 +146,20 @@ int32_t ehttp_gzip_write(ehttp_gzip_t *gzip, const char *buf, int32_t len) { int32_t ehttp_gzip_finish(ehttp_gzip_t *gzip) { if (gzip->state != EHTTP_GZIP_READY) return -1; - gzip->gzip->next_in = NULL; - gzip->gzip->avail_in = 0; + gzip->gzip->next_in = NULL; + gzip->gzip->avail_in = 0; int32_t ret; ret = inflate(gzip->gzip, Z_FINISH); if (ret != Z_STREAM_END) return -1; - int32_t len = (int32_t)(gzip->gzip->next_out - (z_const Bytef*)gzip->chunk); + int32_t len = (int32_t)(gzip->gzip->next_out - (z_const Bytef *)gzip->chunk); gzip->gzip->next_out[0] = '\0'; gzip->callbacks.on_data(gzip, gzip->arg, gzip->chunk, len); - gzip->gzip->next_out = NULL; - gzip->gzip->avail_out = 0; + gzip->gzip->next_out = NULL; + gzip->gzip->avail_out = 0; return 0; } - diff --git a/src/plugins/http/src/httpHandle.c b/src/plugins/http/src/httpHandle.c index 7c56507514..ad79e24061 100644 --- a/src/plugins/http/src/httpHandle.c +++ b/src/plugins/http/src/httpHandle.c @@ -48,7 +48,7 @@ bool httpProcessData(HttpContext* pContext) { /* * httpCloseContextByApp has been called when parsing the error */ - //httpCloseContextByApp(pContext); + // httpCloseContextByApp(pContext); } else { httpProcessRequest(pContext); } diff --git a/src/plugins/http/src/httpJson.c b/src/plugins/http/src/httpJson.c index 132553ffe1..b120496898 100644 --- a/src/plugins/http/src/httpJson.c +++ b/src/plugins/http/src/httpJson.c @@ -44,20 +44,21 @@ int32_t httpWriteBufByFd(struct HttpContext* pContext, const char* buf, int32_t int32_t writeLen = 0; do { - if (pContext->fd > 2){ + if (pContext->fd > 2) { len = (int32_t)taosSend(pContext->fd, buf + writeLen, (size_t)(sz - writeLen), MSG_NOSIGNAL); - } - else { + } else { return sz; } if (len < 0) { - httpDebug("context:%p, fd:%d, socket write errno:%d:%s, times:%d", pContext, pContext->fd, errno, strerror(errno), countWait); + httpDebug("context:%p, fd:%d, socket write errno:%d:%s, times:%d", pContext, pContext->fd, errno, strerror(errno), + countWait); if (++countWait > HTTP_WRITE_RETRY_TIMES) break; taosMsleep(HTTP_WRITE_WAIT_TIME_MS); continue; } else if (len == 0) { - httpDebug("context:%p, fd:%d, socket write errno:%d:%s, connect already closed", pContext, pContext->fd, errno, strerror(errno)); + httpDebug("context:%p, fd:%d, socket write errno:%d:%s, connect already closed", pContext, pContext->fd, errno, + strerror(errno)); break; } else { countWait = 0; @@ -80,7 +81,7 @@ int32_t httpWriteBuf(struct HttpContext* pContext, const char* buf, int32_t sz) return writeSz; } -int32_t httpWriteBufNoTrace(struct HttpContext *pContext, const char *buf, int32_t sz) { +int32_t httpWriteBufNoTrace(struct HttpContext* pContext, const char* buf, int32_t sz) { int32_t writeSz = httpWriteBufByFd(pContext, buf, sz); if (writeSz != sz) { httpError("context:%p, fd:%d, dataSize:%d, writeSize:%d, failed to send response", pContext, pContext->fd, sz, @@ -92,8 +93,8 @@ int32_t httpWriteBufNoTrace(struct HttpContext *pContext, const char *buf, int32 int32_t httpWriteJsonBufBody(JsonBuf* buf, bool isTheLast) { int32_t remain = 0; - char sLen[24]; - int32_t srcLen = (int32_t) (buf->lst - buf->buf); + char sLen[24]; + int32_t srcLen = (int32_t)(buf->lst - buf->buf); if (buf->pContext->fd <= 0) { httpTrace("context:%p, fd:%d, write json body error", buf->pContext, buf->pContext->fd); @@ -113,21 +114,21 @@ int32_t httpWriteJsonBufBody(JsonBuf* buf, bool isTheLast) { httpTrace("context:%p, fd:%d, no data need dump", buf->pContext, buf->pContext->fd); return 0; // there is no data to dump. } else { - int32_t len = sprintf(sLen, "%d\r\n", srcLen); - httpTrace("context:%p, fd:%d, write body, chunkSize:%d, response:\n%s", buf->pContext, buf->pContext->fd, - srcLen, buf->buf); + int32_t len = sprintf(sLen, "%x\r\n", srcLen); + httpTrace("context:%p, fd:%d, write body, chunkSize:%d, response:\n%s", buf->pContext, buf->pContext->fd, srcLen, + buf->buf); httpWriteBufNoTrace(buf->pContext, sLen, len); remain = httpWriteBufNoTrace(buf->pContext, buf->buf, srcLen); } } else { - char compressBuf[JSON_BUFFER_SIZE] = {0}; + char compressBuf[JSON_BUFFER_SIZE] = {0}; int32_t compressBufLen = JSON_BUFFER_SIZE; int32_t ret = httpGzipCompress(buf->pContext, buf->buf, srcLen, compressBuf, &compressBufLen, isTheLast); if (ret == 0) { if (compressBufLen > 0) { int32_t len = sprintf(sLen, "%x\r\n", compressBufLen); - httpTrace("context:%p, fd:%d, write body, chunkSize:%d, compressSize:%d, last:%d, response:\n%s", - buf->pContext, buf->pContext->fd, srcLen, compressBufLen, isTheLast, buf->buf); + httpTrace("context:%p, fd:%d, write body, chunkSize:%d, compressSize:%d, last:%d, response:\n%s", buf->pContext, + buf->pContext->fd, srcLen, compressBufLen, isTheLast, buf->buf); httpWriteBufNoTrace(buf->pContext, sLen, len); remain = httpWriteBufNoTrace(buf->pContext, (const char*)compressBuf, compressBufLen); } else { @@ -154,8 +155,8 @@ void httpWriteJsonBufHead(JsonBuf* buf) { buf->pContext->fd = -1; } - char msg[1024] = {0}; - int32_t len = -1; + char msg[1024] = {0}; + int32_t len = -1; if (buf->pContext->parser->acceptEncodingGzip == 0 || !tsHttpEnableCompress) { len = sprintf(msg, httpRespTemplate[HTTP_RESPONSE_CHUNKED_UN_COMPRESS], httpVersionStr[buf->pContext->parser->httpVersion], @@ -256,16 +257,16 @@ void httpJsonInt64(JsonBuf* buf, int64_t num) { } void httpJsonTimestamp(JsonBuf* buf, int64_t t, bool us) { - char ts[35] = {0}; - struct tm *ptm; - int32_t precision = 1000; + char ts[35] = {0}; + struct tm* ptm; + int32_t precision = 1000; if (us) { precision = 1000000; } time_t tt = t / precision; ptm = localtime(&tt); - int32_t length = (int32_t) strftime(ts, 35, "%Y-%m-%d %H:%M:%S", ptm); + int32_t length = (int32_t)strftime(ts, 35, "%Y-%m-%d %H:%M:%S", ptm); if (us) { length += snprintf(ts + length, 8, ".%06" PRId64, t % precision); } else { @@ -276,9 +277,9 @@ void httpJsonTimestamp(JsonBuf* buf, int64_t t, bool us) { } void httpJsonUtcTimestamp(JsonBuf* buf, int64_t t, bool us) { - char ts[40] = {0}; - struct tm *ptm; - int32_t precision = 1000; + char ts[40] = {0}; + struct tm* ptm; + int32_t precision = 1000; if (us) { precision = 1000000; } diff --git a/src/plugins/http/src/httpParser.c b/src/plugins/http/src/httpParser.c index 331ca079fa..5ebe9df96f 100644 --- a/src/plugins/http/src/httpParser.c +++ b/src/plugins/http/src/httpParser.c @@ -130,7 +130,7 @@ static int32_t httpAppendString(HttpString *str, const char *s, int32_t len) { static void httpClearString(HttpString *str) { if (str->str) { str->str[0] = '\0'; - str->pos = 0; + str->pos = 0; } } @@ -237,7 +237,6 @@ static int32_t httpOnParseHeaderField(HttpParser *parser, const char *key, const } httpTrace("context:%p, fd:%d, keepAlive:%d", pContext, pContext->fd, pContext->parser->keepAlive); } - #if 0 else if (0 == strcasecmp(key, "Content-Encoding")) { if (0 == strcmp(val, "gzip")) { @@ -246,7 +245,7 @@ static int32_t httpOnParseHeaderField(HttpParser *parser, const char *key, const } return 0; } - #endif +#endif else if (0 == strcasecmp(key, "Transfer-Encoding") || 0 == strcasecmp(key, "Content-Encoding")) { if (strstr(val, "gzip")) { @@ -349,7 +348,7 @@ static int32_t httpOnBody(HttpParser *parser, const char *chunk, int32_t len) { newSize = MIN(newSize, HTTP_BUFFER_SIZE); buf->str = realloc(buf->str, newSize); buf->size = newSize; - + if (buf->str == NULL) { httpError("context:%p, fd:%d, failed parse body, realloc %d failed", pContext, pContext->fd, buf->size); httpOnError(parser, 0, TSDB_CODE_HTTP_NO_ENOUGH_MEMORY); @@ -410,9 +409,7 @@ static int32_t httpPopStack(HttpParser *parser) { return 0; } -static void httpClearStack(HttpStack *stack) { - stack->pos = 0; -} +static void httpClearStack(HttpStack *stack) { stack->pos = 0; } static int32_t httpCleanupStack(HttpStack *stack) { free(stack->stacks); @@ -451,7 +448,7 @@ void httpInitParser(HttpParser *parser) { free(parser->key); parser->key = NULL; free(parser->val); parser->val = NULL; free(parser->authContent); parser->authContent = NULL; - + httpClearStack(&parser->stacks); httpClearString(&parser->str); httpClearString(&parser->body); @@ -497,7 +494,7 @@ void httpDestroyParser(HttpParser *parser) { free(parser->key); parser->key = NULL; free(parser->val); parser->val = NULL; free(parser->authContent); parser->authContent = NULL; - + httpCleanupStack(&parser->stacks); httpCleanupString(&parser->str); httpCleanupString(&parser->body); @@ -513,25 +510,36 @@ void httpDestroyParser(HttpParser *parser) { free(parser); } -#define is_token(c) (strchr("!#$%&'*+-.^_`|~", c) || isdigit(c) || isalpha(c)) +#define is_token(c) (strchr("!#$%&'*+-.^_`|~", c) || isdigit(c) || isalpha(c)) char *httpDecodeUrl(const char *enc) { - int32_t ok = 1; + int32_t ok = 1; HttpString str = {0}; while (*enc) { char *p = strchr(enc, '%'); if (!p) break; int32_t hex, cnt; - int32_t n = sscanf(p+1, "%2x%n", &hex, &cnt); - if (n!=1 && cnt !=2) { ok = 0; break; } - if (httpAppendString(&str, enc, (int32_t)(p-enc))) { ok = 0; break; } + int32_t n = sscanf(p + 1, "%2x%n", &hex, &cnt); + if (n != 1 && cnt != 2) { + ok = 0; + break; + } + if (httpAppendString(&str, enc, (int32_t)(p - enc))) { + ok = 0; + break; + } char c = (char)hex; - if (httpAppendString(&str, &c, 1)) { ok = 0; break; } - enc = p+3; + if (httpAppendString(&str, &c, 1)) { + ok = 0; + break; + } + enc = p + 3; } char *dec = NULL; if (ok && *enc) { - if (httpAppendString(&str, enc, (int32_t)strlen(enc))) { ok = 0; } + if (httpAppendString(&str, enc, (int32_t)strlen(enc))) { + ok = 0; + } } if (ok) { dec = str.str; @@ -542,13 +550,13 @@ char *httpDecodeUrl(const char *enc) { } static void httpOnData(ehttp_gzip_t *gzip, void *arg, const char *buf, int32_t len) { - HttpParser *parser = (HttpParser*)arg; + HttpParser *parser = (HttpParser *)arg; httpOnBody(parser, buf, len); } static int32_t httpParserOnBegin(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { if (c == 'G' || c == 'P' || c == 'H' || c == 'D' || c == 'C' || c == 'O' || c == 'T') { if (httpAppendString(&parser->str, &c, 1)) { @@ -570,7 +578,7 @@ static int32_t httpParserOnBegin(HttpParser *parser, HTTP_PARSER_STATE state, co static int32_t httpParserOnRquestOrResponse(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { if (parser->str.pos == 1) { if (c == 'T' && parser->str.str[0] == 'H') { @@ -608,7 +616,7 @@ static int32_t httpParserOnRquestOrResponse(HttpParser *parser, HTTP_PARSER_STAT static int32_t httpParserOnMethod(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { if (isalnum(c) || strchr("!#$%&'*+-.^_`|~", c)) { if (httpAppendString(&parser->str, &c, 1)) { @@ -637,7 +645,7 @@ static int32_t httpParserOnMethod(HttpParser *parser, HTTP_PARSER_STATE state, c static int32_t httpParserOnTarget(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { if (!isspace(c) && c != '\r' && c != '\n') { if (httpAppendString(&parser->str, &c, 1)) { @@ -664,7 +672,7 @@ static int32_t httpParserOnTarget(HttpParser *parser, HTTP_PARSER_STATE state, c static int32_t httpParserOnVersion(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { const char *prefix = "HTTP/1."; int32_t len = (int32_t)strlen(prefix); @@ -727,7 +735,7 @@ static int32_t httpParserOnVersion(HttpParser *parser, HTTP_PARSER_STATE state, static int32_t httpParserOnSp(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { if (c == ' ') { httpPopStack(parser); @@ -742,7 +750,7 @@ static int32_t httpParserOnSp(HttpParser *parser, HTTP_PARSER_STATE state, const static int32_t httpParserOnStatusCode(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { if (isdigit(c)) { if (httpAppendString(&parser->str, &c, 1)) { @@ -767,7 +775,7 @@ static int32_t httpParserOnStatusCode(HttpParser *parser, HTTP_PARSER_STATE stat static int32_t httpParserOnReasonPhrase(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { if (c == '\r') { parser->reasonPhrase = strdup(parser->str.str); @@ -808,10 +816,10 @@ static int32_t httpParserPostProcess(HttpParser *parser) { static int32_t httpParserOnCrlf(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { - const char *s = "\r\n"; - int32_t len = (int32_t)strlen(s); + const char *s = "\r\n"; + int32_t len = (int32_t)strlen(s); if (s[parser->str.pos] != c) { httpError("context:%p, fd:%d, parser state:%d, unexpected char:[%c]%02x", pContext, pContext->fd, state, c, c); ok = -1; @@ -838,7 +846,7 @@ static int32_t httpParserOnCrlf(HttpParser *parser, HTTP_PARSER_STATE state, con static int32_t httpParserOnHeader(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { if (c == '\r') { httpPopStack(parser); @@ -876,7 +884,7 @@ static int32_t httpParserOnHeader(HttpParser *parser, HTTP_PARSER_STATE state, c static int32_t httpParserOnHeaderKey(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { if (isalnum(c) || strchr("!#$%&'*+-.^_`|~", c)) { if (httpAppendString(&parser->str, &c, 1)) { @@ -888,7 +896,7 @@ static int32_t httpParserOnHeaderKey(HttpParser *parser, HTTP_PARSER_STATE state break; } if (c == ':') { - parser->key = strdup(parser->str.str); + parser->key = strdup(parser->str.str); if (!parser->key) { httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); ok = -1; @@ -908,7 +916,7 @@ static int32_t httpParserOnHeaderKey(HttpParser *parser, HTTP_PARSER_STATE state static int32_t httpParserOnHeaderVal(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { if (c != '\r' && c != '\n' && (!isspace(c) || parser->str.pos > 0)) { if (httpAppendString(&parser->str, &c, 1)) { @@ -935,10 +943,10 @@ static int32_t httpParserOnHeaderVal(HttpParser *parser, HTTP_PARSER_STATE state static int32_t httpParserOnChunkSize(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; - int32_t bytes; - int32_t len; - int32_t n; + int32_t ok = 0; + int32_t bytes; + int32_t len; + int32_t n; do { if (isxdigit(c)) { if (httpAppendString(&parser->str, &c, 1)) { @@ -985,7 +993,7 @@ static int32_t httpParserOnChunkSize(HttpParser *parser, HTTP_PARSER_STATE state static int32_t httpParserOnChunk(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { if (httpAppendString(&parser->str, &c, 1)) { httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); @@ -1019,7 +1027,7 @@ static int32_t httpParserOnChunk(HttpParser *parser, HTTP_PARSER_STATE state, co static int32_t httpParserOnEnd(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { ok = -1; httpError("context:%p, fd:%d, parser state:%d, unexpected char:[%c]%02x", pContext, pContext->fd, state, c, c); @@ -1029,8 +1037,8 @@ static int32_t httpParserOnEnd(HttpParser *parser, HTTP_PARSER_STATE state, cons } static int32_t httpParseChar(HttpParser *parser, const char c, int32_t *again) { - HttpContext *pContext = parser->pContext; - int32_t ok = 0; + HttpContext * pContext = parser->pContext; + int32_t ok = 0; HTTP_PARSER_STATE state = httpTopStack(parser); do { if (state == HTTP_PARSER_BEGIN) { @@ -1119,9 +1127,9 @@ static int32_t httpParseChar(HttpParser *parser, const char c, int32_t *again) { int32_t httpParseBuf(HttpParser *parser, const char *buf, int32_t len) { HttpContext *pContext = parser->pContext; - const char *p = buf; - int32_t ret = 0; - int32_t i = 0; + const char * p = buf; + int32_t ret = 0; + int32_t i = 0; while (i < len) { int32_t again = 0; diff --git a/src/plugins/http/src/httpQueue.c b/src/plugins/http/src/httpQueue.c index 7ae54717f3..aebba97fb8 100644 --- a/src/plugins/http/src/httpQueue.c +++ b/src/plugins/http/src/httpQueue.c @@ -38,16 +38,16 @@ typedef struct { } SHttpWorkerPool; typedef struct { - void * param; - void * result; - int32_t code; - int32_t rows; + void * param; + void * result; + int32_t code; + int32_t rows; FHttpResultFp fp; } SHttpResult; static SHttpWorkerPool tsHttpPool; -static taos_qset tsHttpQset; -static taos_queue tsHttpQueue; +static taos_qset tsHttpQset; +static taos_queue tsHttpQueue; void httpDispatchToResultQueue(void *param, TAOS_RES *result, int32_t code, int32_t rows, FHttpResultFp fp) { if (tsHttpQueue != NULL) { @@ -105,7 +105,7 @@ static bool httpAllocateResultQueue() { httpDebug("http result worker:%d is launched, total:%d", pWorker->workerId, tsHttpPool.num); } - httpInfo("http result queue is opened"); + httpInfo("http result queue is opened"); return true; } diff --git a/src/plugins/http/src/httpResp.c b/src/plugins/http/src/httpResp.c index 72604e79b7..37eef2bfad 100644 --- a/src/plugins/http/src/httpResp.c +++ b/src/plugins/http/src/httpResp.c @@ -160,8 +160,9 @@ void httpSendTaosdInvalidSqlErrorResp(HttpContext *pContext, char *errMsg) { if (temp[i] == '\"') { temp[i] = '\''; } else if (temp[i] == '\n') { - temp[i] = ' '; - } else {} + temp[i] = ' '; + } else { + } } httpSendErrorRespImp(pContext, httpCode, "Bad Request", TSDB_CODE_TSC_INVALID_SQL & 0XFFFF, temp); diff --git a/src/plugins/http/src/httpRestHandle.c b/src/plugins/http/src/httpRestHandle.c index 112137b3df..8999fb879e 100644 --- a/src/plugins/http/src/httpRestHandle.c +++ b/src/plugins/http/src/httpRestHandle.c @@ -95,7 +95,6 @@ bool restProcessSqlRequest(HttpContext* pContext, int32_t timestampFmt) { return false; } - /* * for async test * diff --git a/src/plugins/http/src/httpRestJson.c b/src/plugins/http/src/httpRestJson.c index a5b156bffc..baa61117be 100644 --- a/src/plugins/http/src/httpRestJson.c +++ b/src/plugins/http/src/httpRestJson.c @@ -83,7 +83,8 @@ void restStartSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result) httpJsonToken(jsonBuf, JsonArrStt); } -bool restBuildSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows, int32_t timestampFormat) { +bool restBuildSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows, + int32_t timestampFormat) { JsonBuf *jsonBuf = httpMallocJsonBuf(pContext); if (jsonBuf == NULL) return false; @@ -95,7 +96,7 @@ bool restBuildSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, if (row == NULL) { continue; } - int32_t* length = taos_fetch_lengths(result); + int32_t *length = taos_fetch_lengths(result); // data row array begin httpJsonItemToken(jsonBuf); @@ -131,15 +132,17 @@ bool restBuildSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, break; case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: - httpJsonStringForTransMean(jsonBuf, (char*)row[i], length[i]); + httpJsonStringForTransMean(jsonBuf, (char *)row[i], length[i]); break; case TSDB_DATA_TYPE_TIMESTAMP: if (timestampFormat == REST_TIMESTAMP_FMT_LOCAL_STRING) { - httpJsonTimestamp(jsonBuf, *((int64_t *)row[i]), taos_result_precision(result) == TSDB_TIME_PRECISION_MICRO); + httpJsonTimestamp(jsonBuf, *((int64_t *)row[i]), + taos_result_precision(result) == TSDB_TIME_PRECISION_MICRO); } else if (timestampFormat == REST_TIMESTAMP_FMT_TIMESTAMP) { httpJsonInt64(jsonBuf, *((int64_t *)row[i])); } else { - httpJsonUtcTimestamp(jsonBuf, *((int64_t *)row[i]), taos_result_precision(result) == TSDB_TIME_PRECISION_MICRO); + httpJsonUtcTimestamp(jsonBuf, *((int64_t *)row[i]), + taos_result_precision(result) == TSDB_TIME_PRECISION_MICRO); } break; default: @@ -148,8 +151,8 @@ bool restBuildSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, } // data row array end - httpJsonToken(jsonBuf, JsonArrEnd); - cmd->numOfRows ++; + httpJsonToken(jsonBuf, JsonArrEnd); + cmd->numOfRows++; if (pContext->fd <= 0) { httpError("context:%p, fd:%d, user:%s, conn closed, abort retrieve", pContext, pContext->fd, pContext->user); @@ -168,15 +171,15 @@ bool restBuildSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, } bool restBuildSqlTimestampJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows) { - return restBuildSqlJson(pContext,cmd, result, numOfRows, REST_TIMESTAMP_FMT_TIMESTAMP); + return restBuildSqlJson(pContext, cmd, result, numOfRows, REST_TIMESTAMP_FMT_TIMESTAMP); } bool restBuildSqlLocalTimeStringJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows) { - return restBuildSqlJson(pContext,cmd, result, numOfRows, REST_TIMESTAMP_FMT_LOCAL_STRING); + return restBuildSqlJson(pContext, cmd, result, numOfRows, REST_TIMESTAMP_FMT_LOCAL_STRING); } bool restBuildSqlUtcTimeStringJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows) { - return restBuildSqlJson(pContext,cmd, result, numOfRows, REST_TIMESTAMP_FMT_UTC_STRING); + return restBuildSqlJson(pContext, cmd, result, numOfRows, REST_TIMESTAMP_FMT_UTC_STRING); } void restStopSqlJson(HttpContext *pContext, HttpSqlCmd *cmd) { diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index 505b658743..c4a03d44ea 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -27,19 +27,21 @@ static bool httpReadData(HttpContext *pContext); -static void httpStopThread(HttpThread* pThread) { +static void httpStopThread(HttpThread *pThread) { pThread->stop = true; // signal the thread to stop, try graceful method first, // and use pthread_cancel when failed - struct epoll_event event = { .events = EPOLLIN }; - eventfd_t fd = eventfd(1, 0); + struct epoll_event event = {.events = EPOLLIN}; + eventfd_t fd = eventfd(1, 0); if (fd == -1) { - httpError("%s, failed to create eventfd, will call pthread_cancel instead, which may result in data corruption: %s", pThread->label, strerror(errno)); + httpError("%s, failed to create eventfd, will call pthread_cancel instead, which may result in data corruption: %s", + pThread->label, strerror(errno)); pThread->stop = true; pthread_cancel(pThread->thread); } else if (epoll_ctl(pThread->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) { - httpError("%s, failed to call epoll_ctl, will call pthread_cancel instead, which may result in data corruption: %s", pThread->label, strerror(errno)); + httpError("%s, failed to call epoll_ctl, will call pthread_cancel instead, which may result in data corruption: %s", + pThread->label, strerror(errno)); pthread_cancel(pThread->thread); } @@ -61,7 +63,7 @@ void httpCleanUpConnect() { } for (int32_t i = 0; i < pServer->numOfThreads; ++i) { - HttpThread* pThread = pServer->pThreads + i; + HttpThread *pThread = pServer->pThreads + i; if (pThread != NULL) { httpStopThread(pThread); } @@ -71,8 +73,8 @@ void httpCleanUpConnect() { } static void httpProcessHttpData(void *param) { - HttpServer *pServer = &tsHttpServer; - HttpThread *pThread = (HttpThread *)param; + HttpServer * pServer = &tsHttpServer; + HttpThread * pThread = (HttpThread *)param; HttpContext *pContext; int32_t fdNum; @@ -92,8 +94,8 @@ static void httpProcessHttpData(void *param) { pContext = httpGetContext(events[i].data.ptr); if (pContext == NULL) { httpError("context:%p, is already released, close connect", events[i].data.ptr); - //epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, events[i].data.fd, NULL); - //taosClose(events[i].data.fd); + // epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, events[i].data.fd, NULL); + // taosClose(events[i].data.fd); continue; } @@ -200,7 +202,7 @@ static void *httpAcceptHttpConnection(void *arg) { taosCloseSocket(connFd); continue; } -#endif +#endif taosKeepTcpAlive(connFd); taosSetNonblocking(connFd, 1); @@ -210,15 +212,15 @@ static void *httpAcceptHttpConnection(void *arg) { pContext = httpCreateContext(connFd); if (pContext == NULL) { - httpError("fd:%d, ip:%s:%u, no enough resource to allocate http context", connFd, taosInetNtoa(clientAddr.sin_addr), - htons(clientAddr.sin_port)); + httpError("fd:%d, ip:%s:%u, no enough resource to allocate http context", connFd, + taosInetNtoa(clientAddr.sin_addr), htons(clientAddr.sin_port)); taosCloseSocket(connFd); continue; } pContext->pThread = pThread; sprintf(pContext->ipstr, "%s:%u", taosInetNtoa(clientAddr.sin_addr), htons(clientAddr.sin_port)); - + struct epoll_event event; event.events = EPOLLIN | EPOLLPRI | EPOLLWAKEUP | EPOLLERR | EPOLLHUP | EPOLLRDHUP; event.data.ptr = pContext; @@ -276,7 +278,7 @@ bool httpInitConnect() { if (pthread_create(&(pThread->thread), &thattr, (void *)httpProcessHttpData, (void *)(pThread)) != 0) { httpError("http thread:%s, failed to create HTTP process data thread, reason:%s", pThread->label, strerror(errno)); - pthread_mutex_destroy(&(pThread->threadMutex)); + pthread_mutex_destroy(&(pThread->threadMutex)); return false; } pthread_attr_destroy(&thattr); @@ -307,7 +309,7 @@ static bool httpReadData(HttpContext *pContext) { } if (pParser->parsed) { - httpDebug("context:%p, fd:%d, not in ready state, parsed:%d", pContext, pContext->fd, pParser->parsed); + httpDebug("context:%p, fd:%d, not in ready state, parsed:%d", pContext, pContext->fd, pParser->parsed); return false; } diff --git a/src/plugins/http/src/httpSession.c b/src/plugins/http/src/httpSession.c index 35ce0160b2..2e1ee7df2f 100644 --- a/src/plugins/http/src/httpSession.c +++ b/src/plugins/http/src/httpSession.c @@ -35,7 +35,8 @@ void httpCreateSession(HttpContext *pContext, void *taos) { session.refCount = 1; int32_t len = snprintf(session.id, HTTP_SESSION_ID_LEN, "%s.%s", pContext->user, pContext->pass); - pContext->session = taosCachePut(server->sessionCache, session.id, len, &session, sizeof(HttpSession), tsHttpSessionExpire * 1000); + pContext->session = + taosCachePut(server->sessionCache, session.id, len, &session, sizeof(HttpSession), tsHttpSessionExpire * 1000); // void *temp = pContext->session; // taosCacheRelease(server->sessionCache, (void **)&temp, false); @@ -56,7 +57,7 @@ static void httpFetchSessionImp(HttpContext *pContext) { HttpServer *server = &tsHttpServer; pthread_mutex_lock(&server->serverMutex); - char sessionId[HTTP_SESSION_ID_LEN]; + char sessionId[HTTP_SESSION_ID_LEN]; int32_t len = snprintf(sessionId, HTTP_SESSION_ID_LEN, "%s.%s", pContext->user, pContext->pass); pContext->session = taosCacheAcquireByKey(server->sessionCache, sessionId, len); diff --git a/src/plugins/http/src/httpSql.c b/src/plugins/http/src/httpSql.c index cc8e9e86e3..4e9b54b7bd 100644 --- a/src/plugins/http/src/httpSql.c +++ b/src/plugins/http/src/httpSql.c @@ -64,7 +64,7 @@ void httpProcessMultiSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int32 } taos_free_result(result); - + if (singleCmd->cmdReturnType == HTTP_CMD_RETURN_TYPE_WITH_RETURN && encode->stopJsonFp) { (encode->stopJsonFp)(pContext, singleCmd); } @@ -82,7 +82,7 @@ void httpProcessMultiSqlCallBackImp(void *param, TAOS_RES *result, int32_t code, HttpContext *pContext = (HttpContext *)param; if (pContext == NULL) return; - HttpSqlCmds *multiCmds = pContext->multiCmds; + HttpSqlCmds * multiCmds = pContext->multiCmds; HttpEncodeMethod *encode = pContext->encodeMethod; HttpSqlCmd *singleCmd = multiCmds->cmds + multiCmds->pos; @@ -269,8 +269,8 @@ void httpProcessSingleSqlCallBackImp(void *param, TAOS_RES *result, int32_t code pContext->user, tstrerror(code), pObj, taos_errstr(pObj)); httpSendTaosdInvalidSqlErrorResp(pContext, taos_errstr(pObj)); } else { - httpError("context:%p, fd:%d, user:%s, query error, code:%s, sqlObj:%p", pContext, pContext->fd, - pContext->user, tstrerror(code), pObj); + httpError("context:%p, fd:%d, user:%s, query error, code:%s, sqlObj:%p", pContext, pContext->fd, pContext->user, + tstrerror(code), pObj); httpSendErrorResp(pContext, code); } taos_free_result(result); @@ -381,7 +381,7 @@ void httpExecCmd(HttpContext *pContext) { void httpProcessRequestCb(void *param, TAOS_RES *result, int32_t code) { HttpContext *pContext = param; taos_free_result(result); - + if (pContext == NULL) return; if (code < 0) { diff --git a/src/plugins/http/src/httpSystem.c b/src/plugins/http/src/httpSystem.c index df39c080a5..4721451529 100644 --- a/src/plugins/http/src/httpSystem.c +++ b/src/plugins/http/src/httpSystem.c @@ -110,10 +110,8 @@ void httpCleanUpSystem() { pthread_mutex_destroy(&tsHttpServer.serverMutex); tfree(tsHttpServer.pThreads); tsHttpServer.pThreads = NULL; - + tsHttpServer.status = HTTP_SERVER_CLOSED; } -int32_t httpGetReqCount() { - return atomic_exchange_32(&tsHttpServer.requestNum, 0); -} +int32_t httpGetReqCount() { return atomic_exchange_32(&tsHttpServer.requestNum, 0); } diff --git a/src/plugins/http/src/httpTgHandle.c b/src/plugins/http/src/httpTgHandle.c index 7a26cae97c..c1d006ff5a 100644 --- a/src/plugins/http/src/httpTgHandle.c +++ b/src/plugins/http/src/httpTgHandle.c @@ -137,8 +137,8 @@ void tgInitSchemas(int32_t size) { } void tgParseSchemaMetric(cJSON *metric) { - STgSchema schema = {0}; - bool parsedOk = true; + STgSchema schema = {0}; + bool parsedOk = true; // name cJSON *name = cJSON_GetObjectItem(metric, "name"); @@ -186,7 +186,7 @@ void tgParseSchemaMetric(cJSON *metric) { schema.tbName = calloc(tbnameLen + 1, 1); strcpy(schema.tbName, tbname->valuestring); - // fields + // fields cJSON *fields = cJSON_GetObjectItem(metric, "fields"); if (fields == NULL) { goto ParseEnd; @@ -227,14 +227,14 @@ ParseEnd: } } -int32_t tgParseSchema(const char *content, char*fileName) { +int32_t tgParseSchema(const char *content, char *fileName) { cJSON *root = cJSON_Parse(content); if (root == NULL) { httpError("failed to parse telegraf schema file:%s, invalid json format, content:%s", fileName, content); return -1; } int32_t size = 0; - cJSON *metrics = cJSON_GetObjectItem(root, "metrics"); + cJSON * metrics = cJSON_GetObjectItem(root, "metrics"); if (metrics != NULL) { size = cJSON_GetArraySize(metrics); if (size <= 0) { @@ -277,7 +277,7 @@ int32_t tgReadSchema(char *fileName) { rewind(fp); char * content = (char *)calloc(contentSize + 1, 1); int32_t result = (int32_t)fread(content, 1, contentSize, fp); - + if (result != contentSize) { httpError("failed to read telegraf schema file:%s", fileName); fclose(fp); @@ -296,7 +296,7 @@ int32_t tgReadSchema(char *fileName) { } void tgInitHandle(HttpServer *pServer) { - char fileName[TSDB_FILENAME_LEN*2] = {0}; + char fileName[TSDB_FILENAME_LEN * 2] = {0}; sprintf(fileName, "%s/taos.telegraf.cfg", configDir); if (tgReadSchema(fileName) <= 0) { tgFreeSchemas(); @@ -308,9 +308,7 @@ void tgInitHandle(HttpServer *pServer) { httpAddMethod(pServer, &tgDecodeMethod); } -void tgCleanupHandle() { - tgFreeSchemas(); -} +void tgCleanupHandle() { tgFreeSchemas(); } bool tgGetUserFromUrl(HttpContext *pContext) { HttpParser *pParser = pContext->parser; @@ -357,7 +355,7 @@ char *tgGetStableName(char *stname, cJSON *fields, int32_t fieldsSize) { bool schemaMatched = true; for (int32_t f = 0; f < schema->fieldNum; ++f) { char *fieldName = schema->fields[f]; - bool fieldMatched = false; + bool fieldMatched = false; for (int32_t i = 0; i < fieldsSize; i++) { cJSON *field = cJSON_GetArrayItem(fields, i); @@ -469,9 +467,9 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) { } /* - * tag size may be larget than TSDB_COL_NAME_LEN - * we keep the first TSDB_COL_NAME_LEN bytes - */ + * tag size may be larget than TSDB_COL_NAME_LEN + * we keep the first TSDB_COL_NAME_LEN bytes + */ if (0) { if (strlen(tag->string) >= TSDB_COL_NAME_LEN) { httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TAG_NAME_SIZE); @@ -540,9 +538,9 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) { return false; } /* - * tag size may be larget than TSDB_COL_NAME_LEN - * we keep the first TSDB_COL_NAME_LEN bytes - */ + * tag size may be larget than TSDB_COL_NAME_LEN + * we keep the first TSDB_COL_NAME_LEN bytes + */ if (0) { if (strlen(field->string) >= TSDB_COL_NAME_LEN) { httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_FIELD_NAME_SIZE); @@ -578,8 +576,8 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) { table_cmd->cmdType = HTTP_CMD_TYPE_INSERT; // order by tag name - cJSON *orderedTags[TG_MAX_SORT_TAG_SIZE] = {0}; - int32_t orderTagsLen = 0; + cJSON * orderedTags[TG_MAX_SORT_TAG_SIZE] = {0}; + int32_t orderTagsLen = 0; for (int32_t i = 0; i < tagsSize; ++i) { cJSON *tag = cJSON_GetArrayItem(tags, i); orderedTags[orderTagsLen++] = tag; @@ -603,7 +601,8 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) { if (tsTelegrafUseFieldNum == 0) { table_cmd->stable = stable_cmd->stable = httpAddToSqlCmdBuffer(pContext, "%s", stname); } else { - table_cmd->stable = stable_cmd->stable = httpAddToSqlCmdBuffer(pContext, "%s_%d_%d", stname, fieldsSize, orderTagsLen); + table_cmd->stable = stable_cmd->stable = + httpAddToSqlCmdBuffer(pContext, "%s_%d_%d", stname, fieldsSize, orderTagsLen); } table_cmd->stable = stable_cmd->stable = httpShrinkTableName(pContext, table_cmd->stable, httpGetCmdsString(pContext, table_cmd->stable)); @@ -627,9 +626,11 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) { // table name if (tsTelegrafUseFieldNum == 0) { - table_cmd->table = stable_cmd->table = httpAddToSqlCmdBufferNoTerminal(pContext, "%s_%s", stname, host->valuestring); + table_cmd->table = stable_cmd->table = + httpAddToSqlCmdBufferNoTerminal(pContext, "%s_%s", stname, host->valuestring); } else { - table_cmd->table = stable_cmd->table = httpAddToSqlCmdBufferNoTerminal(pContext, "%s_%d_%d_%s", stname, fieldsSize, orderTagsLen, host->valuestring); + table_cmd->table = stable_cmd->table = + httpAddToSqlCmdBufferNoTerminal(pContext, "%s_%d_%d_%s", stname, fieldsSize, orderTagsLen, host->valuestring); } for (int32_t i = 0; i < orderTagsLen; ++i) { cJSON *tag = orderedTags[i]; diff --git a/src/plugins/http/src/httpUtil.c b/src/plugins/http/src/httpUtil.c index 51333d2118..17d61e9e3d 100644 --- a/src/plugins/http/src/httpUtil.c +++ b/src/plugins/http/src/httpUtil.c @@ -160,8 +160,7 @@ bool httpMallocMultiCmds(HttpContext *pContext, int32_t cmdSize, int32_t bufferS free(multiCmds->cmds); multiCmds->cmds = (HttpSqlCmd *)malloc((size_t)cmdSize * sizeof(HttpSqlCmd)); if (multiCmds->cmds == NULL) { - httpError("context:%p, fd:%d, user:%s, malloc cmds:%d error", pContext, pContext->fd, - pContext->user, cmdSize); + httpError("context:%p, fd:%d, user:%s, malloc cmds:%d error", pContext, pContext->fd, pContext->user, cmdSize); return false; } multiCmds->maxSize = (int16_t)cmdSize; @@ -350,38 +349,40 @@ char *httpGetCmdsString(HttpContext *pContext, int32_t pos) { } int32_t httpGzipDeCompress(char *srcData, int32_t nSrcData, char *destData, int32_t *nDestData) { - int32_t err = 0; + int32_t err = 0; z_stream gzipStream = {0}; static char dummyHead[2] = { - 0x8 + 0x7 * 0x10, - (((0x8 + 0x7 * 0x10) * 0x100 + 30) / 31 * 31) & 0xFF, + 0x8 + 0x7 * 0x10, + (((0x8 + 0x7 * 0x10) * 0x100 + 30) / 31 * 31) & 0xFF, }; - gzipStream.zalloc = (alloc_func) 0; - gzipStream.zfree = (free_func) 0; - gzipStream.opaque = (voidpf) 0; - gzipStream.next_in = (Bytef *) srcData; + gzipStream.zalloc = (alloc_func)0; + gzipStream.zfree = (free_func)0; + gzipStream.opaque = (voidpf)0; + gzipStream.next_in = (Bytef *)srcData; gzipStream.avail_in = 0; - gzipStream.next_out = (Bytef *) destData; + gzipStream.next_out = (Bytef *)destData; if (inflateInit2(&gzipStream, 47) != Z_OK) { return -1; } while (gzipStream.total_out < *nDestData && gzipStream.total_in < nSrcData) { - gzipStream.avail_in = gzipStream.avail_out = nSrcData; //1 + gzipStream.avail_in = gzipStream.avail_out = nSrcData; // 1 if ((err = inflate(&gzipStream, Z_NO_FLUSH)) == Z_STREAM_END) { break; } if (err != Z_OK) { if (err == Z_DATA_ERROR) { - gzipStream.next_in = (Bytef *) dummyHead; + gzipStream.next_in = (Bytef *)dummyHead; gzipStream.avail_in = sizeof(dummyHead); if ((err = inflate(&gzipStream, Z_NO_FLUSH)) != Z_OK) { return -2; } - } else return -3; + } else { + return -3; + } } } @@ -394,23 +395,25 @@ int32_t httpGzipDeCompress(char *srcData, int32_t nSrcData, char *destData, int3 } int32_t httpGzipCompressInit(HttpContext *pContext) { - pContext->gzipStream.zalloc = (alloc_func) 0; - pContext->gzipStream.zfree = (free_func) 0; - pContext->gzipStream.opaque = (voidpf) 0; - if (deflateInit2(&pContext->gzipStream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, MAX_WBITS + 16, 8, Z_DEFAULT_STRATEGY) != Z_OK) { + pContext->gzipStream.zalloc = (alloc_func)0; + pContext->gzipStream.zfree = (free_func)0; + pContext->gzipStream.opaque = (voidpf)0; + if (deflateInit2(&pContext->gzipStream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, MAX_WBITS + 16, 8, Z_DEFAULT_STRATEGY) != + Z_OK) { return -1; } return 0; } -int32_t httpGzipCompress(HttpContext *pContext, char *srcData, int32_t nSrcData, char *destData, int32_t *nDestData, bool isTheLast) { +int32_t httpGzipCompress(HttpContext *pContext, char *srcData, int32_t nSrcData, char *destData, int32_t *nDestData, + bool isTheLast) { int32_t err = 0; - int32_t lastTotalLen = (int32_t) (pContext->gzipStream.total_out); - pContext->gzipStream.next_in = (Bytef *) srcData; - pContext->gzipStream.avail_in = (uLong) nSrcData; - pContext->gzipStream.next_out = (Bytef *) destData; - pContext->gzipStream.avail_out = (uLong) (*nDestData); + int32_t lastTotalLen = (int32_t)(pContext->gzipStream.total_out); + pContext->gzipStream.next_in = (Bytef *)srcData; + pContext->gzipStream.avail_in = (uLong)nSrcData; + pContext->gzipStream.next_out = (Bytef *)destData; + pContext->gzipStream.avail_out = (uLong)(*nDestData); while (pContext->gzipStream.avail_in != 0) { if (deflate(&pContext->gzipStream, Z_FULL_FLUSH) != Z_OK) { @@ -442,12 +445,12 @@ int32_t httpGzipCompress(HttpContext *pContext, char *srcData, int32_t nSrcData, } } - *nDestData = (int32_t) (pContext->gzipStream.total_out) - lastTotalLen; + *nDestData = (int32_t)(pContext->gzipStream.total_out) - lastTotalLen; return 0; } -bool httpUrlMatch(HttpContext* pContext, int32_t pos, char* cmp) { - HttpParser* pParser = pContext->parser; +bool httpUrlMatch(HttpContext *pContext, int32_t pos, char *cmp) { + HttpParser *pParser = pContext->parser; if (pos < 0 || pos >= HTTP_MAX_URL) { return false; diff --git a/tests/script/general/http/restful.sim b/tests/script/general/http/restful.sim index a06e899d93..fdde975238 100644 --- a/tests/script/general/http/restful.sim +++ b/tests/script/general/http/restful.sim @@ -1,12 +1,10 @@ system sh/stop_dnodes.sh -sleep 2000 system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/cfg.sh -n dnode1 -c http -v 1 system sh/cfg.sh -n dnode1 -c httpEnableRecordSql -v 1 system sh/exec.sh -n dnode1 -s start -sleep 2000 sql connect print ============================ dnode1 start From 0694e69dda7a590c3ad804ba8d66a67c10ca4b51 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Fri, 22 Jan 2021 15:16:33 +0800 Subject: [PATCH 035/147] test for mail --- tests/pytest/insert/unsigenedTinyint.py | 106 ++++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 tests/pytest/insert/unsigenedTinyint.py diff --git a/tests/pytest/insert/unsigenedTinyint.py b/tests/pytest/insert/unsigenedTinyint.py new file mode 100644 index 0000000000..1c0634b69a --- /dev/null +++ b/tests/pytest/insert/unsigenedTinyint.py @@ -0,0 +1,106 @@ +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + + tdLog.info('=============== step1') + tdLog.info('create table tb (ts timestamp, speed tinyint unsigned)') + tdSql.execute('create table tb (ts timestamp, speed tinyint unsigned)') + tdLog.info("insert into tb values (now, NULL)") + tdSql.execute("insert into tb values (now, NULL)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + tdLog.info('tdSql.checkData(0, 1, null)') + tdSql.checkData(0, 1, None) + tdLog.info('=============== step2') + tdLog.info("insert into tb values (now+1m, -1) -x step2") + tdSql.error("insert into tb values (now+1m, -1) ") + tdLog.info("insert into tb values (now+1m, NULL)") + tdSql.execute("insert into tb values (now+1m, NULL)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(2)') + tdSql.checkRows(2) + tdLog.info('tdSql.checkData(0, 1, null)') + tdSql.checkData(0, 1, None) + tdLog.info('=============== step3') + tdLog.info("insert into tb values (now+2m, 254)") + tdSql.execute("insert into tb values (now+2m, 254)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(3)') + tdSql.checkRows(3) + tdLog.info('tdSql.checkData(0, 1, 254)') + tdSql.checkData(0, 1, 254) + tdLog.info('=============== step4') + tdLog.info("insert into tb values (now+3m, 255) -x step4") + tdSql.error("insert into tb values (now+3m, 255)") + tdLog.info("insert into tb values (now+3m, NULL)") + tdSql.execute("insert into tb values (now+3m, NULL)") + tdLog.info('select * from tb') + tdSql.query('select * from tb') + tdLog.info('tdSql.checkRow(4)') + tdSql.checkRows(4) + tdLog.info('tdSql.checkData(0, 1, null)') + tdSql.checkData(0, 1, None) + tdLog.info('=============== step5') + tdLog.info("insert into tb values (now+4m, a2)") + tdSql.error("insert into tb values (now+4m, a2)") + tdLog.info("insert into tb values (now+4m, 0)") + tdSql.execute("insert into tb values (now+4m, 0)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(5)') + tdSql.checkRows(5) + tdLog.info('tdSql.checkData(0, 1, 0)') + tdSql.checkData(0, 1, 0) + tdLog.info('=============== step6') + tdLog.info("insert into tb values (now+5m, 2a)") + tdSql.error("insert into tb values (now+5m, 2a)") + tdLog.info("insert into tb values (now+5m, 2)") + tdSql.execute("insert into tb values (now+5m, 2)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(6)') + tdSql.checkRows(6) + tdLog.info('tdSql.checkData(0, 1, 2)') + tdSql.checkData(0, 1, 2) + tdLog.info('=============== step7') + tdLog.info("insert into tb values (now+6m, 2a'1)") + tdSql.error("insert into tb values (now+6m, 2a'1)") + tdLog.info("insert into tb values (now+6m, 2)") + tdSql.execute("insert into tb values (now+6m, 2)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(7)') + tdSql.checkRows(7) + tdLog.info('tdSql.checkData(0, 1, 2)') + tdSql.checkData(0, 1, 2) + tdLog.info('drop database db') + tdSql.execute('drop database db') + tdLog.info('show databases') + tdSql.query('show databases') + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) +# convert end + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From b304cd96c0c8d03a43940353e2c6ee9aa86ad33b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 22 Jan 2021 16:11:33 +0800 Subject: [PATCH 036/147] TD-1207 --- src/os/inc/osDef.h | 2 +- src/wal/src/walMgmt.c | 4 ++-- tests/tsim/src/simMain.c | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/os/inc/osDef.h b/src/os/inc/osDef.h index e5b5e0f895..a21b721565 100644 --- a/src/os/inc/osDef.h +++ b/src/os/inc/osDef.h @@ -100,7 +100,7 @@ extern "C" { #elif defined(__GNUC__) && !defined(threadlocal) #define threadlocal __thread #else - #define threadlocal + #define threadlocal __declspec( thread ) #endif #ifdef __cplusplus diff --git a/src/wal/src/walMgmt.c b/src/wal/src/walMgmt.c index 62b066500c..39ce2657aa 100644 --- a/src/wal/src/walMgmt.c +++ b/src/wal/src/walMgmt.c @@ -44,7 +44,7 @@ int32_t walInit() { return code; } - wInfo("wal module is initialized, refId:%d", tsWal.refId); + wInfo("wal module is initialized, rsetId:%d", tsWal.refId); return code; } @@ -203,7 +203,7 @@ static int32_t walCreateThread() { } pthread_attr_destroy(&thAttr); - wDebug("wal thread is launched"); + wDebug("wal thread is launched, thread:0x%08" PRIx64, taosGetPthreadId(tsWal.thread)); return TSDB_CODE_SUCCESS; } diff --git a/tests/tsim/src/simMain.c b/tests/tsim/src/simMain.c index 6c9c492a5a..251b4f80bb 100644 --- a/tests/tsim/src/simMain.c +++ b/tests/tsim/src/simMain.c @@ -64,7 +64,7 @@ int32_t main(int32_t argc, char *argv[]) { simExecuteScript(script); int32_t ret = simExecSuccess ? 0 : -1; - simError("execute result %d", ret); + simInfo("execute result %d", ret); return ret; } \ No newline at end of file From 33228ee5897341e1e6413858d182ad6442be9fbd Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 22 Jan 2021 19:37:17 +0800 Subject: [PATCH 037/147] TD-1207 --- src/dnode/src/dnodeSystem.c | 16 ++++++++++------ src/kit/shell/src/shellMain.c | 2 +- src/os/inc/osSignal.h | 2 +- src/os/src/detail/osSignal.c | 6 ++++++ src/os/src/windows/wSignal.c | 4 +++- src/sync/src/syncArbitrator.c | 4 ++-- tests/tsim/src/simMain.c | 2 +- 7 files changed, 24 insertions(+), 12 deletions(-) diff --git a/src/dnode/src/dnodeSystem.c b/src/dnode/src/dnodeSystem.c index b7ca6f053c..ad5950ab5b 100644 --- a/src/dnode/src/dnodeSystem.c +++ b/src/dnode/src/dnodeSystem.c @@ -20,9 +20,9 @@ #include "dnodeMain.h" static tsem_t exitSem; -static void siguser1Handler(int32_t signum); -static void siguser2Handler(int32_t signum); -static void sigintHandler(int32_t signum); +static void siguser1Handler(int32_t signum, void *sigInfo, void *context); +static void siguser2Handler(int32_t signum, void *sigInfo, void *context); +static void sigintHandler(int32_t signum, void *sigInfo, void *context); int32_t main(int32_t argc, char *argv[]) { int dump_config = 0; @@ -152,11 +152,11 @@ int32_t main(int32_t argc, char *argv[]) { return EXIT_SUCCESS; } -static void siguser1Handler(int32_t signum) { taosCfgDynamicOptions("debugFlag 143"); } +static void siguser1Handler(int32_t signum, void *sigInfo, void *context) { taosCfgDynamicOptions("debugFlag 143"); } -static void siguser2Handler(int32_t signum) { taosCfgDynamicOptions("resetlog"); } +static void siguser2Handler(int32_t signum, void *sigInfo, void *context) { taosCfgDynamicOptions("resetlog"); } -static void sigintHandler(int32_t signum) { +static void sigintHandler(int32_t signum, void *sigInfo, void *context) { // protect the application from receive another signal taosIgnSignal(SIGUSR1); taosIgnSignal(SIGUSR2); @@ -169,6 +169,10 @@ static void sigintHandler(int32_t signum) { // clean the system. dInfo("shut down signal is %d", signum); +#ifndef WINDOWS + dInfo("sender PID:%d cmdline:%s",((siginfo_t *)sigInfo)->si_pid, taosGetCmdlineByPID(sigInfo->si_pid)); +#endif + syslog(LOG_INFO, "Shut down signal is %d", signum); syslog(LOG_INFO, "Shutting down TDengine service..."); diff --git a/src/kit/shell/src/shellMain.c b/src/kit/shell/src/shellMain.c index c7a0dab2dd..49de42796c 100644 --- a/src/kit/shell/src/shellMain.c +++ b/src/kit/shell/src/shellMain.c @@ -21,7 +21,7 @@ pthread_t pid; static tsem_t cancelSem; -void shellQueryInterruptHandler(int32_t signum) { +void shellQueryInterruptHandler(int32_t signum, void *sigInfo, void *context) { tsem_post(&cancelSem); } diff --git a/src/os/inc/osSignal.h b/src/os/inc/osSignal.h index 8016c49ba8..c13cd83178 100644 --- a/src/os/inc/osSignal.h +++ b/src/os/inc/osSignal.h @@ -48,7 +48,7 @@ extern "C" { #define SIGBREAK 1234 #endif -typedef void (*FSignalHandler)(int32_t signum); +typedef void (*FSignalHandler)(int32_t signum, void *sigInfo, void *context); void taosSetSignal(int32_t signum, FSignalHandler sigfp); void taosIgnSignal(int32_t signum); void taosDflSignal(int32_t signum); diff --git a/src/os/src/detail/osSignal.c b/src/os/src/detail/osSignal.c index c97a3343de..e1a0e84e7f 100644 --- a/src/os/src/detail/osSignal.c +++ b/src/os/src/detail/osSignal.c @@ -20,10 +20,16 @@ #include "tulog.h" #ifndef TAOS_OS_FUNC_SIGNAL +typedef void (*FLinuxSignalHandler)(int32_t signum, siginfo_t *sigInfo, void *context); void taosSetSignal(int32_t signum, FSignalHandler sigfp) { struct sigaction act = {{0}}; +#if 1 + act.sa_flags = SA_SIGINFO; + act.sa_sigaction = (FLinuxSignalHandler)sigfp; +#else act.sa_handler = sigfp; +#endif sigaction(signum, &act, NULL); } diff --git a/src/os/src/windows/wSignal.c b/src/os/src/windows/wSignal.c index 3988f0c6e1..9de6b5f343 100644 --- a/src/os/src/windows/wSignal.c +++ b/src/os/src/windows/wSignal.c @@ -18,6 +18,8 @@ #include #include +typedef void (*FWinSignalHandler)(int32_t signum); + void taosSetSignal(int32_t signum, FSignalHandler sigfp) { if (signum == SIGUSR1) return; @@ -25,7 +27,7 @@ void taosSetSignal(int32_t signum, FSignalHandler sigfp) { if (signum == SIGHUP) { SetConsoleCtrlHandler((PHANDLER_ROUTINE)sigfp, TRUE); } else { - signal(signum, sigfp); + signal(signum, (FWinSignalHandler)sigfp); } } diff --git a/src/sync/src/syncArbitrator.c b/src/sync/src/syncArbitrator.c index d65a7b3043..24760ce1a0 100644 --- a/src/sync/src/syncArbitrator.c +++ b/src/sync/src/syncArbitrator.c @@ -27,7 +27,7 @@ #include "syncInt.h" #include "syncTcp.h" -static void arbSignalHandler(int32_t signum); +static void arbSignalHandler(int32_t signum, void *sigInfo, void *context); static void arbProcessIncommingConnection(int32_t connFd, uint32_t sourceIp); static void arbProcessBrokenLink(int64_t rid); static int32_t arbProcessPeerMsg(int64_t rid, void *buffer); @@ -170,7 +170,7 @@ static int32_t arbProcessPeerMsg(int64_t rid, void *buffer) { return 0; } -static void arbSignalHandler(int32_t signum) { +static void arbSignalHandler(int32_t signum, void *sigInfo, void *context) { taosIgnSignal(SIGTERM); taosIgnSignal(SIGINT); taosIgnSignal(SIGABRT); diff --git a/tests/tsim/src/simMain.c b/tests/tsim/src/simMain.c index 251b4f80bb..990ea71413 100644 --- a/tests/tsim/src/simMain.c +++ b/tests/tsim/src/simMain.c @@ -22,7 +22,7 @@ bool simAsyncQuery = false; bool simExecSuccess = false; -void simHandleSignal(int32_t signo) { +void simHandleSignal(int32_t signo, void *sigInfo, void *context) { simSystemCleanUp(); exit(1); } From df96fd1b3dcdb055fdfbdb81284c6d1bc62d1f97 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 22 Jan 2021 19:47:25 +0800 Subject: [PATCH 038/147] compile error --- src/dnode/src/dnodeSystem.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dnode/src/dnodeSystem.c b/src/dnode/src/dnodeSystem.c index ad5950ab5b..33ebc8b64f 100644 --- a/src/dnode/src/dnodeSystem.c +++ b/src/dnode/src/dnodeSystem.c @@ -170,7 +170,7 @@ static void sigintHandler(int32_t signum, void *sigInfo, void *context) { dInfo("shut down signal is %d", signum); #ifndef WINDOWS - dInfo("sender PID:%d cmdline:%s",((siginfo_t *)sigInfo)->si_pid, taosGetCmdlineByPID(sigInfo->si_pid)); + dInfo("sender PID:%d cmdline:%s", ((siginfo_t *)sigInfo)->si_pid, taosGetCmdlineByPID(((siginfo_t *)sigInfo)->si_pid)); #endif syslog(LOG_INFO, "Shut down signal is %d", signum); From 0be7fe43cab50799c2c6937eae72a7c1253706f9 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Sat, 23 Jan 2021 04:25:52 +0800 Subject: [PATCH 039/147] fix bug --- src/common/src/tvariant.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/common/src/tvariant.c b/src/common/src/tvariant.c index 95aa576889..fa95e17f32 100644 --- a/src/common/src/tvariant.c +++ b/src/common/src/tvariant.c @@ -775,7 +775,7 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu return -1; } } else { - wcsncpy((wchar_t *)p, pVariant->wpz, pVariant->nLen); + memcpy(p, pVariant->wpz, pVariant->nLen); newlen = pVariant->nLen; } @@ -867,4 +867,4 @@ int32_t tVariantTypeSetType(tVariant *pVariant, char type) { } return 0; -} \ No newline at end of file +} From d6c7c58be1dde5b4d3a74a36cad497d2277300e2 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Sat, 23 Jan 2021 06:39:38 +0800 Subject: [PATCH 040/147] fix bug --- src/client/src/tscSQLParser.c | 80 +++++++++++++++++++++++++++++++++++ src/common/src/tvariant.c | 4 +- src/inc/taosdef.h | 5 +++ 3 files changed, 87 insertions(+), 2 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index f9eedff7eb..6b24dcd858 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -4300,6 +4300,78 @@ static void doAddJoinTagsColumnsIntoTagList(SSqlCmd* pCmd, SQueryInfo* pQueryInf } } +static int32_t validateTagCondExpr(SSqlCmd* pCmd, tExprNode *p) { + const char *msg1 = "tag type mismatch"; + const char *msg2 = "invalid tag operator"; + const char* msg3 = "not supported filter condition"; + + do { + if (p->nodeType != TSQL_NODE_EXPR) { + break; + } + + if (!p->_node.pLeft || !p->_node.pRight) { + break; + } + + if (IS_ARITHMETIC_OPTR(p->_node.optr)) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + } + + if (!IS_RELATION_OPTR(p->_node.optr)) { + break; + } + + tVariant * vVariant = NULL; + int32_t schemaType = -1; + + if (p->_node.pLeft->nodeType == TSQL_NODE_VALUE && p->_node.pRight->nodeType == TSQL_NODE_COL) { + if (!p->_node.pRight->pSchema) { + break; + } + + vVariant = p->_node.pLeft->pVal->nType; + schemaType = p->_node.pRight->pSchema->type; + } else if (p->_node.pLeft->nodeType == TSQL_NODE_COL && p->_node.pRight->nodeType == TSQL_NODE_VALUE) { + if (!p->_node.pLeft->pSchema) { + break; + } + + vVariant = p->_node.pRight->pVal->nType; + schemaType = p->_node.pLeft->pSchema->type; + } else { + break; + } + + if (schemaType >= TSDB_DATA_TYPE_TINYINT && schemaType <= TSDB_DATA_TYPE_BIGINT) { + schemaType = TSDB_DATA_TYPE_BIGINT; + } else if (schemaType == TSDB_DATA_TYPE_FLOAT || schemaType == TSDB_DATA_TYPE_DOUBLE) { + schemaType = TSDB_DATA_TYPE_DOUBLE; + } + + int32_t retVal = TSDB_CODE_SUCCESS; + if (schemaType == TSDB_DATA_TYPE_BINARY) { + char *tmp = (int64_t)calloc(1, (vVariant->nLen + 1) + TSDB_NCHAR_SIZE); + retVal = tVariantDump(vVariant, tmp, schemaType, false); + free(tmp); + } else if (schemaType == TSDB_DATA_TYPE_NCHAR) { + // pRight->val.nLen + 1 is larger than the actual nchar string length + char *tmp = (int64_t)calloc(1, (vVariant->nLen + 1) * TSDB_NCHAR_SIZE); + retVal = tVariantDump(vVariant, tmp, schemaType, false); + free(tmp); + } else { + double tmp; + retVal = tVariantDump(vVariant, (char*)&tmp, schemaType, false); + } + + if (retVal != TSDB_CODE_SUCCESS) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + } + }while (0); + + return TSDB_CODE_SUCCESS; +} + static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondExpr* pCondExpr, tSQLExpr** pExpr) { int32_t ret = TSDB_CODE_SUCCESS; @@ -4342,6 +4414,10 @@ static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondE tsSetSTableQueryCond(&pQueryInfo->tagCond, uid, &bw); doCompactQueryExpr(pExpr); + if (ret == TSDB_CODE_SUCCESS) { + ret = validateTagCondExpr(pCmd, p); + } + tSqlExprDestroy(p1); tExprTreeDestroy(p, NULL); @@ -4349,6 +4425,10 @@ static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondE if (pQueryInfo->tagCond.pCond != NULL && taosArrayGetSize(pQueryInfo->tagCond.pCond) > 0 && !UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "filter on tag not supported for normal table"); } + + if (ret) { + break; + } } pCondExpr->pTagCond = NULL; diff --git a/src/common/src/tvariant.c b/src/common/src/tvariant.c index 95aa576889..2f2750df4f 100644 --- a/src/common/src/tvariant.c +++ b/src/common/src/tvariant.c @@ -430,7 +430,7 @@ static FORCE_INLINE int32_t convertToInteger(tVariant *pVariant, int64_t *result } errno = 0; - if (IS_SIGNED_NUMERIC_TYPE(pVariant->nType)) { + if (IS_SIGNED_NUMERIC_TYPE(pVariant->nType) || (pVariant->nType == TSDB_DATA_TYPE_BOOL)) { *result = pVariant->i64; } else if (IS_UNSIGNED_NUMERIC_TYPE(pVariant->nType)) { *result = pVariant->u64; @@ -867,4 +867,4 @@ int32_t tVariantTypeSetType(tVariant *pVariant, char type) { } return 0; -} \ No newline at end of file +} diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 6fea049074..9f3c31f225 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -163,6 +163,11 @@ do { \ #define TSDB_BINARY_OP_MULTIPLY 32 #define TSDB_BINARY_OP_DIVIDE 33 #define TSDB_BINARY_OP_REMAINDER 34 + + +#define IS_RELATION_OPTR(op) (((op) >= TSDB_RELATION_LESS) && ((op) <= TSDB_RELATION_IN)) +#define IS_ARITHMETIC_OPTR(op) (((op) >= TSDB_BINARY_OP_ADD) && ((op) <= TSDB_BINARY_OP_REMAINDER)) + #define TS_PATH_DELIMITER_LEN 1 #define TSDB_UNI_LEN 24 From 94eb0e1e71b713d66d00274242d7667408095154 Mon Sep 17 00:00:00 2001 From: freemine Date: Sat, 23 Jan 2021 08:44:32 +0800 Subject: [PATCH 041/147] 1. __APPLE__ 2. tsem_xxx for mac --- src/client/src/tscUtil.c | 4 ++-- src/dnode/src/dnodeSystem.c | 2 +- src/dnode/src/dnodeTelemetry.c | 2 +- src/mnode/src/mnodeCluster.c | 1 + src/os/inc/osDarwin.h | 10 +++++----- src/os/inc/osDef.h | 5 +++-- src/plugins/http/src/httpServer.c | 2 +- src/rpc/src/rpcTcp.c | 13 ++++++++----- src/sync/src/syncTcp.c | 8 ++++---- src/tsdb/inc/tsdbMain.h | 4 ++-- src/tsdb/src/tsdbCommit.c | 4 ++-- src/tsdb/src/tsdbMain.c | 12 ++++++------ src/tsdb/src/tsdbMemTable.c | 8 ++++---- src/util/src/tsocket.c | 2 +- tests/examples/c/epoll.c | 4 ++-- 15 files changed, 43 insertions(+), 38 deletions(-) diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 19a1e99c33..bb2a81db66 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -2520,9 +2520,9 @@ bool tscSetSqlOwner(SSqlObj* pSql) { // set the sql object owner #ifdef __APPLE__ pthread_t threadId = (pthread_t)taosGetSelfPthreadId(); -#else +#else // __APPLE__ uint64_t threadId = taosGetSelfPthreadId(); -#endif +#endif // __APPLE__ if (atomic_val_compare_exchange_64(&pSql->owner, 0, threadId) != 0) { pRes->code = TSDB_CODE_QRY_IN_EXEC; return false; diff --git a/src/dnode/src/dnodeSystem.c b/src/dnode/src/dnodeSystem.c index 893049df2e..7a218f901e 100644 --- a/src/dnode/src/dnodeSystem.c +++ b/src/dnode/src/dnodeSystem.c @@ -164,7 +164,7 @@ static void signal_handler(int32_t signum, siginfo_t *sigInfo, void *context) { dInfo("shut down signal is %d, sender PID:%d", signum, sigInfo->si_pid); #else // __APPLE__ dInfo("shut down signal is %d, sender PID:%d cmdline:%s", signum, sigInfo->si_pid, taosGetCmdlineByPID(sigInfo->si_pid)); -#endif +#endif // __APPLE__ // protect the application from receive another signal struct sigaction act = {{0}}; diff --git a/src/dnode/src/dnodeTelemetry.c b/src/dnode/src/dnodeTelemetry.c index 1f7084c3b5..b2e8eac267 100644 --- a/src/dnode/src/dnodeTelemetry.c +++ b/src/dnode/src/dnodeTelemetry.c @@ -241,7 +241,7 @@ static int sem_timedwait(tsem_t *sem, struct timespec *to) { fprintf(stderr, "%s[%d]%s(): not implemented yet!\n", basename(__FILE__), __LINE__, __func__); abort(); } -#endif +#endif // __APPLE__ static void* telemetryThread(void* param) { struct timespec end = {0}; diff --git a/src/mnode/src/mnodeCluster.c b/src/mnode/src/mnodeCluster.c index 1b153247f0..5dee0e1c26 100644 --- a/src/mnode/src/mnodeCluster.c +++ b/src/mnode/src/mnodeCluster.c @@ -145,6 +145,7 @@ bool taosGetSystemUid(char *uid) { return false; } #endif // __APPLE__ + static int32_t mnodeCreateCluster() { int32_t numOfClusters = sdbGetNumOfRows(tsClusterSdb); if (numOfClusters != 0) return TSDB_CODE_SUCCESS; diff --git a/src/os/inc/osDarwin.h b/src/os/inc/osDarwin.h index 935708eb1b..0a77ed6895 100644 --- a/src/os/inc/osDarwin.h +++ b/src/os/inc/osDarwin.h @@ -75,11 +75,11 @@ extern "C" { #define TAOS_OS_FUNC_FILE_SENDIFLE #define TAOS_OS_FUNC_SEMPHONE - #define tsem_t dispatch_semaphore_t - int tsem_init(dispatch_semaphore_t *sem, int pshared, unsigned int value); - int tsem_wait(dispatch_semaphore_t *sem); - int tsem_post(dispatch_semaphore_t *sem); - int tsem_destroy(dispatch_semaphore_t *sem); + typedef struct tsem_s *tsem_t; + int tsem_init(tsem_t *sem, int pshared, unsigned int value); + int tsem_wait(tsem_t *sem); + int tsem_post(tsem_t *sem); + int tsem_destroy(tsem_t *sem); #define TAOS_OS_FUNC_SOCKET_SETSOCKETOPT #define TAOS_OS_FUNC_STRING_STR2INT64 diff --git a/src/os/inc/osDef.h b/src/os/inc/osDef.h index d718bef6da..f646940d3a 100644 --- a/src/os/inc/osDef.h +++ b/src/os/inc/osDef.h @@ -90,11 +90,12 @@ extern "C" { #ifdef _ISOC11_SOURCE #define threadlocal _Thread_local #elif defined(__APPLE__) - #define threadlocal + #define threadlocal __thread #elif defined(__GNUC__) && !defined(threadlocal) #define threadlocal __thread #else - #define threadlocal + // #define threadlocal + #error please follow with the target platform's thread-local implementation #endif #ifdef __cplusplus diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index 97a5b4bc23..6a8eb556d8 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -33,7 +33,7 @@ static bool httpReadData(HttpContext *pContext); #ifdef __APPLE__ static int sv_dummy = 0; -#endif +#endif // __APPLE__ static void httpStopThread(HttpThread* pThread) { pThread->stop = true; diff --git a/src/rpc/src/rpcTcp.c b/src/rpc/src/rpcTcp.c index ba90f93073..d64bdb42aa 100644 --- a/src/rpc/src/rpcTcp.c +++ b/src/rpc/src/rpcTcp.c @@ -312,9 +312,9 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int num, void * epoll_close(pThreadObj->pollFd); pThreadObj->pollFd = -1; } -#else +#else // __APPLE__ taosCloseSocket(pThreadObj->pollFd); -#endif +#endif // __APPLE__ free(pThreadObj); terrno = TAOS_SYSTEM_ERROR(errno); tError("%s failed to create TCP read data thread(%s)", label, strerror(errno)); @@ -477,7 +477,10 @@ static void *taosProcessTcpData(void *param) { SFdObj *pFdObj; struct epoll_event events[maxEvents]; SRecvInfo recvInfo; - + +#ifdef __APPLE__ + taos_block_sigalrm(); +#endif // __APPLE__ while (1) { int fdNum = epoll_wait(pThreadObj->pollFd, events, maxEvents, TAOS_EPOLL_WAIT_TIME); if (pThreadObj->stop) { @@ -524,9 +527,9 @@ static void *taosProcessTcpData(void *param) { epoll_close(pThreadObj->pollFd); pThreadObj->pollFd = -1; } -#else +#else // __APPLE__ if (pThreadObj->pollFd >=0) taosCloseSocket(pThreadObj->pollFd); -#endif +#endif // __APPLE__ while (pThreadObj->pHead) { SFdObj *pFdObj = pThreadObj->pHead; diff --git a/src/sync/src/syncTcp.c b/src/sync/src/syncTcp.c index 1e5761316a..e15f71e905 100644 --- a/src/sync/src/syncTcp.c +++ b/src/sync/src/syncTcp.c @@ -235,9 +235,9 @@ static void *syncProcessTcpData(void *param) { #ifdef __APPLE__ epoll_close(pThread->pollFd); -#else +#else // __APPLE__ close(pThread->pollFd); -#endif +#endif // __APPLE__ tfree(pThread); tfree(buffer); return NULL; @@ -296,9 +296,9 @@ static SThreadObj *syncGetTcpThread(SPoolObj *pPool) { if (ret != 0) { #ifdef __APPLE__ epoll_close(pThread->pollFd); -#else +#else // __APPLE__ close(pThread->pollFd); -#endif +#endif // __APPLE__ tfree(pThread); return NULL; } diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 984839162e..f3e69d9210 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -235,9 +235,9 @@ typedef struct { STsdbFileH* tsdbFileH; #ifdef __APPLE__ sem_t *readyToCommit; -#else +#else // __APPLE__ sem_t readyToCommit; -#endif +#endif // __APPLE__ pthread_mutex_t mutex; bool repoLocked; int32_t code; // Commit code diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c index 9551d1b3a5..1c8a661c0a 100644 --- a/src/tsdb/src/tsdbCommit.c +++ b/src/tsdb/src/tsdbCommit.c @@ -168,9 +168,9 @@ static void tsdbEndCommit(STsdbRepo *pRepo, int eno) { tsdbUnRefMemTable(pRepo, pIMem); #ifdef __APPLE__ sem_post(pRepo->readyToCommit); -#else +#else // __APPLE__ sem_post(&(pRepo->readyToCommit)); -#endif +#endif // __APPLE__ } static int tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey) { diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 5493188c09..c770f0ff42 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -148,9 +148,9 @@ int tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit) { tsdbAsyncCommit(pRepo); #ifdef __APPLE__ sem_wait(pRepo->readyToCommit); -#else +#else // __APPLE__ sem_wait(&(pRepo->readyToCommit)); -#endif +#endif // __APPLE__ terrno = pRepo->code; } tsdbUnRefMemTable(pRepo, pRepo->mem); @@ -654,14 +654,14 @@ static STsdbRepo *tsdbNewRepo(char *rootDir, STsdbAppH *pAppH, STsdbCfg *pCfg) { terrno = TAOS_SYSTEM_ERROR(code); goto _err; } -#else +#else // __APPLE__ code = sem_init(&(pRepo->readyToCommit), 0, 1); if (code != 0) { code = errno; terrno = TAOS_SYSTEM_ERROR(code); goto _err; } -#endif +#endif // __APPLE__ pRepo->repoLocked = false; @@ -709,9 +709,9 @@ static void tsdbFreeRepo(STsdbRepo *pRepo) { tfree(pRepo->rootDir); #ifdef __APPLE__ sem_close(pRepo->readyToCommit); -#else +#else // __APPLE__ sem_destroy(&(pRepo->readyToCommit)); -#endif +#endif // __APPLE__ pthread_mutex_destroy(&pRepo->mutex); free(pRepo); } diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 202405df00..206a3332d5 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -209,9 +209,9 @@ int tsdbAsyncCommit(STsdbRepo *pRepo) { #ifdef __APPLE__ sem_wait(pRepo->readyToCommit); -#else +#else // __APPLE__ sem_wait(&(pRepo->readyToCommit)); -#endif +#endif // __APPLE__ ASSERT(pRepo->imem == NULL); @@ -236,10 +236,10 @@ int tsdbSyncCommit(TSDB_REPO_T *repo) { #ifdef __APPLE__ sem_wait(pRepo->readyToCommit); sem_post(pRepo->readyToCommit); -#else +#else // __APPLE__ sem_wait(&(pRepo->readyToCommit)); sem_post(&(pRepo->readyToCommit)); -#endif +#endif // __APPLE__ if (pRepo->code != TSDB_CODE_SUCCESS) { terrno = pRepo->code; diff --git a/src/util/src/tsocket.c b/src/util/src/tsocket.c index cfd948265b..4bfd7a855b 100644 --- a/src/util/src/tsocket.c +++ b/src/util/src/tsocket.c @@ -376,7 +376,7 @@ int32_t taosKeepTcpAlive(SOCKET sockFd) { taosCloseSocket(sockFd); return -1; } -#endif +#endif // __APPLE__ int32_t nodelay = 1; if (taosSetSockOpt(sockFd, IPPROTO_TCP, TCP_NODELAY, (void *)&nodelay, sizeof(nodelay)) < 0) { diff --git a/tests/examples/c/epoll.c b/tests/examples/c/epoll.c index eb66156c14..4f65ea478e 100644 --- a/tests/examples/c/epoll.c +++ b/tests/examples/c/epoll.c @@ -15,9 +15,9 @@ #ifdef __APPLE__ #include "eok.h" -#else +#else // __APPLE__ #include -#endif +#endif // __APPLE__ #include #include #include From 4e774660b6de2f0035f8735c3a16c2f3b705e5a2 Mon Sep 17 00:00:00 2001 From: freemine Date: Sat, 23 Jan 2021 08:45:44 +0800 Subject: [PATCH 042/147] tsem_xxx for mac --- src/os/src/darwin/darwinSemphone.c | 88 ++++++++++++++++++++++++------ 1 file changed, 72 insertions(+), 16 deletions(-) diff --git a/src/os/src/darwin/darwinSemphone.c b/src/os/src/darwin/darwinSemphone.c index 97ff543789..d8e60cd2d5 100644 --- a/src/os/src/darwin/darwinSemphone.c +++ b/src/os/src/darwin/darwinSemphone.c @@ -16,25 +16,81 @@ #define _DEFAULT_SOURCE #include "os.h" -int tsem_init(dispatch_semaphore_t *sem, int pshared, unsigned int value) { - *sem = dispatch_semaphore_create(value); - if (*sem == NULL) { - return -1; - } else { +struct tsem_s { + dispatch_semaphore_t sem; + volatile unsigned int valid:1; +}; + +int tsem_wait(tsem_t *sem); +int tsem_post(tsem_t *sem); +int tsem_destroy(tsem_t *sem); +int tsem_init(tsem_t *sem, int pshared, unsigned int value) { + fprintf(stderr, "==%s[%d]%s():[%p]==creating\n", basename(__FILE__), __LINE__, __func__, sem); + if (*sem) { + fprintf(stderr, "==%s[%d]%s():[%p]==already initialized\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } + struct tsem_s *p = (struct tsem_s*)calloc(1, sizeof(*p)); + if (!p) { + fprintf(stderr, "==%s[%d]%s():[%p]==out of memory\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } + + p->sem = dispatch_semaphore_create(value); + if (p->sem == NULL) { + fprintf(stderr, "==%s[%d]%s():[%p]==not created\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } + p->valid = 1; + + *sem = p; + + return 0; +} + +int tsem_wait(tsem_t *sem) { + if (!*sem) { + fprintf(stderr, "==%s[%d]%s():[%p]==not initialized\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } + struct tsem_s *p = *sem; + if (!p->valid) { + fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } + return dispatch_semaphore_wait(p->sem, DISPATCH_TIME_FOREVER); +} + +int tsem_post(tsem_t *sem) { + if (!*sem) { + fprintf(stderr, "==%s[%d]%s():[%p]==not initialized\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } + struct tsem_s *p = *sem; + if (!p->valid) { + fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } + return dispatch_semaphore_signal(p->sem); +} + +int tsem_destroy(tsem_t *sem) { + fprintf(stderr, "==%s[%d]%s():[%p]==destroying\n", basename(__FILE__), __LINE__, __func__, sem); + if (!*sem) { + fprintf(stderr, "==%s[%d]%s():[%p]==not initialized\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); return 0; } -} + struct tsem_s *p = *sem; + if (!p->valid) { + fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } -int tsem_wait(dispatch_semaphore_t *sem) { - dispatch_semaphore_wait(*sem, DISPATCH_TIME_FOREVER); + p->valid = 0; + free(p); + + *sem = NULL; return 0; } -int tsem_post(dispatch_semaphore_t *sem) { - dispatch_semaphore_signal(*sem); - return 0; -} - -int tsem_destroy(dispatch_semaphore_t *sem) { - return 0; -} From 79e4925dc73231190c2395214ada2d540ea30bf0 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 23 Jan 2021 12:00:02 +0800 Subject: [PATCH 043/147] [TD-2696] : fix RFC3339 timestamp bug. --- src/os/src/detail/osTime.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/os/src/detail/osTime.c b/src/os/src/detail/osTime.c index d6877790a8..d9d070218e 100644 --- a/src/os/src/detail/osTime.c +++ b/src/os/src/detail/osTime.c @@ -43,7 +43,7 @@ */ int64_t user_mktime64(const unsigned int year0, const unsigned int mon0, const unsigned int day, const unsigned int hour, - const unsigned int min, const unsigned int sec) + const unsigned int min, const unsigned int sec, int64_t timezone) { unsigned int mon = mon0, year = year0; @@ -61,12 +61,6 @@ int64_t user_mktime64(const unsigned int year0, const unsigned int mon0, res = res*24; res = ((res + hour) * 60 + min) * 60 + sec; -#ifdef _MSC_VER -#if _MSC_VER >= 1900 - int64_t timezone = _timezone; -#endif -#endif - return (res + timezone); } @@ -219,7 +213,7 @@ int32_t parseTimeWithTz(char* timestr, int64_t* time, int32_t timePrec) { /* mktime will be affected by TZ, set by using taos_options */ #ifdef WINDOWS - int64_t seconds = user_mktime64(tm.tm_year+1900, tm.tm_mon+1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); + int64_t seconds = user_mktime64(tm.tm_year+1900, tm.tm_mon+1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec, 0); //int64_t seconds = gmtime(&tm); #else int64_t seconds = timegm(&tm); @@ -276,7 +270,13 @@ int32_t parseLocaltime(char* timestr, int64_t* time, int32_t timePrec) { return -1; } - int64_t seconds = user_mktime64(tm.tm_year+1900, tm.tm_mon+1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); +#ifdef _MSC_VER +#if _MSC_VER >= 1900 + int64_t timezone = _timezone; +#endif +#endif + + int64_t seconds = user_mktime64(tm.tm_year+1900, tm.tm_mon+1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec, timezone); int64_t fraction = 0; @@ -574,4 +574,4 @@ const char* fmtts(int64_t ts) { } return buf; -} \ No newline at end of file +} From b15d8fefaa9f3d35a3e0eade89e5bbb178f097bf Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 23 Jan 2021 12:55:47 +0800 Subject: [PATCH 044/147] [TD-2829] : refine Windows compiling section of README.md --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 566b3e5c8b..fafcdc8321 100644 --- a/README.md +++ b/README.md @@ -133,7 +133,9 @@ cmake .. -G "NMake Makefiles" nmake ``` -If you use the Visual Studio 2019, please open a command window by executing "cmd.exe". +If you use the Visual Studio 2019 or 2017: + +please open a command window by executing "cmd.exe". Please specify "x64" for 64 bits Windows or specify "x86" is for 32 bits Windows when you execute vcvarsall.bat. ``` mkdir debug && cd debug @@ -142,7 +144,7 @@ cmake .. -G "NMake Makefiles" nmake ``` -Or, you can open a command window by clicking Visual Studio 2019 menu "Tools -> Command Line -> Developer Command Prompt" or "Tools -> Command Line -> Developer PowerShell" then execute commands as follows: +Or, you can simply open a command window by clicking Windows Start -> "Visual Studio < 2019 | 2017 >" folder -> "x64 Native Tools Command Prompt for VS < 2019 | 2017 >" or "x86 Native Tools Command Prompt for VS < 2019 | 2017 >" depends what architecture your Windows is, then execute commands as follows: ``` mkdir debug && cd debug cmake .. -G "NMake Makefiles" From ce50e131bc35b23f9db111736c0a55c59135f958 Mon Sep 17 00:00:00 2001 From: freemine Date: Sun, 24 Jan 2021 09:29:13 +0800 Subject: [PATCH 045/147] 1. kqueue rather than SIGALRM for the timer 2. semaphore rather than dispatch_semaphore for tsem_xxx 3. fail-fast comments 4. niche --- src/client/src/tscSql.c | 4 + src/client/src/tscUtil.c | 4 + src/os/src/darwin/darwinSemphone.c | 201 +++++++++++++++++++++++++++-- src/os/src/darwin/darwinTimer.c | 74 +++++++++++ src/os/src/darwin/eok.c | 37 +++--- 5 files changed, 297 insertions(+), 23 deletions(-) diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index fa7c1f83ba..ef9b079cfc 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -295,6 +295,10 @@ void taos_close(TAOS *taos) { tscDebug("%p HB is freed", pHb); taosReleaseRef(tscObjRef, pHb->self); +#ifdef __APPLE__ + // to satisfy later tsem_destroy in taos_free_result + tsem_init(&pHb->rspSem, 0, 0); +#endif // __APPLE__ taos_free_result(pHb); } } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 8dcb7425a8..e1521b4aa1 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1933,6 +1933,10 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, __async_cb_func_t fp, void* param, in } if (tscAddSubqueryInfo(pCmd) != TSDB_CODE_SUCCESS) { +#ifdef __APPLE__ + // to satisfy later tsem_destroy in taos_free_result + tsem_init(&pNew->rspSem, 0, 0); +#endif // __APPLE__ tscFreeSqlObj(pNew); return NULL; } diff --git a/src/os/src/darwin/darwinSemphone.c b/src/os/src/darwin/darwinSemphone.c index d8e60cd2d5..a9e470bd70 100644 --- a/src/os/src/darwin/darwinSemphone.c +++ b/src/os/src/darwin/darwinSemphone.c @@ -13,19 +13,76 @@ * along with this program. If not, see . */ +// fail-fast or let-it-crash philosophy +// https://en.wikipedia.org/wiki/Fail-fast +// https://stackoverflow.com/questions/4393197/erlangs-let-it-crash-philosophy-applicable-elsewhere +// experimentally, we follow log-and-crash here + #define _DEFAULT_SOURCE #include "os.h" +// #define SEM_USE_PTHREAD +// #define SEM_USE_POSIX +#define SEM_USE_SEM + +#ifdef SEM_USE_SEM +#include +#include +#include +#include + +static pthread_t sem_thread; +static pthread_once_t sem_once; +static task_t sem_port; +static volatile int sem_inited = 0; +static semaphore_t sem_exit; + +static void* sem_thread_routine(void *arg) { + (void)arg; + sem_port = mach_task_self(); + kern_return_t ret = semaphore_create(sem_port, &sem_exit, SYNC_POLICY_FIFO, 0); + if (ret != KERN_SUCCESS) { + fprintf(stderr, "==%s[%d]%s()==failed to create sem_exit\n", basename(__FILE__), __LINE__, __func__); + sem_inited = -1; + return NULL; + } + sem_inited = 1; + semaphore_wait(sem_exit); + return NULL; +} + +static void once_init(void) { + int r = 0; + r = pthread_create(&sem_thread, NULL, sem_thread_routine, NULL); + if (r) { + fprintf(stderr, "==%s[%d]%s()==failed to create thread\n", basename(__FILE__), __LINE__, __func__); + return; + } + while (sem_inited==0) { + ; + } +} +#endif + struct tsem_s { +#ifdef SEM_USE_PTHREAD + pthread_mutex_t lock; + pthread_cond_t cond; + volatile int64_t val; +#elif defined(SEM_USE_POSIX) + size_t id; + sem_t *sem; +#elif defined(SEM_USE_SEM) + semaphore_t sem; +#else // SEM_USE_PTHREAD dispatch_semaphore_t sem; +#endif // SEM_USE_PTHREAD + volatile unsigned int valid:1; }; -int tsem_wait(tsem_t *sem); -int tsem_post(tsem_t *sem); -int tsem_destroy(tsem_t *sem); int tsem_init(tsem_t *sem, int pshared, unsigned int value) { - fprintf(stderr, "==%s[%d]%s():[%p]==creating\n", basename(__FILE__), __LINE__, __func__, sem); + // fprintf(stderr, "==%s[%d]%s():[%p]==creating\n", basename(__FILE__), __LINE__, __func__, sem); if (*sem) { fprintf(stderr, "==%s[%d]%s():[%p]==already initialized\n", basename(__FILE__), __LINE__, __func__, sem); abort(); @@ -36,11 +93,61 @@ int tsem_init(tsem_t *sem, int pshared, unsigned int value) { abort(); } +#ifdef SEM_USE_PTHREAD + int r = pthread_mutex_init(&p->lock, NULL); + do { + if (r) break; + r = pthread_cond_init(&p->cond, NULL); + if (r) { + pthread_mutex_destroy(&p->lock); + break; + } + p->val = value; + } while (0); + if (r) { + fprintf(stderr, "==%s[%d]%s():[%p]==not created\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } +#elif defined(SEM_USE_POSIX) + static size_t tick = 0; + do { + size_t id = atomic_add_fetch_64(&tick, 1); + if (id==SEM_VALUE_MAX) { + atomic_store_64(&tick, 0); + id = 0; + } + char name[NAME_MAX-4]; + snprintf(name, sizeof(name), "/t%ld", id); + p->sem = sem_open(name, O_CREAT|O_EXCL, pshared, value); + p->id = id; + if (p->sem!=SEM_FAILED) break; + int e = errno; + if (e==EEXIST) continue; + if (e==EINTR) continue; + fprintf(stderr, "==%s[%d]%s():[%p]==not created[%d]%s\n", basename(__FILE__), __LINE__, __func__, sem, e, strerror(e)); + abort(); + } while (p->sem==SEM_FAILED); +#elif defined(SEM_USE_SEM) + pthread_once(&sem_once, once_init); + if (sem_inited!=1) { + fprintf(stderr, "==%s[%d]%s():[%p]==internal resource init failed\n", basename(__FILE__), __LINE__, __func__, sem); + errno = ENOMEM; + return -1; + } + kern_return_t ret = semaphore_create(sem_port, &p->sem, SYNC_POLICY_FIFO, 0); + if (ret != KERN_SUCCESS) { + fprintf(stderr, "==%s[%d]%s():[%p]==semophore_create failed\n", basename(__FILE__), __LINE__, __func__, sem); + // we fail-fast here, because we have less-doc about semaphore_create for the moment + abort(); + } +#else // SEM_USE_PTHREAD p->sem = dispatch_semaphore_create(value); if (p->sem == NULL) { fprintf(stderr, "==%s[%d]%s():[%p]==not created\n", basename(__FILE__), __LINE__, __func__, sem); abort(); } +#endif // SEM_USE_PTHREAD + p->valid = 1; *sem = p; @@ -58,7 +165,30 @@ int tsem_wait(tsem_t *sem) { fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", basename(__FILE__), __LINE__, __func__, sem); abort(); } +#ifdef SEM_USE_PTHREAD + if (pthread_mutex_lock(&p->lock)) { + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } + p->val -= 1; + if (p->val < 0) { + if (pthread_cond_wait(&p->cond, &p->lock)) { + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } + } + if (pthread_mutex_unlock(&p->lock)) { + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } + return 0; +#elif defined(SEM_USE_POSIX) + return sem_wait(p->sem); +#elif defined(SEM_USE_SEM) + return semaphore_wait(p->sem); +#else // SEM_USE_PTHREAD return dispatch_semaphore_wait(p->sem, DISPATCH_TIME_FOREVER); +#endif // SEM_USE_PTHREAD } int tsem_post(tsem_t *sem) { @@ -71,21 +201,76 @@ int tsem_post(tsem_t *sem) { fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", basename(__FILE__), __LINE__, __func__, sem); abort(); } +#ifdef SEM_USE_PTHREAD + if (pthread_mutex_lock(&p->lock)) { + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } + p->val += 1; + if (p->val <= 0) { + if (pthread_cond_signal(&p->cond)) { + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } + } + if (pthread_mutex_unlock(&p->lock)) { + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } + return 0; +#elif defined(SEM_USE_POSIX) + return sem_post(p->sem); +#elif defined(SEM_USE_SEM) + return semaphore_signal(p->sem); +#else // SEM_USE_PTHREAD return dispatch_semaphore_signal(p->sem); +#endif // SEM_USE_PTHREAD } int tsem_destroy(tsem_t *sem) { - fprintf(stderr, "==%s[%d]%s():[%p]==destroying\n", basename(__FILE__), __LINE__, __func__, sem); + // fprintf(stderr, "==%s[%d]%s():[%p]==destroying\n", basename(__FILE__), __LINE__, __func__, sem); if (!*sem) { - fprintf(stderr, "==%s[%d]%s():[%p]==not initialized\n", basename(__FILE__), __LINE__, __func__, sem); - abort(); + // fprintf(stderr, "==%s[%d]%s():[%p]==not initialized\n", basename(__FILE__), __LINE__, __func__, sem); + // abort(); return 0; } struct tsem_s *p = *sem; if (!p->valid) { - fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", basename(__FILE__), __LINE__, __func__, sem); + // fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", basename(__FILE__), __LINE__, __func__, sem); + // abort(); + return 0; + } +#ifdef SEM_USE_PTHREAD + if (pthread_mutex_lock(&p->lock)) { + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); abort(); } + p->valid = 0; + if (pthread_cond_destroy(&p->cond)) { + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } + if (pthread_mutex_unlock(&p->lock)) { + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } + if (pthread_mutex_destroy(&p->lock)) { + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } +#elif defined(SEM_USE_POSIX) + char name[NAME_MAX-4]; + snprintf(name, sizeof(name), "/t%ld", p->id); + int r = sem_unlink(name); + if (r) { + int e = errno; + fprintf(stderr, "==%s[%d]%s():[%p]==unlink failed[%d]%s\n", basename(__FILE__), __LINE__, __func__, sem, e, strerror(e)); + abort(); + } +#elif defined(SEM_USE_SEM) + semaphore_destroy(sem_port, p->sem); +#else // SEM_USE_PTHREAD +#endif // SEM_USE_PTHREAD p->valid = 0; free(p); diff --git a/src/os/src/darwin/darwinTimer.c b/src/os/src/darwin/darwinTimer.c index 121e2e9427..ee1becc91a 100644 --- a/src/os/src/darwin/darwinTimer.c +++ b/src/os/src/darwin/darwinTimer.c @@ -13,9 +13,82 @@ * along with this program. If not, see . */ +// fail-fast or let-it-crash philosophy +// https://en.wikipedia.org/wiki/Fail-fast +// https://stackoverflow.com/questions/4393197/erlangs-let-it-crash-philosophy-applicable-elsewhere +// experimentally, we follow log-and-crash here + #define _DEFAULT_SOURCE #include "os.h" +#if 1 +#include + +static void (*timer_callback)(int); +static int timer_ms = 0; +static pthread_t timer_thread; +static int timer_kq = -1; +static volatile int timer_stop = 0; + +static void* timer_routine(void *arg) { + (void)arg; + + int r = 0; + struct timespec to = {0}; + to.tv_sec = timer_ms / 1000; + to.tv_nsec = (timer_ms % 1000) * 1000000; + while (!timer_stop) { + struct kevent64_s kev[10] = {0}; + r = kevent64(timer_kq, NULL, 0, kev, sizeof(kev)/sizeof(kev[0]), 0, &to); + if (r!=0) { + fprintf(stderr, "==%s[%d]%s()==kevent64 failed\n", basename(__FILE__), __LINE__, __func__); + abort(); + } + timer_callback(SIGALRM); // just mock + } + + return NULL; +} + +int taosInitTimer(void (*callback)(int), int ms) { + int r = 0; + timer_ms = ms; + timer_callback = callback; + + timer_kq = kqueue(); + if (timer_kq==-1) { + fprintf(stderr, "==%s[%d]%s()==failed to create timer kq\n", basename(__FILE__), __LINE__, __func__); + // since no caller of this func checks the return value for the moment + abort(); + } + + r = pthread_create(&timer_thread, NULL, timer_routine, NULL); + if (r) { + fprintf(stderr, "==%s[%d]%s()==failed to create timer thread\n", basename(__FILE__), __LINE__, __func__); + // since no caller of this func checks the return value for the moment + abort(); + } + return 0; +} + +void taosUninitTimer() { + int r = 0; + timer_stop = 1; + r = pthread_join(timer_thread, NULL); + if (r) { + fprintf(stderr, "==%s[%d]%s()==failed to join timer thread\n", basename(__FILE__), __LINE__, __func__); + // since no caller of this func checks the return value for the moment + abort(); + } + close(timer_kq); + timer_kq = -1; +} + +void taos_block_sigalrm(void) { + // we don't know if there's any specific API for SIGALRM to deliver to specific thread + // this implementation relies on kqueue rather than SIGALRM +} +#else int taosInitTimer(void (*callback)(int), int ms) { signal(SIGALRM, callback); @@ -46,4 +119,5 @@ void taos_block_sigalrm(void) { already_set = 1; } } +#endif diff --git a/src/os/src/darwin/eok.c b/src/os/src/darwin/eok.c index 9c6a263a40..f731aaf330 100644 --- a/src/os/src/darwin/eok.c +++ b/src/os/src/darwin/eok.c @@ -13,13 +13,18 @@ * along with this program. If not, see . */ +// fail-fast or let-it-crash philosophy +// https://en.wikipedia.org/wiki/Fail-fast +// https://stackoverflow.com/questions/4393197/erlangs-let-it-crash-philosophy-applicable-elsewhere +// experimentally, we follow log-and-crash here + #include "eok.h" #include "os.h" #include -#define LET_IT_BE +// #define BALANCE_CHECK_WHEN_CLOSE #ifdef ENABLE_LOG #define D(fmt, ...) fprintf(stderr, "%s[%d]%s(): " fmt "\n", basename(__FILE__), __LINE__, __func__, ##__VA_ARGS__) @@ -99,16 +104,16 @@ struct eok_event_s { typedef struct eoks_s eoks_t; struct eoks_s { pthread_mutex_t lock; - ep_over_kq_t **eoks; // note: this memory leaks when process terminates - int neoks; // we can add an extra api to let user clean - ep_over_kq_t *eoks_free; // currently, we just keep it simple stupid + ep_over_kq_t **eoks; // note: this memory leaks when process terminates + int neoks; // we can add an extra api to let user clean + ep_over_kq_t *eoks_free_list; // currently, we just keep it simple stupid }; static eoks_t eoks = { - .lock = PTHREAD_MUTEX_INITIALIZER, - .eoks = NULL, - .neoks = 0, - .eoks_free = NULL, + .lock = PTHREAD_MUTEX_INITIALIZER, + .eoks = NULL, + .neoks = 0, + .eoks_free_list = NULL, }; #ifdef ENABLE_LOG @@ -760,9 +765,9 @@ static ep_over_kq_t* eoks_alloc(void) { A(0==pthread_mutex_lock(&eoks.lock), ""); do { - if (eoks.eoks_free) { - eok = eoks.eoks_free; - eoks.eoks_free = eok->next; + if (eoks.eoks_free_list) { + eok = eoks.eoks_free_list; + eoks.eoks_free_list = eok->next; A(eoks.eoks, "internal logic error"); A(eok->idx>=0 && eok->idxidx)==NULL, "internal logic error"); @@ -820,10 +825,12 @@ static void eoks_free(ep_over_kq_t *eok) { A(eok->waiting==0, "internal logic error"); eok_event_t *ev = eok->evs_head; + int sv_closed = 0; while (ev) { eok_event_t *next = ev->next; if (ev->fd==eok->sv[0]) { // fd is critical system resource + A(sv_closed==0, "internal logic error"); close(eok->sv[0]); eok->sv[0] = -1; close(eok->sv[1]); @@ -832,11 +839,11 @@ static void eoks_free(ep_over_kq_t *eok) { } else { // user forget calling epoll_ctl(EPOLL_CTL_DEL) before calling epoll_close/close? // calling close(ev->fd) here smells really bad -#ifdef LET_IT_BE +#ifndef BALANCE_CHECK_WHEN_CLOSE // we just let it be and reclaim ev eok_free_ev(eok, ev); #else - // panic otherwise, if LET_IT_BE not defined + // panic otherwise, if BALANCE_CHECK_WHEN_CLOSE is defined A(eok->evs_head==NULL && eok->evs_tail==NULL && eok->evs_count==0, "epfd[%d] fd[%d]: internal logic error: have you epoll_ctl(EPOLL_CTL_DEL) everything before calling epoll_close?", eok->idx, ev->fd); @@ -861,8 +868,8 @@ static void eoks_free(ep_over_kq_t *eok) { close(eok->kq); eok->kq = -1; } - eok->next = eoks.eoks_free; - eoks.eoks_free = eok; + eok->next = eoks.eoks_free_list; + eoks.eoks_free_list = eok; *(eoks.eoks + eok->idx) = NULL; } while (0); A(0==pthread_mutex_unlock(&eoks.lock), ""); From 15b7973e48caf52b7a22e621fe9fa654bfc6a8a6 Mon Sep 17 00:00:00 2001 From: freemine Date: Sun, 24 Jan 2021 09:44:47 +0800 Subject: [PATCH 046/147] typo --- src/os/inc/osDef.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/os/inc/osDef.h b/src/os/inc/osDef.h index f646940d3a..e15c45ad14 100644 --- a/src/os/inc/osDef.h +++ b/src/os/inc/osDef.h @@ -95,7 +95,7 @@ extern "C" { #define threadlocal __thread #else // #define threadlocal - #error please follow with the target platform's thread-local implementation + #error please follow with the thread-local implementation on the target platform #endif #ifdef __cplusplus From a368b7b40afccb1bf6b4568f2db3c06354aa496b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 24 Jan 2021 10:24:44 +0800 Subject: [PATCH 047/147] TD-1207 change int32_t to socket --- src/common/inc/tdataformat.h | 22 +++++++++---------- src/dnode/src/dnodeCheck.c | 16 +++++++------- src/dnode/src/dnodeTelemetry.c | 2 +- src/os/inc/osFile.h | 2 +- src/os/inc/osSocket.h | 7 ++++-- src/os/inc/osWindows.h | 6 +++++ src/os/src/darwin/darwinFile.c | 2 +- src/os/src/darwin/darwinSocket.c | 2 +- src/os/src/detail/osFile.c | 2 +- src/os/src/detail/osSocket.c | 4 ++-- src/os/src/linux/linuxEnv.c | 4 ---- src/os/src/windows/wFile.c | 2 +- src/os/src/windows/wSocket.c | 4 ++-- src/os/src/windows/wSysinfo.c | 2 +- src/plugins/http/inc/httpContext.h | 2 +- src/plugins/http/inc/httpInt.h | 7 +++--- src/plugins/http/src/httpContext.c | 10 ++++++--- src/plugins/http/src/httpServer.c | 12 +++++----- src/rpc/src/rpcTcp.c | 35 ++++++++++++------------------ src/rpc/src/rpcUdp.c | 2 +- src/sync/inc/syncInt.h | 4 ++-- src/sync/inc/syncTcp.h | 4 ++-- src/sync/src/syncArbitrator.c | 6 ++--- src/sync/src/syncMain.c | 8 +++---- src/sync/src/syncTcp.c | 24 +++++++------------- src/util/inc/tsocket.h | 24 +++++++++----------- src/util/src/tnettest.c | 21 +++++++++--------- src/util/src/tsocket.c | 30 ++++++++++++------------- 28 files changed, 128 insertions(+), 138 deletions(-) diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index b6ad99f02a..4999fbb9bf 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -27,23 +27,23 @@ extern "C" { #endif -#define STR_TO_VARSTR(x, str) \ - do { \ - VarDataLenT __len = (int32_t)strlen(str); \ - *(VarDataLenT *)(x) = __len; \ - memcpy(varDataVal(x), (str), __len); \ +#define STR_TO_VARSTR(x, str) \ + do { \ + VarDataLenT __len = (int32_t)strlen(str); \ + *(VarDataLenT *)(x) = __len; \ + memcpy(varDataVal(x), (str), __len); \ } while (0); -#define STR_WITH_MAXSIZE_TO_VARSTR(x, str, _maxs) \ - do { \ +#define STR_WITH_MAXSIZE_TO_VARSTR(x, str, _maxs) \ + do { \ char *_e = stpncpy(varDataVal(x), (str), (_maxs)-VARSTR_HEADER_SIZE); \ - varDataSetLen(x, (_e - (x)-VARSTR_HEADER_SIZE)); \ + varDataSetLen(x, (_e - (x)-VARSTR_HEADER_SIZE)); \ } while (0) #define STR_WITH_SIZE_TO_VARSTR(x, str, _size) \ do { \ - *(VarDataLenT *)(x) = (int32_t)(_size); \ - memcpy(varDataVal(x), (str), (_size)); \ + *(VarDataLenT *)(x) = (int32_t)(_size); \ + memcpy(varDataVal(x), (str), (_size)); \ } while (0); // ----------------- TSDB COLUMN DEFINITION @@ -156,7 +156,7 @@ static FORCE_INLINE int tkeyComparFn(const void *tkey1, const void *tkey2) { * +----------+----------+---------------------------------+---------------------------------+ * | len | sversion | First part | Second part | * +----------+----------+---------------------------------+---------------------------------+ - * + * * NOTE: timestamp in this row structure is TKEY instead of TSKEY */ typedef void *SDataRow; diff --git a/src/dnode/src/dnodeCheck.c b/src/dnode/src/dnodeCheck.c index 2c8dcfdcc9..94d2360950 100644 --- a/src/dnode/src/dnodeCheck.c +++ b/src/dnode/src/dnodeCheck.c @@ -29,11 +29,11 @@ typedef struct { static SCheckItem tsCheckItem[TSDB_CHECK_ITEM_MAX] = {{0}}; int64_t tsMinFreeMemSizeForStart = 0; -static int32_t bindTcpPort(int32_t port) { - int32_t serverSocket; +static int32_t bindTcpPort(int16_t port) { + SOCKET serverSocket; struct sockaddr_in server_addr; - if ((serverSocket = ( int32_t)socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0) { + if ((serverSocket = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0) { dError("socket() fail: %s", strerror(errno)); return -1; } @@ -59,11 +59,11 @@ static int32_t bindTcpPort(int32_t port) { return 0; } -static int32_t bindUdpPort(int32_t port) { - int32_t serverSocket; +static int32_t bindUdpPort(int16_t port) { + SOCKET serverSocket; struct sockaddr_in server_addr; - if ((serverSocket = (int32_t)socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0) { + if ((serverSocket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0) { dError("socket() fail: %s", strerror(errno)); return -1; } @@ -85,9 +85,9 @@ static int32_t bindUdpPort(int32_t port) { static int32_t dnodeCheckNetwork() { int32_t ret; - int32_t startPort = tsServerPort; + int16_t startPort = tsServerPort; - for (int32_t port = startPort; port < startPort + 12; port++) { + for (int16_t port = startPort; port < startPort + 12; port++) { ret = bindTcpPort(port); if (0 != ret) { dError("failed to tcp bind port %d, quit", port); diff --git a/src/dnode/src/dnodeTelemetry.c b/src/dnode/src/dnodeTelemetry.c index d1b7127dda..7a9cf45431 100644 --- a/src/dnode/src/dnodeTelemetry.c +++ b/src/dnode/src/dnodeTelemetry.c @@ -200,7 +200,7 @@ static void sendTelemetryReport() { dTrace("failed to get IP address of " TELEMETRY_SERVER ", reason:%s", strerror(errno)); return; } - int32_t fd = taosOpenTcpClientSocket(ip, TELEMETRY_PORT, 0); + SOCKET fd = taosOpenTcpClientSocket(ip, TELEMETRY_PORT, 0); if (fd < 0) { dTrace("failed to create socket for telemetry, reason:%s", strerror(errno)); return; diff --git a/src/os/inc/osFile.h b/src/os/inc/osFile.h index d72f9cefb9..19cc78472c 100644 --- a/src/os/inc/osFile.h +++ b/src/os/inc/osFile.h @@ -39,7 +39,7 @@ int32_t taosRenameFile(char *fullPath, char *suffix, char delimiter, char **dstP } // TAOS_OS_FUNC_FILE_SENDIFLE -int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t *offset, int64_t size); +int64_t taosSendFile(SOCKET dfd, int32_t sfd, int64_t *offset, int64_t size); int64_t taosFSendFile(FILE *outfile, FILE *infile, int64_t *offset, int64_t size); #ifdef TAOS_RANDOM_FILE_FAIL diff --git a/src/os/inc/osSocket.h b/src/os/inc/osSocket.h index 9683adf45a..111fca712c 100644 --- a/src/os/inc/osSocket.h +++ b/src/os/inc/osSocket.h @@ -37,6 +37,9 @@ extern "C" { #ifndef TAOS_OS_DEF_EPOLL #define TAOS_EPOLL_WAIT_TIME 500 + typedef int32_t SOCKET; + typedef SOCKET EpollFd; + #define EpollClose(pollFd) taosCloseSocket(pollFd) #endif #ifdef TAOS_RANDOM_NETWORK_FAIL @@ -57,13 +60,13 @@ extern "C" { #endif // TAOS_OS_FUNC_SOCKET -int32_t taosSetNonblocking(int32_t sock, int32_t on); +int32_t taosSetNonblocking(SOCKET sock, int32_t on); void taosIgnSIGPIPE(); void taosBlockSIGPIPE(); void taosSetMaskSIGPIPE(); // TAOS_OS_FUNC_SOCKET_SETSOCKETOPT -int32_t taosSetSockOpt(int32_t socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen); +int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen); // TAOS_OS_FUNC_SOCKET_INET uint32_t taosInetAddr(char *ipAddr); diff --git a/src/os/inc/osWindows.h b/src/os/inc/osWindows.h index a984a5ba88..1f3b1b02e3 100644 --- a/src/os/inc/osWindows.h +++ b/src/os/inc/osWindows.h @@ -93,6 +93,12 @@ typedef SOCKET eventfd_t; #define TAOS_OS_DEF_EPOLL #define TAOS_EPOLL_WAIT_TIME 100 + typedef SOCKET EpollFd; + #define EpollClose(pollFd) epoll_close(pollFd) + +#ifndef EPOLLWAKEUP + #define EPOLLWAKEUP (1u << 29) +#endif #define TAOS_OS_DEF_ZU #define PRIzu "ld" diff --git a/src/os/src/darwin/darwinFile.c b/src/os/src/darwin/darwinFile.c index 00c1d8532f..1e77cd68d8 100644 --- a/src/os/src/darwin/darwinFile.c +++ b/src/os/src/darwin/darwinFile.c @@ -51,7 +51,7 @@ int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t co return writeLen; } -int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t* offset, int64_t count) { +int64_t taosSendFile(SOCKET dfd, int32_t sfd, int64_t* offset, int64_t count) { lseek(sfd, (int32_t)(*offset), 0); int64_t writeLen = 0; uint8_t buffer[_SEND_FILE_STEP_] = { 0 }; diff --git a/src/os/src/darwin/darwinSocket.c b/src/os/src/darwin/darwinSocket.c index 9137d4b125..69a4666d34 100644 --- a/src/os/src/darwin/darwinSocket.c +++ b/src/os/src/darwin/darwinSocket.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "os.h" -int taosSetSockOpt(int32_t socketfd, int level, int optname, void *optval, int optlen) { +int taosSetSockOpt(SOCKET socketfd, int level, int optname, void *optval, int optlen) { if (level == SOL_SOCKET && optname == SO_SNDBUF) { return 0; } diff --git a/src/os/src/detail/osFile.c b/src/os/src/detail/osFile.c index 9560ce59bb..bb68622731 100644 --- a/src/os/src/detail/osFile.c +++ b/src/os/src/detail/osFile.c @@ -121,7 +121,7 @@ int64_t taosLSeekImp(int32_t fd, int64_t offset, int32_t whence) { #ifndef TAOS_OS_FUNC_FILE_SENDIFLE -int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t *offset, int64_t size) { +int64_t taosSendFile(SOCKET dfd, int32_t sfd, int64_t *offset, int64_t size) { int64_t leftbytes = size; int64_t sentbytes; diff --git a/src/os/src/detail/osSocket.c b/src/os/src/detail/osSocket.c index f929b3e717..d03e8086bf 100644 --- a/src/os/src/detail/osSocket.c +++ b/src/os/src/detail/osSocket.c @@ -19,7 +19,7 @@ #ifndef TAOS_OS_FUNC_SOCKET -int32_t taosSetNonblocking(int32_t sock, int32_t on) { +int32_t taosSetNonblocking(SOCKET sock, int32_t on) { int32_t flags = 0; if ((flags = fcntl(sock, F_GETFL, 0)) < 0) { uError("fcntl(F_GETFL) error: %d (%s)\n", errno, strerror(errno)); @@ -67,7 +67,7 @@ void taosSetMaskSIGPIPE() { #ifndef TAOS_OS_FUNC_SOCKET_SETSOCKETOPT -int32_t taosSetSockOpt(int32_t socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen) { +int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen) { return setsockopt(socketfd, level, optname, optval, (socklen_t)optlen); } diff --git a/src/os/src/linux/linuxEnv.c b/src/os/src/linux/linuxEnv.c index abcdabcfd5..e3eadbc94b 100644 --- a/src/os/src/linux/linuxEnv.c +++ b/src/os/src/linux/linuxEnv.c @@ -43,7 +43,6 @@ void osInit() { char cmdline[1024]; char* taosGetCmdlineByPID(int pid) { -#if 0 sprintf(cmdline, "/proc/%d/cmdline", pid); FILE* f = fopen(cmdline, "r"); if (f) { @@ -55,7 +54,4 @@ char* taosGetCmdlineByPID(int pid) { fclose(f); } return cmdline; -#else - return ""; -#endif } diff --git a/src/os/src/windows/wFile.c b/src/os/src/windows/wFile.c index e388e5fc67..4ad195dc6f 100644 --- a/src/os/src/windows/wFile.c +++ b/src/os/src/windows/wFile.c @@ -76,7 +76,7 @@ int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t co return writeLen; } -int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t *offset, int64_t count) { +int64_t taosSendFile(SOCKET dfd, int32_t sfd, int64_t *offset, int64_t count) { if (offset != NULL) lseek(sfd, (int32_t)(*offset), 0); int64_t writeLen = 0; diff --git a/src/os/src/windows/wSocket.c b/src/os/src/windows/wSocket.c index 679842a7e9..4e6ee15e77 100644 --- a/src/os/src/windows/wSocket.c +++ b/src/os/src/windows/wSocket.c @@ -34,7 +34,7 @@ void taosWinSocketInit() { } } -int32_t taosSetNonblocking(int32_t sock, int32_t on) { +int32_t taosSetNonblocking(SOCKET sock, int32_t on) { u_long mode; if (on) { mode = 1; @@ -50,7 +50,7 @@ void taosIgnSIGPIPE() {} void taosBlockSIGPIPE() {} void taosSetMaskSIGPIPE() {} -int32_t taosSetSockOpt(int32_t socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen) { +int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen) { if (level == SOL_SOCKET && optname == TCP_KEEPCNT) { return 0; } diff --git a/src/os/src/windows/wSysinfo.c b/src/os/src/windows/wSysinfo.c index 9a79f5a6bf..082aaaf5d8 100644 --- a/src/os/src/windows/wSysinfo.c +++ b/src/os/src/windows/wSysinfo.c @@ -66,7 +66,7 @@ bool taosGetSysMemory(float *memoryUsedMB) { bool taosGetProcMemory(float *memoryUsedMB) { unsigned bytes_used = 0; -#if defined(_WIN32) && defined(_MSC_VER) +#if defined(_WIN64) && defined(_MSC_VER) PROCESS_MEMORY_COUNTERS pmc; HANDLE cur_proc = GetCurrentProcess(); diff --git a/src/plugins/http/inc/httpContext.h b/src/plugins/http/inc/httpContext.h index 260858c5cc..b016da2dd3 100644 --- a/src/plugins/http/inc/httpContext.h +++ b/src/plugins/http/inc/httpContext.h @@ -22,7 +22,7 @@ bool httpInitContexts(); void httpCleanupContexts(); const char *httpContextStateStr(HttpContextState state); -HttpContext *httpCreateContext(int32_t fd); +HttpContext *httpCreateContext(SOCKET fd); bool httpInitContext(HttpContext *pContext); HttpContext *httpGetContext(void * pContext); void httpReleaseContext(HttpContext *pContext, bool clearRes); diff --git a/src/plugins/http/inc/httpInt.h b/src/plugins/http/inc/httpInt.h index 73e9d30924..634468f3cc 100644 --- a/src/plugins/http/inc/httpInt.h +++ b/src/plugins/http/inc/httpInt.h @@ -16,6 +16,7 @@ #ifndef TDENGINE_HTTP_INT_H #define TDENGINE_HTTP_INT_H +#include "os.h" #include #include "pthread.h" #include "semaphore.h" @@ -140,7 +141,7 @@ typedef enum { typedef struct HttpContext { int32_t refCount; - int32_t fd; + SOCKET fd; uint32_t accessTimes; uint32_t lastAccessTime; int32_t state; @@ -167,7 +168,7 @@ typedef struct HttpThread { HttpContext * pHead; pthread_mutex_t threadMutex; bool stop; - int32_t pollFd; + EpollFd pollFd; int32_t numOfContexts; int32_t threadId; char label[HTTP_LABEL_SIZE]; @@ -180,7 +181,7 @@ typedef struct HttpServer { uint16_t serverPort; int8_t stop; int8_t reserve; - int32_t fd; + SOCKET fd; int32_t numOfThreads; int32_t methodScannerLen; int32_t requestNum; diff --git a/src/plugins/http/src/httpContext.c b/src/plugins/http/src/httpContext.c index db77c479c0..f71a84a5af 100644 --- a/src/plugins/http/src/httpContext.c +++ b/src/plugins/http/src/httpContext.c @@ -35,14 +35,18 @@ static void httpRemoveContextFromEpoll(HttpContext *pContext) { HttpThread *pThread = pContext->pThread; if (pContext->fd >= 0) { epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, pContext->fd, NULL); - int32_t fd = atomic_val_compare_exchange_32(&pContext->fd, pContext->fd, -1); +#ifdef WINDOWS + SOCKET fd = atomic_val_compare_exchange_32(&pContext->fd, pContext->fd, -1); +#else + SOCKET fd = atomic_val_compare_exchange_64(&pContext->fd, pContext->fd, -1); +#endif taosCloseSocket(fd); } } static void httpDestroyContext(void *data) { HttpContext *pContext = *(HttpContext **)data; - if (pContext->fd > 0) taosClose(pContext->fd); + if (pContext->fd > 0) taosCloseSocket(pContext->fd); HttpThread *pThread = pContext->pThread; httpRemoveContextFromEpoll(pContext); @@ -106,7 +110,7 @@ bool httpAlterContextState(HttpContext *pContext, HttpContextState srcState, Htt return (atomic_val_compare_exchange_32(&pContext->state, srcState, destState) == srcState); } -HttpContext *httpCreateContext(int32_t fd) { +HttpContext *httpCreateContext(SOCKET fd) { HttpContext *pContext = calloc(1, sizeof(HttpContext)); if (pContext == NULL) return NULL; diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index c4a03d44ea..932f3e3808 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -50,7 +50,7 @@ static void httpStopThread(HttpThread *pThread) { taosCloseSocket(fd); } - taosCloseSocket(pThread->pollFd); + EpollClose(pThread->pollFd); pthread_mutex_destroy(&(pThread->threadMutex)); } @@ -152,7 +152,7 @@ static void httpProcessHttpData(void *param) { } static void *httpAcceptHttpConnection(void *arg) { - int32_t connFd = -1; + SOCKET connFd = -1; struct sockaddr_in clientAddr; int32_t threadId = 0; HttpServer * pServer = &tsHttpServer; @@ -175,7 +175,7 @@ static void *httpAcceptHttpConnection(void *arg) { while (1) { socklen_t addrlen = sizeof(clientAddr); - connFd = (int32_t)accept(pServer->fd, (struct sockaddr *)&clientAddr, &addrlen); + connFd = accept(pServer->fd, (struct sockaddr *)&clientAddr, &addrlen); if (pServer->stop) { httpDebug("http server:%s socket stop, exiting...", pServer->label); break; @@ -227,7 +227,7 @@ static void *httpAcceptHttpConnection(void *arg) { if (epoll_ctl(pThread->pollFd, EPOLL_CTL_ADD, connFd, &event) < 0) { httpError("context:%p, fd:%d, ip:%s, thread:%s, failed to add http fd for epoll, error:%s", pContext, connFd, pContext->ipstr, pThread->label, strerror(errno)); - taosClose(pContext->fd); + taosCloseSocket(pContext->fd); httpReleaseContext(pContext, true); continue; } @@ -265,8 +265,8 @@ bool httpInitConnect() { return false; } - pThread->pollFd = (int32_t)epoll_create(HTTP_MAX_EVENTS); // size does not matter - if (pThread->pollFd < 0) { + pThread->pollFd = (EpollFd)epoll_create(HTTP_MAX_EVENTS); // size does not matter + if (pThread->pollFd <= 0) { httpError("http thread:%s, failed to create HTTP epoll", pThread->label); pthread_mutex_destroy(&(pThread->threadMutex)); return false; diff --git a/src/rpc/src/rpcTcp.c b/src/rpc/src/rpcTcp.c index 1d2dbe281e..acd81e94a4 100644 --- a/src/rpc/src/rpcTcp.c +++ b/src/rpc/src/rpcTcp.c @@ -21,21 +21,14 @@ #include "rpcLog.h" #include "rpcHead.h" #include "rpcTcp.h" -#ifdef WINDOWS -#include "wepoll.h" -#endif - -#ifndef EPOLLWAKEUP - #define EPOLLWAKEUP (1u << 29) -#endif typedef struct SFdObj { void *signature; - int32_t fd; // TCP socket FD - int closedByApp; // 1: already closed by App + SOCKET fd; // TCP socket FD void *thandle; // handle from upper layer, like TAOS uint32_t ip; uint16_t port; + int16_t closedByApp; // 1: already closed by App struct SThreadObj *pThreadObj; struct SFdObj *prev; struct SFdObj *next; @@ -47,7 +40,7 @@ typedef struct SThreadObj { pthread_mutex_t mutex; uint32_t ip; bool stop; - int32_t pollFd; + EpollFd pollFd; int numOfFds; int threadId; char label[TSDB_LABEL_LEN]; @@ -56,7 +49,7 @@ typedef struct SThreadObj { } SThreadObj; typedef struct { - int32_t fd; + SOCKET fd; uint32_t ip; uint16_t port; int8_t stop; @@ -69,7 +62,7 @@ typedef struct { } SServerObj; static void *taosProcessTcpData(void *param); -static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, int32_t fd); +static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, SOCKET fd); static void taosFreeFdObj(SFdObj *pFdObj); static void taosReportBrokenLink(SFdObj *pFdObj); static void *taosAcceptTcpConnection(void *arg); @@ -134,7 +127,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread break; } - pThreadObj->pollFd = (int32_t)epoll_create(10); // size does not matter + pThreadObj->pollFd = (EpollFd)epoll_create(10); // size does not matter if (pThreadObj->pollFd < 0) { tError("%s failed to create TCP epoll", label); code = -1; @@ -227,7 +220,7 @@ void taosCleanUpTcpServer(void *handle) { } static void *taosAcceptTcpConnection(void *arg) { - int32_t connFd = -1; + SOCKET connFd = -1; struct sockaddr_in caddr; int threadId = 0; SThreadObj *pThreadObj; @@ -238,7 +231,7 @@ static void *taosAcceptTcpConnection(void *arg) { while (1) { socklen_t addrlen = sizeof(caddr); - connFd = (int32_t)accept(pServerObj->fd, (struct sockaddr *)&caddr, &addrlen); + connFd = accept(pServerObj->fd, (struct sockaddr *)&caddr, &addrlen); if (pServerObj->stop) { tDebug("%s TCP server stop accepting new connections", pServerObj->label); break; @@ -306,7 +299,7 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int num, void * return NULL; } - pThreadObj->pollFd = (int32_t)epoll_create(10); // size does not matter + pThreadObj->pollFd = (EpollFd)epoll_create(10); // size does not matter if (pThreadObj->pollFd < 0) { tError("%s failed to create TCP client epoll", label); free(pThreadObj); @@ -321,7 +314,7 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int num, void * int code = pthread_create(&(pThreadObj->thread), &thattr, taosProcessTcpData, (void *)(pThreadObj)); pthread_attr_destroy(&thattr); if (code != 0) { - taosCloseSocket(pThreadObj->pollFd); + EpollClose(pThreadObj->pollFd); free(pThreadObj); terrno = TAOS_SYSTEM_ERROR(errno); tError("%s failed to create TCP read data thread(%s)", label, strerror(errno)); @@ -351,8 +344,8 @@ void taosCleanUpTcpClient(void *chandle) { void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port) { SThreadObj * pThreadObj = shandle; - int32_t fd = taosOpenTcpClientSocket(ip, port, pThreadObj->ip); - if (fd < 0) return NULL; + SOCKET fd = taosOpenTcpClientSocket(ip, port, pThreadObj->ip); + if (fd <= 0) return NULL; struct sockaddr_in sin; uint16_t localPort = 0; @@ -526,7 +519,7 @@ static void *taosProcessTcpData(void *param) { if (pThreadObj->stop) break; } - if (pThreadObj->pollFd >=0) taosCloseSocket(pThreadObj->pollFd); + if (pThreadObj->pollFd >=0) EpollClose(pThreadObj->pollFd); while (pThreadObj->pHead) { SFdObj *pFdObj = pThreadObj->pHead; @@ -541,7 +534,7 @@ static void *taosProcessTcpData(void *param) { return NULL; } -static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, int32_t fd) { +static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, SOCKET fd) { struct epoll_event event; SFdObj *pFdObj = (SFdObj *)calloc(sizeof(SFdObj), 1); diff --git a/src/rpc/src/rpcUdp.c b/src/rpc/src/rpcUdp.c index 862d9b1f5c..2599bca075 100644 --- a/src/rpc/src/rpcUdp.c +++ b/src/rpc/src/rpcUdp.c @@ -31,7 +31,7 @@ typedef struct { int index; - int32_t fd; + SOCKET fd; uint16_t port; // peer port uint16_t localPort; // local port char label[TSDB_LABEL_LEN]; // copy from udpConnSet; diff --git a/src/sync/inc/syncInt.h b/src/sync/inc/syncInt.h index eef687d647..e43140d4e6 100644 --- a/src/sync/inc/syncInt.h +++ b/src/sync/inc/syncInt.h @@ -82,8 +82,8 @@ typedef struct SsyncPeer { uint64_t sversion; // track the peer version in retrieve process uint64_t lastFileVer; // track the file version while retrieve uint64_t lastWalVer; // track the wal version while retrieve - int32_t syncFd; - int32_t peerFd; // forward FD + SOCKET syncFd; + SOCKET peerFd; // forward FD int32_t numOfRetrieves; // number of retrieves tried int32_t fileChanged; // a flag to indicate file is changed during retrieving process int32_t refCount; diff --git a/src/sync/inc/syncTcp.h b/src/sync/inc/syncTcp.h index d4674fee6b..b322c3440c 100644 --- a/src/sync/inc/syncTcp.h +++ b/src/sync/inc/syncTcp.h @@ -27,12 +27,12 @@ typedef struct { int32_t bufferSize; void (*processBrokenLink)(int64_t handleId); int32_t (*processIncomingMsg)(int64_t handleId, void *buffer); - void (*processIncomingConn)(int32_t fd, uint32_t ip); + void (*processIncomingConn)(SOCKET fd, uint32_t ip); } SPoolInfo; void *syncOpenTcpThreadPool(SPoolInfo *pInfo); void syncCloseTcpThreadPool(void *); -void *syncAllocateTcpConn(void *, int64_t rid, int32_t connFd); +void *syncAllocateTcpConn(void *, int64_t rid, SOCKET connFd); void syncFreeTcpConn(void *); #ifdef __cplusplus diff --git a/src/sync/src/syncArbitrator.c b/src/sync/src/syncArbitrator.c index 24760ce1a0..9fb6b0ddb7 100644 --- a/src/sync/src/syncArbitrator.c +++ b/src/sync/src/syncArbitrator.c @@ -28,7 +28,7 @@ #include "syncTcp.h" static void arbSignalHandler(int32_t signum, void *sigInfo, void *context); -static void arbProcessIncommingConnection(int32_t connFd, uint32_t sourceIp); +static void arbProcessIncommingConnection(SOCKET connFd, uint32_t sourceIp); static void arbProcessBrokenLink(int64_t rid); static int32_t arbProcessPeerMsg(int64_t rid, void *buffer); static tsem_t tsArbSem; @@ -36,7 +36,7 @@ static void * tsArbTcpPool; typedef struct { char id[TSDB_EP_LEN + 24]; - int32_t nodeFd; + SOCKET nodeFd; void * pConn; } SNodeConn; @@ -106,7 +106,7 @@ int32_t main(int32_t argc, char *argv[]) { return 0; } -static void arbProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { +static void arbProcessIncommingConnection(SOCKET connFd, uint32_t sourceIp) { char ipstr[24]; tinet_ntoa(ipstr, sourceIp); sDebug("peer TCP connection from ip:%s", ipstr); diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index 606809e82b..d698432176 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -45,7 +45,7 @@ static void syncCheckPeerConnection(void *param, void *tmrId); static int32_t syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack, int8_t type, uint16_t tranId); static void syncProcessBrokenLink(int64_t rid); static int32_t syncProcessPeerMsg(int64_t rid, void *buffer); -static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp); +static void syncProcessIncommingConnection(SOCKET connFd, uint32_t sourceIp); static void syncRemovePeer(SSyncPeer *pPeer); static void syncAddArbitrator(SSyncNode *pNode); static void syncFreeNode(void *); @@ -1114,8 +1114,8 @@ static void syncSetupPeerConnection(SSyncPeer *pPeer) { return; } - int32_t connFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0); - if ((int32_t)connFd < 0) { + SOCKET connFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0); + if (connFd <= 0) { sDebug("%s, failed to open tcp socket since %s", pPeer->id, strerror(errno)); taosTmrReset(syncCheckPeerConnection, SYNC_CHECK_INTERVAL, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer); return; @@ -1179,7 +1179,7 @@ static void syncCreateRestoreDataThread(SSyncPeer *pPeer) { } } -static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { +static void syncProcessIncommingConnection(SOCKET connFd, uint32_t sourceIp) { char ipstr[24]; int32_t i; diff --git a/src/sync/src/syncTcp.c b/src/sync/src/syncTcp.c index 1210caa84c..50c52623f7 100644 --- a/src/sync/src/syncTcp.c +++ b/src/sync/src/syncTcp.c @@ -24,18 +24,10 @@ #include "syncInt.h" #include "syncTcp.h" -#ifdef WINDOWS -#include "wepoll.h" -#endif - -#ifndef EPOLLWAKEUP - #define EPOLLWAKEUP (1u << 29) -#endif - typedef struct SThreadObj { pthread_t thread; bool stop; - int32_t pollFd; + SOCKET pollFd; int32_t numOfFds; struct SPoolObj *pPool; } SThreadObj; @@ -45,14 +37,14 @@ typedef struct SPoolObj { SThreadObj **pThread; pthread_t thread; int32_t nextId; - int32_t acceptFd; // FD for accept new connection + SOCKET acceptFd; // FD for accept new connection int8_t stop; } SPoolObj; typedef struct { SThreadObj *pThread; int64_t handleId; - int32_t fd; + SOCKET fd; int32_t closedByApp; } SConnObj; @@ -128,7 +120,7 @@ void syncCloseTcpThreadPool(void *param) { tfree(pPool); } -void *syncAllocateTcpConn(void *param, int64_t rid, int32_t connFd) { +void *syncAllocateTcpConn(void *param, int64_t rid, SOCKET connFd) { struct epoll_event event; SPoolObj *pPool = param; @@ -249,7 +241,7 @@ static void *syncProcessTcpData(void *param) { sDebug("%p TCP epoll thread exits", pThread); - taosCloseSocket(pThread->pollFd); + EpollClose(pThread->pollFd); tfree(pThread); tfree(buffer); return NULL; @@ -264,13 +256,13 @@ static void *syncAcceptPeerTcpConnection(void *argv) { while (1) { struct sockaddr_in clientAddr; socklen_t addrlen = sizeof(clientAddr); - int32_t connFd = (int32_t)accept(pPool->acceptFd, (struct sockaddr *)&clientAddr, &addrlen); + SOCKET connFd = accept(pPool->acceptFd, (struct sockaddr *)&clientAddr, &addrlen); if (pPool->stop) { sDebug("%p TCP server accept is stopped", pPool); break; } - if ((int32_t)connFd < 0) { + if (connFd < 0) { if (errno == EINVAL) { sDebug("%p TCP server accept is exiting...", pPool); break; @@ -298,7 +290,7 @@ static SThreadObj *syncGetTcpThread(SPoolObj *pPool) { if (pThread == NULL) return NULL; pThread->pPool = pPool; - pThread->pollFd = (int32_t)epoll_create(10); // size does not matter + pThread->pollFd = (EpollFd)epoll_create(10); // size does not matter if (pThread->pollFd < 0) { tfree(pThread); return NULL; diff --git a/src/util/inc/tsocket.h b/src/util/inc/tsocket.h index fc0d71c4d8..35b591b61e 100644 --- a/src/util/inc/tsocket.h +++ b/src/util/inc/tsocket.h @@ -24,21 +24,17 @@ extern "C" { #include "wepoll.h" #endif -#ifndef EPOLLWAKEUP - #define EPOLLWAKEUP (1u << 29) -#endif +int32_t taosReadn(SOCKET sock, char *buffer, int32_t len); +int32_t taosWriteMsg(SOCKET fd, void *ptr, int32_t nbytes); +int32_t taosReadMsg(SOCKET fd, void *ptr, int32_t nbytes); +int32_t taosNonblockwrite(SOCKET fd, char *ptr, int32_t nbytes); +int32_t taosCopyFds(SOCKET sfd, int32_t dfd, int64_t len); +int32_t taosSetNonblocking(SOCKET sock, int32_t on); -int32_t taosReadn(int32_t sock, char *buffer, int32_t len); -int32_t taosWriteMsg(int32_t fd, void *ptr, int32_t nbytes); -int32_t taosReadMsg(int32_t fd, void *ptr, int32_t nbytes); -int32_t taosNonblockwrite(int32_t fd, char *ptr, int32_t nbytes); -int32_t taosCopyFds(int32_t sfd, int32_t dfd, int64_t len); -int32_t taosSetNonblocking(int32_t sock, int32_t on); - -int32_t taosOpenUdpSocket(uint32_t localIp, uint16_t localPort); -int32_t taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp); -int32_t taosOpenTcpServerSocket(uint32_t ip, uint16_t port); -int32_t taosKeepTcpAlive(int32_t sockFd); +SOCKET taosOpenUdpSocket(uint32_t localIp, uint16_t localPort); +SOCKET taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp); +SOCKET taosOpenTcpServerSocket(uint32_t ip, uint16_t port); +int32_t taosKeepTcpAlive(SOCKET sockFd); int32_t taosGetFqdn(char *); uint32_t taosGetIpv4FromFqdn(const char *); diff --git a/src/util/src/tnettest.c b/src/util/src/tnettest.c index 70c38d36dc..fe6dfb493d 100644 --- a/src/util/src/tnettest.c +++ b/src/util/src/tnettest.c @@ -39,7 +39,7 @@ typedef struct { static void *taosNetBindUdpPort(void *sarg) { STestInfo *pinfo = (STestInfo *)sarg; int32_t port = pinfo->port; - int32_t serverSocket; + SOCKET serverSocket; char buffer[BUFFER_SIZE]; int32_t iDataNum; socklen_t sin_size; @@ -48,7 +48,7 @@ static void *taosNetBindUdpPort(void *sarg) { struct sockaddr_in server_addr; struct sockaddr_in clientAddr; - if ((serverSocket = (int32_t)socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0) { + if ((serverSocket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0) { uError("failed to create UDP socket since %s", strerror(errno)); return NULL; } @@ -106,12 +106,12 @@ static void *taosNetBindTcpPort(void *sarg) { STestInfo *pinfo = sarg; int32_t port = pinfo->port; - int32_t serverSocket; + SOCKET serverSocket; int32_t addr_len = sizeof(clientAddr); - int32_t client; + SOCKET client; char buffer[BUFFER_SIZE]; - if ((serverSocket = (int32_t)socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0) { + if ((serverSocket = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0) { uError("failed to create TCP socket since %s", strerror(errno)); return NULL; } @@ -133,7 +133,6 @@ static void *taosNetBindTcpPort(void *sarg) { return NULL; } - if (taosKeepTcpAlive(serverSocket) < 0) { uError("failed to set tcp server keep-alive option since %s", strerror(errno)); taosCloseSocket(serverSocket); @@ -148,7 +147,7 @@ static void *taosNetBindTcpPort(void *sarg) { uInfo("TCP server at port:%d is listening", port); while (1) { - client = (int32_t)accept(serverSocket, (struct sockaddr *)&clientAddr, (socklen_t *)&addr_len); + client = accept(serverSocket, (struct sockaddr *)&clientAddr, (socklen_t *)&addr_len); if (client < 0) { uDebug("TCP: failed to accept at port:%d since %s", port, strerror(errno)); continue; @@ -178,10 +177,10 @@ static void *taosNetBindTcpPort(void *sarg) { } static int32_t taosNetCheckTcpPort(STestInfo *info) { - int32_t clientSocket; + SOCKET clientSocket; char buffer[BUFFER_SIZE] = {0}; - if ((clientSocket = (int32_t)socket(AF_INET, SOCK_STREAM, 0)) < 0) { + if ((clientSocket = socket(AF_INET, SOCK_STREAM, 0)) < 0) { uError("failed to create TCP client socket since %s", strerror(errno)); return -1; } @@ -226,14 +225,14 @@ static int32_t taosNetCheckTcpPort(STestInfo *info) { } static int32_t taosNetCheckUdpPort(STestInfo *info) { - int32_t clientSocket; + SOCKET clientSocket; char buffer[BUFFER_SIZE] = {0}; int32_t iDataNum = 0; int32_t bufSize = 1024000; struct sockaddr_in serverAddr; - if ((clientSocket = (int32_t)socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0) { + if ((clientSocket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0) { uError("failed to create udp client socket since %s", strerror(errno)); return -1; } diff --git a/src/util/src/tsocket.c b/src/util/src/tsocket.c index d41c1518b8..3636698162 100644 --- a/src/util/src/tsocket.c +++ b/src/util/src/tsocket.c @@ -102,7 +102,7 @@ uint32_t ip2uint(const char *const ip_addr) { return *((uint32_t *)ip); } -int32_t taosWriteMsg(int32_t fd, void *buf, int32_t nbytes) { +int32_t taosWriteMsg(SOCKET fd, void *buf, int32_t nbytes) { int32_t nleft, nwritten; char * ptr = (char *)buf; @@ -128,7 +128,7 @@ int32_t taosWriteMsg(int32_t fd, void *buf, int32_t nbytes) { return (nbytes - nleft); } -int32_t taosReadMsg(int32_t fd, void *buf, int32_t nbytes) { +int32_t taosReadMsg(SOCKET fd, void *buf, int32_t nbytes) { int32_t nleft, nread; char * ptr = (char *)buf; @@ -159,7 +159,7 @@ int32_t taosReadMsg(int32_t fd, void *buf, int32_t nbytes) { return (nbytes - nleft); } -int32_t taosNonblockwrite(int32_t fd, char *ptr, int32_t nbytes) { +int32_t taosNonblockwrite(SOCKET fd, char *ptr, int32_t nbytes) { taosSetNonblocking(fd, 1); int32_t nleft, nwritten, nready; @@ -201,7 +201,7 @@ int32_t taosNonblockwrite(int32_t fd, char *ptr, int32_t nbytes) { return (nbytes - nleft); } -int32_t taosReadn(int32_t fd, char *ptr, int32_t nbytes) { +int32_t taosReadn(SOCKET fd, char *ptr, int32_t nbytes) { int32_t nread, nready, nleft = nbytes; fd_set fset; @@ -239,9 +239,9 @@ int32_t taosReadn(int32_t fd, char *ptr, int32_t nbytes) { return (nbytes - nleft); } -int32_t taosOpenUdpSocket(uint32_t ip, uint16_t port) { +SOCKET taosOpenUdpSocket(uint32_t ip, uint16_t port) { struct sockaddr_in localAddr; - int32_t sockFd; + SOCKET sockFd; int32_t bufSize = 1024000; uDebug("open udp socket:0x%x:%hu", ip, port); @@ -251,7 +251,7 @@ int32_t taosOpenUdpSocket(uint32_t ip, uint16_t port) { localAddr.sin_addr.s_addr = ip; localAddr.sin_port = (uint16_t)htons(port); - if ((sockFd = (int32_t)socket(AF_INET, SOCK_DGRAM, 0)) <= 2) { + if ((sockFd = socket(AF_INET, SOCK_DGRAM, 0)) <= 2) { uError("failed to open udp socket: %d (%s)", errno, strerror(errno)); taosCloseSocketNoCheck(sockFd); return -1; @@ -279,13 +279,13 @@ int32_t taosOpenUdpSocket(uint32_t ip, uint16_t port) { return sockFd; } -int32_t taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clientIp) { - int32_t sockFd = 0; +SOCKET taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clientIp) { + SOCKET sockFd = 0; int32_t ret; struct sockaddr_in serverAddr, clientAddr; int32_t bufSize = 1024 * 1024; - sockFd = (int32_t)socket(PF_INET, SOCK_STREAM, IPPROTO_TCP); + sockFd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP); if (sockFd <= 2) { uError("failed to open the socket: %d (%s)", errno, strerror(errno)); @@ -346,7 +346,7 @@ int32_t taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t cli return sockFd; } -int32_t taosKeepTcpAlive(int32_t sockFd) { +int32_t taosKeepTcpAlive(SOCKET sockFd) { int32_t alive = 1; if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_KEEPALIVE, (void *)&alive, sizeof(alive)) < 0) { uError("fd:%d setsockopt SO_KEEPALIVE failed: %d (%s)", sockFd, errno, strerror(errno)); @@ -394,9 +394,9 @@ int32_t taosKeepTcpAlive(int32_t sockFd) { return 0; } -int32_t taosOpenTcpServerSocket(uint32_t ip, uint16_t port) { +SOCKET taosOpenTcpServerSocket(uint32_t ip, uint16_t port) { struct sockaddr_in serverAdd; - int32_t sockFd; + SOCKET sockFd; int32_t reuse; uDebug("open tcp server socket:0x%x:%hu", ip, port); @@ -406,7 +406,7 @@ int32_t taosOpenTcpServerSocket(uint32_t ip, uint16_t port) { serverAdd.sin_addr.s_addr = ip; serverAdd.sin_port = (uint16_t)htons(port); - if ((sockFd = (int32_t)socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) <= 2) { + if ((sockFd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) <= 2) { uError("failed to open TCP socket: %d (%s)", errno, strerror(errno)); taosCloseSocketNoCheck(sockFd); return -1; @@ -449,7 +449,7 @@ void tinet_ntoa(char *ipstr, uint32_t ip) { #define COPY_SIZE 32768 // sendfile shall be used -int32_t taosCopyFds(int32_t sfd, int32_t dfd, int64_t len) { +int32_t taosCopyFds(SOCKET sfd, int32_t dfd, int64_t len) { int64_t leftLen; int32_t readLen, writeLen; char temp[COPY_SIZE]; From d6a79a43167e6721fd5fd330739e42cc076ee517 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 24 Jan 2021 11:53:33 +0800 Subject: [PATCH 048/147] compile error in v2019 --- src/plugins/http/src/httpParser.c | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/src/plugins/http/src/httpParser.c b/src/plugins/http/src/httpParser.c index 5ebe9df96f..0de437c761 100644 --- a/src/plugins/http/src/httpParser.c +++ b/src/plugins/http/src/httpParser.c @@ -271,18 +271,15 @@ static int32_t httpOnParseHeaderField(HttpParser *parser, const char *key, const } else if (0 == strcasecmp(key, "Authorization")) { - char * t = NULL; - char * s = NULL; + char t[6] = {0}; + char s[129] = {0}; int32_t bytes = 0; - int32_t n = sscanf(val, "%ms %ms%n", &t, &s, &bytes); - if (n == 2 && t && s && bytes == strlen(val)) { + int32_t n = sscanf(val, "%5s %128s%n", t, s, &bytes); + if (n == 2 && t[0] && s[0] && bytes == strlen(val)) { if (strcmp(t, "Basic") == 0) { free(parser->authContent); - parser->authContent = s; + parser->authContent = strdup(s); parser->authType = HTTP_BASIC_AUTH; - s = NULL; - free(t); - free(s); httpTrace("context:%p, fd:%d, basic auth:%s", pContext, pContext->fd, parser->authContent); int32_t ok = httpParseBasicAuthToken(pContext, parser->authContent, (int32_t)strlen(parser->authContent)); if (ok != 0) { @@ -292,11 +289,8 @@ static int32_t httpOnParseHeaderField(HttpParser *parser, const char *key, const return 0; } else if (strcmp(t, "Taosd") == 0) { free(parser->authContent); - parser->authContent = s; + parser->authContent = strdup(s); parser->authType = HTTP_TAOSD_AUTH; - s = NULL; - free(t); - free(s); httpTrace("context:%p, fd:%d, taosd auth:%s", pContext, pContext->fd, parser->authContent); int32_t ok = httpParseTaosdAuthToken(pContext, parser->authContent, (int32_t)strlen(parser->authContent)); if (ok != 0) { @@ -308,16 +302,12 @@ static int32_t httpOnParseHeaderField(HttpParser *parser, const char *key, const parser->authType = HTTP_INVALID_AUTH; httpError("context:%p, fd:%d, invalid auth, t:%s s:%s", pContext, pContext->fd, t, s); httpOnError(parser, 0, TSDB_CODE_HTTP_INVALID_AUTH_TYPE); - free(t); - free(s); return -1; } } else { parser->authType = HTTP_INVALID_AUTH; httpError("context:%p, fd:%d, parse auth failed, t:%s s:%s", pContext, pContext->fd, t, s); httpOnError(parser, 0, TSDB_CODE_HTTP_INVALID_AUTH_FORMAT); - free(t); - free(s); return -1; } } From d95effe1fbee479d6ece65979694be0a8721672b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 24 Jan 2021 17:31:56 +0800 Subject: [PATCH 049/147] TD-1207 --- tests/script/general/http/grafana.sim | 2 +- tests/script/general/http/restful_full.sim | 8 +++++++- tests/script/unique/http/admin.sim | 4 ++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/script/general/http/grafana.sim b/tests/script/general/http/grafana.sim index 128994640d..ca2f62a368 100644 --- a/tests/script/general/http/grafana.sim +++ b/tests/script/general/http/grafana.sim @@ -84,7 +84,7 @@ endi system_content curl -H 'Authorization: Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJ3d3cudGFvc2RhdGEuY29tIiwicGFzcyI6InRhb3NkYXRhIiwic3ViIjoicm9vdCJ9.xPv3b5odlR7YF8G_QWASjIRbMtA5v4ItToJ35fFgi' -d 'show databases' 127.0.0.1:7111/grafana/root/1/login print 6-> $system_content -if $system_content != @{"status":"error","code":4386,"desc":"invalid type of Authorization"}@ then +if $system_content != @{"status":"error","code":4387,"desc":"invalid format of Authorization"}@ then return -1 endi diff --git a/tests/script/general/http/restful_full.sim b/tests/script/general/http/restful_full.sim index 69f8206347..12f8cc5c38 100644 --- a/tests/script/general/http/restful_full.sim +++ b/tests/script/general/http/restful_full.sim @@ -57,12 +57,18 @@ if $system_content != @{"status":"error","code":3,"desc":"Authentication failure endi #8 -system_content curl -H 'Authorization: Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJ3d3cudGFvc2RhdGEuY29tIiwicGFzcyI6InRhb3NkYXRhIiwic3ViIjoicm9vdCJ9.xPv3b5odlR7YF8G_QWASjIRbMtA5v4ItToJ35fFgi' -d 'show databases' 127.0.0.1:7111/rest/login/root/1 +system_content curl -H 'Authorization: Beare eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9' -d 'show databases' 127.0.0.1:7111/rest/login/root/1 print 8-> $system_content if $system_content != @{"status":"error","code":4386,"desc":"invalid type of Authorization"}@ then return -1 endi +system_content curl -H 'Authorization: Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJ3d3cudGFvc2RhdGEuY29tIiwicGFzcyI6InRhb3NkYXRhIiwic3ViIjoicm9vdCJ9.xPv3b5odlR7YF8G_QWASjIRbMtA5v4ItToJ35fFgi' -d 'show databases' 127.0.0.1:7111/rest/login/root/1 +print 8-> $system_content +if $system_content != @{"status":"error","code":4387,"desc":"invalid format of Authorization"}@ then + return -1 +endi + system_content curl -H 'Authorization: eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJ3d3cudGFvc2RhdGEuY29tIiwicGFzcyI6InRhb3NkYXRhIiwic3ViIjoicm9vdCJ9.xPv3b5odlR7YF8G_QWASjIRbMtA5v4ItToJ35fFgi' -d 'show databases' 127.0.0.1:7111/rest/login/root/1 print 9-> $system_content if $system_content != @{"status":"error","code":4387,"desc":"invalid format of Authorization"}@ then diff --git a/tests/script/unique/http/admin.sim b/tests/script/unique/http/admin.sim index acc28a4e13..1d67a7f86b 100644 --- a/tests/script/unique/http/admin.sim +++ b/tests/script/unique/http/admin.sim @@ -69,13 +69,13 @@ endi system_content curl -H 'Authorization: Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.' -d 'show databases' 127.0.0.1:7111/admin/login/root/1 print 7-> $system_content -if $system_content != @{"status":"error","code":4386,"desc":"invalid type of Authorization"}@ then +if $system_content != @{"status":"error","code":4387,"desc":"invalid format of Authorization"}@ then return -1 endi system_content curl -H 'Authorization: Taosd eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJ3d3cudGFvc2RhdGEuY29tIiwicGFzcyI6InRhb3NkYXRhIiwic3ViIjoicm9vdCJ9.xPv3b5odlR7YF8G_QWASjIRbMtA5v4ItToJ35fFgi' 127.0.0.1:7111/admin/login/root/1 print 8-> $system_content -if $system_content != @{"status":"error","code":4389,"desc":"invalid taosd Authorization"}@ then +if $system_content != @{"status":"error","code":4387,"desc":"invalid format of Authorization"}@ then return -1 endi From 8f488a0f99c3a9a7955f5427a5514c856a18c938 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 24 Jan 2021 17:57:55 +0800 Subject: [PATCH 050/147] compile error in 2019 --- src/plugins/http/src/httpUtil.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/http/src/httpUtil.c b/src/plugins/http/src/httpUtil.c index 17d61e9e3d..7f1e2a94d1 100644 --- a/src/plugins/http/src/httpUtil.c +++ b/src/plugins/http/src/httpUtil.c @@ -37,7 +37,7 @@ void httpTimeToString(time_t t, char *buf, int32_t buflen) { time_t tt = t / 1000; ptm = localtime(&tt); strftime(ts, 31, "%Y-%m-%d %H:%M:%S", ptm); - sprintf(buf, "%s.%03ld", ts, t % 1000); + sprintf(buf, "%s.%03" PRId64, ts, t % 1000); } int32_t httpAddToSqlCmdBuffer(HttpContext *pContext, const char *const format, ...) { From 39bd90915a620750ac3d4c81276d5bb5e0280031 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 25 Jan 2021 11:30:41 +0800 Subject: [PATCH 051/147] TD-1207 compile error in windows --- cmake/define.inc | 3 +++ src/common/inc/tdataformat.h | 18 +++++++++--------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/cmake/define.inc b/cmake/define.inc index 5d4d94ff42..9ee7b34e6a 100755 --- a/cmake/define.inc +++ b/cmake/define.inc @@ -139,6 +139,9 @@ IF (TD_WINDOWS) SET(CMAKE_GENERATOR "NMake Makefiles" CACHE INTERNAL "" FORCE) IF (NOT TD_GODLL) SET(COMMON_FLAGS "/nologo /WX /wd4018 /wd2220 /Oi /Oy- /Gm- /EHsc /MT /GS /Gy /fp:precise /Zc:wchar_t /Zc:forScope /Gd /errorReport:prompt /analyze-") + IF (MSVC AND (MSVC_VERSION GREATER_EQUAL 1900)) + SET(COMMON_FLAGS "${COMMON_FLAGS} /Wv:18") + ENDIF () SET(DEBUG_FLAGS "/Zi /W3 /GL") SET(RELEASE_FLAGS "/W0 /O3 /GL") ENDIF () diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index 4999fbb9bf..e842030b4c 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -27,11 +27,11 @@ extern "C" { #endif -#define STR_TO_VARSTR(x, str) \ - do { \ - VarDataLenT __len = (int32_t)strlen(str); \ - *(VarDataLenT *)(x) = __len; \ - memcpy(varDataVal(x), (str), __len); \ +#define STR_TO_VARSTR(x, str) \ + do { \ + VarDataLenT __len = (VarDataLenT)strlen(str); \ + *(VarDataLenT *)(x) = __len; \ + memcpy(varDataVal(x), (str), __len); \ } while (0); #define STR_WITH_MAXSIZE_TO_VARSTR(x, str, _maxs) \ @@ -40,10 +40,10 @@ extern "C" { varDataSetLen(x, (_e - (x)-VARSTR_HEADER_SIZE)); \ } while (0) -#define STR_WITH_SIZE_TO_VARSTR(x, str, _size) \ - do { \ - *(VarDataLenT *)(x) = (int32_t)(_size); \ - memcpy(varDataVal(x), (str), (_size)); \ +#define STR_WITH_SIZE_TO_VARSTR(x, str, _size) \ + do { \ + *(VarDataLenT *)(x) = (VarDataLenT)(_size); \ + memcpy(varDataVal(x), (str), (_size)); \ } while (0); // ----------------- TSDB COLUMN DEFINITION From 9efd58c62a006ab53f115167c2afad47db5e641b Mon Sep 17 00:00:00 2001 From: slguan Date: Mon, 25 Jan 2021 11:53:55 +0800 Subject: [PATCH 052/147] TD-2837 --- packaging/tools/install_client.sh | 2 +- packaging/tools/make_install.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packaging/tools/install_client.sh b/packaging/tools/install_client.sh index 84ca3b5131..d52428dc83 100755 --- a/packaging/tools/install_client.sh +++ b/packaging/tools/install_client.sh @@ -21,7 +21,7 @@ else cd ${script_dir} script_dir="$(pwd)" data_dir="/var/lib/taos" - log_dir=~/TDengineLog + log_dir=~/TDengine/log fi log_link_dir="/usr/local/taos/log" diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh index 0727fe2a1e..474b6f4619 100755 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -24,7 +24,7 @@ data_dir="/var/lib/taos" if [ "$osType" != "Darwin" ]; then log_dir="/var/log/taos" else - log_dir=~/TDengineLog + log_dir=~/TDengine/log fi data_link_dir="/usr/local/taos/data" From 7625e803d165778ac95627ec1afec949f5264c22 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 25 Jan 2021 13:23:55 +0800 Subject: [PATCH 053/147] TD-2837 --- CMakeLists.txt | 2 +- src/balance/CMakeLists.txt | 8 +-- src/cq/CMakeLists.txt | 24 ++----- src/dnode/CMakeLists.txt | 101 +++++++++-------------------- src/mnode/CMakeLists.txt | 23 ++----- src/os/inc/osDarwin.h | 9 +-- src/plugins/http/src/httpServer.c | 19 +++--- src/plugins/monitor/CMakeLists.txt | 22 ++----- src/rpc/src/rpcTcp.c | 14 +--- src/sync/src/syncTcp.c | 8 --- src/vnode/CMakeLists.txt | 12 +--- src/wal/CMakeLists.txt | 14 +--- 12 files changed, 68 insertions(+), 188 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7ac06c165d..315036d115 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -30,7 +30,7 @@ MESSAGE(STATUS "Community directory: " ${TD_COMMUNITY_DIR}) INCLUDE(cmake/input.inc) INCLUDE(cmake/platform.inc) -IF (TD_WINDOWS) +IF (TD_WINDOWS OR TD_DARWIN) SET(TD_SOMODE_STATIC TRUE) ENDIF () diff --git a/src/balance/CMakeLists.txt b/src/balance/CMakeLists.txt index 17eb34666e..bcb3769087 100644 --- a/src/balance/CMakeLists.txt +++ b/src/balance/CMakeLists.txt @@ -8,10 +8,4 @@ INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/inc) INCLUDE_DIRECTORIES(inc) AUX_SOURCE_DIRECTORY(src SRC) -IF (TD_LINUX OR TD_WINDOWS) - ADD_LIBRARY(balance ${SRC}) -ENDIF () - -IF (TD_DARWIN) - ADD_LIBRARY(balance ${SRC}) -ENDIF () +ADD_LIBRARY(balance ${SRC}) diff --git a/src/cq/CMakeLists.txt b/src/cq/CMakeLists.txt index d5d9116074..73d5eebd6d 100644 --- a/src/cq/CMakeLists.txt +++ b/src/cq/CMakeLists.txt @@ -6,22 +6,10 @@ INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR}/src SRC) -IF (TD_LINUX OR TD_WINDOWS) - ADD_LIBRARY(tcq ${SRC}) - IF (TD_SOMODE_STATIC) - TARGET_LINK_LIBRARIES(tcq tutil common taos_static) - ELSE () - TARGET_LINK_LIBRARIES(tcq tutil common taos) - ENDIF () - ADD_SUBDIRECTORY(test) -ENDIF () - -IF (TD_DARWIN) - ADD_LIBRARY(tcq ${SRC}) - IF (TD_SOMODE_STATIC) - TARGET_LINK_LIBRARIES(tcq tutil common taos_static) - ELSE () - TARGET_LINK_LIBRARIES(tcq tutil common taos) - ENDIF () - ADD_SUBDIRECTORY(test) +ADD_LIBRARY(tcq ${SRC}) +IF (TD_SOMODE_STATIC) + TARGET_LINK_LIBRARIES(tcq tutil common taos_static) +ELSE () + TARGET_LINK_LIBRARIES(tcq tutil common taos) ENDIF () +ADD_SUBDIRECTORY(test) diff --git a/src/dnode/CMakeLists.txt b/src/dnode/CMakeLists.txt index 8e902e1114..14ec98b8f8 100644 --- a/src/dnode/CMakeLists.txt +++ b/src/dnode/CMakeLists.txt @@ -10,79 +10,38 @@ INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/inc) INCLUDE_DIRECTORIES(inc) AUX_SOURCE_DIRECTORY(src SRC) -IF (TD_LINUX OR TD_WINDOWS) - ADD_EXECUTABLE(taosd ${SRC}) - TARGET_LINK_LIBRARIES(taosd mnode monitor http tsdb twal vnode cJson lz4 balance sync) +ADD_EXECUTABLE(taosd ${SRC}) +TARGET_LINK_LIBRARIES(taosd mnode monitor http tsdb twal vnode cJson lz4 balance sync) - IF (TD_SOMODE_STATIC) - TARGET_LINK_LIBRARIES(taosd taos_static) - ELSE () - TARGET_LINK_LIBRARIES(taosd taos) - ENDIF () - - IF (TD_ACCOUNT) - TARGET_LINK_LIBRARIES(taosd account) - ENDIF () - - IF (TD_GRANT) - TARGET_LINK_LIBRARIES(taosd grant) - ENDIF () - - IF (TD_LINUX AND TD_MQTT) - TARGET_LINK_LIBRARIES(taosd mqtt) - ENDIF () - - SET(PREPARE_ENV_CMD "prepare_env_cmd") - SET(PREPARE_ENV_TARGET "prepare_env_target") - ADD_CUSTOM_COMMAND(OUTPUT ${PREPARE_ENV_CMD} - POST_BUILD - COMMAND echo "make test directory" - DEPENDS taosd - COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/cfg/ - COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/log/ - COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/data/ - COMMAND ${CMAKE_COMMAND} -E echo dataDir ${TD_TESTS_OUTPUT_DIR}/data > ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg - COMMAND ${CMAKE_COMMAND} -E echo logDir ${TD_TESTS_OUTPUT_DIR}/log >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg - COMMAND ${CMAKE_COMMAND} -E echo charset UTF-8 >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg - COMMENT "prepare taosd environment") - ADD_CUSTOM_TARGET(${PREPARE_ENV_TARGET} ALL WORKING_DIRECTORY ${TD_EXECUTABLE_OUTPUT_PATH} DEPENDS ${PREPARE_ENV_CMD}) +IF (TD_SOMODE_STATIC) + TARGET_LINK_LIBRARIES(taosd taos_static) +ELSE () + TARGET_LINK_LIBRARIES(taosd taos) ENDIF () -IF (TD_DARWIN) - ADD_EXECUTABLE(taosd ${SRC}) - TARGET_LINK_LIBRARIES(taosd mnode monitor http tsdb twal vnode cJson lz4 balance sync) - - IF (TD_SOMODE_STATIC) - TARGET_LINK_LIBRARIES(taosd taos_static) - ELSE () - TARGET_LINK_LIBRARIES(taosd taos) - ENDIF () - - IF (TD_ACCOUNT) - TARGET_LINK_LIBRARIES(taosd account) - ENDIF () - - IF (TD_GRANT) - TARGET_LINK_LIBRARIES(taosd grant) - ENDIF () - - IF (TD_MQTT) - TARGET_LINK_LIBRARIES(taosd mqtt) - ENDIF () - - # SET(PREPARE_ENV_CMD "prepare_env_cmd") - # SET(PREPARE_ENV_TARGET "prepare_env_target") - # ADD_CUSTOM_COMMAND(OUTPUT ${PREPARE_ENV_CMD} - # POST_BUILD - # COMMAND echo "make test directory" - # DEPENDS taosd - # COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/cfg/ - # COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/log/ - # COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/data/ - # COMMAND ${CMAKE_COMMAND} -E echo dataDir ${TD_TESTS_OUTPUT_DIR}/data > ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg - # COMMAND ${CMAKE_COMMAND} -E echo logDir ${TD_TESTS_OUTPUT_DIR}/log >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg - # COMMAND ${CMAKE_COMMAND} -E echo charset UTF-8 >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg - # COMMENT "prepare taosd environment") - # ADD_CUSTOM_TARGET(${PREPARE_ENV_TARGET} ALL WORKING_DIRECTORY ${TD_EXECUTABLE_OUTPUT_PATH} DEPENDS ${PREPARE_ENV_CMD}) +IF (TD_ACCOUNT) + TARGET_LINK_LIBRARIES(taosd account) ENDIF () +IF (TD_GRANT) + TARGET_LINK_LIBRARIES(taosd grant) +ENDIF () + +IF ((TD_LINUX OR TD_WINDOWS) AND TD_MQTT) + TARGET_LINK_LIBRARIES(taosd mqtt) +ENDIF () + +SET(PREPARE_ENV_CMD "prepare_env_cmd") +SET(PREPARE_ENV_TARGET "prepare_env_target") +ADD_CUSTOM_COMMAND(OUTPUT ${PREPARE_ENV_CMD} + POST_BUILD + COMMAND echo "make test directory" + DEPENDS taosd + COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/cfg/ + COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/log/ + COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/data/ + COMMAND ${CMAKE_COMMAND} -E echo dataDir ${TD_TESTS_OUTPUT_DIR}/data > ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg + COMMAND ${CMAKE_COMMAND} -E echo logDir ${TD_TESTS_OUTPUT_DIR}/log >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg + COMMAND ${CMAKE_COMMAND} -E echo charset UTF-8 >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg + COMMENT "prepare taosd environment") +ADD_CUSTOM_TARGET(${PREPARE_ENV_TARGET} ALL WORKING_DIRECTORY ${TD_EXECUTABLE_OUTPUT_PATH} DEPENDS ${PREPARE_ENV_CMD}) diff --git a/src/mnode/CMakeLists.txt b/src/mnode/CMakeLists.txt index 7b25b1bec8..fffc82c6ef 100644 --- a/src/mnode/CMakeLists.txt +++ b/src/mnode/CMakeLists.txt @@ -1,23 +1,10 @@ CMAKE_MINIMUM_REQUIRED(VERSION 3.5) PROJECT(TDengine) -IF (TD_LINUX OR TD_WINDOWS) - INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) - INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/dnode/inc) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/dnode/inc) - INCLUDE_DIRECTORIES(inc) - AUX_SOURCE_DIRECTORY(src SRC) - - ADD_LIBRARY(mnode ${SRC}) -ENDIF () - -IF (TD_DARWIN) - INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) - INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/dnode/inc) - - INCLUDE_DIRECTORIES(inc) - AUX_SOURCE_DIRECTORY(src SRC) - - ADD_LIBRARY(mnode ${SRC}) -ENDIF () +INCLUDE_DIRECTORIES(inc) +AUX_SOURCE_DIRECTORY(src SRC) +ADD_LIBRARY(mnode ${SRC}) diff --git a/src/os/inc/osDarwin.h b/src/os/inc/osDarwin.h index 0a77ed6895..13f93456f2 100644 --- a/src/os/inc/osDarwin.h +++ b/src/os/inc/osDarwin.h @@ -106,10 +106,11 @@ int64_t tsosStr2int64(char *str); void taos_block_sigalrm(void); - - - - +#define TAOS_OS_DEF_EPOLL + #define TAOS_EPOLL_WAIT_TIME 500 + typedef int32_t SOCKET; + typedef SOCKET EpollFd; + #define EpollClose(pollFd) epoll_close(pollFd) #ifdef __cplusplus } diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index a0e5bede5a..a5f40fdc4c 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -48,7 +48,7 @@ static void httpStopThread(HttpThread *pThread) { pThread->stop = true; r = epoll_ctl(pThread->pollFd, EPOLL_CTL_ADD, sv[0], &ev); if (r) break; - if (1!=send(sv[1], "1", 1, 0)) { + if (1 != send(sv[1], "1", 1, 0)) { r = -1; break; } @@ -56,7 +56,7 @@ static void httpStopThread(HttpThread *pThread) { if (r) { pthread_cancel(pThread->thread); } -#else +#else struct epoll_event event = {.events = EPOLLIN}; eventfd_t fd = eventfd(1, 0); if (fd == -1) { @@ -69,29 +69,26 @@ static void httpStopThread(HttpThread *pThread) { pThread->label, strerror(errno)); pthread_cancel(pThread->thread); } -#endif // __APPLE__ +#endif // __APPLE__ pthread_join(pThread->thread, NULL); + #ifdef __APPLE__ - if (sv[0]!=-1) { + if (sv[0] != -1) { close(sv[0]); sv[0] = -1; } - if (sv[1]!=-1) { + if (sv[1] != -1) { close(sv[1]); sv[1] = -1; } -#else // __APPLE__ +#else // __APPLE__ if (fd != -1) { taosCloseSocket(fd); } -#endif // __APPLE__ +#endif // __APPLE__ -#ifdef __APPLE__ - epoll_close(pThread->pollFd); -#else EpollClose(pThread->pollFd); -#endif pthread_mutex_destroy(&(pThread->threadMutex)); } diff --git a/src/plugins/monitor/CMakeLists.txt b/src/plugins/monitor/CMakeLists.txt index 0e0bc19736..abab07e0cd 100644 --- a/src/plugins/monitor/CMakeLists.txt +++ b/src/plugins/monitor/CMakeLists.txt @@ -6,22 +6,10 @@ INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) AUX_SOURCE_DIRECTORY(./src SRC) -IF (TD_LINUX OR TD_WINDOWS) - ADD_LIBRARY(monitor ${SRC}) +ADD_LIBRARY(monitor ${SRC}) - IF (TD_SOMODE_STATIC) - TARGET_LINK_LIBRARIES(monitor taos_static) - ELSE () - TARGET_LINK_LIBRARIES(monitor taos) - ENDIF () -ENDIF () - -IF (TD_DARWIN) - ADD_LIBRARY(monitor ${SRC}) - - IF (TD_SOMODE_STATIC) - TARGET_LINK_LIBRARIES(monitor taos_static) - ELSE () - TARGET_LINK_LIBRARIES(monitor taos) - ENDIF () +IF (TD_SOMODE_STATIC) + TARGET_LINK_LIBRARIES(monitor taos_static) +ELSE () + TARGET_LINK_LIBRARIES(monitor taos) ENDIF () diff --git a/src/rpc/src/rpcTcp.c b/src/rpc/src/rpcTcp.c index cb8611bdfb..4fe41b1709 100644 --- a/src/rpc/src/rpcTcp.c +++ b/src/rpc/src/rpcTcp.c @@ -314,14 +314,8 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int num, void * int code = pthread_create(&(pThreadObj->thread), &thattr, taosProcessTcpData, (void *)(pThreadObj)); pthread_attr_destroy(&thattr); if (code != 0) { -#ifdef __APPLE__ - if (pThreadObj->pollFd!=-1) { - epoll_close(pThreadObj->pollFd); - pThreadObj->pollFd = -1; - } -#else // __APPLE__ EpollClose(pThreadObj->pollFd); -#endif // __APPLE__ + pThreadObj->pollFd = -1; free(pThreadObj); terrno = TAOS_SYSTEM_ERROR(errno); tError("%s failed to create TCP read data thread(%s)", label, strerror(errno)); @@ -529,14 +523,10 @@ static void *taosProcessTcpData(void *param) { if (pThreadObj->stop) break; } -#ifdef __APPLE__ if (pThreadObj->pollFd >=0) { - epoll_close(pThreadObj->pollFd); + EpollClose(pThreadObj->pollFd); pThreadObj->pollFd = -1; } -#else // __APPLE__ - if (pThreadObj->pollFd >=0) EpollClose(pThreadObj->pollFd); -#endif while (pThreadObj->pHead) { SFdObj *pFdObj = pThreadObj->pHead; diff --git a/src/sync/src/syncTcp.c b/src/sync/src/syncTcp.c index fb3a016791..22bdc7e74e 100644 --- a/src/sync/src/syncTcp.c +++ b/src/sync/src/syncTcp.c @@ -241,11 +241,7 @@ static void *syncProcessTcpData(void *param) { sDebug("%p TCP epoll thread exits", pThread); -#ifdef __APPLE__ - epoll_close(pThread->pollFd); -#else // __APPLE__ EpollClose(pThread->pollFd); -#endif // __APPLE__ tfree(pThread); tfree(buffer); return NULL; @@ -307,11 +303,7 @@ static SThreadObj *syncGetTcpThread(SPoolObj *pPool) { pthread_attr_destroy(&thattr); if (ret != 0) { -#ifdef __APPLE__ - epoll_close(pThread->pollFd); -#else // __APPLE__ EpollClose(pThread->pollFd); -#endif tfree(pThread); return NULL; } diff --git a/src/vnode/CMakeLists.txt b/src/vnode/CMakeLists.txt index 07106598a9..5d77d48ebf 100644 --- a/src/vnode/CMakeLists.txt +++ b/src/vnode/CMakeLists.txt @@ -10,13 +10,5 @@ INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/inc) INCLUDE_DIRECTORIES(inc) AUX_SOURCE_DIRECTORY(src SRC) -IF (TD_LINUX OR TD_WINDOWS) - ADD_LIBRARY(vnode ${SRC}) - TARGET_LINK_LIBRARIES(vnode tsdb tcq) -ENDIF () - -IF (TD_DARWIN) - ADD_LIBRARY(vnode ${SRC}) - TARGET_LINK_LIBRARIES(vnode tsdb tcq) -ENDIF () - +ADD_LIBRARY(vnode ${SRC}) +TARGET_LINK_LIBRARIES(vnode tsdb tcq) diff --git a/src/wal/CMakeLists.txt b/src/wal/CMakeLists.txt index e77d3cd329..6f35cb9ba7 100644 --- a/src/wal/CMakeLists.txt +++ b/src/wal/CMakeLists.txt @@ -4,14 +4,6 @@ PROJECT(TDengine) INCLUDE_DIRECTORIES(inc) AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR}/src SRC) -IF (TD_LINUX OR TD_WINDOWS) - ADD_LIBRARY(twal ${SRC}) - TARGET_LINK_LIBRARIES(twal tutil common) - ADD_SUBDIRECTORY(test) -ENDIF () - -IF (TD_DARWIN) - ADD_LIBRARY(twal ${SRC}) - TARGET_LINK_LIBRARIES(twal tutil common) - ADD_SUBDIRECTORY(test) -ENDIF () +ADD_LIBRARY(twal ${SRC}) +TARGET_LINK_LIBRARIES(twal tutil common) +ADD_SUBDIRECTORY(test) From d5a4e0c36876d8287eb3af805c8b21b3571a8d40 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Mon, 25 Jan 2021 13:59:58 +0800 Subject: [PATCH 054/147] [TD-2840]Add email notification for jenkins --- Jenkinsfile | 78 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index 73bb832d8e..74cef8f954 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -176,5 +176,83 @@ pipeline { } } } + post { + success { + emailext ( + subject: "PR-result: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'", + body: ''' + + + + + + + + + + + + +

+ 构建信息 +
+
    +
    +
  • 构建名称>>分支：${PROJECT_NAME}
    • +
  • 构建结果： Successful
    • +
  • 构建编号：${BUILD_NUMBER}
    • +
  • 触发用户：${CAUSE}
    • +
  • 提交信息：${CHANGE_TITLE}
    • +
  • 构建地址：${BUILD_URL}
    • +
  • 构建日志：${BUILD_URL}console
    • +
  • 变更集：${JELLY_SCRIPT}
    • +
    +
+
+ + ''', + to: "${CHANGE_AUTHOR_EMAIL}", + from: "support@taosdata.com" + ) + } + failure { + emailext ( + subject: "PR-result: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'", + body: ''' + + + + + + + + + + + + +

+ 构建信息 +
+
    +
    +
  • 构建名称>>分支：${PROJECT_NAME}
    • +
  • 构建结果： Successful
    • +
  • 构建编号：${BUILD_NUMBER}
    • +
  • 触发用户：${CAUSE}
    • +
  • 提交信息：${CHANGE_TITLE}
    • +
  • 构建地址：${BUILD_URL}
    • +
  • 构建日志：${BUILD_URL}console
    • +
  • 变更集：${JELLY_SCRIPT}
    • +
    +
+
+ + ''', + to: "${CHANGE_AUTHOR_EMAIL}", + from: "support@taosdata.com" + ) + } + } } From 7ef5e333b612fac4bb8f5df5c538c6eb259ae148 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Mon, 25 Jan 2021 14:16:44 +0800 Subject: [PATCH 055/147] fix some error --- Jenkinsfile | 4 +-- tests/pytest/tt.py | 27 +++++++++++++++++++ .../general/connection/test_old_data.sim | 6 ++--- 3 files changed, 32 insertions(+), 5 deletions(-) create mode 100644 tests/pytest/tt.py diff --git a/Jenkinsfile b/Jenkinsfile index 74cef8f954..0f07aeb659 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -211,7 +211,7 @@ pipeline { ''', - to: "${CHANGE_AUTHOR_EMAIL}", + to: "${env.CHANGE_AUTHOR_EMAIL}", from: "support@taosdata.com" ) } @@ -249,7 +249,7 @@ pipeline { ''', - to: "${CHANGE_AUTHOR_EMAIL}", + to: "${env.CHANGE_AUTHOR_EMAIL}", from: "support@taosdata.com" ) } diff --git a/tests/pytest/tt.py b/tests/pytest/tt.py new file mode 100644 index 0000000000..70bc70f581 --- /dev/null +++ b/tests/pytest/tt.py @@ -0,0 +1,27 @@ +import taos +import datetime +import random +# host= '127.0.0.1' +# user = 'root' +# password = 'taosdata' +# conn = taos.connect( +# host, +# user, +# password, +# ) +# cl = conn.cursor() +# cl.execute("select first(_c0),last(_c0) from test.st0 " ) +# dd1=datetime.datetime.now()+ datetime.timedelta(days=5) +# sql = 'select count(*) from test.st0 where ts <= ' + "'" + str(dd1) + "'" +# print(sql) +# cl.execute(sql) +# for data in cl: +# print(data[0]) +# # print(cl[0],cl[1]) +# # d1 = data[0] +# # d2 = data[1] + +# # print(d1+(d2-d1)/2) +# print(random.randrange(-100,100)) +# random.randrange(-100,100) +print(random.getrandbits(1)) \ No newline at end of file diff --git a/tests/script/general/connection/test_old_data.sim b/tests/script/general/connection/test_old_data.sim index 83df850f0b..c7cb1eeb77 100644 --- a/tests/script/general/connection/test_old_data.sim +++ b/tests/script/general/connection/test_old_data.sim @@ -27,6 +27,6 @@ if $rows != 7 then endi -system sh/exec.sh -n dnode1 -s stop -x SIGINT -system sh/exec.sh -n dnode2 -s stop -x SIGINT -system sh/exec.sh -n dnode3 -s stop -x SIGINT \ No newline at end of file +# system sh/exec.sh -n dnode1 -s stop -x SIGINT +# system sh/exec.sh -n dnode2 -s stop -x SIGINT +# system sh/exec.sh -n dnode3 -s stop -x SIGINT \ No newline at end of file From 85fbc21f7dfdb07cf49bc8a52e8b49f95a463a96 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Mon, 25 Jan 2021 14:23:55 +0800 Subject: [PATCH 056/147] revert --- Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0f07aeb659..516b179dce 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -176,7 +176,8 @@ pipeline { } } } - post { + post { + success { emailext ( subject: "PR-result: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'", From c906dd5c1e2c3b9340a50fac0610b4d282d565a3 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Mon, 25 Jan 2021 14:26:13 +0800 Subject: [PATCH 057/147] revert --- tests/pytest/tt.py | 27 ------------------- .../general/connection/test_old_data.sim | 6 ++--- 2 files changed, 3 insertions(+), 30 deletions(-) delete mode 100644 tests/pytest/tt.py diff --git a/tests/pytest/tt.py b/tests/pytest/tt.py deleted file mode 100644 index 70bc70f581..0000000000 --- a/tests/pytest/tt.py +++ /dev/null @@ -1,27 +0,0 @@ -import taos -import datetime -import random -# host= '127.0.0.1' -# user = 'root' -# password = 'taosdata' -# conn = taos.connect( -# host, -# user, -# password, -# ) -# cl = conn.cursor() -# cl.execute("select first(_c0),last(_c0) from test.st0 " ) -# dd1=datetime.datetime.now()+ datetime.timedelta(days=5) -# sql = 'select count(*) from test.st0 where ts <= ' + "'" + str(dd1) + "'" -# print(sql) -# cl.execute(sql) -# for data in cl: -# print(data[0]) -# # print(cl[0],cl[1]) -# # d1 = data[0] -# # d2 = data[1] - -# # print(d1+(d2-d1)/2) -# print(random.randrange(-100,100)) -# random.randrange(-100,100) -print(random.getrandbits(1)) \ No newline at end of file diff --git a/tests/script/general/connection/test_old_data.sim b/tests/script/general/connection/test_old_data.sim index c7cb1eeb77..83df850f0b 100644 --- a/tests/script/general/connection/test_old_data.sim +++ b/tests/script/general/connection/test_old_data.sim @@ -27,6 +27,6 @@ if $rows != 7 then endi -# system sh/exec.sh -n dnode1 -s stop -x SIGINT -# system sh/exec.sh -n dnode2 -s stop -x SIGINT -# system sh/exec.sh -n dnode3 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT \ No newline at end of file From c745ba0b81308ba9185848cd00a54e29e5827188 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Mon, 25 Jan 2021 14:51:57 +0800 Subject: [PATCH 058/147] fix bug --- src/client/src/tscSQLParser.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 6b24dcd858..3ffb48b5ca 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -4301,9 +4301,8 @@ static void doAddJoinTagsColumnsIntoTagList(SSqlCmd* pCmd, SQueryInfo* pQueryInf } static int32_t validateTagCondExpr(SSqlCmd* pCmd, tExprNode *p) { - const char *msg1 = "tag type mismatch"; - const char *msg2 = "invalid tag operator"; - const char* msg3 = "not supported filter condition"; + const char *msg1 = "invalid tag operator"; + const char* msg2 = "not supported filter condition"; do { if (p->nodeType != TSQL_NODE_EXPR) { @@ -4315,7 +4314,7 @@ static int32_t validateTagCondExpr(SSqlCmd* pCmd, tExprNode *p) { } if (IS_ARITHMETIC_OPTR(p->_node.optr)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } if (!IS_RELATION_OPTR(p->_node.optr)) { @@ -4330,14 +4329,14 @@ static int32_t validateTagCondExpr(SSqlCmd* pCmd, tExprNode *p) { break; } - vVariant = p->_node.pLeft->pVal->nType; + vVariant = p->_node.pLeft->pVal; schemaType = p->_node.pRight->pSchema->type; } else if (p->_node.pLeft->nodeType == TSQL_NODE_COL && p->_node.pRight->nodeType == TSQL_NODE_VALUE) { if (!p->_node.pLeft->pSchema) { break; } - vVariant = p->_node.pRight->pVal->nType; + vVariant = p->_node.pRight->pVal; schemaType = p->_node.pLeft->pSchema->type; } else { break; @@ -4351,12 +4350,12 @@ static int32_t validateTagCondExpr(SSqlCmd* pCmd, tExprNode *p) { int32_t retVal = TSDB_CODE_SUCCESS; if (schemaType == TSDB_DATA_TYPE_BINARY) { - char *tmp = (int64_t)calloc(1, (vVariant->nLen + 1) + TSDB_NCHAR_SIZE); + char *tmp = calloc(1, (vVariant->nLen + 1) + TSDB_NCHAR_SIZE); retVal = tVariantDump(vVariant, tmp, schemaType, false); free(tmp); } else if (schemaType == TSDB_DATA_TYPE_NCHAR) { // pRight->val.nLen + 1 is larger than the actual nchar string length - char *tmp = (int64_t)calloc(1, (vVariant->nLen + 1) * TSDB_NCHAR_SIZE); + char *tmp = calloc(1, (vVariant->nLen + 1) * TSDB_NCHAR_SIZE); retVal = tVariantDump(vVariant, tmp, schemaType, false); free(tmp); } else { @@ -4365,7 +4364,7 @@ static int32_t validateTagCondExpr(SSqlCmd* pCmd, tExprNode *p) { } if (retVal != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } }while (0); From 12f9959702a263b6142e1e496adb1742223bd790 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Mon, 25 Jan 2021 14:53:05 +0800 Subject: [PATCH 059/147] fix bug --- src/client/src/tscSQLParser.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 3ffb48b5ca..0ccc54ff0d 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -4350,7 +4350,7 @@ static int32_t validateTagCondExpr(SSqlCmd* pCmd, tExprNode *p) { int32_t retVal = TSDB_CODE_SUCCESS; if (schemaType == TSDB_DATA_TYPE_BINARY) { - char *tmp = calloc(1, (vVariant->nLen + 1) + TSDB_NCHAR_SIZE); + char *tmp = calloc(1, vVariant->nLen + TSDB_NCHAR_SIZE); retVal = tVariantDump(vVariant, tmp, schemaType, false); free(tmp); } else if (schemaType == TSDB_DATA_TYPE_NCHAR) { From 5d976a26a5d8457bc78f667a66385559e017b900 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Mon, 25 Jan 2021 17:00:22 +0800 Subject: [PATCH 060/147] fix bug --- src/client/src/tscSQLParser.c | 9 ++++++--- src/client/src/tscServer.c | 4 +++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 0ccc54ff0d..a23c9b75bb 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -921,16 +921,19 @@ int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SStrToken* pTableNam return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } } else { // get current DB name first, and then set it into path - char* t = getCurrentDBName(pSql); + char* t = cloneCurrentDBName(pSql); if (strlen(t) == 0) { return TSDB_CODE_TSC_DB_NOT_SELECTED; } code = tNameFromString(&pTableMetaInfo->name, t, T_NAME_ACCT | T_NAME_DB); if (code != 0) { + free(t); return TSDB_CODE_TSC_DB_NOT_SELECTED; } + free(t); + if (pTableName->n >= TSDB_TABLE_NAME_LEN) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } @@ -1244,8 +1247,8 @@ static bool has(SArray* pFieldList, int32_t startIdx, const char* name) { static char* getAccountId(SSqlObj* pSql) { return pSql->pTscObj->acctId; } -static char* getCurrentDBName(SSqlObj* pSql) { - return pSql->pTscObj->db; +static char* cloneCurrentDBName(SSqlObj* pSql) { + return strdup(pSql->pTscObj->db); } /* length limitation, strstr cannot be applied */ diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 8403876566..1ea19d5198 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -2165,7 +2165,9 @@ int tscProcessUseDbRsp(SSqlObj *pSql) { } int tscProcessDropDbRsp(SSqlObj *pSql) { - pSql->pTscObj->db[0] = 0; + //TODO LOCK DB WHEN MODIFY IT + //pSql->pTscObj->db[0] = 0; + taosHashEmpty(tscTableMetaInfo); return 0; } From acaefc098b6db21ad8a471f925c1e7489f62c9d2 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 25 Jan 2021 17:00:54 +0800 Subject: [PATCH 061/147] [TD-2832]: use httpDecodeUrl to parse reserved characters after percent-encoding --- src/plugins/http/src/httpParser.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/plugins/http/src/httpParser.c b/src/plugins/http/src/httpParser.c index b844834537..7fdd0a64f6 100644 --- a/src/plugins/http/src/httpParser.c +++ b/src/plugins/http/src/httpParser.c @@ -537,7 +537,7 @@ char *httpDecodeUrl(const char *enc) { dec = str.str; str.str = NULL; } - httpCleanupString(&str); + //httpCleanupString(&str); return dec; } @@ -648,7 +648,7 @@ static int32_t httpParserOnTarget(HttpParser *parser, HTTP_PARSER_STATE state, c } break; } - parser->target = strdup(parser->str.str); + parser->target = httpDecodeUrl(parser->str.str); if (!parser->target) { httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); ok = -1; @@ -717,6 +717,10 @@ static int32_t httpParserOnVersion(HttpParser *parser, HTTP_PARSER_STATE state, if (parser->method) { ok = httpOnRequestLine(parser, parser->method, parser->target, parser->version); + if (parser->target) { + free(parser->target); + parser->target = NULL; + } } httpClearString(&parser->str); From a2da0001454a70448dec54c54a7b3d82013710d0 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 25 Jan 2021 17:10:07 +0800 Subject: [PATCH 062/147] [TD-2839]: fix tsOfflineThreshold code comments --- src/common/src/tglobal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index bda4dce593..4961bd41d8 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -137,7 +137,7 @@ int32_t tsTableIncStepPerVnode = TSDB_TABLES_STEP; int8_t tsEnableBalance = 1; int8_t tsAlternativeRole = 0; int32_t tsBalanceInterval = 300; // seconds -int32_t tsOfflineThreshold = 86400 * 100; // seconds 10days +int32_t tsOfflineThreshold = 86400 * 100; // seconds 100 days int32_t tsMnodeEqualVnodeNum = 4; int8_t tsEnableFlowCtrl = 1; int8_t tsEnableSlaveQuery = 1; From 0369b0bfa3c07ce5163fe3aab425bf50c91d8df9 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Mon, 25 Jan 2021 17:48:04 +0800 Subject: [PATCH 063/147] add lock for db access --- src/client/src/tscLocal.c | 3 +++ src/client/src/tscSQLParser.c | 8 ++++++-- src/client/src/tscServer.c | 21 ++++++++++++++++++--- 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index 4b1ab47730..bb015bce3d 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -749,7 +749,10 @@ static int32_t tscProcessCurrentUser(SSqlObj *pSql) { static int32_t tscProcessCurrentDB(SSqlObj *pSql) { char db[TSDB_DB_NAME_LEN] = {0}; + + pthread_mutex_lock(&pSql->pTscObj->mutex); extractDBName(pSql->pTscObj->db, db); + pthread_mutex_unlock(&pSql->pTscObj->mutex); SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, pSql->cmd.clauseIndex); diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index a23c9b75bb..fb48dc4519 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -60,7 +60,7 @@ static int32_t setShowInfo(SSqlObj* pSql, SSqlInfo* pInfo); static char* getAccountId(SSqlObj* pSql); static bool has(SArray* pFieldList, int32_t startIdx, const char* name); -static char* getCurrentDBName(SSqlObj* pSql); +static char* cloneCurrentDBName(SSqlObj* pSql); static bool hasSpecifyDB(SStrToken* pTableName); static bool validateTableColumnInfo(SArray* pFieldList, SSqlCmd* pCmd); static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pCmd); @@ -1248,7 +1248,11 @@ static bool has(SArray* pFieldList, int32_t startIdx, const char* name) { static char* getAccountId(SSqlObj* pSql) { return pSql->pTscObj->acctId; } static char* cloneCurrentDBName(SSqlObj* pSql) { - return strdup(pSql->pTscObj->db); + pthread_mutex_lock(&pSql->pTscObj->mutex); + char *p = strdup(pSql->pTscObj->db); + pthread_mutex_unlock(&pSql->pTscObj->mutex); + + return p; } /* length limitation, strstr cannot be applied */ diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 1ea19d5198..6035a270c5 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1250,8 +1250,10 @@ int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) { STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); - if (tNameIsEmpty(&pTableMetaInfo->name)) { - tstrncpy(pShowMsg->db, pObj->db, sizeof(pShowMsg->db)); + if (tNameIsEmpty(&pTableMetaInfo->name)) { + pthread_mutex_lock(&pObj->mutex); + tstrncpy(pShowMsg->db, pObj->db, sizeof(pShowMsg->db)); + pthread_mutex_unlock(&pObj->mutex); } else { tNameGetFullDbName(&pTableMetaInfo->name, pShowMsg->db); } @@ -1611,9 +1613,14 @@ int tscBuildConnectMsg(SSqlObj *pSql, SSqlInfo *pInfo) { // TODO refactor full_name char *db; // ugly code to move the space + + pthread_mutex_lock(&pObj->mutex); db = strstr(pObj->db, TS_PATH_DELIMITER); + db = (db == NULL) ? pObj->db : db + 1; tstrncpy(pConnect->db, db, sizeof(pConnect->db)); + pthread_mutex_unlock(&pObj->mutex); + tstrncpy(pConnect->clientVersion, version, sizeof(pConnect->clientVersion)); tstrncpy(pConnect->msgVersion, "", sizeof(pConnect->msgVersion)); @@ -2131,10 +2138,13 @@ int tscProcessConnectRsp(SSqlObj *pSql) { SConnectRsp *pConnect = (SConnectRsp *)pRes->pRsp; tstrncpy(pObj->acctId, pConnect->acctId, sizeof(pObj->acctId)); // copy acctId from response + + pthread_mutex_lock(&pObj->mutex); int32_t len = sprintf(temp, "%s%s%s", pObj->acctId, TS_PATH_DELIMITER, pObj->db); assert(len <= sizeof(pObj->db)); tstrncpy(pObj->db, temp, sizeof(pObj->db)); + pthread_mutex_unlock(&pObj->mutex); if (pConnect->epSet.numOfEps > 0) { tscEpSetHtons(&pConnect->epSet); @@ -2161,7 +2171,12 @@ int tscProcessConnectRsp(SSqlObj *pSql) { int tscProcessUseDbRsp(SSqlObj *pSql) { STscObj * pObj = pSql->pTscObj; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); - return tNameExtractFullName(&pTableMetaInfo->name, pObj->db); + + pthread_mutex_lock(&pObj->mutex); + int ret = tNameExtractFullName(&pTableMetaInfo->name, pObj->db); + pthread_mutex_unlock(&pObj->mutex); + + return ret; } int tscProcessDropDbRsp(SSqlObj *pSql) { From 054a700a2943d320aab4e868d315b7be9ba0efd7 Mon Sep 17 00:00:00 2001 From: slguan Date: Mon, 25 Jan 2021 18:43:34 +0800 Subject: [PATCH 064/147] TD-2837 --- src/mnode/src/mnodeCluster.c | 8 -------- src/os/src/darwin/darwinSysInfo.c | 8 +++++++- src/plugins/http/inc/httpUtil.h | 2 +- src/plugins/http/src/httpUtil.c | 4 ++-- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/src/mnode/src/mnodeCluster.c b/src/mnode/src/mnodeCluster.c index 94ec59b1b0..169d2ebd9d 100644 --- a/src/mnode/src/mnodeCluster.c +++ b/src/mnode/src/mnodeCluster.c @@ -138,14 +138,6 @@ void mnodeDecClusterRef(SClusterObj *pCluster) { sdbDecRef(tsClusterSdb, pCluster); } -#ifdef __APPLE__ -bool taosGetSystemUid(char *uid) { - fprintf(stderr, "%s[%d]%s(): not implemented yet!\n", basename(__FILE__), __LINE__, __func__); - abort(); - return false; -} -#endif // __APPLE__ - static int32_t mnodeCreateCluster() { int64_t numOfClusters = sdbGetNumOfRows(tsClusterSdb); if (numOfClusters != 0) return TSDB_CODE_SUCCESS; diff --git a/src/os/src/darwin/darwinSysInfo.c b/src/os/src/darwin/darwinSysInfo.c index ad3facdcee..5fa61b07ac 100644 --- a/src/os/src/darwin/darwinSysInfo.c +++ b/src/os/src/darwin/darwinSysInfo.c @@ -103,4 +103,10 @@ int taosSystem(const char *cmd) { void taosSetCoreDump() {} -char *taosGetCmdlineByPID(int pid) { return ""; } \ No newline at end of file +char *taosGetCmdlineByPID(int pid) { return ""; } + +bool taosGetSystemUid(char *uid) { + fprintf(stderr, "%s[%d]%s(): not implemented yet!\n", basename(__FILE__), __LINE__, __func__); + abort(); + return false; +} \ No newline at end of file diff --git a/src/plugins/http/inc/httpUtil.h b/src/plugins/http/inc/httpUtil.h index 61cd50a77a..54c95b6980 100644 --- a/src/plugins/http/inc/httpUtil.h +++ b/src/plugins/http/inc/httpUtil.h @@ -17,7 +17,7 @@ #define TDENGINE_HTTP_UTIL_H bool httpCheckUsedbSql(char *sql); -void httpTimeToString(time_t t, char *buf, int32_t buflen); +void httpTimeToString(int32_t t, char *buf, int32_t buflen); bool httpUrlMatch(HttpContext *pContext, int32_t pos, char *cmp); bool httpParseRequest(HttpContext *pContext); diff --git a/src/plugins/http/src/httpUtil.c b/src/plugins/http/src/httpUtil.c index 7f1e2a94d1..a84ae9f617 100644 --- a/src/plugins/http/src/httpUtil.c +++ b/src/plugins/http/src/httpUtil.c @@ -29,7 +29,7 @@ bool httpCheckUsedbSql(char *sql) { return false; } -void httpTimeToString(time_t t, char *buf, int32_t buflen) { +void httpTimeToString(int32_t t, char *buf, int32_t buflen) { memset(buf, 0, (size_t)buflen); char ts[32] = {0}; @@ -37,7 +37,7 @@ void httpTimeToString(time_t t, char *buf, int32_t buflen) { time_t tt = t / 1000; ptm = localtime(&tt); strftime(ts, 31, "%Y-%m-%d %H:%M:%S", ptm); - sprintf(buf, "%s.%03" PRId64, ts, t % 1000); + sprintf(buf, "%s.%03d", ts, t % 1000); } int32_t httpAddToSqlCmdBuffer(HttpContext *pContext, const char *const format, ...) { From e19272903d747b49817e6cc4bbda801428fc9068 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 25 Jan 2021 19:03:14 +0800 Subject: [PATCH 065/147] [TD-2839]: set tsOfflineThreshold max <- 86400 * 365 --- src/common/src/tglobal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 4961bd41d8..4a9b4c3e43 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -550,7 +550,7 @@ static void doInitGlobalConfig(void) { cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; cfg.minValue = 3; - cfg.maxValue = 7200000; + cfg.maxValue = 86400 * 365; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_SECOND; taosInitConfigOption(cfg); From e3f623076134b8a2351de44a048e89f330263221 Mon Sep 17 00:00:00 2001 From: freemine Date: Mon, 25 Jan 2021 19:27:46 +0800 Subject: [PATCH 066/147] typo: value in semaphore_create --- src/os/src/darwin/darwinSemphone.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/os/src/darwin/darwinSemphone.c b/src/os/src/darwin/darwinSemphone.c index a9e470bd70..6df44a52b3 100644 --- a/src/os/src/darwin/darwinSemphone.c +++ b/src/os/src/darwin/darwinSemphone.c @@ -134,7 +134,7 @@ int tsem_init(tsem_t *sem, int pshared, unsigned int value) { errno = ENOMEM; return -1; } - kern_return_t ret = semaphore_create(sem_port, &p->sem, SYNC_POLICY_FIFO, 0); + kern_return_t ret = semaphore_create(sem_port, &p->sem, SYNC_POLICY_FIFO, value); if (ret != KERN_SUCCESS) { fprintf(stderr, "==%s[%d]%s():[%p]==semophore_create failed\n", basename(__FILE__), __LINE__, __func__, sem); // we fail-fast here, because we have less-doc about semaphore_create for the moment From b13a524d39f89af0d84efaeec9e4bebebe2f2b52 Mon Sep 17 00:00:00 2001 From: freemine Date: Mon, 25 Jan 2021 19:55:26 +0800 Subject: [PATCH 067/147] sizeof(time_t) is platform-dependent --- src/plugins/http/src/httpUtil.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/http/src/httpUtil.c b/src/plugins/http/src/httpUtil.c index 7f1e2a94d1..abf02232e4 100644 --- a/src/plugins/http/src/httpUtil.c +++ b/src/plugins/http/src/httpUtil.c @@ -37,7 +37,7 @@ void httpTimeToString(time_t t, char *buf, int32_t buflen) { time_t tt = t / 1000; ptm = localtime(&tt); strftime(ts, 31, "%Y-%m-%d %H:%M:%S", ptm); - sprintf(buf, "%s.%03" PRId64, ts, t % 1000); + sprintf(buf, "%s.%03" PRId64, ts, (long long)(t % 1000)); } int32_t httpAddToSqlCmdBuffer(HttpContext *pContext, const char *const format, ...) { From 17de46555c2d8ffbb8fd70af6ecc6a635416f718 Mon Sep 17 00:00:00 2001 From: freemine Date: Mon, 25 Jan 2021 20:03:38 +0800 Subject: [PATCH 068/147] cast time_t to int64_t to satisfy PRId64 --- src/plugins/http/src/httpUtil.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/http/src/httpUtil.c b/src/plugins/http/src/httpUtil.c index abf02232e4..4fdd755b40 100644 --- a/src/plugins/http/src/httpUtil.c +++ b/src/plugins/http/src/httpUtil.c @@ -37,7 +37,7 @@ void httpTimeToString(time_t t, char *buf, int32_t buflen) { time_t tt = t / 1000; ptm = localtime(&tt); strftime(ts, 31, "%Y-%m-%d %H:%M:%S", ptm); - sprintf(buf, "%s.%03" PRId64, ts, (long long)(t % 1000)); + sprintf(buf, "%s.%03" PRId64, ts, (int64_t)(t % 1000)); } int32_t httpAddToSqlCmdBuffer(HttpContext *pContext, const char *const format, ...) { From 7960dbe1089dd389c3270af3734470d02aa0a7bb Mon Sep 17 00:00:00 2001 From: freemine Date: Mon, 25 Jan 2021 20:45:48 +0800 Subject: [PATCH 069/147] 1. sem_xxx -> tsem_xxx 2. SystemUid --- src/mnode/src/mnodeCluster.c | 8 -------- src/os/src/darwin/darwinSysInfo.c | 13 ++++++++++++- src/tsdb/inc/tsdbMain.h | 6 +----- src/tsdb/src/tsdbCommit.c | 6 +----- src/tsdb/src/tsdbMain.c | 23 +++-------------------- src/tsdb/src/tsdbMemTable.c | 15 +++------------ 6 files changed, 20 insertions(+), 51 deletions(-) diff --git a/src/mnode/src/mnodeCluster.c b/src/mnode/src/mnodeCluster.c index 94ec59b1b0..169d2ebd9d 100644 --- a/src/mnode/src/mnodeCluster.c +++ b/src/mnode/src/mnodeCluster.c @@ -138,14 +138,6 @@ void mnodeDecClusterRef(SClusterObj *pCluster) { sdbDecRef(tsClusterSdb, pCluster); } -#ifdef __APPLE__ -bool taosGetSystemUid(char *uid) { - fprintf(stderr, "%s[%d]%s(): not implemented yet!\n", basename(__FILE__), __LINE__, __func__); - abort(); - return false; -} -#endif // __APPLE__ - static int32_t mnodeCreateCluster() { int64_t numOfClusters = sdbGetNumOfRows(tsClusterSdb); if (numOfClusters != 0) return TSDB_CODE_SUCCESS; diff --git a/src/os/src/darwin/darwinSysInfo.c b/src/os/src/darwin/darwinSysInfo.c index ad3facdcee..cf799d7f55 100644 --- a/src/os/src/darwin/darwinSysInfo.c +++ b/src/os/src/darwin/darwinSysInfo.c @@ -103,4 +103,15 @@ int taosSystem(const char *cmd) { void taosSetCoreDump() {} -char *taosGetCmdlineByPID(int pid) { return ""; } \ No newline at end of file +char *taosGetCmdlineByPID(int pid) { + return "[not supported yet]"; +} + +bool taosGetSystemUid(char *uid) { + uuid_t uuid = {0}; + uuid_generate(uuid); + // it's caller's responsibility to make enough space for `uid`, that's 36-char + 1-null + uuid_unparse(uuid, uid); + return true; +} + diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index f3e69d9210..05335b45d5 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -233,11 +233,7 @@ typedef struct { SMemTable* mem; SMemTable* imem; STsdbFileH* tsdbFileH; -#ifdef __APPLE__ - sem_t *readyToCommit; -#else // __APPLE__ - sem_t readyToCommit; -#endif // __APPLE__ + tsem_t readyToCommit; pthread_mutex_t mutex; bool repoLocked; int32_t code; // Commit code diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c index b3f7f51327..cd8358e5e3 100644 --- a/src/tsdb/src/tsdbCommit.c +++ b/src/tsdb/src/tsdbCommit.c @@ -166,11 +166,7 @@ static void tsdbEndCommit(STsdbRepo *pRepo, int eno) { pRepo->imem = NULL; tsdbUnlockRepo(pRepo); tsdbUnRefMemTable(pRepo, pIMem); -#ifdef __APPLE__ - sem_post(pRepo->readyToCommit); -#else // __APPLE__ - sem_post(&(pRepo->readyToCommit)); -#endif // __APPLE__ + tsem_post(&(pRepo->readyToCommit)); } static int tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey) { diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index adab932bb6..cadbfa91cf 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -146,11 +146,7 @@ int tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit) { if (toCommit) { tsdbAsyncCommit(pRepo); -#ifdef __APPLE__ - sem_wait(pRepo->readyToCommit); -#else // __APPLE__ - sem_wait(&(pRepo->readyToCommit)); -#endif // __APPLE__ + tsem_wait(&(pRepo->readyToCommit)); terrno = pRepo->code; } tsdbUnRefMemTable(pRepo, pRepo->mem); @@ -647,21 +643,12 @@ static STsdbRepo *tsdbNewRepo(char *rootDir, STsdbAppH *pAppH, STsdbCfg *pCfg) { goto _err; } -#ifdef __APPLE__ - pRepo->readyToCommit = sem_open(NULL, O_CREAT, 0644, 1); - if (pRepo->readyToCommit==SEM_FAILED) { - code = errno; - terrno = TAOS_SYSTEM_ERROR(code); - goto _err; - } -#else // __APPLE__ - code = sem_init(&(pRepo->readyToCommit), 0, 1); + code = tsem_init(&(pRepo->readyToCommit), 0, 1); if (code != 0) { code = errno; terrno = TAOS_SYSTEM_ERROR(code); goto _err; } -#endif // __APPLE__ pRepo->repoLocked = false; @@ -707,11 +694,7 @@ static void tsdbFreeRepo(STsdbRepo *pRepo) { // tsdbFreeMemTable(pRepo->mem); // tsdbFreeMemTable(pRepo->imem); tfree(pRepo->rootDir); -#ifdef __APPLE__ - sem_close(pRepo->readyToCommit); -#else // __APPLE__ - sem_destroy(&(pRepo->readyToCommit)); -#endif // __APPLE__ + tsem_destroy(&(pRepo->readyToCommit)); pthread_mutex_destroy(&pRepo->mutex); free(pRepo); } diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 206a3332d5..42bbebe5f7 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -207,11 +207,7 @@ void *tsdbAllocBytes(STsdbRepo *pRepo, int bytes) { int tsdbAsyncCommit(STsdbRepo *pRepo) { if (pRepo->mem == NULL) return 0; -#ifdef __APPLE__ - sem_wait(pRepo->readyToCommit); -#else // __APPLE__ - sem_wait(&(pRepo->readyToCommit)); -#endif // __APPLE__ + tsem_wait(&(pRepo->readyToCommit)); ASSERT(pRepo->imem == NULL); @@ -233,13 +229,8 @@ int tsdbSyncCommit(TSDB_REPO_T *repo) { STsdbRepo *pRepo = (STsdbRepo *)repo; tsdbAsyncCommit(pRepo); -#ifdef __APPLE__ - sem_wait(pRepo->readyToCommit); - sem_post(pRepo->readyToCommit); -#else // __APPLE__ - sem_wait(&(pRepo->readyToCommit)); - sem_post(&(pRepo->readyToCommit)); -#endif // __APPLE__ + tsem_wait(&(pRepo->readyToCommit)); + tsem_post(&(pRepo->readyToCommit)); if (pRepo->code != TSDB_CODE_SUCCESS) { terrno = pRepo->code; From 33f099b57c31419ec3736bcf926a5d69816ec760 Mon Sep 17 00:00:00 2001 From: zyyang Date: Tue, 26 Jan 2021 10:19:34 +0800 Subject: [PATCH 070/147] [TD-2827]: fix JDBC-RESTful resultSet and restSetMetaData bugs --- .../java/com/taosdata/jdbc/TSDBConstants.java | 4 +- .../java/com/taosdata/jdbc/TSDBResultSet.java | 1974 +++++++++-------- .../taosdata/jdbc/rs/RestfulResultSet.java | 304 ++- .../jdbc/rs/RestfulResultSetMetaData.java | 120 +- .../taosdata/jdbc/rs/RestfulStatement.java | 135 +- .../jdbc/utils/SqlSyntaxValidator.java | 42 +- .../taosdata/jdbc/rs/AuthenticationTest.java | 3 +- .../com/taosdata/jdbc/rs/RestfulJDBCTest.java | 105 +- .../jdbc/rs/RestfulResultSetMetaDataTest.java | 223 ++ .../jdbc/rs/RestfulResultSetTest.java | 588 +++++ .../java/com/taosdata/jdbc/rs/SQLTest.java | 172 +- .../com/taosdata/jdbc/utils/SQLExecutor.java | 74 + .../com/taosdata/example/JdbcRestfulDemo.java | 8 +- tests/examples/JDBC/connectionPools/pom.xml | 3 +- 14 files changed, 2418 insertions(+), 1337 deletions(-) create mode 100644 src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetMetaDataTest.java create mode 100644 src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetTest.java create mode 100644 src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SQLExecutor.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java index 4f4911aad9..8ffdd7b97a 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java @@ -22,6 +22,8 @@ public abstract class TSDBConstants { public static final String DEFAULT_PORT = "6200"; public static final String UNSUPPORT_METHOD_EXCEPTIONZ_MSG = "this operation is NOT supported currently!"; public static final String INVALID_VARIABLES = "invalid variables"; + public static final String RESULT_SET_IS_CLOSED = "resultSet is closed."; + public static Map DATATYPE_MAP = null; public static final long JNI_NULL_POINTER = 0L; @@ -74,7 +76,7 @@ public abstract class TSDBConstants { } static { - DATATYPE_MAP = new HashMap(); + DATATYPE_MAP = new HashMap<>(); DATATYPE_MAP.put(1, "BOOL"); DATATYPE_MAP.put(2, "TINYINT"); DATATYPE_MAP.put(3, "SMALLINT"); diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSet.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSet.java index a8a8b3ca87..d06922c680 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSet.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSet.java @@ -40,1060 +40,1070 @@ import java.util.List; import java.util.Map; public class TSDBResultSet implements ResultSet { - private TSDBJNIConnector jniConnector = null; - - private long resultSetPointer = 0L; - private List columnMetaDataList = new ArrayList(); - - private TSDBResultSetRowData rowData; - private TSDBResultSetBlockData blockData; - - private boolean batchFetch = false; - private boolean lastWasNull = false; - private final int COLUMN_INDEX_START_VALUE = 1; - - private int rowIndex = 0; - - public TSDBJNIConnector getJniConnector() { - return jniConnector; - } - - public void setJniConnector(TSDBJNIConnector jniConnector) { - this.jniConnector = jniConnector; - } - - public long getResultSetPointer() { - return resultSetPointer; - } - - public void setResultSetPointer(long resultSetPointer) { - this.resultSetPointer = resultSetPointer; - } - - public void setBatchFetch(boolean batchFetch) { - this.batchFetch = batchFetch; - } - - public Boolean getBatchFetch() { - return this.batchFetch; - } - - public List getColumnMetaDataList() { - return columnMetaDataList; - } - - public void setColumnMetaDataList(List columnMetaDataList) { - this.columnMetaDataList = columnMetaDataList; - } - - public TSDBResultSetRowData getRowData() { - return rowData; - } - - public void setRowData(TSDBResultSetRowData rowData) { - this.rowData = rowData; - } - - public boolean isLastWasNull() { - return lastWasNull; - } - - public void setLastWasNull(boolean lastWasNull) { - this.lastWasNull = lastWasNull; - } - - public TSDBResultSet() { - - } - - public TSDBResultSet(TSDBJNIConnector connector, long resultSetPointer) throws SQLException { - this.jniConnector = connector; - this.resultSetPointer = resultSetPointer; - int code = this.jniConnector.getSchemaMetaData(this.resultSetPointer, this.columnMetaDataList); - if (code == TSDBConstants.JNI_CONNECTION_NULL) { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); - } else if (code == TSDBConstants.JNI_RESULT_SET_NULL) { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_RESULT_SET_NULL)); - } else if (code == TSDBConstants.JNI_NUM_OF_FIELDS_0) { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_NUM_OF_FIELDS_0)); - } - - this.rowData = new TSDBResultSetRowData(this.columnMetaDataList.size()); - this.blockData = new TSDBResultSetBlockData(this.columnMetaDataList, this.columnMetaDataList.size()); - } - - public T unwrap(Class iface) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean isWrapperFor(Class iface) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean next() throws SQLException { - if (this.getBatchFetch()) { - if (this.blockData.forward()) { - return true; - } - - int code = this.jniConnector.fetchBlock(this.resultSetPointer, this.blockData); - this.blockData.reset(); - - if (code == TSDBConstants.JNI_CONNECTION_NULL) { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); - } else if (code == TSDBConstants.JNI_RESULT_SET_NULL) { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_RESULT_SET_NULL)); - } else if (code == TSDBConstants.JNI_NUM_OF_FIELDS_0) { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_NUM_OF_FIELDS_0)); - } else if (code == TSDBConstants.JNI_FETCH_END) { - return false; - } - - return true; - } else { - if (rowData != null) { - this.rowData.clear(); - } - - int code = this.jniConnector.fetchRow(this.resultSetPointer, this.rowData); - if (code == TSDBConstants.JNI_CONNECTION_NULL) { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); - } else if (code == TSDBConstants.JNI_RESULT_SET_NULL) { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_RESULT_SET_NULL)); - } else if (code == TSDBConstants.JNI_NUM_OF_FIELDS_0) { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_NUM_OF_FIELDS_0)); - } else if (code == TSDBConstants.JNI_FETCH_END) { - return false; - } else { - return true; - } - } - } - - public void close() throws SQLException { - if (this.jniConnector != null) { - int code = this.jniConnector.freeResultSet(this.resultSetPointer); - if (code == TSDBConstants.JNI_CONNECTION_NULL) { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); - } else if (code == TSDBConstants.JNI_RESULT_SET_NULL) { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_RESULT_SET_NULL)); - } - } - } - - public boolean wasNull() throws SQLException { - return this.lastWasNull; - } - - public String getString(int columnIndex) throws SQLException { - String res = null; - int colIndex = getTrueColumnIndex(columnIndex); - - if (!this.getBatchFetch()) { - this.lastWasNull = this.rowData.wasNull(colIndex); - if (!lastWasNull) { - res = this.rowData.getString(colIndex, this.columnMetaDataList.get(colIndex).getColType()); - } - return res; - } else { - return this.blockData.getString(colIndex); - } - } - - public boolean getBoolean(int columnIndex) throws SQLException { - boolean res = false; - int colIndex = getTrueColumnIndex(columnIndex); - - if (!this.getBatchFetch()) { - this.lastWasNull = this.rowData.wasNull(colIndex); - if (!lastWasNull) { - res = this.rowData.getBoolean(colIndex, this.columnMetaDataList.get(colIndex).getColType()); - } - } else { - return this.blockData.getBoolean(colIndex); - } - - return res; - } - - public byte getByte(int columnIndex) throws SQLException { - byte res = 0; - int colIndex = getTrueColumnIndex(columnIndex); - - if (!this.getBatchFetch()) { - this.lastWasNull = this.rowData.wasNull(colIndex); - if (!lastWasNull) { - res = (byte) this.rowData.getInt(colIndex, this.columnMetaDataList.get(colIndex).getColType()); - } - return res; - } else { - return (byte) this.blockData.getInt(colIndex); - } - } - - public short getShort(int columnIndex) throws SQLException { - short res = 0; - int colIndex = getTrueColumnIndex(columnIndex); - - if (!this.getBatchFetch()) { - this.lastWasNull = this.rowData.wasNull(colIndex); - if (!lastWasNull) { - res = (short) this.rowData.getInt(colIndex, this.columnMetaDataList.get(colIndex).getColType()); - } - return res; - } else { - return (short) this.blockData.getInt(colIndex); - } - } - - public int getInt(int columnIndex) throws SQLException { - int res = 0; - int colIndex = getTrueColumnIndex(columnIndex); - - if (!this.getBatchFetch()) { - this.lastWasNull = this.rowData.wasNull(colIndex); - if (!lastWasNull) { - res = this.rowData.getInt(colIndex, this.columnMetaDataList.get(colIndex).getColType()); - } - return res; - } else { - return this.blockData.getInt(colIndex); - } - - } - - public long getLong(int columnIndex) throws SQLException { - long res = 0l; - int colIndex = getTrueColumnIndex(columnIndex); - - if (!this.getBatchFetch()) { - this.lastWasNull = this.rowData.wasNull(colIndex); - if (!lastWasNull) { - res = this.rowData.getLong(colIndex, this.columnMetaDataList.get(colIndex).getColType()); - } - return res; - } else { - return this.blockData.getLong(colIndex); - } - } - - public float getFloat(int columnIndex) throws SQLException { - float res = 0; - int colIndex = getTrueColumnIndex(columnIndex); - - if (!this.getBatchFetch()) { - this.lastWasNull = this.rowData.wasNull(colIndex); - if (!lastWasNull) { - res = this.rowData.getFloat(colIndex, this.columnMetaDataList.get(colIndex).getColType()); - } - return res; - } else { - return (float) this.blockData.getDouble(colIndex); - } - } - - public double getDouble(int columnIndex) throws SQLException { - double res = 0; - int colIndex = getTrueColumnIndex(columnIndex); - - if (!this.getBatchFetch()) { - this.lastWasNull = this.rowData.wasNull(colIndex); - if (!lastWasNull) { - res = this.rowData.getDouble(colIndex, this.columnMetaDataList.get(colIndex).getColType()); - } - return res; - } else { - return this.blockData.getDouble(colIndex); - } - } - - /* - * (non-Javadoc) - * - * @see java.sql.ResultSet#getBigDecimal(int, int) - * - * @deprecated Use {@code getBigDecimal(int columnIndex)} or {@code - * getBigDecimal(String columnLabel)} - */ - @Deprecated - public BigDecimal getBigDecimal(int columnIndex, int scale) throws SQLException { - return new BigDecimal(getLong(columnIndex)); - } - - public byte[] getBytes(int columnIndex) throws SQLException { - return getString(columnIndex).getBytes(); - } - - public Date getDate(int columnIndex) throws SQLException { - int colIndex = getTrueColumnIndex(columnIndex); - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Time getTime(int columnIndex) throws SQLException { - int colIndex = getTrueColumnIndex(columnIndex); - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Timestamp getTimestamp(int columnIndex) throws SQLException { - Timestamp res = null; - int colIndex = getTrueColumnIndex(columnIndex); - - if (!this.getBatchFetch()) { - this.lastWasNull = this.rowData.wasNull(colIndex); - if (!lastWasNull) { - res = this.rowData.getTimestamp(colIndex); - } - return res; - } else { - return this.blockData.getTimestamp(columnIndex); - } - } - - public InputStream getAsciiStream(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - /* - * (non-Javadoc) - * - * @see java.sql.ResultSet#getUnicodeStream(int) - * - * * @deprecated use getCharacterStream in place of - * getUnicodeStream - */ - @Deprecated - public InputStream getUnicodeStream(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public InputStream getBinaryStream(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public String getString(String columnLabel) throws SQLException { - return this.getString(this.findColumn(columnLabel)); - } - - public boolean getBoolean(String columnLabel) throws SQLException { - return this.getBoolean(this.findColumn(columnLabel)); - } - - public byte getByte(String columnLabel) throws SQLException { - return this.getByte(this.findColumn(columnLabel)); - } - - public short getShort(String columnLabel) throws SQLException { - return this.getShort(this.findColumn(columnLabel)); - } - - public int getInt(String columnLabel) throws SQLException { - return this.getInt(this.findColumn(columnLabel)); - } - - public long getLong(String columnLabel) throws SQLException { - return this.getLong(this.findColumn(columnLabel)); - } - - public float getFloat(String columnLabel) throws SQLException { - return this.getFloat(this.findColumn(columnLabel)); - } - - public double getDouble(String columnLabel) throws SQLException { - return this.getDouble(this.findColumn(columnLabel)); - } - - /* - * used by spark - */ - @Deprecated - public BigDecimal getBigDecimal(String columnLabel, int scale) throws SQLException { - return this.getBigDecimal(this.findColumn(columnLabel), scale); - } - - public byte[] getBytes(String columnLabel) throws SQLException { - return this.getBytes(this.findColumn(columnLabel)); - } - - public Date getDate(String columnLabel) throws SQLException { - return this.getDate(this.findColumn(columnLabel)); - } - - public Time getTime(String columnLabel) throws SQLException { - return this.getTime(this.findColumn(columnLabel)); - } - - public Timestamp getTimestamp(String columnLabel) throws SQLException { - return this.getTimestamp(this.findColumn(columnLabel)); - } - - public InputStream getAsciiStream(String columnLabel) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Deprecated - public InputStream getUnicodeStream(String columnLabel) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public InputStream getBinaryStream(String columnLabel) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public SQLWarning getWarnings() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void clearWarnings() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public String getCursorName() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public ResultSetMetaData getMetaData() throws SQLException { - return new TSDBResultSetMetaData(this.columnMetaDataList); - } - - public Object getObject(int columnIndex) throws SQLException { - int colIndex = getTrueColumnIndex(columnIndex); - - if (!this.getBatchFetch()) { - this.lastWasNull = this.rowData.wasNull(colIndex); - return this.rowData.get(colIndex); - } else { - return this.blockData.get(colIndex); - } - } - - public Object getObject(String columnLabel) throws SQLException { - return this.getObject(this.findColumn(columnLabel)); - } - - public int findColumn(String columnLabel) throws SQLException { - Iterator colMetaDataIt = this.columnMetaDataList.iterator(); - while (colMetaDataIt.hasNext()) { - ColumnMetaData colMetaData = colMetaDataIt.next(); - if (colMetaData.getColName() != null && colMetaData.getColName().equalsIgnoreCase(columnLabel)) { - return colMetaData.getColIndex() + 1; - } - } - throw new SQLException(TSDBConstants.INVALID_VARIABLES); - } - - public Reader getCharacterStream(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Reader getCharacterStream(String columnLabel) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - /* - * used by spark - */ - public BigDecimal getBigDecimal(int columnIndex) throws SQLException { - int colIndex = getTrueColumnIndex(columnIndex); - - if (!this.getBatchFetch()) { - this.lastWasNull = this.rowData.wasNull(colIndex); - return new BigDecimal(this.rowData.getLong(colIndex, this.columnMetaDataList.get(colIndex).getColType())); - } else { - return new BigDecimal(this.blockData.getLong(colIndex)); - } - } - - public BigDecimal getBigDecimal(String columnLabel) throws SQLException { - return this.getBigDecimal(this.findColumn(columnLabel)); - } - - public boolean isBeforeFirst() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean isAfterLast() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean isFirst() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean isLast() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void beforeFirst() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void afterLast() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean first() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean last() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public int getRow() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean absolute(int row) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean relative(int rows) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean previous() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void setFetchDirection(int direction) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public int getFetchDirection() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void setFetchSize(int rows) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public int getFetchSize() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public int getType() throws SQLException { - return ResultSet.TYPE_FORWARD_ONLY; - } - - public int getConcurrency() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean rowUpdated() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean rowInserted() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean rowDeleted() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateNull(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + private TSDBJNIConnector jniConnector = null; - public void updateBoolean(int columnIndex, boolean x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + private long resultSetPointer = 0L; + private List columnMetaDataList = new ArrayList(); + + private TSDBResultSetRowData rowData; + private TSDBResultSetBlockData blockData; + + private boolean batchFetch = false; + private boolean lastWasNull = false; + private final int COLUMN_INDEX_START_VALUE = 1; + + private int rowIndex = 0; + + public TSDBJNIConnector getJniConnector() { + return jniConnector; + } + + public void setJniConnector(TSDBJNIConnector jniConnector) { + this.jniConnector = jniConnector; + } + + public long getResultSetPointer() { + return resultSetPointer; + } + + public void setResultSetPointer(long resultSetPointer) { + this.resultSetPointer = resultSetPointer; + } + + public void setBatchFetch(boolean batchFetch) { + this.batchFetch = batchFetch; + } + + public Boolean getBatchFetch() { + return this.batchFetch; + } + + public List getColumnMetaDataList() { + return columnMetaDataList; + } + + public void setColumnMetaDataList(List columnMetaDataList) { + this.columnMetaDataList = columnMetaDataList; + } + + public TSDBResultSetRowData getRowData() { + return rowData; + } + + public void setRowData(TSDBResultSetRowData rowData) { + this.rowData = rowData; + } + + public boolean isLastWasNull() { + return lastWasNull; + } + + public void setLastWasNull(boolean lastWasNull) { + this.lastWasNull = lastWasNull; + } + + public TSDBResultSet() { + + } + + public TSDBResultSet(TSDBJNIConnector connector, long resultSetPointer) throws SQLException { + this.jniConnector = connector; + this.resultSetPointer = resultSetPointer; + int code = this.jniConnector.getSchemaMetaData(this.resultSetPointer, this.columnMetaDataList); + if (code == TSDBConstants.JNI_CONNECTION_NULL) { + throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); + } else if (code == TSDBConstants.JNI_RESULT_SET_NULL) { + throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_RESULT_SET_NULL)); + } else if (code == TSDBConstants.JNI_NUM_OF_FIELDS_0) { + throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_NUM_OF_FIELDS_0)); + } + + this.rowData = new TSDBResultSetRowData(this.columnMetaDataList.size()); + this.blockData = new TSDBResultSetBlockData(this.columnMetaDataList, this.columnMetaDataList.size()); + } + + public T unwrap(Class iface) throws SQLException { + if (isClosed()) + throw new SQLException(TSDBConstants.WrapErrMsg(TSDBConstants.RESULT_SET_IS_CLOSED)); + + try { + return iface.cast(this); + } catch (ClassCastException cce) { + throw new SQLException("Unable to unwrap to " + iface.toString()); + } + } + + public boolean isWrapperFor(Class iface) throws SQLException { + if (isClosed()) + throw new SQLException(TSDBConstants.WrapErrMsg(TSDBConstants.RESULT_SET_IS_CLOSED)); + + return iface.isInstance(this); + } + + public boolean next() throws SQLException { + if (this.getBatchFetch()) { + if (this.blockData.forward()) { + return true; + } + + int code = this.jniConnector.fetchBlock(this.resultSetPointer, this.blockData); + this.blockData.reset(); + + if (code == TSDBConstants.JNI_CONNECTION_NULL) { + throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); + } else if (code == TSDBConstants.JNI_RESULT_SET_NULL) { + throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_RESULT_SET_NULL)); + } else if (code == TSDBConstants.JNI_NUM_OF_FIELDS_0) { + throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_NUM_OF_FIELDS_0)); + } else if (code == TSDBConstants.JNI_FETCH_END) { + return false; + } + + return true; + } else { + if (rowData != null) { + this.rowData.clear(); + } + + int code = this.jniConnector.fetchRow(this.resultSetPointer, this.rowData); + if (code == TSDBConstants.JNI_CONNECTION_NULL) { + throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); + } else if (code == TSDBConstants.JNI_RESULT_SET_NULL) { + throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_RESULT_SET_NULL)); + } else if (code == TSDBConstants.JNI_NUM_OF_FIELDS_0) { + throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_NUM_OF_FIELDS_0)); + } else if (code == TSDBConstants.JNI_FETCH_END) { + return false; + } else { + return true; + } + } + } + + public void close() throws SQLException { + if (this.jniConnector != null) { + int code = this.jniConnector.freeResultSet(this.resultSetPointer); + if (code == TSDBConstants.JNI_CONNECTION_NULL) { + throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); + } else if (code == TSDBConstants.JNI_RESULT_SET_NULL) { + throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_RESULT_SET_NULL)); + } + } + } + + public boolean wasNull() throws SQLException { + return this.lastWasNull; + } + + public String getString(int columnIndex) throws SQLException { + String res = null; + int colIndex = getTrueColumnIndex(columnIndex); + + if (!this.getBatchFetch()) { + this.lastWasNull = this.rowData.wasNull(colIndex); + if (!lastWasNull) { + res = this.rowData.getString(colIndex, this.columnMetaDataList.get(colIndex).getColType()); + } + return res; + } else { + return this.blockData.getString(colIndex); + } + } + + public boolean getBoolean(int columnIndex) throws SQLException { + boolean res = false; + int colIndex = getTrueColumnIndex(columnIndex); + + if (!this.getBatchFetch()) { + this.lastWasNull = this.rowData.wasNull(colIndex); + if (!lastWasNull) { + res = this.rowData.getBoolean(colIndex, this.columnMetaDataList.get(colIndex).getColType()); + } + } else { + return this.blockData.getBoolean(colIndex); + } + + return res; + } + + public byte getByte(int columnIndex) throws SQLException { + byte res = 0; + int colIndex = getTrueColumnIndex(columnIndex); + + if (!this.getBatchFetch()) { + this.lastWasNull = this.rowData.wasNull(colIndex); + if (!lastWasNull) { + res = (byte) this.rowData.getInt(colIndex, this.columnMetaDataList.get(colIndex).getColType()); + } + return res; + } else { + return (byte) this.blockData.getInt(colIndex); + } + } + + public short getShort(int columnIndex) throws SQLException { + short res = 0; + int colIndex = getTrueColumnIndex(columnIndex); + + if (!this.getBatchFetch()) { + this.lastWasNull = this.rowData.wasNull(colIndex); + if (!lastWasNull) { + res = (short) this.rowData.getInt(colIndex, this.columnMetaDataList.get(colIndex).getColType()); + } + return res; + } else { + return (short) this.blockData.getInt(colIndex); + } + } + + public int getInt(int columnIndex) throws SQLException { + int res = 0; + int colIndex = getTrueColumnIndex(columnIndex); + + if (!this.getBatchFetch()) { + this.lastWasNull = this.rowData.wasNull(colIndex); + if (!lastWasNull) { + res = this.rowData.getInt(colIndex, this.columnMetaDataList.get(colIndex).getColType()); + } + return res; + } else { + return this.blockData.getInt(colIndex); + } + + } + + public long getLong(int columnIndex) throws SQLException { + long res = 0l; + int colIndex = getTrueColumnIndex(columnIndex); + + if (!this.getBatchFetch()) { + this.lastWasNull = this.rowData.wasNull(colIndex); + if (!lastWasNull) { + res = this.rowData.getLong(colIndex, this.columnMetaDataList.get(colIndex).getColType()); + } + return res; + } else { + return this.blockData.getLong(colIndex); + } + } + + public float getFloat(int columnIndex) throws SQLException { + float res = 0; + int colIndex = getTrueColumnIndex(columnIndex); + + if (!this.getBatchFetch()) { + this.lastWasNull = this.rowData.wasNull(colIndex); + if (!lastWasNull) { + res = this.rowData.getFloat(colIndex, this.columnMetaDataList.get(colIndex).getColType()); + } + return res; + } else { + return (float) this.blockData.getDouble(colIndex); + } + } + + public double getDouble(int columnIndex) throws SQLException { + double res = 0; + int colIndex = getTrueColumnIndex(columnIndex); + + if (!this.getBatchFetch()) { + this.lastWasNull = this.rowData.wasNull(colIndex); + if (!lastWasNull) { + res = this.rowData.getDouble(colIndex, this.columnMetaDataList.get(colIndex).getColType()); + } + return res; + } else { + return this.blockData.getDouble(colIndex); + } + } + + /* + * (non-Javadoc) + * + * @see java.sql.ResultSet#getBigDecimal(int, int) + * + * @deprecated Use {@code getBigDecimal(int columnIndex)} or {@code + * getBigDecimal(String columnLabel)} + */ + @Deprecated + public BigDecimal getBigDecimal(int columnIndex, int scale) throws SQLException { + return new BigDecimal(getLong(columnIndex)); + } + + public byte[] getBytes(int columnIndex) throws SQLException { + return getString(columnIndex).getBytes(); + } + + public Date getDate(int columnIndex) throws SQLException { + int colIndex = getTrueColumnIndex(columnIndex); + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public Time getTime(int columnIndex) throws SQLException { + int colIndex = getTrueColumnIndex(columnIndex); + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public Timestamp getTimestamp(int columnIndex) throws SQLException { + Timestamp res = null; + int colIndex = getTrueColumnIndex(columnIndex); + + if (!this.getBatchFetch()) { + this.lastWasNull = this.rowData.wasNull(colIndex); + if (!lastWasNull) { + res = this.rowData.getTimestamp(colIndex); + } + return res; + } else { + return this.blockData.getTimestamp(columnIndex); + } + } + + public InputStream getAsciiStream(int columnIndex) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + /* + * (non-Javadoc) + * + * @see java.sql.ResultSet#getUnicodeStream(int) + * + * * @deprecated use getCharacterStream in place of + * getUnicodeStream + */ + @Deprecated + public InputStream getUnicodeStream(int columnIndex) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public InputStream getBinaryStream(int columnIndex) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public String getString(String columnLabel) throws SQLException { + return this.getString(this.findColumn(columnLabel)); + } + + public boolean getBoolean(String columnLabel) throws SQLException { + return this.getBoolean(this.findColumn(columnLabel)); + } + + public byte getByte(String columnLabel) throws SQLException { + return this.getByte(this.findColumn(columnLabel)); + } + + public short getShort(String columnLabel) throws SQLException { + return this.getShort(this.findColumn(columnLabel)); + } + + public int getInt(String columnLabel) throws SQLException { + return this.getInt(this.findColumn(columnLabel)); + } + + public long getLong(String columnLabel) throws SQLException { + return this.getLong(this.findColumn(columnLabel)); + } + + public float getFloat(String columnLabel) throws SQLException { + return this.getFloat(this.findColumn(columnLabel)); + } + + public double getDouble(String columnLabel) throws SQLException { + return this.getDouble(this.findColumn(columnLabel)); + } + + /* + * used by spark + */ + @Deprecated + public BigDecimal getBigDecimal(String columnLabel, int scale) throws SQLException { + return this.getBigDecimal(this.findColumn(columnLabel), scale); + } + + public byte[] getBytes(String columnLabel) throws SQLException { + return this.getBytes(this.findColumn(columnLabel)); + } + + public Date getDate(String columnLabel) throws SQLException { + return this.getDate(this.findColumn(columnLabel)); + } + + public Time getTime(String columnLabel) throws SQLException { + return this.getTime(this.findColumn(columnLabel)); + } + + public Timestamp getTimestamp(String columnLabel) throws SQLException { + return this.getTimestamp(this.findColumn(columnLabel)); + } + + public InputStream getAsciiStream(String columnLabel) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + @Deprecated + public InputStream getUnicodeStream(String columnLabel) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public InputStream getBinaryStream(String columnLabel) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public SQLWarning getWarnings() throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void clearWarnings() throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public String getCursorName() throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public ResultSetMetaData getMetaData() throws SQLException { + return new TSDBResultSetMetaData(this.columnMetaDataList); + } + + public Object getObject(int columnIndex) throws SQLException { + int colIndex = getTrueColumnIndex(columnIndex); + + if (!this.getBatchFetch()) { + this.lastWasNull = this.rowData.wasNull(colIndex); + return this.rowData.get(colIndex); + } else { + return this.blockData.get(colIndex); + } + } + + public Object getObject(String columnLabel) throws SQLException { + return this.getObject(this.findColumn(columnLabel)); + } + + public int findColumn(String columnLabel) throws SQLException { + Iterator colMetaDataIt = this.columnMetaDataList.iterator(); + while (colMetaDataIt.hasNext()) { + ColumnMetaData colMetaData = colMetaDataIt.next(); + if (colMetaData.getColName() != null && colMetaData.getColName().equalsIgnoreCase(columnLabel)) { + return colMetaData.getColIndex() + 1; + } + } + throw new SQLException(TSDBConstants.INVALID_VARIABLES); + } + + public Reader getCharacterStream(int columnIndex) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public Reader getCharacterStream(String columnLabel) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + /* + * used by spark + */ + public BigDecimal getBigDecimal(int columnIndex) throws SQLException { + int colIndex = getTrueColumnIndex(columnIndex); + + if (!this.getBatchFetch()) { + this.lastWasNull = this.rowData.wasNull(colIndex); + return new BigDecimal(this.rowData.getLong(colIndex, this.columnMetaDataList.get(colIndex).getColType())); + } else { + return new BigDecimal(this.blockData.getLong(colIndex)); + } + } + + public BigDecimal getBigDecimal(String columnLabel) throws SQLException { + return this.getBigDecimal(this.findColumn(columnLabel)); + } + + public boolean isBeforeFirst() throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public boolean isAfterLast() throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public boolean isFirst() throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public boolean isLast() throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void beforeFirst() throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void afterLast() throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public boolean first() throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public boolean last() throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public int getRow() throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public boolean absolute(int row) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public boolean relative(int rows) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public boolean previous() throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void setFetchDirection(int direction) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public int getFetchDirection() throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void setFetchSize(int rows) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public int getFetchSize() throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public int getType() throws SQLException { + return ResultSet.TYPE_FORWARD_ONLY; + } + + public int getConcurrency() throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public boolean rowUpdated() throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public boolean rowInserted() throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public boolean rowDeleted() throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void updateNull(int columnIndex) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public void updateByte(int columnIndex, byte x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public void updateBoolean(int columnIndex, boolean x) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public void updateShort(int columnIndex, short x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public void updateByte(int columnIndex, byte x) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public void updateInt(int columnIndex, int x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public void updateShort(int columnIndex, short x) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public void updateLong(int columnIndex, long x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public void updateInt(int columnIndex, int x) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public void updateFloat(int columnIndex, float x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public void updateLong(int columnIndex, long x) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public void updateDouble(int columnIndex, double x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public void updateFloat(int columnIndex, float x) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public void updateBigDecimal(int columnIndex, BigDecimal x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public void updateDouble(int columnIndex, double x) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public void updateString(int columnIndex, String x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public void updateBigDecimal(int columnIndex, BigDecimal x) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public void updateBytes(int columnIndex, byte[] x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public void updateString(int columnIndex, String x) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public void updateDate(int columnIndex, Date x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public void updateBytes(int columnIndex, byte[] x) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public void updateTime(int columnIndex, Time x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public void updateDate(int columnIndex, Date x) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public void updateTimestamp(int columnIndex, Timestamp x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public void updateTime(int columnIndex, Time x) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public void updateAsciiStream(int columnIndex, InputStream x, int length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public void updateTimestamp(int columnIndex, Timestamp x) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public void updateBinaryStream(int columnIndex, InputStream x, int length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public void updateAsciiStream(int columnIndex, InputStream x, int length) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public void updateCharacterStream(int columnIndex, Reader x, int length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public void updateBinaryStream(int columnIndex, InputStream x, int length) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public void updateObject(int columnIndex, Object x, int scaleOrLength) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public void updateCharacterStream(int columnIndex, Reader x, int length) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public void updateObject(int columnIndex, Object x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public void updateObject(int columnIndex, Object x, int scaleOrLength) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public void updateNull(String columnLabel) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public void updateObject(int columnIndex, Object x) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public void updateBoolean(String columnLabel, boolean x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public void updateNull(String columnLabel) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public void updateByte(String columnLabel, byte x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public void updateBoolean(String columnLabel, boolean x) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public void updateShort(String columnLabel, short x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public void updateByte(String columnLabel, byte x) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public void updateInt(String columnLabel, int x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public void updateShort(String columnLabel, short x) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public void updateLong(String columnLabel, long x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public void updateInt(String columnLabel, int x) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public void updateFloat(String columnLabel, float x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public void updateLong(String columnLabel, long x) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public void updateDouble(String columnLabel, double x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public void updateFloat(String columnLabel, float x) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public void updateBigDecimal(String columnLabel, BigDecimal x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public void updateDouble(String columnLabel, double x) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public void updateString(String columnLabel, String x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public void updateBigDecimal(String columnLabel, BigDecimal x) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public void updateBytes(String columnLabel, byte[] x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public void updateString(String columnLabel, String x) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public void updateDate(String columnLabel, Date x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public void updateBytes(String columnLabel, byte[] x) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public void updateTime(String columnLabel, Time x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public void updateDate(String columnLabel, Date x) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public void updateTimestamp(String columnLabel, Timestamp x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public void updateTime(String columnLabel, Time x) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public void updateAsciiStream(String columnLabel, InputStream x, int length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public void updateTimestamp(String columnLabel, Timestamp x) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public void updateBinaryStream(String columnLabel, InputStream x, int length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public void updateAsciiStream(String columnLabel, InputStream x, int length) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public void updateCharacterStream(String columnLabel, Reader reader, int length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public void updateBinaryStream(String columnLabel, InputStream x, int length) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public void updateObject(String columnLabel, Object x, int scaleOrLength) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public void updateCharacterStream(String columnLabel, Reader reader, int length) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public void updateObject(String columnLabel, Object x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public void updateObject(String columnLabel, Object x, int scaleOrLength) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public void insertRow() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public void updateObject(String columnLabel, Object x) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public void updateRow() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public void insertRow() throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public void deleteRow() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public void updateRow() throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public void refreshRow() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public void deleteRow() throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public void cancelRowUpdates() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public void refreshRow() throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public void moveToInsertRow() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public void cancelRowUpdates() throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public void moveToCurrentRow() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public void moveToInsertRow() throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public Statement getStatement() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public void moveToCurrentRow() throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public Object getObject(int columnIndex, Map> map) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public Statement getStatement() throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public Ref getRef(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public Object getObject(int columnIndex, Map> map) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public Blob getBlob(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public Ref getRef(int columnIndex) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public Clob getClob(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public Blob getBlob(int columnIndex) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public Array getArray(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public Clob getClob(int columnIndex) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public Object getObject(String columnLabel, Map> map) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public Array getArray(int columnIndex) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public Ref getRef(String columnLabel) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public Object getObject(String columnLabel, Map> map) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public Blob getBlob(String columnLabel) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public Ref getRef(String columnLabel) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public Clob getClob(String columnLabel) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public Blob getBlob(String columnLabel) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public Array getArray(String columnLabel) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public Clob getClob(String columnLabel) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public Date getDate(int columnIndex, Calendar cal) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public Array getArray(String columnLabel) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public Date getDate(String columnLabel, Calendar cal) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public Date getDate(int columnIndex, Calendar cal) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public Time getTime(int columnIndex, Calendar cal) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public Date getDate(String columnLabel, Calendar cal) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public Time getTime(String columnLabel, Calendar cal) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public Time getTime(int columnIndex, Calendar cal) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public Timestamp getTimestamp(int columnIndex, Calendar cal) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public Time getTime(String columnLabel, Calendar cal) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public Timestamp getTimestamp(String columnLabel, Calendar cal) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public Timestamp getTimestamp(int columnIndex, Calendar cal) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public URL getURL(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public Timestamp getTimestamp(String columnLabel, Calendar cal) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public URL getURL(String columnLabel) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public URL getURL(int columnIndex) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public void updateRef(int columnIndex, Ref x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateRef(String columnLabel, Ref x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateBlob(int columnIndex, Blob x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateBlob(String columnLabel, Blob x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateClob(int columnIndex, Clob x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateClob(String columnLabel, Clob x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateArray(int columnIndex, Array x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateArray(String columnLabel, Array x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public RowId getRowId(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public RowId getRowId(String columnLabel) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateRowId(int columnIndex, RowId x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateRowId(String columnLabel, RowId x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public int getHoldability() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean isClosed() throws SQLException { - boolean isClosed = true; - if (jniConnector != null) { - isClosed = jniConnector.isResultsetClosed(); - } - return isClosed; - } - - public void updateNString(int columnIndex, String nString) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateNString(String columnLabel, String nString) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateNClob(int columnIndex, NClob nClob) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateNClob(String columnLabel, NClob nClob) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public NClob getNClob(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public NClob getNClob(String columnLabel) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public SQLXML getSQLXML(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public SQLXML getSQLXML(String columnLabel) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateSQLXML(int columnIndex, SQLXML xmlObject) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateSQLXML(String columnLabel, SQLXML xmlObject) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public String getNString(int columnIndex) throws SQLException { - int colIndex = getTrueColumnIndex(columnIndex); - return (String) rowData.get(colIndex); - } - - public String getNString(String columnLabel) throws SQLException { - return (String) this.getString(columnLabel); - } - - public Reader getNCharacterStream(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Reader getNCharacterStream(String columnLabel) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateNCharacterStream(int columnIndex, Reader x, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateNCharacterStream(String columnLabel, Reader reader, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateAsciiStream(int columnIndex, InputStream x, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateBinaryStream(int columnIndex, InputStream x, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateCharacterStream(int columnIndex, Reader x, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateAsciiStream(String columnLabel, InputStream x, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateBinaryStream(String columnLabel, InputStream x, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateCharacterStream(String columnLabel, Reader reader, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateBlob(int columnIndex, InputStream inputStream, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateBlob(String columnLabel, InputStream inputStream, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateClob(int columnIndex, Reader reader, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateClob(String columnLabel, Reader reader, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateNClob(int columnIndex, Reader reader, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateNClob(String columnLabel, Reader reader, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateNCharacterStream(int columnIndex, Reader x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateNCharacterStream(String columnLabel, Reader reader) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateAsciiStream(int columnIndex, InputStream x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateBinaryStream(int columnIndex, InputStream x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateCharacterStream(int columnIndex, Reader x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateAsciiStream(String columnLabel, InputStream x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateBinaryStream(String columnLabel, InputStream x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateCharacterStream(String columnLabel, Reader reader) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public URL getURL(String columnLabel) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public void updateBlob(int columnIndex, InputStream inputStream) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public void updateRef(int columnIndex, Ref x) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void updateRef(String columnLabel, Ref x) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void updateBlob(int columnIndex, Blob x) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void updateBlob(String columnLabel, Blob x) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void updateClob(int columnIndex, Clob x) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void updateClob(String columnLabel, Clob x) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void updateArray(int columnIndex, Array x) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void updateArray(String columnLabel, Array x) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public RowId getRowId(int columnIndex) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public RowId getRowId(String columnLabel) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void updateRowId(int columnIndex, RowId x) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void updateRowId(String columnLabel, RowId x) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public int getHoldability() throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public boolean isClosed() throws SQLException { + boolean isClosed = true; + if (jniConnector != null) { + isClosed = jniConnector.isResultsetClosed(); + } + return isClosed; + } + + public void updateNString(int columnIndex, String nString) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void updateNString(String columnLabel, String nString) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void updateNClob(int columnIndex, NClob nClob) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void updateNClob(String columnLabel, NClob nClob) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public NClob getNClob(int columnIndex) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public NClob getNClob(String columnLabel) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public SQLXML getSQLXML(int columnIndex) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public SQLXML getSQLXML(String columnLabel) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void updateSQLXML(int columnIndex, SQLXML xmlObject) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void updateSQLXML(String columnLabel, SQLXML xmlObject) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public String getNString(int columnIndex) throws SQLException { + int colIndex = getTrueColumnIndex(columnIndex); + return (String) rowData.get(colIndex); + } + + public String getNString(String columnLabel) throws SQLException { + return (String) this.getString(columnLabel); + } + + public Reader getNCharacterStream(int columnIndex) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public Reader getNCharacterStream(String columnLabel) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void updateNCharacterStream(int columnIndex, Reader x, long length) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void updateNCharacterStream(String columnLabel, Reader reader, long length) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void updateAsciiStream(int columnIndex, InputStream x, long length) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void updateBinaryStream(int columnIndex, InputStream x, long length) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void updateCharacterStream(int columnIndex, Reader x, long length) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void updateAsciiStream(String columnLabel, InputStream x, long length) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void updateBinaryStream(String columnLabel, InputStream x, long length) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void updateCharacterStream(String columnLabel, Reader reader, long length) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void updateBlob(int columnIndex, InputStream inputStream, long length) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void updateBlob(String columnLabel, InputStream inputStream, long length) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void updateClob(int columnIndex, Reader reader, long length) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void updateClob(String columnLabel, Reader reader, long length) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void updateNClob(int columnIndex, Reader reader, long length) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void updateNClob(String columnLabel, Reader reader, long length) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void updateNCharacterStream(int columnIndex, Reader x) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void updateNCharacterStream(String columnLabel, Reader reader) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void updateAsciiStream(int columnIndex, InputStream x) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void updateBinaryStream(int columnIndex, InputStream x) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void updateCharacterStream(int columnIndex, Reader x) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void updateAsciiStream(String columnLabel, InputStream x) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void updateBinaryStream(String columnLabel, InputStream x) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void updateCharacterStream(String columnLabel, Reader reader) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public void updateBlob(String columnLabel, InputStream inputStream) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public void updateBlob(int columnIndex, InputStream inputStream) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public void updateClob(int columnIndex, Reader reader) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public void updateBlob(String columnLabel, InputStream inputStream) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public void updateClob(String columnLabel, Reader reader) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public void updateClob(int columnIndex, Reader reader) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public void updateNClob(int columnIndex, Reader reader) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public void updateClob(String columnLabel, Reader reader) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public void updateNClob(String columnLabel, Reader reader) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public void updateNClob(int columnIndex, Reader reader) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public T getObject(int columnIndex, Class type) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public void updateNClob(String columnLabel, Reader reader) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - public T getObject(String columnLabel, Class type) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + public T getObject(int columnIndex, Class type) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - private int getTrueColumnIndex(int columnIndex) throws SQLException { - if (columnIndex < this.COLUMN_INDEX_START_VALUE) { - throw new SQLException("Column Index out of range, " + columnIndex + " < " + this.COLUMN_INDEX_START_VALUE); - } - - int numOfCols = this.columnMetaDataList.size(); - if (columnIndex > numOfCols) { - throw new SQLException("Column Index out of range, " + columnIndex + " > " + numOfCols); - } + public T getObject(String columnLabel, Class type) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } - return columnIndex - 1; - } + private int getTrueColumnIndex(int columnIndex) throws SQLException { + if (columnIndex < this.COLUMN_INDEX_START_VALUE) { + throw new SQLException("Column Index out of range, " + columnIndex + " < " + this.COLUMN_INDEX_START_VALUE); + } + + int numOfCols = this.columnMetaDataList.size(); + if (columnIndex > numOfCols) { + throw new SQLException("Column Index out of range, " + columnIndex + " > " + numOfCols); + } + + return columnIndex - 1; + } } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java index 1aa3d5b3ce..38d3f2b6aa 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java @@ -15,28 +15,28 @@ import java.util.List; import java.util.Map; public class RestfulResultSet implements ResultSet { - private static final String RESULT_SET_IS_CLOSED = "resultSet is closed."; private volatile boolean isClosed; private int pos = -1; private final String database; private final Statement statement; // data - private ArrayList> resultSet; + private ArrayList> resultSet = new ArrayList<>(); // meta - private ArrayList columnNames; - private ArrayList columns; + private ArrayList columnNames = new ArrayList<>(); + private ArrayList columns = new ArrayList<>(); private RestfulResultSetMetaData metaData; /** - * 由一个result的Json构造结果集，对应执行show databases,show tables等这些语句，返回结果集，但无法获取结果集对应的meta，统一当成String处理 + * 由一个result的Json构造结果集，对应执行show databases, show tables等这些语句，返回结果集，但无法获取结果集对应的meta，统一当成String处理 + * + * @param resultJson: 包含data信息的结果集，有sql返回的结果集 ***/ public RestfulResultSet(String database, Statement statement, JSONObject resultJson) { this.database = database; this.statement = statement; // row data JSONArray data = resultJson.getJSONArray("data"); - resultSet = new ArrayList<>(); int columnIndex = 0; for (; columnIndex < data.size(); columnIndex++) { ArrayList oneRow = new ArrayList<>(); @@ -48,15 +48,13 @@ public class RestfulResultSet implements ResultSet { } // column only names - columnNames = new ArrayList<>(); - columns = new ArrayList<>(); JSONArray head = resultJson.getJSONArray("head"); for (int i = 0; i < head.size(); i++) { String name = head.getString(i); columnNames.add(name); columns.add(new Field(name, "", 0, "")); } - this.metaData = new RestfulResultSetMetaData(this.database, columns); + this.metaData = new RestfulResultSetMetaData(this.database, columns, this); } /** @@ -78,7 +76,7 @@ public class RestfulResultSet implements ResultSet { } } this.columns = newColumns; - this.metaData = new RestfulResultSetMetaData(this.database, this.columns); + this.metaData = new RestfulResultSetMetaData(this.database, this.columns, this); } public Field findField(String columnName, List fieldJsonList) { @@ -91,7 +89,6 @@ public class RestfulResultSet implements ResultSet { } } } - return null; } @@ -112,10 +109,9 @@ public class RestfulResultSet implements ResultSet { @Override public boolean next() throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); - - if (pos < resultSet.size() - 1) { - pos++; + throw new SQLException(TSDBConstants.WrapErrMsg(TSDBConstants.RESULT_SET_IS_CLOSED)); + pos++; + if (pos <= resultSet.size() - 1) { return true; } return false; @@ -131,37 +127,38 @@ public class RestfulResultSet implements ResultSet { @Override public boolean wasNull() throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); + throw new SQLException(TSDBConstants.WrapErrMsg(TSDBConstants.RESULT_SET_IS_CLOSED)); return resultSet.isEmpty(); } @Override public String getString(int columnIndex) throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); + throw new SQLException(TSDBConstants.WrapErrMsg(TSDBConstants.RESULT_SET_IS_CLOSED)); if (columnIndex > resultSet.get(pos).size()) { throw new SQLException(TSDBConstants.WrapErrMsg("Column Index out of range, " + columnIndex + " > " + resultSet.get(pos).size())); } - return resultSet.get(pos).get(columnIndex - 1).toString(); + + columnIndex = getTrueColumnIndex(columnIndex); + return resultSet.get(pos).get(columnIndex).toString(); } + @Override public boolean getBoolean(int columnIndex) throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); + throw new SQLException(TSDBConstants.WrapErrMsg(TSDBConstants.RESULT_SET_IS_CLOSED)); - String result = getString(columnIndex); - if (!(result.equals("true") || result.equals("false"))) { - throw new SQLException("not boolean value"); - } - return result.equals("true"); + columnIndex = getTrueColumnIndex(columnIndex); + int result = getInt(columnIndex); + return result == 0 ? false : true; } @Override public byte getByte(int columnIndex) throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); + throw new SQLException(TSDBConstants.WrapErrMsg(TSDBConstants.RESULT_SET_IS_CLOSED)); throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); } @@ -169,40 +166,55 @@ public class RestfulResultSet implements ResultSet { @Override public short getShort(int columnIndex) throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); - - return Short.parseShort(getString(columnIndex)); + throw new SQLException(TSDBConstants.WrapErrMsg(TSDBConstants.RESULT_SET_IS_CLOSED)); + columnIndex = getTrueColumnIndex(columnIndex); + return Short.parseShort(resultSet.get(pos).get(columnIndex).toString()); } @Override public int getInt(int columnIndex) throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); - return Integer.parseInt(getString(columnIndex)); + throw new SQLException(TSDBConstants.WrapErrMsg(TSDBConstants.RESULT_SET_IS_CLOSED)); + columnIndex = getTrueColumnIndex(columnIndex); + return Integer.parseInt(resultSet.get(pos).get(columnIndex).toString()); } @Override public long getLong(int columnIndex) throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); - - return Long.parseLong(getString(columnIndex)); + throw new SQLException(TSDBConstants.WrapErrMsg(TSDBConstants.RESULT_SET_IS_CLOSED)); + columnIndex = getTrueColumnIndex(columnIndex); + return Long.parseLong(resultSet.get(pos).get(columnIndex).toString()); } @Override public float getFloat(int columnIndex) throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); - - return Float.parseFloat(getString(columnIndex)); + throw new SQLException(TSDBConstants.WrapErrMsg(TSDBConstants.RESULT_SET_IS_CLOSED)); + columnIndex = getTrueColumnIndex(columnIndex); + return Float.parseFloat(resultSet.get(pos).get(columnIndex).toString()); } @Override public double getDouble(int columnIndex) throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); + throw new SQLException(TSDBConstants.WrapErrMsg(TSDBConstants.RESULT_SET_IS_CLOSED)); - return Double.parseDouble(getString(columnIndex)); + columnIndex = getTrueColumnIndex(columnIndex); + return Double.parseDouble(resultSet.get(pos).get(columnIndex).toString()); + } + + private int getTrueColumnIndex(int columnIndex) throws SQLException { + if (columnIndex < 1) { + throw new SQLException("Column Index out of range, " + columnIndex + " < 1"); + } + + int numOfCols = resultSet.get(pos).size(); + if (columnIndex > numOfCols) { + throw new SQLException("Column Index out of range, " + columnIndex + " > " + numOfCols); + } + + return columnIndex - 1; } /*******************************************************************************************************************/ @@ -210,7 +222,7 @@ public class RestfulResultSet implements ResultSet { @Override public BigDecimal getBigDecimal(int columnIndex, int scale) throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); + throw new SQLException(TSDBConstants.WrapErrMsg(TSDBConstants.RESULT_SET_IS_CLOSED)); throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); } @@ -218,7 +230,7 @@ public class RestfulResultSet implements ResultSet { @Override public byte[] getBytes(int columnIndex) throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); + throw new SQLException(TSDBConstants.WrapErrMsg(TSDBConstants.RESULT_SET_IS_CLOSED)); throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); } @@ -226,7 +238,7 @@ public class RestfulResultSet implements ResultSet { @Override public Date getDate(int columnIndex) throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); + throw new SQLException(TSDBConstants.WrapErrMsg(TSDBConstants.RESULT_SET_IS_CLOSED)); throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); } @@ -234,7 +246,7 @@ public class RestfulResultSet implements ResultSet { @Override public Time getTime(int columnIndex) throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); + throw new SQLException(TSDBConstants.WrapErrMsg(TSDBConstants.RESULT_SET_IS_CLOSED)); throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); } @@ -242,17 +254,18 @@ public class RestfulResultSet implements ResultSet { @Override public Timestamp getTimestamp(int columnIndex) throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); + throw new SQLException(TSDBConstants.WrapErrMsg(TSDBConstants.RESULT_SET_IS_CLOSED)); - String strDate = getString(columnIndex); - strDate = strDate.substring(1, strDate.length() - 1); + columnIndex = getTrueColumnIndex(columnIndex); + String strDate = resultSet.get(pos).get(columnIndex).toString(); +// strDate = strDate.substring(1, strDate.length() - 1); return Timestamp.valueOf(strDate); } @Override public InputStream getAsciiStream(int columnIndex) throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); + throw new SQLException(TSDBConstants.WrapErrMsg(TSDBConstants.RESULT_SET_IS_CLOSED)); throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); } @@ -260,7 +273,7 @@ public class RestfulResultSet implements ResultSet { @Override public InputStream getUnicodeStream(int columnIndex) throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); + throw new SQLException(TSDBConstants.WrapErrMsg(TSDBConstants.RESULT_SET_IS_CLOSED)); throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); } @@ -268,17 +281,16 @@ public class RestfulResultSet implements ResultSet { @Override public InputStream getBinaryStream(int columnIndex) throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); + throw new SQLException(TSDBConstants.WrapErrMsg(TSDBConstants.RESULT_SET_IS_CLOSED)); throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); } + /*************************************************************************************************************/ + @Override public String getString(String columnLabel) throws SQLException { - if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); - - return getString(findColumn(columnLabel) + 1); + return getString(findColumn(columnLabel)); } @Override @@ -338,53 +350,44 @@ public class RestfulResultSet implements ResultSet { @Override public Timestamp getTimestamp(String columnLabel) throws SQLException { - if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); - return Timestamp.valueOf(getString(findColumn(columnLabel))); + return getTimestamp(findColumn(columnLabel)); } @Override public InputStream getAsciiStream(String columnLabel) throws SQLException { - if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); - - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + return getAsciiStream(findColumn(columnLabel)); } @Override public InputStream getUnicodeStream(String columnLabel) throws SQLException { - if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); - - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + return getUnicodeStream(findColumn(columnLabel)); } @Override public InputStream getBinaryStream(String columnLabel) throws SQLException { - if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); - - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + return getBinaryStream(findColumn(columnLabel)); } + /*************************************************************************************************************/ + @Override public SQLWarning getWarnings() throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); + throw new SQLException(TSDBConstants.WrapErrMsg(TSDBConstants.RESULT_SET_IS_CLOSED)); return null; } @Override public void clearWarnings() throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); + throw new SQLException(TSDBConstants.WrapErrMsg(TSDBConstants.RESULT_SET_IS_CLOSED)); return; } @Override public String getCursorName() throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); + throw new SQLException(TSDBConstants.WrapErrMsg(TSDBConstants.RESULT_SET_IS_CLOSED)); throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); } @@ -392,7 +395,7 @@ public class RestfulResultSet implements ResultSet { @Override public ResultSetMetaData getMetaData() throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); + throw new SQLException(TSDBConstants.WrapErrMsg(TSDBConstants.RESULT_SET_IS_CLOSED)); return this.metaData; } @@ -400,7 +403,7 @@ public class RestfulResultSet implements ResultSet { @Override public Object getObject(int columnIndex) throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); + throw new SQLException(TSDBConstants.WrapErrMsg(TSDBConstants.RESULT_SET_IS_CLOSED)); throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); } @@ -413,15 +416,18 @@ public class RestfulResultSet implements ResultSet { @Override public int findColumn(String columnLabel) throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); + throw new SQLException(TSDBConstants.WrapErrMsg(TSDBConstants.RESULT_SET_IS_CLOSED)); - return columnNames.indexOf(columnLabel); + int columnIndex = columnNames.indexOf(columnLabel); + if (columnIndex == -1) + throw new SQLException("cannot find Column in resultSet"); + return columnIndex + 1; } @Override public Reader getCharacterStream(int columnIndex) throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); + throw new SQLException(TSDBConstants.WrapErrMsg(TSDBConstants.RESULT_SET_IS_CLOSED)); throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); } @@ -429,7 +435,7 @@ public class RestfulResultSet implements ResultSet { @Override public Reader getCharacterStream(String columnLabel) throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); + throw new SQLException(TSDBConstants.WrapErrMsg(TSDBConstants.RESULT_SET_IS_CLOSED)); throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); } @@ -437,7 +443,7 @@ public class RestfulResultSet implements ResultSet { @Override public BigDecimal getBigDecimal(int columnIndex) throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); + throw new SQLException(TSDBConstants.WrapErrMsg(TSDBConstants.RESULT_SET_IS_CLOSED)); throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); } @@ -445,7 +451,7 @@ public class RestfulResultSet implements ResultSet { @Override public BigDecimal getBigDecimal(String columnLabel) throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); + throw new SQLException(TSDBConstants.WrapErrMsg(TSDBConstants.RESULT_SET_IS_CLOSED)); throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); } @@ -453,78 +459,132 @@ public class RestfulResultSet implements ResultSet { @Override public boolean isBeforeFirst() throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); - - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw new SQLException(TSDBConstants.WrapErrMsg(TSDBConstants.RESULT_SET_IS_CLOSED)); + return this.pos == -1 && this.resultSet.size() != 0; } @Override public boolean isAfterLast() throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw new SQLException(TSDBConstants.WrapErrMsg(TSDBConstants.RESULT_SET_IS_CLOSED)); + + return this.pos >= resultSet.size() && this.resultSet.size() != 0; } @Override public boolean isFirst() throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); + throw new SQLException(TSDBConstants.WrapErrMsg(TSDBConstants.RESULT_SET_IS_CLOSED)); - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + return this.pos == 0; } @Override public boolean isLast() throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); - - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw new SQLException(TSDBConstants.WrapErrMsg(TSDBConstants.RESULT_SET_IS_CLOSED)); + if (this.resultSet.size() == 0) + return false; + return this.pos == (this.resultSet.size() - 1); } @Override public void beforeFirst() throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); + throw new SQLException(TSDBConstants.WrapErrMsg(TSDBConstants.RESULT_SET_IS_CLOSED)); - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + synchronized (this) { + if (this.resultSet.size() > 0) { + this.pos = -1; + } + } } @Override public void afterLast() throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); + throw new SQLException(TSDBConstants.WrapErrMsg(TSDBConstants.RESULT_SET_IS_CLOSED)); + synchronized (this) { + if (this.resultSet.size() > 0) { + this.pos = this.resultSet.size(); + } + } - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); } @Override public boolean first() throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); + throw new SQLException(TSDBConstants.WrapErrMsg(TSDBConstants.RESULT_SET_IS_CLOSED)); - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (this.resultSet.size() == 0) + return false; + + synchronized (this) { + this.pos = 0; + } + return true; } @Override public boolean last() throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); - - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw new SQLException(TSDBConstants.WrapErrMsg(TSDBConstants.RESULT_SET_IS_CLOSED)); + if (this.resultSet.size() == 0) + return false; + synchronized (this) { + this.pos = this.resultSet.size() - 1; + } + return true; } @Override public int getRow() throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); - - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw new SQLException(TSDBConstants.WrapErrMsg(TSDBConstants.RESULT_SET_IS_CLOSED)); + int row; + synchronized (this) { + if (this.pos < 0 || this.pos >= this.resultSet.size()) + return 0; + row = this.pos + 1; + } + return row; } @Override public boolean absolute(int row) throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); + throw new SQLException(TSDBConstants.WrapErrMsg(TSDBConstants.RESULT_SET_IS_CLOSED)); + +// if (this.resultSet.size() == 0) +// return false; +// +// if (row == 0) { +// beforeFirst(); +// return false; +// } else if (row == 1) { +// return first(); +// } else if (row == -1) { +// return last(); +// } else if (row > this.resultSet.size()) { +// afterLast(); +// return false; +// } else { +// if (row < 0) { +// // adjust to reflect after end of result set +// int newRowPosition = this.resultSet.size() + row + 1; +// if (newRowPosition <= 0) { +// beforeFirst(); +// return false; +// } else { +// return absolute(newRowPosition); +// } +// } else { +// row--; // adjust for index difference +// this.pos = row; +// return true; +// } +// } throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); } @@ -532,7 +592,7 @@ public class RestfulResultSet implements ResultSet { @Override public boolean relative(int rows) throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); + throw new SQLException(TSDBConstants.WrapErrMsg(TSDBConstants.RESULT_SET_IS_CLOSED)); throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); } @@ -540,7 +600,7 @@ public class RestfulResultSet implements ResultSet { @Override public boolean previous() throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); + throw new SQLException(TSDBConstants.WrapErrMsg(TSDBConstants.RESULT_SET_IS_CLOSED)); throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); } @@ -548,17 +608,18 @@ public class RestfulResultSet implements ResultSet { @Override public void setFetchDirection(int direction) throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); - if (direction != ResultSet.FETCH_REVERSE || direction != ResultSet.FETCH_REVERSE || direction != ResultSet.FETCH_UNKNOWN) + throw new SQLException(TSDBConstants.WrapErrMsg(TSDBConstants.RESULT_SET_IS_CLOSED)); + if ((direction != ResultSet.FETCH_FORWARD) && (direction != ResultSet.FETCH_REVERSE) && (direction != ResultSet.FETCH_UNKNOWN)) throw new SQLException(TSDBConstants.INVALID_VARIABLES); - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (!(getType() == ResultSet.TYPE_FORWARD_ONLY && direction == ResultSet.FETCH_FORWARD)) + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); } @Override public int getFetchDirection() throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); + throw new SQLException(TSDBConstants.WrapErrMsg(TSDBConstants.RESULT_SET_IS_CLOSED)); return ResultSet.FETCH_FORWARD; } @@ -566,17 +627,17 @@ public class RestfulResultSet implements ResultSet { @Override public void setFetchSize(int rows) throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); + throw new SQLException(TSDBConstants.WrapErrMsg(TSDBConstants.RESULT_SET_IS_CLOSED)); if (rows < 0) throw new SQLException(TSDBConstants.INVALID_VARIABLES); - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); } @Override public int getFetchSize() throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); + throw new SQLException(TSDBConstants.WrapErrMsg(TSDBConstants.RESULT_SET_IS_CLOSED)); return this.resultSet.size(); } @@ -834,7 +895,7 @@ public class RestfulResultSet implements ResultSet { @Override public Statement getStatement() throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); + throw new SQLException(TSDBConstants.WrapErrMsg(TSDBConstants.RESULT_SET_IS_CLOSED)); return this.statement; } @@ -992,14 +1053,15 @@ public class RestfulResultSet implements ResultSet { @Override public int getHoldability() throws SQLException { + if (isClosed()) + throw new SQLException(TSDBConstants.WrapErrMsg(TSDBConstants.RESULT_SET_IS_CLOSED)); + return ResultSet.HOLD_CURSORS_OVER_COMMIT; } @Override public boolean isClosed() throws SQLException { - return false; - //TODO: SQLFeature Not Supported -// throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + return isClosed; } @Override @@ -1224,11 +1286,21 @@ public class RestfulResultSet implements ResultSet { @Override public T unwrap(Class iface) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw new SQLException(TSDBConstants.WrapErrMsg(TSDBConstants.RESULT_SET_IS_CLOSED)); + + try { + return iface.cast(this); + } catch (ClassCastException cce) { + throw new SQLException("Unable to unwrap to " + iface.toString()); + } } @Override public boolean isWrapperFor(Class iface) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw new SQLException(TSDBConstants.WrapErrMsg(TSDBConstants.RESULT_SET_IS_CLOSED)); + + return iface.isInstance(this); } } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSetMetaData.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSetMetaData.java index 1af3088b17..9252ed23a6 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSetMetaData.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSetMetaData.java @@ -1,17 +1,22 @@ package com.taosdata.jdbc.rs; +import com.taosdata.jdbc.TSDBConstants; + import java.sql.ResultSetMetaData; import java.sql.SQLException; +import java.sql.Timestamp; import java.util.ArrayList; public class RestfulResultSetMetaData implements ResultSetMetaData { private final String database; private ArrayList fields; + private final RestfulResultSet resultSet; - public RestfulResultSetMetaData(String database, ArrayList fields) { + public RestfulResultSetMetaData(String database, ArrayList fields, RestfulResultSet resultSet) { this.database = database; this.fields = fields; + this.resultSet = resultSet; } @Override @@ -26,13 +31,12 @@ public class RestfulResultSetMetaData implements ResultSetMetaData { @Override public boolean isCaseSensitive(int column) throws SQLException { - //TODO return false; } @Override public boolean isSearchable(int column) throws SQLException { - return false; + return true; } @Override @@ -42,17 +46,30 @@ public class RestfulResultSetMetaData implements ResultSetMetaData { @Override public int isNullable(int column) throws SQLException { + if (column == 1) + return ResultSetMetaData.columnNoNulls; return ResultSetMetaData.columnNullable; } @Override public boolean isSigned(int column) throws SQLException { - return false; + String type = this.fields.get(column - 1).type.toUpperCase(); + switch (type) { + case "TINYINT": + case "SMALLINT": + case "INT": + case "BIGINT": + case "FLOAT": + case "DOUBLE": + return true; + default: + return false; + } } @Override public int getColumnDisplaySize(int column) throws SQLException { - return 0; + return this.fields.get(column - 1).length; } @Override @@ -62,27 +79,46 @@ public class RestfulResultSetMetaData implements ResultSetMetaData { @Override public String getColumnName(int column) throws SQLException { - return null; + return fields.get(column - 1).name; } @Override public String getSchemaName(int column) throws SQLException { - return this.database; + return ""; } @Override public int getPrecision(int column) throws SQLException { - return 0; + String type = this.fields.get(column - 1).type.toUpperCase(); + switch (type) { + case "FLOAT": + return 5; + case "DOUBLE": + return 9; + case "BINARY": + case "NCHAR": + return this.fields.get(column - 1).length; + default: + return 0; + } } @Override public int getScale(int column) throws SQLException { - return 0; + String type = this.fields.get(column - 1).type.toUpperCase(); + switch (type) { + case "FLOAT": + return 5; + case "DOUBLE": + return 9; + default: + return 0; + } } @Override public String getTableName(int column) throws SQLException { - return null; + return ""; } @Override @@ -92,17 +128,42 @@ public class RestfulResultSetMetaData implements ResultSetMetaData { @Override public int getColumnType(int column) throws SQLException { - return 0; + String type = this.fields.get(column - 1).type.toUpperCase(); + switch (type) { + case "BOOL": + return java.sql.Types.BOOLEAN; + case "TINYINT": + return java.sql.Types.TINYINT; + case "SMALLINT": + return java.sql.Types.SMALLINT; + case "INT": + return java.sql.Types.INTEGER; + case "BIGINT": + return java.sql.Types.BIGINT; + case "FLOAT": + return java.sql.Types.FLOAT; + case "DOUBLE": + return java.sql.Types.DOUBLE; + case "BINARY": + return java.sql.Types.BINARY; + case "TIMESTAMP": + return java.sql.Types.TIMESTAMP; + case "NCHAR": + return java.sql.Types.NCHAR; + } + throw new SQLException(TSDBConstants.INVALID_VARIABLES); + } @Override public String getColumnTypeName(int column) throws SQLException { - return null; + String type = fields.get(column - 1).type; + return type.toUpperCase(); } @Override public boolean isReadOnly(int column) throws SQLException { - return false; + return true; } @Override @@ -117,16 +178,43 @@ public class RestfulResultSetMetaData implements ResultSetMetaData { @Override public String getColumnClassName(int column) throws SQLException { - return null; + String type = this.fields.get(column - 1).type; + String columnClassName = ""; + switch (type) { + case "BOOL": + return Boolean.class.getName(); + case "TINYINT": + case "SMALLINT": + return Short.class.getName(); + case "INT": + return Integer.class.getName(); + case "BIGINT": + return Long.class.getName(); + case "FLOAT": + return Float.class.getName(); + case "DOUBLE": + return Double.class.getName(); + case "TIMESTAMP": + return Timestamp.class.getName(); + case "BINARY": + case "NCHAR": + return String.class.getName(); + } + return columnClassName; } @Override public T unwrap(Class iface) throws SQLException { - return null; + try { + return iface.cast(this); + } catch (ClassCastException cce) { + throw new SQLException("Unable to unwrap to " + iface.toString()); + } } @Override public boolean isWrapperFor(Class iface) throws SQLException { - return false; + return iface.isInstance(this); } + } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java index 8b2276fbb0..56d4318fb4 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java @@ -65,37 +65,18 @@ public class RestfulStatement implements Statement { public ResultSet executeQuery(String sql) throws SQLException { if (isClosed()) throw new SQLException("statement already closed"); - if (!SqlSyntaxValidator.isSelectSql(sql)) - throw new SQLException("not a select sql for executeQuery: " + sql); + if (!SqlSyntaxValidator.isValidForExecuteQuery(sql)) + throw new SQLException("not a valid sql for executeQuery: " + sql); final String url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sql"; - // row data - String result = HttpClientPoolUtil.execute(url, sql); - JSONObject resultJson = JSON.parseObject(result); - if (resultJson.getString("status").equals("error")) { - throw new SQLException(TSDBConstants.WrapErrMsg("SQL execution error: " + resultJson.getString("desc") + "\n" + "error code: " + resultJson.getString("code"))); + if (SqlSyntaxValidator.isDatabaseUnspecifiedQuery(sql)) { + return executeOneQuery(url, sql); } - // parse table name from sql - String[] tableIdentifiers = parseTableIdentifier(sql); - if (tableIdentifiers != null) { - List fieldJsonList = new ArrayList<>(); - for (String tableIdentifier : tableIdentifiers) { - // field meta - String fields = HttpClientPoolUtil.execute(url, "DESCRIBE " + tableIdentifier); - JSONObject fieldJson = JSON.parseObject(fields); - if (fieldJson.getString("status").equals("error")) { - throw new SQLException(TSDBConstants.WrapErrMsg("SQL execution error: " + fieldJson.getString("desc") + "\n" + "error code: " + fieldJson.getString("code"))); - } - fieldJsonList.add(fieldJson); - } - this.resultSet = new RestfulResultSet(database, this, resultJson, fieldJsonList); - } else { - this.resultSet = new RestfulResultSet(database, this, resultJson); - } - - this.affectedRows = 0; - return resultSet; + if (this.database == null || this.database.isEmpty()) + throw new SQLException("Database not specified or available"); + HttpClientPoolUtil.execute(url, "use " + this.database); + return executeOneQuery(url, sql); } @Override @@ -105,19 +86,15 @@ public class RestfulStatement implements Statement { if (!SqlSyntaxValidator.isValidForExecuteUpdate(sql)) throw new SQLException("not a valid sql for executeUpdate: " + sql); - if (this.database == null) - throw new SQLException("Database not specified or available"); - - final String url = "http://" + conn.getHost().trim() + ":" + conn.getPort() + "/rest/sql"; -// HttpClientPoolUtil.execute(url, "use " + conn.getDatabase()); - String result = HttpClientPoolUtil.execute(url, sql); - JSONObject jsonObject = JSON.parseObject(result); - if (jsonObject.getString("status").equals("error")) { - throw new SQLException(TSDBConstants.WrapErrMsg("SQL execution error: " + jsonObject.getString("desc") + "\n" + "error code: " + jsonObject.getString("code"))); + final String url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sql"; + if (SqlSyntaxValidator.isDatabaseUnspecifiedUpdate(sql)) { + return executeOneUpdate(url, sql); } - this.resultSet = null; - this.affectedRows = Integer.parseInt(jsonObject.getString("rows")); - return this.affectedRows; + + if (this.database == null || this.database.isEmpty()) + throw new SQLException("Database not specified or available"); + HttpClientPoolUtil.execute(url, "use " + this.database); + return executeOneUpdate(url, sql); } @Override @@ -209,37 +186,77 @@ public class RestfulStatement implements Statement { @Override public boolean execute(String sql) throws SQLException { - if (isClosed()) { + if (isClosed()) throw new SQLException("Invalid method call on a closed statement."); - } + if (!SqlSyntaxValidator.isValidForExecute(sql)) + throw new SQLException("not a valid sql for execute: " + sql); + //如果执行了use操作应该将当前Statement的catalog设置为新的database + final String url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sql"; if (SqlSyntaxValidator.isUseSql(sql)) { + HttpClientPoolUtil.execute(url, sql); this.database = sql.trim().replace("use", "").trim(); this.conn.setCatalog(this.database); - } - if (this.database == null) - throw new SQLException("Database not specified or available"); - - if (SqlSyntaxValidator.isSelectSql(sql)) { - executeQuery(sql); - } else if (SqlSyntaxValidator.isShowSql(sql) || SqlSyntaxValidator.isDescribeSql(sql)) { - final String url = "http://" + conn.getHost().trim() + ":" + conn.getPort() + "/rest/sql"; - if (!SqlSyntaxValidator.isShowDatabaseSql(sql)) { - HttpClientPoolUtil.execute(url, "use " + conn.getDatabase()); - } - String result = HttpClientPoolUtil.execute(url, sql); - JSONObject resultJson = JSON.parseObject(result); - if (resultJson.getString("status").equals("error")) { - throw new SQLException(TSDBConstants.WrapErrMsg("SQL execution error: " + resultJson.getString("desc") + "\n" + "error code: " + resultJson.getString("code"))); - } - this.resultSet = new RestfulResultSet(database, this, resultJson); + } else if (SqlSyntaxValidator.isDatabaseUnspecifiedQuery(sql)) { + executeOneQuery(url, sql); + } else if (SqlSyntaxValidator.isDatabaseUnspecifiedUpdate(sql)) { + executeOneUpdate(url, sql); } else { - executeUpdate(sql); + if (SqlSyntaxValidator.isValidForExecuteQuery(sql)) { + executeQuery(sql); + } else { + executeUpdate(sql); + } } return true; } + private ResultSet executeOneQuery(String url, String sql) throws SQLException { + if (!SqlSyntaxValidator.isValidForExecuteQuery(sql)) + throw new SQLException("not a select sql for executeQuery: " + sql); + + // row data + String result = HttpClientPoolUtil.execute(url, sql); + JSONObject resultJson = JSON.parseObject(result); + if (resultJson.getString("status").equals("error")) { + throw new SQLException(TSDBConstants.WrapErrMsg("SQL execution error: " + resultJson.getString("desc") + "\n" + "error code: " + resultJson.getString("code"))); + } + // parse table name from sql + String[] tableIdentifiers = parseTableIdentifier(sql); + if (tableIdentifiers != null) { + List fieldJsonList = new ArrayList<>(); + for (String tableIdentifier : tableIdentifiers) { + // field meta + String fields = HttpClientPoolUtil.execute(url, "DESCRIBE " + tableIdentifier); + JSONObject fieldJson = JSON.parseObject(fields); + if (fieldJson.getString("status").equals("error")) { + throw new SQLException(TSDBConstants.WrapErrMsg("SQL execution error: " + fieldJson.getString("desc") + "\n" + "error code: " + fieldJson.getString("code"))); + } + fieldJsonList.add(fieldJson); + } + this.resultSet = new RestfulResultSet(database, this, resultJson, fieldJsonList); + } else { + this.resultSet = new RestfulResultSet(database, this, resultJson); + } + this.affectedRows = 0; + return resultSet; + } + + private int executeOneUpdate(String url, String sql) throws SQLException { + if (!SqlSyntaxValidator.isValidForExecuteUpdate(sql)) + throw new SQLException("not a valid sql for executeUpdate: " + sql); + + String result = HttpClientPoolUtil.execute(url, sql); + JSONObject jsonObject = JSON.parseObject(result); + if (jsonObject.getString("status").equals("error")) { + throw new SQLException(TSDBConstants.WrapErrMsg("SQL execution error: " + jsonObject.getString("desc") + "\n" + "error code: " + jsonObject.getString("code"))); + } + this.resultSet = null; + this.affectedRows = Integer.parseInt(jsonObject.getString("rows")); + return this.affectedRows; + } + @Override public ResultSet getResultSet() throws SQLException { if (isClosed()) diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/SqlSyntaxValidator.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/SqlSyntaxValidator.java index f0d9234616..251ca2af01 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/SqlSyntaxValidator.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/SqlSyntaxValidator.java @@ -20,8 +20,11 @@ import java.sql.Connection; public class SqlSyntaxValidator { - private static final String[] updateSQL = {"insert", "update", "delete", "create", "alter", "drop", "show", "describe", "use", "import"}; - private static final String[] querySQL = {"select"}; + private static final String[] SQL = {"select", "insert", "import", "create", "use", "alter", "drop", "set", "show", "describe"}; + private static final String[] updateSQL = {"insert", "import", "create", "use", "alter", "drop", "set"}; + private static final String[] querySQL = {"select", "show", "describe"}; + + private static final String[] databaseUnspecifiedShow = {"databases", "dnodes", "mnodes", "variables"}; private TSDBConnection tsdbConnection; @@ -37,8 +40,38 @@ public class SqlSyntaxValidator { return false; } + public static boolean isValidForExecuteQuery(String sql) { + for (String prefix : querySQL) { + if (sql.trim().toLowerCase().startsWith(prefix)) + return true; + } + return false; + } + + public static boolean isValidForExecute(String sql) { + for (String prefix : SQL) { + if (sql.trim().toLowerCase().startsWith(prefix)) + return true; + } + return false; + } + + public static boolean isDatabaseUnspecifiedQuery(String sql) { + for (String databaseObj : databaseUnspecifiedShow) { + if (sql.trim().toLowerCase().matches("show\\s+" + databaseObj + ".*")) + return true; + } + return false; + } + + public static boolean isDatabaseUnspecifiedUpdate(String sql) { + sql = sql.trim().toLowerCase(); + return sql.matches("create\\s+database.*") || sql.startsWith("set") || sql.matches("drop\\s+database.*"); + } + public static boolean isUseSql(String sql) { - return sql.trim().toLowerCase().startsWith("use") || sql.trim().toLowerCase().matches("create\\s*database.*") || sql.toLowerCase().toLowerCase().matches("drop\\s*database.*"); + return sql.trim().toLowerCase().startsWith("use"); +// || sql.trim().toLowerCase().matches("create\\s*database.*") || sql.toLowerCase().toLowerCase().matches("drop\\s*database.*"); } public static boolean isShowSql(String sql) { @@ -58,8 +91,9 @@ public class SqlSyntaxValidator { return sql.trim().toLowerCase().startsWith("select"); } - public static boolean isShowDatabaseSql(String sql) { return sql.trim().toLowerCase().matches("show\\s*databases"); } + + } diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/AuthenticationTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/AuthenticationTest.java index 918714da22..60ed747670 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/AuthenticationTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/AuthenticationTest.java @@ -7,7 +7,8 @@ import java.sql.*; public class AuthenticationTest { - private static final String host = "127.0.0.1"; +// private static final String host = "127.0.0.1"; + private static final String host = "master"; private static final String user = "root"; private static final String password = "123456"; private Connection conn; diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java index ee4b2ed4dd..c8da366158 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java @@ -9,44 +9,33 @@ import java.util.Random; @FixMethodOrder(MethodSorters.NAME_ASCENDING) public class RestfulJDBCTest { - private static final String host = "127.0.0.1"; - private Connection connection; - - @Before - public void before() throws ClassNotFoundException, SQLException { - Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); - connection = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/restful_test?user=root&password=taosdata"); - } - - @After - public void after() throws SQLException { - if (connection != null) - connection.close(); - } - + // private static final String host = "127.0.0.1"; + private static final String host = "master"; + private static Connection connection; + private Random random = new Random(System.currentTimeMillis()); /** - * 查询所有log.log + * select * from log.log **/ - @Test - public void testCase001() { - try { - Statement statement = connection.createStatement(); - ResultSet resultSet = statement.executeQuery("select * from log.log"); - ResultSetMetaData metaData = resultSet.getMetaData(); - while (resultSet.next()) { - for (int i = 1; i <= metaData.getColumnCount(); i++) { - String column = metaData.getColumnLabel(i); - String value = resultSet.getString(i); - System.out.print(column + ":" + value + "\t"); - } - System.out.println(); - } - statement.close(); - } catch (SQLException e) { - e.printStackTrace(); - } - } +// @Test +// public void testCase001() { +// try { +// Statement statement = connection.createStatement(); +// ResultSet resultSet = statement.executeQuery("select * from log.log"); +// ResultSetMetaData metaData = resultSet.getMetaData(); +// while (resultSet.next()) { +// for (int i = 1; i <= metaData.getColumnCount(); i++) { +// String column = metaData.getColumnLabel(i); +// String value = resultSet.getString(i); +// System.out.print(column + ":" + value + "\t"); +// } +// System.out.println(); +// } +// statement.close(); +// } catch (SQLException e) { +// e.printStackTrace(); +// } +// } /** * create database @@ -85,7 +74,6 @@ public class RestfulJDBCTest { } } - private Random random = new Random(System.currentTimeMillis()); @Test public void testCase005() { @@ -105,5 +93,50 @@ public class RestfulJDBCTest { } } + @Test + public void testCase006() { + try (Statement stmt = connection.createStatement()) { + ResultSet rs = stmt.executeQuery("select * from weather"); + while (rs.next()) { + System.out.print("ts: " + rs.getTimestamp("ts")); + System.out.print(", temperature: " + rs.getString("temperature")); + System.out.print(", humidity: " + rs.getString("humidity")); + System.out.println(", location: " + rs.getString("location")); + } + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void testCase007() { + try (Statement stmt = connection.createStatement()) { + ResultSet rs = stmt.executeQuery("select * from weather"); + ResultSetMetaData meta = rs.getMetaData(); + while (rs.next()) { + int columnCount = meta.getColumnCount(); + for (int i = 1; i <= columnCount; i++) { + String columnLabel = meta.getColumnLabel(i); + String value = rs.getString(i); + System.out.print(columnLabel + ": " + value+"\t"); + } + System.out.println(); + } + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @BeforeClass + public static void before() throws ClassNotFoundException, SQLException { + Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); + connection = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/restful_test?user=root&password=taosdata"); + } + + @AfterClass + public static void after() throws SQLException { + if (connection != null) + connection.close(); + } } diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetMetaDataTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetMetaDataTest.java new file mode 100644 index 0000000000..80bd9087bf --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetMetaDataTest.java @@ -0,0 +1,223 @@ +package com.taosdata.jdbc.rs; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.sql.*; + +import static org.junit.Assert.*; + +public class RestfulResultSetMetaDataTest { + + // private static final String host = "127.0.0.1"; + private static final String host = "master"; + + private static Connection conn; + private static Statement stmt; + private static ResultSet rs; + private static ResultSetMetaData meta; + + @Test + public void getColumnCount() throws SQLException { + Assert.assertEquals(10, meta.getColumnCount()); + } + + @Test + public void isAutoIncrement() throws SQLException { + Assert.assertFalse(meta.isAutoIncrement(1)); + Assert.assertFalse(meta.isAutoIncrement(2)); + Assert.assertFalse(meta.isAutoIncrement(3)); + Assert.assertFalse(meta.isAutoIncrement(4)); + Assert.assertFalse(meta.isAutoIncrement(5)); + Assert.assertFalse(meta.isAutoIncrement(6)); + Assert.assertFalse(meta.isAutoIncrement(7)); + Assert.assertFalse(meta.isAutoIncrement(8)); + Assert.assertFalse(meta.isAutoIncrement(9)); + Assert.assertFalse(meta.isAutoIncrement(10)); + } + + @Test + public void isCaseSensitive() throws SQLException { + Assert.assertFalse(meta.isCaseSensitive(1)); + } + + @Test + public void isSearchable() throws SQLException { + Assert.assertTrue(meta.isSearchable(1)); + } + + @Test + public void isCurrency() throws SQLException { + Assert.assertFalse(meta.isCurrency(1)); + } + + @Test + public void isNullable() throws SQLException { + Assert.assertEquals(ResultSetMetaData.columnNoNulls, meta.isNullable(1)); + Assert.assertEquals(ResultSetMetaData.columnNullable, meta.isNullable(2)); + Assert.assertEquals(ResultSetMetaData.columnNullable, meta.isNullable(3)); + Assert.assertEquals(ResultSetMetaData.columnNullable, meta.isNullable(4)); + Assert.assertEquals(ResultSetMetaData.columnNullable, meta.isNullable(5)); + Assert.assertEquals(ResultSetMetaData.columnNullable, meta.isNullable(6)); + Assert.assertEquals(ResultSetMetaData.columnNullable, meta.isNullable(7)); + Assert.assertEquals(ResultSetMetaData.columnNullable, meta.isNullable(8)); + Assert.assertEquals(ResultSetMetaData.columnNullable, meta.isNullable(9)); + Assert.assertEquals(ResultSetMetaData.columnNullable, meta.isNullable(10)); + } + + @Test + public void isSigned() throws SQLException { + Assert.assertFalse(meta.isSigned(1)); + } + + @Test + public void getColumnDisplaySize() throws SQLException { + Assert.assertEquals(64, meta.getColumnDisplaySize(10)); + } + + @Test + public void getColumnLabel() throws SQLException { + Assert.assertEquals("f1", meta.getColumnLabel(1)); + } + + @Test + public void getColumnName() throws SQLException { + Assert.assertEquals("f1", meta.getColumnName(1)); + } + + @Test + public void getSchemaName() throws SQLException { + Assert.assertEquals("", meta.getSchemaName(1)); + } + + @Test + public void getPrecision() throws SQLException { + Assert.assertEquals(0, meta.getPrecision(1)); + } + + @Test + public void getScale() throws SQLException { + Assert.assertEquals(0, meta.getScale(1)); + } + + @Test + public void getTableName() throws SQLException { + Assert.assertEquals("", meta.getTableName(1)); + } + + @Test + public void getCatalogName() throws SQLException { + Assert.assertEquals("restful_test", meta.getCatalogName(1)); + Assert.assertEquals("restful_test", meta.getCatalogName(2)); + Assert.assertEquals("restful_test", meta.getCatalogName(3)); + Assert.assertEquals("restful_test", meta.getCatalogName(4)); + Assert.assertEquals("restful_test", meta.getCatalogName(5)); + Assert.assertEquals("restful_test", meta.getCatalogName(6)); + Assert.assertEquals("restful_test", meta.getCatalogName(7)); + Assert.assertEquals("restful_test", meta.getCatalogName(8)); + Assert.assertEquals("restful_test", meta.getCatalogName(9)); + Assert.assertEquals("restful_test", meta.getCatalogName(10)); + } + + @Test + public void getColumnType() throws SQLException { + Assert.assertEquals(Types.TIMESTAMP, meta.getColumnType(1)); + Assert.assertEquals(Types.INTEGER, meta.getColumnType(2)); + Assert.assertEquals(Types.BIGINT, meta.getColumnType(3)); + Assert.assertEquals(Types.FLOAT, meta.getColumnType(4)); + Assert.assertEquals(Types.DOUBLE, meta.getColumnType(5)); + Assert.assertEquals(Types.BINARY, meta.getColumnType(6)); + Assert.assertEquals(Types.SMALLINT, meta.getColumnType(7)); + Assert.assertEquals(Types.TINYINT, meta.getColumnType(8)); + Assert.assertEquals(Types.BOOLEAN, meta.getColumnType(9)); + Assert.assertEquals(Types.NCHAR, meta.getColumnType(10)); + } + + @Test + public void getColumnTypeName() throws SQLException { + Assert.assertEquals("TIMESTAMP", meta.getColumnTypeName(1)); + Assert.assertEquals("INT", meta.getColumnTypeName(2)); + Assert.assertEquals("BIGINT", meta.getColumnTypeName(3)); + Assert.assertEquals("FLOAT", meta.getColumnTypeName(4)); + Assert.assertEquals("DOUBLE", meta.getColumnTypeName(5)); + Assert.assertEquals("BINARY", meta.getColumnTypeName(6)); + Assert.assertEquals("SMALLINT", meta.getColumnTypeName(7)); + Assert.assertEquals("TINYINT", meta.getColumnTypeName(8)); + Assert.assertEquals("BOOL", meta.getColumnTypeName(9)); + Assert.assertEquals("NCHAR", meta.getColumnTypeName(10)); + } + + @Test + public void isReadOnly() throws SQLException { + Assert.assertTrue(meta.isReadOnly(1)); + } + + @Test + public void isWritable() throws SQLException { + Assert.assertFalse(meta.isWritable(1)); + } + + @Test + public void isDefinitelyWritable() throws SQLException { + Assert.assertFalse(meta.isDefinitelyWritable(1)); + } + + @Test + public void getColumnClassName() throws SQLException { + Assert.assertEquals(Timestamp.class.getName(), meta.getColumnClassName(1)); + Assert.assertEquals(Integer.class.getName(), meta.getColumnClassName(2)); + Assert.assertEquals(Long.class.getName(), meta.getColumnClassName(3)); + Assert.assertEquals(Float.class.getName(), meta.getColumnClassName(4)); + Assert.assertEquals(Double.class.getName(), meta.getColumnClassName(5)); + Assert.assertEquals(String.class.getName(), meta.getColumnClassName(6)); + Assert.assertEquals(Short.class.getName(), meta.getColumnClassName(7)); + Assert.assertEquals(Short.class.getName(), meta.getColumnClassName(8)); + Assert.assertEquals(Boolean.class.getName(), meta.getColumnClassName(9)); + Assert.assertEquals(String.class.getName(), meta.getColumnClassName(10)); + } + + @Test + public void unwrap() throws SQLException { + Assert.assertNotNull(meta.unwrap(RestfulResultSetMetaData.class)); + } + + @Test + public void isWrapperFor() throws SQLException { + Assert.assertTrue(meta.isWrapperFor(RestfulResultSetMetaData.class)); + } + + @BeforeClass + public static void beforeClass() { + try { + Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); + conn = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/restful_test?user=root&password=taosdata"); + stmt = conn.createStatement(); + stmt.execute("create database if not exists restful_test"); + stmt.execute("use restful_test"); + stmt.execute("drop table if exists weather"); + stmt.execute("create table if not exists weather(f1 timestamp, f2 int, f3 bigint, f4 float, f5 double, f6 binary(64), f7 smallint, f8 tinyint, f9 bool, f10 nchar(64))"); + stmt.execute("insert into restful_test.weather values('2021-01-01 00:00:00.000', 1, 100, 3.1415, 3.1415926, 'abc', 10, 10, true, '涛思数据')"); + rs = stmt.executeQuery("select * from restful_test.weather"); + rs.next(); + meta = rs.getMetaData(); + } catch (ClassNotFoundException | SQLException e) { + e.printStackTrace(); + } + } + + @AfterClass + public static void afterClass() { + try { + if (rs != null) + rs.close(); + if (stmt != null) + stmt.close(); + if (conn != null) + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } +} \ No newline at end of file diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetTest.java new file mode 100644 index 0000000000..70fdd35d6a --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetTest.java @@ -0,0 +1,588 @@ +package com.taosdata.jdbc.rs; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.InputStream; +import java.io.Reader; +import java.math.BigDecimal; +import java.sql.*; + +public class RestfulResultSetTest { + + // private static final String host = "127.0.0.1"; + private static final String host = "master"; + + private static Connection conn; + private static Statement stmt; + private static ResultSet rs; + + @Test + public void wasNull() throws SQLException { + Assert.assertFalse(rs.wasNull()); + } + + @Test + public void getString() throws SQLException { + String f10 = rs.getString("f10"); + Assert.assertEquals("涛思数据", f10); + f10 = rs.getString(10); + Assert.assertEquals("涛思数据", f10); + } + + @Test + public void getBoolean() throws SQLException { + Boolean f9 = rs.getBoolean("f9"); + Assert.assertEquals(true, f9); + f9 = rs.getBoolean(9); + Assert.assertEquals(true, f9); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getByte() throws SQLException { + rs.getByte(1); + } + + @Test + public void getShort() throws SQLException { + short f7 = rs.getShort("f7"); + Assert.assertEquals(10, f7); + f7 = rs.getShort(7); + Assert.assertEquals(10, f7); + } + + @Test + public void getInt() throws SQLException { + int f2 = rs.getInt("f2"); + Assert.assertEquals(1, f2); + f2 = rs.getInt(2); + Assert.assertEquals(1, f2); + } + + @Test + public void getLong() throws SQLException { + long f3 = rs.getLong("f3"); + Assert.assertEquals(100, f3); + f3 = rs.getLong(3); + Assert.assertEquals(100, f3); + } + + @Test + public void getFloat() throws SQLException { + float f4 = rs.getFloat("f4"); + Assert.assertEquals(3.1415f, f4, 0f); + f4 = rs.getFloat(4); + Assert.assertEquals(3.1415f, f4, 0f); + } + + @Test + public void getDouble() throws SQLException { + double f5 = rs.getDouble("f5"); + Assert.assertEquals(3.1415926, f5, 0.0); + f5 = rs.getDouble(5); + Assert.assertEquals(3.1415926, f5, 0.0); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getBigDecimal() throws SQLException { + rs.getBigDecimal("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getBytes() throws SQLException { + rs.getBytes("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getDate() throws SQLException { + rs.getDate("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getTime() throws SQLException { + rs.getTime("f1"); + } + + @Test + public void getTimestamp() throws SQLException { + Timestamp f1 = rs.getTimestamp("f1"); + Assert.assertEquals("2021-01-01 00:00:00.0", f1.toString()); + f1 = rs.getTimestamp(1); + Assert.assertEquals("2021-01-01 00:00:00.0", f1.toString()); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getAsciiStream() throws SQLException { + rs.getAsciiStream("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getUnicodeStream() throws SQLException { + rs.getUnicodeStream("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getBinaryStream() throws SQLException { + rs.getBinaryStream("f1"); + } + + @Test + public void getWarnings() throws SQLException { + Assert.assertNull(rs.getWarnings()); + } + + @Test + public void clearWarnings() throws SQLException { + rs.clearWarnings(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getCursorName() throws SQLException { + rs.getCursorName(); + } + + @Test + public void getMetaData() throws SQLException { + ResultSetMetaData meta = rs.getMetaData(); + Assert.assertNotNull(meta); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getObject() throws SQLException { + rs.getObject("f1"); + } + + @Test(expected = SQLException.class) + public void findColumn() throws SQLException { + int columnIndex = rs.findColumn("f1"); + Assert.assertEquals(1, columnIndex); + columnIndex = rs.findColumn("f2"); + Assert.assertEquals(2, columnIndex); + columnIndex = rs.findColumn("f3"); + Assert.assertEquals(3, columnIndex); + columnIndex = rs.findColumn("f4"); + Assert.assertEquals(4, columnIndex); + columnIndex = rs.findColumn("f5"); + Assert.assertEquals(5, columnIndex); + columnIndex = rs.findColumn("f6"); + Assert.assertEquals(6, columnIndex); + columnIndex = rs.findColumn("f7"); + Assert.assertEquals(7, columnIndex); + columnIndex = rs.findColumn("f8"); + Assert.assertEquals(8, columnIndex); + columnIndex = rs.findColumn("f9"); + Assert.assertEquals(9, columnIndex); + columnIndex = rs.findColumn("f10"); + Assert.assertEquals(10, columnIndex); + + rs.findColumn("f11"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getCharacterStream() throws SQLException { + rs.getCharacterStream(1); + } + + @Test + public void isBeforeFirst() throws SQLException { + Assert.assertFalse(rs.isBeforeFirst()); + rs.beforeFirst(); + Assert.assertTrue(rs.isBeforeFirst()); + rs.next(); + } + + @Test + public void isAfterLast() throws SQLException { + Assert.assertFalse(rs.isAfterLast()); + } + + @Test + public void isFirst() throws SQLException { + Assert.assertTrue(rs.isFirst()); + } + + @Test + public void isLast() throws SQLException { + Assert.assertTrue(rs.isLast()); + } + + @Test + public void beforeFirst() throws SQLException { + rs.beforeFirst(); + Assert.assertTrue(rs.isBeforeFirst()); + rs.next(); + } + + @Test + public void afterLast() throws SQLException { + rs.afterLast(); + Assert.assertTrue(rs.isAfterLast()); + rs.first(); + } + + @Test + public void first() throws SQLException { + rs.first(); + Assert.assertEquals("2021-01-01 00:00:00.0", rs.getTimestamp("f1").toString()); + } + + @Test + public void last() throws SQLException { + rs.last(); + Assert.assertEquals("2021-01-01 00:00:00.0", rs.getTimestamp("f1").toString()); + } + + @Test + public void getRow() throws SQLException { + int row = rs.getRow(); + Assert.assertEquals(1, row); + rs.beforeFirst(); + row = rs.getRow(); + Assert.assertEquals(0, row); + rs.first(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void absolute() throws SQLException { + rs.absolute(-1); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void relative() throws SQLException { + rs.relative(-1); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void previous() throws SQLException { + rs.previous(); + } + + @Test(expected = SQLException.class) + public void setFetchDirection() throws SQLException { + rs.setFetchDirection(ResultSet.FETCH_FORWARD); + rs.setFetchDirection(ResultSet.FETCH_UNKNOWN); + } + + @Test + public void getFetchDirection() throws SQLException { + Assert.assertEquals(ResultSet.FETCH_FORWARD, rs.getFetchDirection()); + } + + @Test(expected = SQLException.class) + public void setFetchSize() throws SQLException { + rs.setFetchSize(0); + } + + @Test + public void getFetchSize() throws SQLException { + Assert.assertEquals(1, rs.getFetchSize()); + } + + @Test + public void getType() throws SQLException { + Assert.assertEquals(ResultSet.TYPE_FORWARD_ONLY, rs.getType()); + } + + @Test + public void getConcurrency() throws SQLException { + Assert.assertEquals(ResultSet.CONCUR_READ_ONLY, rs.getConcurrency()); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void rowUpdated() throws SQLException { + rs.rowUpdated(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void rowInserted() throws SQLException { + rs.rowInserted(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void rowDeleted() throws SQLException { + rs.rowDeleted(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateNull() throws SQLException { + rs.updateNull("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateBoolean() throws SQLException { + rs.updateBoolean(1, false); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateByte() throws SQLException { + rs.updateByte(1, new Byte("0")); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateShort() throws SQLException { + rs.updateShort(1, new Short("0")); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateInt() throws SQLException { + rs.updateInt(1, 1); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateLong() throws SQLException { + rs.updateLong(1, 1l); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateFloat() throws SQLException { + rs.updateFloat(1, 1f); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateDouble() throws SQLException { + rs.updateDouble(1, 1.0); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateBigDecimal() throws SQLException { + rs.updateBigDecimal(1, new BigDecimal(1)); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateString() throws SQLException { + rs.updateString(1, "abc"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateBytes() throws SQLException { + rs.updateBytes(1, new byte[]{}); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateDate() throws SQLException { + rs.updateDate(1, new Date(System.currentTimeMillis())); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateTime() throws SQLException { + rs.updateTime(1, new Time(System.currentTimeMillis())); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateTimestamp() throws SQLException { + rs.updateTimestamp(1, new Timestamp(System.currentTimeMillis())); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateAsciiStream() throws SQLException { + rs.updateAsciiStream(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateBinaryStream() throws SQLException { + rs.updateBinaryStream(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateCharacterStream() throws SQLException { + rs.updateCharacterStream(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateObject() throws SQLException { + rs.updateObject(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void insertRow() throws SQLException { + rs.insertRow(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateRow() throws SQLException { + rs.updateRow(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void deleteRow() throws SQLException { + rs.deleteRow(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void refreshRow() throws SQLException { + rs.refreshRow(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void cancelRowUpdates() throws SQLException { + rs.cancelRowUpdates(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void moveToInsertRow() throws SQLException { + rs.moveToInsertRow(); + } + + @Test + public void getStatement() throws SQLException { + Statement stmt = rs.getStatement(); + Assert.assertNotNull(stmt); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void moveToCurrentRow() throws SQLException { + rs.moveToCurrentRow(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getRef() throws SQLException { + rs.getRef(1); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getBlob() throws SQLException { + rs.getBlob("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getClob() throws SQLException { + rs.getClob("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getArray() throws SQLException { + rs.getArray("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getURL() throws SQLException { + rs.getURL("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateRef() throws SQLException { + rs.updateRef("f1", null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateBlob() throws SQLException { + rs.updateBlob(1, (InputStream) null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateClob() throws SQLException { + rs.updateClob(1, (Reader) null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateArray() throws SQLException { + rs.updateArray(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getRowId() throws SQLException { + rs.getRowId("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateRowId() throws SQLException { + rs.updateRowId(1, null); + } + + @Test + public void getHoldability() throws SQLException { + Assert.assertEquals(ResultSet.HOLD_CURSORS_OVER_COMMIT, rs.getHoldability()); + } + + @Test + public void isClosed() throws SQLException { + Assert.assertFalse(rs.isClosed()); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateNString() throws SQLException { + rs.updateNString(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateNClob() throws SQLException { + rs.updateNClob(1, (Reader) null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getNClob() throws SQLException { + rs.getNClob("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getSQLXML() throws SQLException { + rs.getSQLXML("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateSQLXML() throws SQLException { + rs.updateSQLXML(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getNString() throws SQLException { + rs.getNString("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getNCharacterStream() throws SQLException { + rs.getNCharacterStream("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateNCharacterStream() throws SQLException { + rs.updateNCharacterStream(1, null); + } + + @Test + public void unwrap() throws SQLException { + RestfulResultSet unwrap = rs.unwrap(RestfulResultSet.class); + Assert.assertNotNull(unwrap); + } + + @Test + public void isWrapperFor() throws SQLException { + Assert.assertTrue(rs.isWrapperFor(RestfulResultSet.class)); + } + + @BeforeClass + public static void beforeClass() { + try { + Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); + conn = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/restful_test?user=root&password=taosdata"); + stmt = conn.createStatement(); + stmt.execute("create database if not exists restful_test"); + stmt.execute("use restful_test"); + stmt.execute("drop table if exists weather"); + stmt.execute("create table if not exists weather(f1 timestamp, f2 int, f3 bigint, f4 float, f5 double, f6 binary(64), f7 smallint, f8 tinyint, f9 bool, f10 nchar(64))"); + stmt.execute("insert into restful_test.weather values('2021-01-01 00:00:00.000', 1, 100, 3.1415, 3.1415926, 'abc', 10, 10, true, '涛思数据')"); + rs = stmt.executeQuery("select * from restful_test.weather"); + rs.next(); + } catch (ClassNotFoundException | SQLException e) { + e.printStackTrace(); + } + + } + + @AfterClass + public static void afterClass() { + try { + if (rs != null) + rs.close(); + if (stmt != null) + stmt.close(); + if (conn != null) + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + +} \ No newline at end of file diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/SQLTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/SQLTest.java index ad6ed43ac7..090c8f146b 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/SQLTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/SQLTest.java @@ -1,5 +1,6 @@ package com.taosdata.jdbc.rs; +import com.taosdata.jdbc.utils.SQLExecutor; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.FixMethodOrder; @@ -10,379 +11,316 @@ import java.sql.*; @FixMethodOrder(MethodSorters.NAME_ASCENDING) public class SQLTest { - private static final String host = "127.0.0.1"; - +// private static final String host = "127.0.0.1"; + private static final String host = "master"; private static Connection connection; @Test public void testCase001() { String sql = "create database if not exists restful_test"; - execute(sql); + SQLExecutor.execute(connection, sql); } @Test public void testCase002() { String sql = "use restful_test"; - execute(sql); + SQLExecutor.execute(connection, sql); } @Test public void testCase003() { String sql = "show databases"; - executeWithResult(sql); + SQLExecutor.executeWithResult(connection, sql); } @Test public void testCase004() { String sql = "show tables"; - executeWithResult(sql); + SQLExecutor.executeWithResult(connection, sql); } @Test public void testCase005() { String sql = "show stables"; - executeWithResult(sql); + SQLExecutor.executeWithResult(connection, sql); } @Test public void testCase006() { String sql = "show dnodes"; - executeWithResult(sql); + SQLExecutor.executeWithResult(connection, sql); } @Test public void testCase007() { String sql = "show vgroups"; - executeWithResult(sql); + SQLExecutor.executeWithResult(connection, sql); } @Test public void testCase008() { String sql = "drop table if exists restful_test.weather"; - execute(sql); + SQLExecutor.execute(connection, sql); } @Test public void testCase009() { String sql = "create table if not exists restful_test.weather(ts timestamp, temperature float) tags(location nchar(64))"; - execute(sql); + SQLExecutor.execute(connection, sql); } @Test public void testCase010() { String sql = "create table t1 using restful_test.weather tags('北京')"; - execute(sql); + SQLExecutor.execute(connection, sql); } @Test public void testCase011() { String sql = "insert into restful_test.t1 values(now, 22.22)"; - executeUpdate(sql); + SQLExecutor.executeUpdate(connection, sql); } @Test public void testCase012() { String sql = "insert into restful_test.t1 values('2020-01-01 00:00:00.000', 22.22)"; - executeUpdate(sql); + SQLExecutor.executeUpdate(connection, sql); } @Test public void testCase013() { String sql = "insert into restful_test.t1 values('2020-01-01 00:01:00.000', 22.22),('2020-01-01 00:02:00.000', 22.22)"; - executeUpdate(sql); + SQLExecutor.executeUpdate(connection, sql); } @Test public void testCase014() { String sql = "insert into restful_test.t2 using weather tags('上海') values('2020-01-01 00:03:00.000', 22.22)"; - executeUpdate(sql); + SQLExecutor.executeUpdate(connection, sql); } @Test public void testCase015() { String sql = "insert into restful_test.t2 using weather tags('上海') values('2020-01-01 00:01:00.000', 22.22),('2020-01-01 00:02:00.000', 22.22)"; - executeUpdate(sql); + SQLExecutor.executeUpdate(connection, sql); } @Test public void testCase016() { String sql = "insert into t1 values('2020-01-01 01:0:00.000', 22.22),('2020-01-01 02:00:00.000', 22.22) t2 values('2020-01-01 01:0:00.000', 33.33),('2020-01-01 02:00:00.000', 33.33)"; - executeUpdate(sql); + SQLExecutor.executeUpdate(connection, sql); } @Test public void testCase017() { String sql = "Insert into t3 using weather tags('广东') values('2020-01-01 01:0:00.000', 22.22),('2020-01-01 02:00:00.000', 22.22) t4 using weather tags('天津') values('2020-01-01 01:0:00.000', 33.33),('2020-01-01 02:00:00.000', 33.33)"; - executeUpdate(sql); + SQLExecutor.executeUpdate(connection, sql); } @Test public void testCase018() { String sql = "select * from restful_test.t1"; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase019() { String sql = "select * from restful_test.weather"; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase020() { String sql = "select ts, temperature from restful_test.t1"; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase021() { String sql = "select ts, temperature from restful_test.weather"; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase022() { String sql = "select temperature, ts from restful_test.t1"; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase023() { String sql = "select temperature, ts from restful_test.weather"; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase024() { String sql = "import into restful_test.t5 using weather tags('石家庄') values('2020-01-01 00:01:00.000', 22.22)"; - executeUpdate(sql); + SQLExecutor.executeUpdate(connection, sql); } @Test public void testCase025() { String sql = "import into restful_test.t6 using weather tags('沈阳') values('2020-01-01 00:01:00.000', 22.22),('2020-01-01 00:02:00.000', 22.22)"; - executeUpdate(sql); + SQLExecutor.executeUpdate(connection, sql); } @Test public void testCase026() { String sql = "import into restful_test.t7 using weather tags('长沙') values('2020-01-01 00:01:00.000', 22.22) restful_test.t8 using weather tags('吉林') values('2020-01-01 00:01:00.000', 22.22)"; - executeUpdate(sql); + SQLExecutor.executeUpdate(connection, sql); } @Test public void testCase027() { String sql = "import into restful_test.t9 using weather tags('武汉') values('2020-01-01 00:01:00.000', 22.22) ,('2020-01-02 00:01:00.000', 22.22) restful_test.t10 using weather tags('哈尔滨') values('2020-01-01 00:01:00.000', 22.22),('2020-01-02 00:01:00.000', 22.22)"; - executeUpdate(sql); + SQLExecutor.executeUpdate(connection, sql); } @Test public void testCase028() { String sql = "select location, temperature, ts from restful_test.weather where temperature > 1"; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase029() { String sql = "select location, temperature, ts from restful_test.weather where temperature < 1"; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase030() { String sql = "select location, temperature, ts from restful_test.weather where ts > now"; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase031() { String sql = "select location, temperature, ts from restful_test.weather where ts < now"; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase032() { String sql = "select count(*) from restful_test.weather"; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase033() { String sql = "select first(*) from restful_test.weather"; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase034() { String sql = "select last(*) from restful_test.weather"; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase035() { String sql = "select last_row(*) from restful_test.weather"; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase036() { String sql = "select ts, ts as primary_key from restful_test.weather"; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase037() { String sql = "select database()"; - execute("use restful_test"); - executeQuery(sql); + SQLExecutor.execute(connection, "use restful_test"); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase038() { String sql = "select client_version()"; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase039() { String sql = "select server_status()"; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase040() { String sql = "select server_status() as status"; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase041() { String sql = "select tbname, location from restful_test.weather"; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase042() { String sql = "select count(tbname) from restful_test.weather"; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase043() { String sql = "select * from restful_test.weather where ts < now - 1h"; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase044() { String sql = "select * from restful_test.weather where ts < now - 1h and location like '%'"; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase045() { String sql = "select * from restful_test.weather where ts < now - 1h order by ts"; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase046() { String sql = "select last(*) from restful_test.weather where ts < now - 1h group by tbname order by tbname"; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase047() { String sql = "select * from restful_test.weather limit 2"; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase048() { String sql = "select * from restful_test.weather limit 2 offset 5"; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase049() { String sql = "select * from restful_test.t1, restful_test.t3 where t1.ts = t3.ts "; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase050() { String sql = "select * from restful_test.t1, restful_test.t3 where t1.ts = t3.ts and t1.location = t3.location"; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase051() { String sql = "select * from restful_test.t1 tt, restful_test.t3 yy where tt.ts = yy.ts"; - executeQuery(sql); - } - - private void executeUpdate(String sql) { - try (Statement statement = connection.createStatement()) { - long start = System.currentTimeMillis(); - int affectedRows = statement.executeUpdate(sql); - long end = System.currentTimeMillis(); - System.out.println("[ affected rows : " + affectedRows + " ] time cost: " + (end - start) + " ms, execute statement ====> " + sql); - } catch (SQLException e) { - e.printStackTrace(); - } - } - - private void executeWithResult(String sql) { - try (Statement statement = connection.createStatement()) { - statement.execute(sql); - ResultSet resultSet = statement.getResultSet(); - printResult(resultSet); - } catch (SQLException e) { - e.printStackTrace(); - } - } - - private void execute(String sql) { - try (Statement statement = connection.createStatement()) { - long start = System.currentTimeMillis(); - boolean execute = statement.execute(sql); - long end = System.currentTimeMillis(); - printSql(sql, execute, (end - start)); - } catch (SQLException e) { - System.out.println("ERROR execute SQL ===> " + sql); - e.printStackTrace(); - } - } - - private static void printSql(String sql, boolean succeed, long cost) { - System.out.println("[ " + (succeed ? "OK" : "ERROR!") + " ] time cost: " + cost + " ms, execute statement ====> " + sql); - } - - private void executeQuery(String sql) { - try (Statement statement = connection.createStatement()) { - long start = System.currentTimeMillis(); - ResultSet resultSet = statement.executeQuery(sql); - long end = System.currentTimeMillis(); - printSql(sql, true, (end - start)); - printResult(resultSet); - } catch (SQLException e) { - System.out.println("ERROR execute SQL ===> " + sql); - e.printStackTrace(); - } - } - - private static void printResult(ResultSet resultSet) throws SQLException { - ResultSetMetaData metaData = resultSet.getMetaData(); - while (resultSet.next()) { - StringBuilder sb = new StringBuilder(); - for (int i = 1; i <= metaData.getColumnCount(); i++) { - String columnLabel = metaData.getColumnLabel(i); - String value = resultSet.getString(i); - sb.append(columnLabel + ": " + value + "\t"); - } - System.out.println(sb.toString()); - } + SQLExecutor.executeQuery(connection, sql); } @BeforeClass diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SQLExecutor.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SQLExecutor.java new file mode 100644 index 0000000000..bf034bf458 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SQLExecutor.java @@ -0,0 +1,74 @@ +package com.taosdata.jdbc.utils; + +import java.sql.*; + +public class SQLExecutor { + + // insert, import + public static void executeUpdate(Connection connection, String sql) { + try (Statement statement = connection.createStatement()) { + long start = System.currentTimeMillis(); + int affectedRows = statement.executeUpdate(sql); + long end = System.currentTimeMillis(); + System.out.println("[ affected rows : " + affectedRows + " ] time cost: " + (end - start) + " ms, execute statement ====> " + sql); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + // show databases, show tables, show stables + public static void executeWithResult(Connection connection, String sql) { + try (Statement statement = connection.createStatement()) { + statement.execute(sql); + ResultSet resultSet = statement.getResultSet(); + printResult(resultSet); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + // use database, create database, create table, drop table... + public static void execute(Connection connection, String sql) { + try (Statement statement = connection.createStatement()) { + long start = System.currentTimeMillis(); + boolean execute = statement.execute(sql); + long end = System.currentTimeMillis(); + printSql(sql, execute, (end - start)); + } catch (SQLException e) { + System.out.println("ERROR execute SQL ===> " + sql); + e.printStackTrace(); + } + } + + // select + public static void executeQuery(Connection connection, String sql) { + try (Statement statement = connection.createStatement()) { + long start = System.currentTimeMillis(); + ResultSet resultSet = statement.executeQuery(sql); + long end = System.currentTimeMillis(); + printSql(sql, true, (end - start)); + printResult(resultSet); + } catch (SQLException e) { + System.out.println("ERROR execute SQL ===> " + sql); + e.printStackTrace(); + } + } + + private static void printSql(String sql, boolean succeed, long cost) { + System.out.println("[ " + (succeed ? "OK" : "ERROR!") + " ] time cost: " + cost + " ms, execute statement ====> " + sql); + } + + private static void printResult(ResultSet resultSet) throws SQLException { + ResultSetMetaData metaData = resultSet.getMetaData(); + while (resultSet.next()) { + StringBuilder sb = new StringBuilder(); + for (int i = 1; i <= metaData.getColumnCount(); i++) { + String columnLabel = metaData.getColumnLabel(i); + String value = resultSet.getString(i); + sb.append(columnLabel + ": " + value + "\t"); + } + System.out.println(sb.toString()); + } + } + +} diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JdbcRestfulDemo.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JdbcRestfulDemo.java index 880bcbfc6f..91639e85f3 100644 --- a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JdbcRestfulDemo.java +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JdbcRestfulDemo.java @@ -14,13 +14,14 @@ public class JdbcRestfulDemo { String url = "jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata"; Properties properties = new Properties(); - properties.setProperty("charset", "UTF-8"); - properties.setProperty("locale", "en_US.UTF-8"); - properties.setProperty("timezone", "UTC-8"); +// properties.setProperty("charset", "UTF-8"); +// properties.setProperty("locale", "en_US.UTF-8"); +// properties.setProperty("timezone", "UTC-8"); Connection conn = DriverManager.getConnection(url, properties); Statement stmt = conn.createStatement(); + stmt.execute("drop database if exists restful_test"); stmt.execute("create database if not exists restful_test"); stmt.execute("use restful_test"); stmt.execute("create table restful_test.weather(ts timestamp, temperature float) tags(location nchar(64))"); @@ -34,6 +35,7 @@ public class JdbcRestfulDemo { System.out.println(); } + rs.close(); stmt.close(); conn.close(); } catch (ClassNotFoundException e) { diff --git a/tests/examples/JDBC/connectionPools/pom.xml b/tests/examples/JDBC/connectionPools/pom.xml index 2793f0a83d..fbee256067 100644 --- a/tests/examples/JDBC/connectionPools/pom.xml +++ b/tests/examples/JDBC/connectionPools/pom.xml @@ -12,7 +12,7 @@ com.taosdata.jdbc taos-jdbcdriver - 2.0.11 + 2.0.17 @@ -44,7 +44,6 @@ c3p0 0.9.5.4 - log4j From 26df651d56315dbd9d784d5f71ca0dd00937c470 Mon Sep 17 00:00:00 2001 From: zyyang Date: Tue, 26 Jan 2021 10:43:39 +0800 Subject: [PATCH 071/147] change --- .../java/com/taosdata/jdbc/rs/RestfulResultSetMetaData.java | 1 - .../java/com/taosdata/jdbc/rs/RestfulResultSetMetaDataTest.java | 2 -- 2 files changed, 3 deletions(-) diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSetMetaData.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSetMetaData.java index 9252ed23a6..44a02f486b 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSetMetaData.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSetMetaData.java @@ -152,7 +152,6 @@ public class RestfulResultSetMetaData implements ResultSetMetaData { return java.sql.Types.NCHAR; } throw new SQLException(TSDBConstants.INVALID_VARIABLES); - } @Override diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetMetaDataTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetMetaDataTest.java index 80bd9087bf..0af6aa431c 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetMetaDataTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetMetaDataTest.java @@ -7,8 +7,6 @@ import org.junit.Test; import java.sql.*; -import static org.junit.Assert.*; - public class RestfulResultSetMetaDataTest { // private static final String host = "127.0.0.1"; From da6f1548b37902794e2d0b0fc064ce5bb8101114 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Tue, 26 Jan 2021 10:51:41 +0800 Subject: [PATCH 072/147] [TD-2831]add email file in pkg --- packaging/deb/makedeb.sh | 1 + packaging/rpm/tdengine.spec | 1 + 2 files changed, 2 insertions(+) diff --git a/packaging/deb/makedeb.sh b/packaging/deb/makedeb.sh index 431093be95..a2f58619df 100755 --- a/packaging/deb/makedeb.sh +++ b/packaging/deb/makedeb.sh @@ -43,6 +43,7 @@ mkdir -p ${pkg_dir}${install_home_path}/include mkdir -p ${pkg_dir}${install_home_path}/init.d mkdir -p ${pkg_dir}${install_home_path}/script +echo "" > ${pkg_dir}${install_home_path}/email cp ${compile_dir}/../packaging/cfg/taos.cfg ${pkg_dir}${install_home_path}/cfg cp ${compile_dir}/../packaging/deb/taosd ${pkg_dir}${install_home_path}/init.d cp ${compile_dir}/../packaging/tools/post.sh ${pkg_dir}${install_home_path}/script diff --git a/packaging/rpm/tdengine.spec b/packaging/rpm/tdengine.spec index 6f012aa80e..a143d0afdb 100644 --- a/packaging/rpm/tdengine.spec +++ b/packaging/rpm/tdengine.spec @@ -51,6 +51,7 @@ mkdir -p %{buildroot}%{homepath}/include mkdir -p %{buildroot}%{homepath}/init.d mkdir -p %{buildroot}%{homepath}/script +echo "" > %{buildroot}%{homepath}/email cp %{_compiledir}/../packaging/cfg/taos.cfg %{buildroot}%{homepath}/cfg cp %{_compiledir}/../packaging/rpm/taosd %{buildroot}%{homepath}/init.d cp %{_compiledir}/../packaging/tools/post.sh %{buildroot}%{homepath}/script From cdb0a639bbe8b6f218e3026cfddf0e9778cafd88 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Tue, 26 Jan 2021 10:54:06 +0800 Subject: [PATCH 073/147] fix sql empty issue --- src/query/src/qExecutor.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 63b2f3ea15..6140b9824c 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -6424,6 +6424,11 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, pMsg += pQueryMsg->tbnameCondLen; } + //skip ts buf + if ((pQueryMsg->tsOffset + pQueryMsg->tsLen) > 0) { + pMsg = (char *)pQueryMsg + pQueryMsg->tsOffset + pQueryMsg->tsLen; + } + *sql = strndup(pMsg, pQueryMsg->sqlstrLen); if (!validateQuerySourceCols(pQueryMsg, *pExpr, *tagCols)) { From 4a669b9b31e1cd7f612dd46bd3d246756471dd05 Mon Sep 17 00:00:00 2001 From: freemine Date: Tue, 26 Jan 2021 10:58:10 +0800 Subject: [PATCH 074/147] how to do a pressure-test upon eok: refer to comments in tests/examples/c/epoll.c --- tests/examples/c/epoll.c | 157 +++++++++++++++++---------------------- 1 file changed, 69 insertions(+), 88 deletions(-) diff --git a/tests/examples/c/epoll.c b/tests/examples/c/epoll.c index 4f65ea478e..20ab395158 100644 --- a/tests/examples/c/epoll.c +++ b/tests/examples/c/epoll.c @@ -13,6 +13,12 @@ * along with this program. If not, see . */ +// how to use to do a pressure-test upon eok +// tester: cat /dev/urandom | nc -c +// testee: ./debug/build/bin/epoll -l > /dev/null +// compare against: nc -l > /dev/null +// monitor and compare : glances + #ifdef __APPLE__ #include "eok.h" #else // __APPLE__ @@ -68,55 +74,62 @@ static int ep_dummy = 0; static ep_t* ep_create(void); static void ep_destroy(ep_t *ep); static void* routine(void* arg); -static int open_connect(unsigned short port); static int open_listen(unsigned short port); -typedef struct client_s client_t; -struct client_s { +typedef struct fde_s fde_t; +struct fde_s { int skt; - void (*on_event)(ep_t *ep, struct epoll_event *events, client_t *client); - volatile unsigned int state; // 1: listenning; 2: connected + void (*on_event)(ep_t *ep, struct epoll_event *events, fde_t *client); }; -static void echo_event(ep_t *ep, struct epoll_event *ev, client_t *client); +static void listen_event(ep_t *ep, struct epoll_event *ev, fde_t *client); +static void null_event(ep_t *ep, struct epoll_event *ev, fde_t *client); + +#define usage(arg0, fmt, ...) do { \ + if (fmt[0]) { \ + fprintf(stderr, "" fmt "\n", ##__VA_ARGS__); \ + } \ + fprintf(stderr, "usage:\n"); \ + fprintf(stderr, " %s -l : specify listenning port\n", arg0); \ +} while (0) int main(int argc, char *argv[]) { + char *prg = basename(argv[0]); + if (argc==1) { + usage(prg, ""); + return 0; + } ep_t* ep = ep_create(); A(ep, "failed"); - int skt = open_connect(6789); - if (skt!=-1) { - client_t *client = (client_t*)calloc(1, sizeof(*client)); - if (client) { + for (int i=1; i=argc) { + usage(prg, "expecting after -l, but got nothing"); + return 1; // confirmed potential leakage + } + arg = argv[i]; + int port = atoi(arg); + int skt = open_listen(port); + if (skt==-1) continue; + fde_t *client = (fde_t*)calloc(1, sizeof(*client)); + if (!client) { + E("out of memory"); + close(skt); + continue; + } client->skt = skt; - client->on_event = echo_event; - client->state = 2; + client->on_event = listen_event; struct epoll_event ev = {0}; ev.events = EPOLLIN | EPOLLERR | EPOLLHUP | EPOLLRDHUP; ev.data.ptr = client; A(0==epoll_ctl(ep->ep, EPOLL_CTL_ADD, skt, &ev), ""); + continue; } + usage(prg, "unknown argument: [%s]", arg); + return 1; } - skt = open_listen(0); - if (skt!=-1) { - client_t *client = (client_t*)calloc(1, sizeof(*client)); - if (client) { - client->skt = skt; - client->on_event = echo_event; - client->state = 1; - struct epoll_event ev = {0}; - ev.events = EPOLLIN | EPOLLERR | EPOLLHUP | EPOLLRDHUP; - ev.data.ptr = client; - A(0==epoll_ctl(ep->ep, EPOLL_CTL_ADD, skt, &ev), ""); - } - } - // char c = '\0'; - // while ((c=getchar())!=EOF) { - // switch (c) { - // case 'q': break; - // default: continue; - // } - // } - // getchar(); char *line = NULL; size_t linecap = 0; ssize_t linelen; @@ -205,7 +218,7 @@ static void* routine(void* arg) { continue; } A(ev->data.ptr, "internal logic error"); - client_t *client = (client_t*)ev->data.ptr; + fde_t *client = (fde_t*)ev->data.ptr; client->on_event(ep, ev, client); continue; } @@ -223,7 +236,7 @@ static int open_listen(unsigned short port) { do { struct sockaddr_in si = {0}; si.sin_family = AF_INET; - si.sin_addr.s_addr = inet_addr("127.0.0.1"); + si.sin_addr.s_addr = inet_addr("0.0.0.0"); si.sin_port = htons(port); r = bind(skt, (struct sockaddr*)&si, sizeof(si)); if (r) { @@ -249,63 +262,31 @@ static int open_listen(unsigned short port) { return -1; } -static int open_connect(unsigned short port) { - int r = 0; - int skt = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); - if (skt==-1) { - E("socket() failed"); - return -1; +static void listen_event(ep_t *ep, struct epoll_event *ev, fde_t *client) { + A(ev->events & EPOLLIN, "internal logic error"); + struct sockaddr_in si = {0}; + socklen_t silen = sizeof(si); + int skt = accept(client->skt, (struct sockaddr*)&si, &silen); + A(skt!=-1, "internal logic error"); + fde_t *server = (fde_t*)calloc(1, sizeof(*server)); + if (!server) { + close(skt); + return; } - do { - struct sockaddr_in si = {0}; - si.sin_family = AF_INET; - si.sin_addr.s_addr = inet_addr("127.0.0.1"); - si.sin_port = htons(port); - r = connect(skt, (struct sockaddr*)&si, sizeof(si)); - if (r) { - E("connect(%u) failed", port); - break; - } - memset(&si, 0, sizeof(si)); - socklen_t len = sizeof(si); - r = getsockname(skt, (struct sockaddr *)&si, &len); - if (r) { - E("getsockname() failed"); - } - A(len==sizeof(si), "internal logic error"); - D("connected: %d", ntohs(si.sin_port)); - return skt; - } while (0); - close(skt); - return -1; + server->skt = skt; + server->on_event = null_event; + struct epoll_event ee = {0}; + ee.events = EPOLLIN | EPOLLERR | EPOLLHUP | EPOLLRDHUP; + ee.data.ptr = server; + A(0==epoll_ctl(ep->ep, EPOLL_CTL_ADD, skt, &ee), ""); } -static void echo_event(ep_t *ep, struct epoll_event *ev, client_t *client) { +static void null_event(ep_t *ep, struct epoll_event *ev, fde_t *client) { if (ev->events & EPOLLIN) { - if (client->state==1) { - struct sockaddr_in si = {0}; - socklen_t silen = sizeof(si); - int skt = accept(client->skt, (struct sockaddr*)&si, &silen); - if (skt!=-1) { - client_t *server = (client_t*)calloc(1, sizeof(*server)); - if (server) { - server->skt = skt; - server->on_event = echo_event; - server->state = 2; - struct epoll_event ev = {0}; - ev.events = EPOLLIN | EPOLLERR | EPOLLHUP | EPOLLRDHUP; - ev.data.ptr = server; - A(0==epoll_ctl(ep->ep, EPOLL_CTL_ADD, skt, &ev), ""); - } - } - } - if (client->state==2) { - char buf[4]; - int n = recv(client->skt, buf, sizeof(buf)-1, 0); - A(n>=0 && nevents, buf); - } + char buf[8192]; + int n = recv(client->skt, buf, sizeof(buf), 0); + A(n>=0 && n<=sizeof(buf), "internal logic error:[%d]", n); + A(n==fwrite(buf, 1, n, stdout), "internal logic error"); } if (ev->events & (EPOLLERR | EPOLLHUP | EPOLLRDHUP)) { A(0==pthread_mutex_lock(&ep->lock), ""); From b89054644f05a27574812be94d603a028f0e339e Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 26 Jan 2021 10:58:49 +0800 Subject: [PATCH 075/147] [TD-2845]: new -V option to taos.exe --- src/kit/shell/src/shellWindows.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/kit/shell/src/shellWindows.c b/src/kit/shell/src/shellWindows.c index 7cb6c75302..bb3d91d8ff 100644 --- a/src/kit/shell/src/shellWindows.c +++ b/src/kit/shell/src/shellWindows.c @@ -19,6 +19,10 @@ extern char configDir[]; +void printVersion() { + printf("version: %s\n", version); +} + void printHelp() { char indent[10] = " "; printf("taos shell is used to test the TDengine database\n"); @@ -51,6 +55,8 @@ void printHelp() { printf("%s%s%s\n", indent, indent, "Net role when network connectivity test, default is startup, options: client|server|rpc|startup."); printf("%s%s\n", indent, "-l"); printf("%s%s%s\n", indent, indent, "Packet length used for net test, default is 1000 bytes."); + printf("%s%s\n", indent, "-V"); + printf("%s%s%s\n", indent, indent, "Print program version."); exit(EXIT_SUCCESS); } @@ -145,7 +151,6 @@ void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { exit(EXIT_FAILURE); } } - // For time zone else if (strcmp(argv[i], "-n") == 0) { if (i < argc - 1) { arguments->netTestRole = argv[++i]; @@ -154,7 +159,6 @@ void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { exit(EXIT_FAILURE); } } - // For time zone else if (strcmp(argv[i], "-l") == 0) { if (i < argc - 1) { arguments->pktLen = atoi(argv[++i]); @@ -163,6 +167,10 @@ void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { exit(EXIT_FAILURE); } } + else if (strcmp(argv[i], "-V") == 0) { + printVersion(); + exit(EXIT_SUCCESS); + } // For temperory command TODO else if (strcmp(argv[i], "--help") == 0) { printHelp(); From 8fe4e9ca40954a2017a065827bb4efd6f29bc118 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 26 Jan 2021 11:02:22 +0800 Subject: [PATCH 076/147] taos shell: exit --help option with EXIT_SUCCESS --- src/kit/shell/src/shellWindows.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kit/shell/src/shellWindows.c b/src/kit/shell/src/shellWindows.c index bb3d91d8ff..8e8b541246 100644 --- a/src/kit/shell/src/shellWindows.c +++ b/src/kit/shell/src/shellWindows.c @@ -174,7 +174,7 @@ void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { // For temperory command TODO else if (strcmp(argv[i], "--help") == 0) { printHelp(); - exit(EXIT_FAILURE); + exit(EXIT_SUCCESS); } else { fprintf(stderr, "wrong options\n"); printHelp(); From dd3115358ad28e191870dd27e03af511bc3c2fee Mon Sep 17 00:00:00 2001 From: zyyang Date: Tue, 26 Jan 2021 11:07:23 +0800 Subject: [PATCH 077/147] change --- cmake/install.inc | 2 +- src/connector/jdbc/CMakeLists.txt | 2 +- src/connector/jdbc/deploy-pom.xml | 2 +- src/connector/jdbc/pom.xml | 2 +- .../taosdata/jdbc/rs/AuthenticationTest.java | 4 +- .../com/taosdata/jdbc/rs/RestfulJDBCTest.java | 44 +++++++++---------- .../jdbc/rs/RestfulResultSetMetaDataTest.java | 4 +- .../jdbc/rs/RestfulResultSetTest.java | 4 +- .../java/com/taosdata/jdbc/rs/SQLTest.java | 4 +- 9 files changed, 34 insertions(+), 34 deletions(-) diff --git a/cmake/install.inc b/cmake/install.inc index a5b01f43cb..2f0404334c 100755 --- a/cmake/install.inc +++ b/cmake/install.inc @@ -32,7 +32,7 @@ ELSEIF (TD_WINDOWS) #INSTALL(TARGETS taos RUNTIME DESTINATION driver) #INSTALL(TARGETS shell RUNTIME DESTINATION .) IF (TD_MVN_INSTALLED) - INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.17-dist.jar DESTINATION connector/jdbc) + INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.18-dist.jar DESTINATION connector/jdbc) ENDIF () ELSEIF (TD_DARWIN) SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh") diff --git a/src/connector/jdbc/CMakeLists.txt b/src/connector/jdbc/CMakeLists.txt index b64161e2e4..59b09c8695 100644 --- a/src/connector/jdbc/CMakeLists.txt +++ b/src/connector/jdbc/CMakeLists.txt @@ -8,7 +8,7 @@ IF (TD_MVN_INSTALLED) ADD_CUSTOM_COMMAND(OUTPUT ${JDBC_CMD_NAME} POST_BUILD COMMAND mvn -Dmaven.test.skip=true install -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml - COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.17-dist.jar ${LIBRARY_OUTPUT_PATH} + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.18-dist.jar ${LIBRARY_OUTPUT_PATH} COMMAND mvn -Dmaven.test.skip=true clean -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml COMMENT "build jdbc driver") ADD_CUSTOM_TARGET(${JDBC_TARGET_NAME} ALL WORKING_DIRECTORY ${EXECUTABLE_OUTPUT_PATH} DEPENDS ${JDBC_CMD_NAME}) diff --git a/src/connector/jdbc/deploy-pom.xml b/src/connector/jdbc/deploy-pom.xml index 1f03c3c6fe..34b0a3c6d3 100755 --- a/src/connector/jdbc/deploy-pom.xml +++ b/src/connector/jdbc/deploy-pom.xml @@ -5,7 +5,7 @@ com.taosdata.jdbc taos-jdbcdriver - 2.0.17 + 2.0.18 jar JDBCDriver diff --git a/src/connector/jdbc/pom.xml b/src/connector/jdbc/pom.xml index 6be0ca036e..bc90cae0db 100755 --- a/src/connector/jdbc/pom.xml +++ b/src/connector/jdbc/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.taosdata.jdbc taos-jdbcdriver - 2.0.17 + 2.0.18 jar JDBCDriver https://github.com/taosdata/TDengine/tree/master/src/connector/jdbc diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/AuthenticationTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/AuthenticationTest.java index 60ed747670..11d4a14353 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/AuthenticationTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/AuthenticationTest.java @@ -7,8 +7,8 @@ import java.sql.*; public class AuthenticationTest { -// private static final String host = "127.0.0.1"; - private static final String host = "master"; + private static final String host = "127.0.0.1"; + // private static final String host = "master"; private static final String user = "root"; private static final String password = "123456"; private Connection conn; diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java index c8da366158..185c0306f5 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java @@ -9,33 +9,33 @@ import java.util.Random; @FixMethodOrder(MethodSorters.NAME_ASCENDING) public class RestfulJDBCTest { - // private static final String host = "127.0.0.1"; - private static final String host = "master"; + private static final String host = "127.0.0.1"; + // private static final String host = "master"; private static Connection connection; private Random random = new Random(System.currentTimeMillis()); /** * select * from log.log **/ -// @Test -// public void testCase001() { -// try { -// Statement statement = connection.createStatement(); -// ResultSet resultSet = statement.executeQuery("select * from log.log"); -// ResultSetMetaData metaData = resultSet.getMetaData(); -// while (resultSet.next()) { -// for (int i = 1; i <= metaData.getColumnCount(); i++) { -// String column = metaData.getColumnLabel(i); -// String value = resultSet.getString(i); -// System.out.print(column + ":" + value + "\t"); -// } -// System.out.println(); -// } -// statement.close(); -// } catch (SQLException e) { -// e.printStackTrace(); -// } -// } + @Test + public void testCase001() { + try { + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery("select * from log.log"); + ResultSetMetaData metaData = resultSet.getMetaData(); + while (resultSet.next()) { + for (int i = 1; i <= metaData.getColumnCount(); i++) { + String column = metaData.getColumnLabel(i); + String value = resultSet.getString(i); + System.out.print(column + ":" + value + "\t"); + } + System.out.println(); + } + statement.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } /** * create database @@ -118,7 +118,7 @@ public class RestfulJDBCTest { for (int i = 1; i <= columnCount; i++) { String columnLabel = meta.getColumnLabel(i); String value = rs.getString(i); - System.out.print(columnLabel + ": " + value+"\t"); + System.out.print(columnLabel + ": " + value + "\t"); } System.out.println(); } diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetMetaDataTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetMetaDataTest.java index 0af6aa431c..13973d8b6b 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetMetaDataTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetMetaDataTest.java @@ -9,8 +9,8 @@ import java.sql.*; public class RestfulResultSetMetaDataTest { - // private static final String host = "127.0.0.1"; - private static final String host = "master"; + private static final String host = "127.0.0.1"; +// private static final String host = "master"; private static Connection conn; private static Statement stmt; diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetTest.java index 70fdd35d6a..a14d09588d 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetTest.java @@ -12,8 +12,8 @@ import java.sql.*; public class RestfulResultSetTest { - // private static final String host = "127.0.0.1"; - private static final String host = "master"; + private static final String host = "127.0.0.1"; +// private static final String host = "master"; private static Connection conn; private static Statement stmt; diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/SQLTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/SQLTest.java index 090c8f146b..313abfd278 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/SQLTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/SQLTest.java @@ -11,8 +11,8 @@ import java.sql.*; @FixMethodOrder(MethodSorters.NAME_ASCENDING) public class SQLTest { -// private static final String host = "127.0.0.1"; - private static final String host = "master"; + private static final String host = "127.0.0.1"; + // private static final String host = "master"; private static Connection connection; @Test From 6a68c53719eca4d81b585be686fcedd5337b7e3f Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Tue, 26 Jan 2021 11:08:33 +0800 Subject: [PATCH 078/147] fix bug --- src/inc/taoserror.h | 2 ++ src/query/src/qExecutor.c | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index b6cf1b5f1f..ebed2caaa6 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -256,6 +256,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_HAS_RSP, 0, 0x0708, "Query shou TAOS_DEFINE_ERROR(TSDB_CODE_QRY_IN_EXEC, 0, 0x0709, "Multiple retrieval of this query") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW, 0, 0x070A, "Too many time window in query") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_ENOUGH_BUFFER, 0, 0x070B, "Query buffer limit has reached") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INCONSISTAN, 0, 0x070C, "File inconsistance in replica") + // grant TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_EXPIRED, 0, 0x0800, "License expired") diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 63b2f3ea15..fad9435413 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -1398,13 +1398,13 @@ static int32_t doTSJoinFilter(SQueryRuntimeEnv *pRuntimeEnv, int32_t offset) { if (key < elem.ts) { return TS_JOIN_TS_NOT_EQUALS; } else if (key > elem.ts) { - assert(false); + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_INCONSISTAN); } } else { if (key > elem.ts) { return TS_JOIN_TS_NOT_EQUALS; } else if (key < elem.ts) { - assert(false); + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_INCONSISTAN); } } From 650bef8d955165e52552e3bb4fd0854ae807b6d5 Mon Sep 17 00:00:00 2001 From: zyyang <69311263+zyyang-taosdata@users.noreply.github.com> Date: Tue, 26 Jan 2021 11:11:59 +0800 Subject: [PATCH 079/147] Update connector-java-ch.md --- documentation20/webdocs/markdowndocs/connector-java-ch.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation20/webdocs/markdowndocs/connector-java-ch.md b/documentation20/webdocs/markdowndocs/connector-java-ch.md index b8390e7af5..3d067ad206 100644 --- a/documentation20/webdocs/markdowndocs/connector-java-ch.md +++ b/documentation20/webdocs/markdowndocs/connector-java-ch.md @@ -2,7 +2,7 @@ TDengine 提供了遵循 JDBC 标准（3.0）API 规范的 `taos-jdbcdriver` 实现，可在 maven 的中央仓库 [Sonatype Repository][1] 搜索下载。 -`taos-jdbcdriver` 的实现包括 2 种形式： JDBC-JNI 和 JDBC-RESTful（taos-jdbcdriver-2.0.17 开始支持 JDBC-RESTful）。 JDBC-JNI 通过调用客户端 libtaos.so（或 taos.dll ）的本地方法实现， JDBC-RESTful 则在内部封装了 RESTful 接口实现。 +`taos-jdbcdriver` 的实现包括 2 种形式： JDBC-JNI 和 JDBC-RESTful（taos-jdbcdriver-2.0.18 开始支持 JDBC-RESTful）。 JDBC-JNI 通过调用客户端 libtaos.so（或 taos.dll ）的本地方法实现， JDBC-RESTful 则在内部封装了 RESTful 接口实现。 ![tdengine-connector](../assets/tdengine-jdbc-connector.png) @@ -67,7 +67,7 @@ maven 项目中使用如下 pom.xml 配置即可： com.taosdata.jdbc taos-jdbcdriver - 2.0.17 + 2.0.18 ``` From 7e1b6b610d63082c2d07d116a8255228c056f4ed Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 26 Jan 2021 13:07:27 +0800 Subject: [PATCH 080/147] [TD-2793]: fix -p option of taos.exe MUST have a space between -p and is_use_passwd = true; + if (i < argc - 1 && argv[i + 1][0] != '-') { + arguments->password = argv[++i]; + } } // for management port else if (strcmp(argv[i], "-P") == 0) { From 16069460f1fbf09802072d6d958c29605307ed66 Mon Sep 17 00:00:00 2001 From: zyyang Date: Tue, 26 Jan 2021 14:03:06 +0800 Subject: [PATCH 081/147] [TD-2672]: optimized the JDBC test cases --- .../taosdata/jdbc/TSDBPreparedStatement.java | 4 +- .../java/com/taosdata/jdbc/TSDBStatement.java | 10 +- .../taosdata/jdbc/PreparedStatementTest.java | 50 ++--- .../java/com/taosdata/jdbc/SubscribeTest.java | 6 +- .../jdbc/TSDBPreparedStatementTest.java | 186 ++++++++++++++++++ 5 files changed, 217 insertions(+), 39 deletions(-) create mode 100644 src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java index 230943fd53..c6b41ce004 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java @@ -100,7 +100,6 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat * order to process those supported SQLs. */ private void preprocessSql() { - /***** For processing some of Spark SQLs*****/ // should replace it first this.rawSql = this.rawSql.replaceAll("or (.*) is null", ""); @@ -149,7 +148,6 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat rawSql = rawSql.replace(matcher.group(1), tableFullName); } /***** for inner queries *****/ - } /** @@ -196,7 +194,7 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat @Override public void setNull(int parameterIndex, int sqlType) throws SQLException { - setObject(parameterIndex, new String("NULL")); + setObject(parameterIndex, "NULL"); } @Override diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java index 381f1d3622..4c900b5fdc 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java @@ -52,12 +52,18 @@ public class TSDBStatement implements Statement { this.isClosed = false; } + @Override public T unwrap(Class iface) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + try { + return iface.cast(this); + } catch (ClassCastException cce) { + throw new SQLException("Unable to unwrap to " + iface.toString()); + } } + @Override public boolean isWrapperFor(Class iface) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + return iface.isInstance(this); } public ResultSet executeQuery(String sql) throws SQLException { diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/PreparedStatementTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/PreparedStatementTest.java index f52c50ff1a..a49ad05d05 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/PreparedStatementTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/PreparedStatementTest.java @@ -1,9 +1,7 @@ package com.taosdata.jdbc; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.FixMethodOrder; -import org.junit.Test; +import org.junit.*; +import org.junit.runners.MethodSorters; import java.sql.*; import java.util.Properties; @@ -11,14 +9,13 @@ import java.util.Properties; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -@FixMethodOrder() -public class PreparedStatementTest extends BaseTest { - static Connection connection = null; - static PreparedStatement statement = null; +@FixMethodOrder(value = MethodSorters.NAME_ASCENDING) +public class PreparedStatementTest { + static Connection connection; + static TSDBPreparedStatement statement; static String dbName = "test"; static String tName = "t0"; static String host = "localhost"; - static ResultSet resSet = null; @BeforeClass public static void createConnection() throws SQLException { @@ -28,19 +25,16 @@ public class PreparedStatementTest extends BaseTest { return; } Properties properties = new Properties(); - properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host); properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); - String sql = "drop database if exists " + dbName; statement = (TSDBPreparedStatement) connection.prepareStatement(sql); - } @Test - public void createTableAndQuery() throws SQLException { + public void case001_createTableAndQuery() throws SQLException { long ts = System.currentTimeMillis(); statement.executeUpdate("create database if not exists " + dbName); @@ -48,47 +42,40 @@ public class PreparedStatementTest extends BaseTest { statement.executeUpdate("insert into " + dbName + "." + tName + " values (" + ts + ", 1)"); PreparedStatement selectStatement = connection.prepareStatement("select * from " + dbName + "." + tName); - ResultSet resultSet = selectStatement.executeQuery(); assertTrue(null != resultSet); boolean isClosed = statement.isClosed(); assertEquals(false, isClosed); + selectStatement.close(); } @Test - public void testPreparedStatement() throws SQLException { + public void case002_testPreparedStatement() throws SQLException { long ts = System.currentTimeMillis() + 20000; - PreparedStatement saveStatement = connection - .prepareStatement("insert into " + dbName + "." + tName + " values (" + ts + ", 1)"); + PreparedStatement saveStatement = connection.prepareStatement("insert into " + dbName + "." + tName + " values (" + ts + ", 1)"); int affectedRows = saveStatement.executeUpdate(); assertTrue(1 == affectedRows); + saveStatement.close(); } @Test - public void testSavedPreparedStatement() throws SQLException { + public void case003_testSavedPreparedStatement() throws SQLException { long ts = System.currentTimeMillis(); - - TSDBPreparedStatement saveStatement = (TSDBPreparedStatement) connection - .prepareStatement("insert into " + dbName + "." + tName + " values (?, ?)"); - + TSDBPreparedStatement saveStatement = (TSDBPreparedStatement) connection.prepareStatement("insert into " + dbName + "." + tName + " values (?, ?)"); saveStatement.setObject(1, ts + 10000); saveStatement.setObject(2, 3); int rows = saveStatement.executeUpdate(); assertEquals(1, rows); + saveStatement.close(); } @Test - public void testUnsupport() { - // if(null == resSet) { - // return; - // } - TSDBPreparedStatement tsdbStatement = (TSDBPreparedStatement) statement; - try { - tsdbStatement.unwrap(null); - } catch (SQLException e) { - } + public void case004_testUnsupport() throws SQLException { + TSDBPreparedStatement tsdbStatement = statement; + + Assert.assertNotNull(tsdbStatement.unwrap(TSDBPreparedStatement.class)); try { tsdbStatement.isWrapperFor(null); } catch (SQLException e) { @@ -180,6 +167,7 @@ public class PreparedStatementTest extends BaseTest { try { tsdbStatement.closeOnCompletion(); } catch (SQLException e) { + } try { tsdbStatement.isCloseOnCompletion(); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java index 0a71c77d1d..1b09f6d502 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java @@ -10,9 +10,9 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.Properties; -public class SubscribeTest extends BaseTest { - Connection connection = null; - Statement statement = null; +public class SubscribeTest { + Connection connection; + Statement statement; String dbName = "test"; String tName = "t0"; String host = "localhost"; diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java new file mode 100644 index 0000000000..0e4ed67bf9 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java @@ -0,0 +1,186 @@ +package com.taosdata.jdbc; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; + +public class TSDBPreparedStatementTest { + private static final String host = "127.0.0.1"; + private static Connection conn; + + @Test + public void executeQuery() { + + } + + @Test + public void executeUpdate() { + + } + + @Test + public void setNull() { + } + + @Test + public void setBoolean() { + } + + @Test + public void setByte() { + } + + @Test + public void setShort() { + } + + @Test + public void setInt() { + } + + @Test + public void setLong() { + } + + @Test + public void setFloat() { + } + + @Test + public void setDouble() { + } + + @Test + public void setBigDecimal() { + } + + @Test + public void setString() { + } + + @Test + public void setBytes() { + } + + @Test + public void setDate() { + } + + @Test + public void setTime() { + } + + @Test + public void setTimestamp() { + } + + @Test + public void setAsciiStream() { + } + + @Test + public void setUnicodeStream() { + } + + @Test + public void setBinaryStream() { + } + + @Test + public void clearParameters() { + } + + @Test + public void setObject() { + + } + + @Test + public void execute() { + } + + @Test + public void addBatch() { + + } + + @Test + public void setCharacterStream() { + } + + @Test + public void setRef() { + } + + @Test + public void setBlob() { + } + + @Test + public void setClob() { + } + + @Test + public void setArray() { + } + + @Test + public void getMetaData() { + } + + @Test + public void setURL() { + } + + @Test + public void getParameterMetaData() { + } + + @Test + public void setRowId() { + } + + @Test + public void setNString() { + } + + @Test + public void setNCharacterStream() { + } + + @Test + public void setNClob() { + + } + + @Test + public void setSQLXML() { + + } + + + @BeforeClass + public static void beforeClass() { + try { + Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); + conn = DriverManager.getConnection("jdbc:TAOS://" + host + ":6030/jdbc_test?user=root&password=taosdata"); + } catch (ClassNotFoundException | SQLException e) { + e.printStackTrace(); + } + } + + @AfterClass + public static void afterClass() { + try { + if (conn != null) + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + +} \ No newline at end of file From c96e0ea42f31bc0c589a2836cb187eec2fb6ddf8 Mon Sep 17 00:00:00 2001 From: zyyang Date: Tue, 26 Jan 2021 14:10:19 +0800 Subject: [PATCH 082/147] change --- .../taosdata/jdbc/PreparedStatementTest.java | 55 +++++++++---------- 1 file changed, 26 insertions(+), 29 deletions(-) diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/PreparedStatementTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/PreparedStatementTest.java index a49ad05d05..8c9b258dcd 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/PreparedStatementTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/PreparedStatementTest.java @@ -73,104 +73,101 @@ public class PreparedStatementTest { @Test public void case004_testUnsupport() throws SQLException { - TSDBPreparedStatement tsdbStatement = statement; - Assert.assertNotNull(tsdbStatement.unwrap(TSDBPreparedStatement.class)); + Assert.assertNotNull(statement.unwrap(TSDBPreparedStatement.class)); + Assert.assertTrue(statement.isWrapperFor(TSDBPreparedStatement.class)); + try { - tsdbStatement.isWrapperFor(null); + statement.getMaxFieldSize(); } catch (SQLException e) { } try { - tsdbStatement.getMaxFieldSize(); + statement.setMaxFieldSize(0); } catch (SQLException e) { } try { - tsdbStatement.setMaxFieldSize(0); + statement.setEscapeProcessing(true); } catch (SQLException e) { } try { - tsdbStatement.setEscapeProcessing(true); + statement.cancel(); } catch (SQLException e) { } try { - tsdbStatement.cancel(); + statement.getWarnings(); } catch (SQLException e) { } try { - tsdbStatement.getWarnings(); + statement.clearWarnings(); } catch (SQLException e) { } try { - tsdbStatement.clearWarnings(); + statement.setCursorName(null); } catch (SQLException e) { } try { - tsdbStatement.setCursorName(null); + statement.getMoreResults(); } catch (SQLException e) { } try { - tsdbStatement.getMoreResults(); + statement.setFetchDirection(0); } catch (SQLException e) { } try { - tsdbStatement.setFetchDirection(0); + statement.getFetchDirection(); } catch (SQLException e) { } try { - tsdbStatement.getFetchDirection(); + statement.getResultSetConcurrency(); } catch (SQLException e) { } try { - tsdbStatement.getResultSetConcurrency(); + statement.getResultSetType(); } catch (SQLException e) { } try { - tsdbStatement.getResultSetType(); + statement.getConnection(); } catch (SQLException e) { } try { - tsdbStatement.getConnection(); + statement.getMoreResults(); } catch (SQLException e) { } try { - tsdbStatement.getMoreResults(); + statement.getGeneratedKeys(); } catch (SQLException e) { } try { - tsdbStatement.getGeneratedKeys(); + statement.executeUpdate(null, 0); } catch (SQLException e) { } try { - tsdbStatement.executeUpdate(null, 0); + statement.executeUpdate(null, new int[]{0}); } catch (SQLException e) { } try { - tsdbStatement.executeUpdate(null, new int[]{0}); + statement.executeUpdate(null, new String[]{"str1", "str2"}); } catch (SQLException e) { } try { - tsdbStatement.executeUpdate(null, new String[]{"str1", "str2"}); + statement.getResultSetHoldability(); } catch (SQLException e) { } try { - tsdbStatement.getResultSetHoldability(); + statement.setPoolable(true); } catch (SQLException e) { } try { - tsdbStatement.setPoolable(true); + statement.isPoolable(); } catch (SQLException e) { } try { - tsdbStatement.isPoolable(); - } catch (SQLException e) { - } - try { - tsdbStatement.closeOnCompletion(); + statement.closeOnCompletion(); } catch (SQLException e) { } try { - tsdbStatement.isCloseOnCompletion(); + statement.isCloseOnCompletion(); } catch (SQLException e) { } } From 8630797f46ec2702d436f947d74a31f112b51c4e Mon Sep 17 00:00:00 2001 From: zyyang Date: Tue, 26 Jan 2021 14:27:13 +0800 Subject: [PATCH 083/147] change --- .../java/com/taosdata/jdbc/QueryDataTest.java | 83 +++++++++---------- 1 file changed, 37 insertions(+), 46 deletions(-) diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/QueryDataTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/QueryDataTest.java index 611e21e887..e6b2eda271 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/QueryDataTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/QueryDataTest.java @@ -6,76 +6,67 @@ import org.junit.Test; import java.sql.*; import java.util.Properties; -import java.util.Random; import static org.junit.Assert.assertEquals; -import java.util.Properties; -import java.util.concurrent.Executors; -import java.util.concurrent.*; -import static org.junit.Assert.assertTrue; +public class QueryDataTest { -public class QueryDataTest extends BaseTest { - - static Connection connection = null; - static Statement statement = null; + static Connection connection; + static Statement statement; static String dbName = "test"; static String stbName = "meters"; - static String host = "localhost"; - static int numOfTables = 30; - final static int numOfRecordsPerTable = 1000; - static long ts = 1496732686000l; - final static String tablePrefix = "t"; + static String host = "127.0.0.1"; @Before - public void createDatabase() throws SQLException { + public void createDatabase() { try { Class.forName("com.taosdata.jdbc.TSDBDriver"); - } catch (ClassNotFoundException e) { + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); + + statement = connection.createStatement(); + statement.executeUpdate("drop database if exists " + dbName); + statement.executeUpdate("create database if not exists " + dbName); + statement.executeUpdate("use " + dbName); + + String createTableSql = "create table " + stbName + "(ts timestamp, name binary(64))"; + statement.executeUpdate(createTableSql); + + } catch (ClassNotFoundException | SQLException e) { return; } - - Properties properties = new Properties(); - properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host); - properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); - - statement = connection.createStatement(); - statement.executeUpdate("drop database if exists " + dbName); - statement.executeUpdate("create database if not exists " + dbName); - statement.executeUpdate("use " + dbName); - - String createTableSql = "create table " + stbName + "(ts timestamp, name binary(6))"; - statement.executeUpdate(createTableSql); } - - @Test - public void testQueryBinaryData() throws SQLException{ - - String insertSql = "insert into " + stbName + " values(now, 'taosda')"; - System.out.println(insertSql); + @Test + public void testQueryBinaryData() throws SQLException { + String insertSql = "insert into " + stbName + " values(now, 'taosdata')"; + System.out.println(insertSql); statement.executeUpdate(insertSql); String querySql = "select * from " + stbName; - ResultSet rs = statement.executeQuery(querySql); + ResultSet rs = statement.executeQuery(querySql); - while(rs.next()) { + while (rs.next()) { String name = rs.getString(2) + "001"; System.out.println("name = " + name); assertEquals(name, "taosda001"); } - rs.close(); + rs.close(); } - @After - public void close() throws Exception { - statement.close(); - connection.close(); - Thread.sleep(10); + public void close() { + try { + if (statement != null) + statement.close(); + if (connection != null) + connection.close(); + } catch (SQLException e) { + e.printStackTrace(); + } } - + } \ No newline at end of file From 42c802bda2fc912657bfaf0baf02a9d9e8ec118b Mon Sep 17 00:00:00 2001 From: zyyang Date: Tue, 26 Jan 2021 14:30:29 +0800 Subject: [PATCH 084/147] change --- .../jdbc/src/test/java/com/taosdata/jdbc/QueryDataTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/QueryDataTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/QueryDataTest.java index e6b2eda271..37fbc28487 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/QueryDataTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/QueryDataTest.java @@ -50,9 +50,9 @@ public class QueryDataTest { ResultSet rs = statement.executeQuery(querySql); while (rs.next()) { - String name = rs.getString(2) + "001"; + String name = rs.getString(2); System.out.println("name = " + name); - assertEquals(name, "taosda001"); + assertEquals("taosdata", name); } rs.close(); } From 5d0605b933dfd37f18f1b46119e2ab28727e097a Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 26 Jan 2021 14:42:04 +0800 Subject: [PATCH 085/147] [TD-109] : support mac compile with xcode11.4 in travis CI. redefine MSG_NOSIGNAL --- src/os/inc/osDarwin.h | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/os/inc/osDarwin.h b/src/os/inc/osDarwin.h index 13f93456f2..14b8ccf53c 100644 --- a/src/os/inc/osDarwin.h +++ b/src/os/inc/osDarwin.h @@ -91,7 +91,12 @@ extern "C" { typedef int(*__compar_fn_t)(const void *, const void *); // for send function in tsocket.c -// #define MSG_NOSIGNAL 0 +#if defined(MSG_NOSIGNAL) +#undef MSG_NOSIGNAL +#endif + +#define MSG_NOSIGNAL 0 + #define SO_NO_CHECK 0x1234 #define SOL_TCP 0x1234 #define TCP_KEEPIDLE 0x1234 From efa75445a34c37e2c9ab6d30cb04574ef054bf1d Mon Sep 17 00:00:00 2001 From: zyyang Date: Tue, 26 Jan 2021 14:43:56 +0800 Subject: [PATCH 086/147] change --- .../test/java/com/taosdata/jdbc/BaseTest.java | 33 --------- .../java/com/taosdata/jdbc/ResultSetTest.java | 56 +++++++-------- .../java/com/taosdata/jdbc/StatementTest.java | 33 ++++----- .../java/com/taosdata/jdbc/SubscribeTest.java | 53 ++++++++------- .../jdbc/TSDBDatabaseMetaDataTest.java | 61 +++++++++-------- .../com/taosdata/jdbc/TSDBDriverTest.java | 68 +++++-------------- .../taosdata/jdbc/cases/BatchInsertTest.java | 45 ++++++++---- .../com/taosdata/jdbc/lib/TSDBCommon.java | 47 ------------- 8 files changed, 152 insertions(+), 244 deletions(-) delete mode 100644 src/connector/jdbc/src/test/java/com/taosdata/jdbc/BaseTest.java delete mode 100644 src/connector/jdbc/src/test/java/com/taosdata/jdbc/lib/TSDBCommon.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/BaseTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/BaseTest.java deleted file mode 100644 index ce3735c128..0000000000 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/BaseTest.java +++ /dev/null @@ -1,33 +0,0 @@ -package com.taosdata.jdbc; - -import com.taosdata.jdbc.utils.TDNodes; - -import org.junit.AfterClass; -import org.junit.BeforeClass; - -public abstract class BaseTest { - - private static boolean testCluster = false; - private static TDNodes nodes = new TDNodes(); - - @BeforeClass - public static void setupEnv() { - try { - if (nodes.getTDNode(1).getTaosdPid() != null) { - System.out.println("Kill taosd before running JDBC test"); - nodes.getTDNode(1).setRunning(1); - nodes.stop(1); - } - nodes.setTestCluster(testCluster); - nodes.deploy(1); - nodes.start(1); - } catch (Exception e) { - e.printStackTrace(); - } - } - - @AfterClass - public static void cleanUpEnv() { - nodes.stop(1); - } -} \ No newline at end of file diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ResultSetTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ResultSetTest.java index 8067c547df..b70bcd3c23 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ResultSetTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ResultSetTest.java @@ -13,42 +13,37 @@ import java.util.Properties; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -public class ResultSetTest extends BaseTest { - static Connection connection = null; - static Statement statement = null; +public class ResultSetTest { + static Connection connection; + static Statement statement; static String dbName = "test"; static String tName = "t0"; static String host = "localhost"; - static ResultSet resSet = null; + static ResultSet resSet; @BeforeClass - public static void createDatabaseAndTable() throws SQLException { + public static void createDatabaseAndTable() { try { Class.forName("com.taosdata.jdbc.TSDBDriver"); - } catch (ClassNotFoundException e) { + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); + statement = connection.createStatement(); + statement.executeUpdate("drop database if exists " + dbName); + statement.executeUpdate("create database if not exists " + dbName); + statement.execute("use " + dbName); + statement.executeUpdate("create table if not exists " + dbName + "." + tName + " (ts timestamp, k1 int, k2 bigint, k3 float, k4 double, k5 binary(30), k6 smallint, k7 bool, k8 nchar(20))"); + } catch (ClassNotFoundException | SQLException e) { return; } - Properties properties = new Properties(); - properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host); - properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); - - statement = connection.createStatement(); - statement.executeUpdate("drop database if exists " + dbName); - statement.executeUpdate("create database if not exists " + dbName); - statement.executeUpdate("create table if not exists " + dbName + "." + tName + - " (ts timestamp, k1 int, k2 bigint, k3 float, k4 double, k5 binary(30), k6 smallint, k7 bool, k8 nchar(20))"); - - statement.executeQuery("use " + dbName); } @Test public void testResultSet() { - String sql = null; + String sql; long ts = 1496732686000l; int v1 = 2147483600; long v2 = ts + 1000; @@ -815,13 +810,18 @@ public class ResultSetTest extends BaseTest { assertEquals(res.length, 2); statement.clearBatch(); } - @AfterClass - public static void close() throws Exception { - statement.executeUpdate("drop database " + dbName); - statement.close(); - connection.close(); - Thread.sleep(10); + @AfterClass + public static void close() { + try { + statement.executeUpdate("drop database " + dbName); + if (statement != null) + statement.close(); + if (connection != null) + connection.close(); + } catch (SQLException e) { + e.printStackTrace(); + } } } diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StatementTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StatementTest.java index a79512984e..f78f899294 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StatementTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StatementTest.java @@ -16,23 +16,22 @@ public class StatementTest { static String dbName = "test"; static String tName = "t0"; static String host = "localhost"; - static ResultSet resSet = null; @BeforeClass public static void createConnection() throws SQLException { try { Class.forName("com.taosdata.jdbc.TSDBDriver"); + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/?user=root&password=taosdata", properties); + statement = connection.createStatement(); + statement.executeUpdate("drop database if exists " + dbName); + } catch (ClassNotFoundException e) { return; } - Properties properties = new Properties(); - properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/?user=root&password=taosdata", properties); - - statement = connection.createStatement(); - statement.executeUpdate("drop database if exists " + dbName); } @Test @@ -49,7 +48,6 @@ public class StatementTest { } catch (SQLException e) { e.printStackTrace(); } - } @Test @@ -173,12 +171,15 @@ public class StatementTest { } @AfterClass - public static void close() throws Exception { - if (!statement.isClosed()) { - statement.executeUpdate("drop database if exists " + dbName); - statement.close(); - connection.close(); - Thread.sleep(10); + public static void close() { + try { + statement.execute("drop database if exists " + dbName); + if (statement != null) + statement.close(); + if (connection != null) + connection.close(); + } catch (SQLException e) { + e.printStackTrace(); } } } diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java index 1b09f6d502..1d8ff08db6 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java @@ -19,27 +19,28 @@ public class SubscribeTest { String topic = "test"; @Before - public void createDatabase() throws SQLException { + public void createDatabase() { try { Class.forName("com.taosdata.jdbc.TSDBDriver"); - } catch (ClassNotFoundException e) { - return; - } - Properties properties = new Properties(); - properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host); - properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); - statement = connection.createStatement(); - statement.executeUpdate("create database if not exists " + dbName); - statement.executeUpdate("create table if not exists " + dbName + "." + tName + " (ts timestamp, k int, v int)"); - long ts = System.currentTimeMillis(); - for (int i = 0; i < 2; i++) { - ts += i; - String sql = "insert into " + dbName + "." + tName + " values (" + ts + ", " + (100 + i) + ", " + i + ")"; - statement.executeUpdate(sql); + statement = connection.createStatement(); + statement.executeUpdate("create database if not exists " + dbName); + statement.executeUpdate("create table if not exists " + dbName + "." + tName + " (ts timestamp, k int, v int)"); + long ts = System.currentTimeMillis(); + for (int i = 0; i < 2; i++) { + ts += i; + String sql = "insert into " + dbName + "." + tName + " values (" + ts + ", " + (100 + i) + ", " + i + ")"; + statement.executeUpdate(sql); + } + + } catch (ClassNotFoundException | SQLException e) { + return; } } @@ -79,10 +80,16 @@ public class SubscribeTest { } @After - public void close() throws Exception { - statement.executeQuery("drop database " + dbName); - statement.close(); - connection.close(); - Thread.sleep(10); + public void close() { + try { + statement.executeQuery("drop database " + dbName); + if (statement != null) + statement.close(); + if (connection != null) + connection.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } } \ No newline at end of file diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDatabaseMetaDataTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDatabaseMetaDataTest.java index 8066b38573..e3f0d1b8b2 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDatabaseMetaDataTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDatabaseMetaDataTest.java @@ -6,37 +6,9 @@ import java.sql.*; import java.util.Properties; public class TSDBDatabaseMetaDataTest { - private TSDBDatabaseMetaData metaData; private static final String host = "127.0.0.1"; - private Connection connection; - - @BeforeClass - public void before() { - try { - Class.forName("com.taosdata.jdbc.TSDBDriver"); - Properties properties = new Properties(); - properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host); - properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata", properties); - metaData = connection.getMetaData().unwrap(TSDBDatabaseMetaData.class); - } catch (ClassNotFoundException e) { - e.printStackTrace(); - } catch (SQLException e) { - e.printStackTrace(); - } - } - - @AfterClass - public void after() { - try { - if (connection != null) - connection.close(); - } catch (SQLException e) { - e.printStackTrace(); - } - } + private static Connection connection; + private static TSDBDatabaseMetaData metaData; @Test public void unwrap() throws SQLException { @@ -975,4 +947,33 @@ public class TSDBDatabaseMetaDataTest { public void generatedKeyAlwaysReturned() throws SQLException { Assert.assertFalse(metaData.generatedKeyAlwaysReturned()); } + + @BeforeClass + public static void before() { + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata", properties); + metaData = connection.getMetaData().unwrap(TSDBDatabaseMetaData.class); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @AfterClass + public static void after() { + try { + if (connection != null) + connection.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + } \ No newline at end of file diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java index 8adcdefb29..445723f501 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java @@ -3,9 +3,6 @@ package com.taosdata.jdbc; import org.junit.BeforeClass; import org.junit.Test; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; import java.sql.*; import java.util.Properties; @@ -27,54 +24,15 @@ public class TSDBDriverTest { "jdbc:TAOS://:/test", "jdbc:TAOS://localhost:0/?user=root&password=taosdata" }; - private static boolean islibLoaded = false; - private static boolean isTaosdActived; private Connection conn; - @BeforeClass - public static void before() { - String osName = System.getProperty("os.name").toLowerCase(); - if (!osName.equals("linux")) - return; - // try to load taos lib - try { - System.loadLibrary("taos"); - islibLoaded = true; - } catch (UnsatisfiedLinkError error) { - System.out.println("load tdengine lib failed."); - error.printStackTrace(); - } - // check taosd is activated - try { - String[] cmd = {"/bin/bash", "-c", "ps -ef | grep taosd | grep -v \"grep\""}; - Process exec = Runtime.getRuntime().exec(cmd); - BufferedReader reader = new BufferedReader(new InputStreamReader(exec.getInputStream())); - int lineCnt = 0; - while (reader.readLine() != null) { - lineCnt++; - } - if (lineCnt > 0) - isTaosdActived = true; - } catch (IOException e) { - e.printStackTrace(); - } - - try { - Class.forName("com.taosdata.jdbc.TSDBDriver"); - } catch (ClassNotFoundException e) { - e.printStackTrace(); - } - } - @Test public void testConnectWithJdbcURL() { final String url = "jdbc:TAOS://localhost:6030/log?user=root&password=taosdata"; try { - if (islibLoaded && isTaosdActived) { - conn = DriverManager.getConnection(url); - assertNotNull("failure - connection should not be null", conn); - } + conn = DriverManager.getConnection(url); + assertNotNull("failure - connection should not be null", conn); } catch (SQLException e) { e.printStackTrace(); fail("failure - should not throw Exception"); @@ -89,10 +47,8 @@ public class TSDBDriverTest { connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); try { - if (islibLoaded && isTaosdActived) { - conn = DriverManager.getConnection(jdbcUrl, connProps); - assertNotNull("failure - connection should not be null", conn); - } + conn = DriverManager.getConnection(jdbcUrl, connProps); + assertNotNull("failure - connection should not be null", conn); } catch (SQLException e) { e.printStackTrace(); fail("failure - should not throw Exception"); @@ -107,10 +63,8 @@ public class TSDBDriverTest { connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); try { - if (islibLoaded && isTaosdActived) { - conn = DriverManager.getConnection(jdbcUrl, connProps); - assertNotNull("failure - connection should not be null", conn); - } + conn = DriverManager.getConnection(jdbcUrl, connProps); + assertNotNull("failure - connection should not be null", conn); } catch (SQLException e) { e.printStackTrace(); fail("failure - should not throw Exception"); @@ -207,4 +161,14 @@ public class TSDBDriverTest { assertNull("failure - getParentLogger should be be null", new TSDBDriver().getParentLogger()); } + @BeforeClass + public static void before() { + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } + } + + } \ No newline at end of file diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchInsertTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchInsertTest.java index 9608c4985d..66749d057c 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchInsertTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchInsertTest.java @@ -1,14 +1,13 @@ package com.taosdata.jdbc.cases; +import com.taosdata.jdbc.TSDBDriver; import com.taosdata.jdbc.lib.TSDBCommon; import org.junit.After; import org.junit.Before; import org.junit.Test; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; +import java.sql.*; +import java.util.Properties; import java.util.Random; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -18,30 +17,47 @@ import static org.junit.Assert.assertEquals; public class BatchInsertTest { - static String host = "localhost"; + static String host = "127.0.0.1"; static String dbName = "test"; static String stbName = "meters"; static int numOfTables = 30; final static int numOfRecordsPerTable = 1000; static long ts = 1496732686000l; final static String tablePrefix = "t"; - private Connection connection; @Before public void before() { try { - connection = TSDBCommon.getConn(host); - TSDBCommon.createDatabase(connection, dbName); - TSDBCommon.createStable(connection, stbName); - TSDBCommon.createTables(connection, numOfTables, stbName, tablePrefix); + Class.forName("com.taosdata.jdbc.TSDBDriver"); + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); + + Statement statement = connection.createStatement(); + statement.executeUpdate("drop database if exists " + dbName); + statement.executeUpdate("create database if not exists " + dbName); + statement.executeUpdate("use " + dbName); + // create stable + String createTableSql = "create table " + stbName + "(ts timestamp, f1 int, f2 int, f3 int) tags(areaid int, loc binary(20))"; + statement.executeUpdate(createTableSql); + // create tables + for(int i = 0; i < numOfTables; i++) { + String loc = i % 2 == 0 ? "beijing" : "shanghai"; + String createSubTalbesSql = "create table " + tablePrefix + i + " using " + stbName + " tags(" + i + ", '" + loc + "')"; + statement.executeUpdate(createSubTalbesSql); + } + statement.close(); } catch (Exception e) { e.printStackTrace(); } } @Test - public void testBatchInsert(){ + public void testBatchInsert() { ExecutorService executorService = Executors.newFixedThreadPool(numOfTables); for (int i = 0; i < numOfTables; i++) { final int index = i; @@ -63,7 +79,7 @@ public class BatchInsertTest { statement.addBatch(sb.toString()); statement.executeBatch(); long endTime = System.currentTimeMillis(); - System.out.println("Thread " + index + " takes " + (endTime - startTime) + " microseconds"); + System.out.println("Thread " + index + " takes " + (endTime - startTime) + " microseconds"); connection.commit(); statement.close(); } catch (Exception e) { @@ -80,7 +96,7 @@ public class BatchInsertTest { e.printStackTrace(); } - try{ + try { Statement statement = connection.createStatement(); ResultSet rs = statement.executeQuery("select * from meters"); int num = 0; @@ -89,7 +105,7 @@ public class BatchInsertTest { } assertEquals(num, numOfTables * numOfRecordsPerTable); rs.close(); - }catch (Exception e){ + } catch (Exception e) { e.printStackTrace(); } } @@ -102,7 +118,6 @@ public class BatchInsertTest { } catch (SQLException e) { e.printStackTrace(); } - } } diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/lib/TSDBCommon.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/lib/TSDBCommon.java deleted file mode 100644 index 0e2613d617..0000000000 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/lib/TSDBCommon.java +++ /dev/null @@ -1,47 +0,0 @@ -package com.taosdata.jdbc.lib; - -import com.taosdata.jdbc.TSDBDriver; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.Properties; - -public class TSDBCommon { - - public static Connection getConn(String host) throws SQLException, ClassNotFoundException { - Class.forName("com.taosdata.jdbc.TSDBDriver"); - Properties properties = new Properties(); - properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host); - properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - return DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); - } - - public static void createDatabase(Connection connection, String dbName) throws SQLException { - Statement statement = connection.createStatement(); - statement.executeUpdate("drop database if exists " + dbName); - statement.executeUpdate("create database if not exists " + dbName); - statement.executeUpdate("use " + dbName); - statement.close(); - } - - public static void createStable(Connection connection, String stbName) throws SQLException { - Statement statement = connection.createStatement(); - String createTableSql = "create table " + stbName + "(ts timestamp, f1 int, f2 int, f3 int) tags(areaid int, loc binary(20))"; - statement.executeUpdate(createTableSql); - statement.close(); - } - - public static void createTables(Connection connection, int numOfTables, String stbName,String tablePrefix) throws SQLException { - Statement statement = connection.createStatement(); - for(int i = 0; i < numOfTables; i++) { - String loc = i % 2 == 0 ? "beijing" : "shanghai"; - String createSubTalbesSql = "create table " + tablePrefix + i + " using " + stbName + " tags(" + i + ", '" + loc + "')"; - statement.executeUpdate(createSubTalbesSql); - } - statement.close(); - } -} From 8678883050502c3f3c0b4183543b12d0a7b89f56 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 26 Jan 2021 14:44:15 +0800 Subject: [PATCH 087/147] [TD-109] : support mac compile with xcode11.4 in travis CI. enable in .travis.yml --- .travis.yml | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9aac6c597c..d814a465e6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -270,19 +270,20 @@ matrix: fi - make > /dev/null -# - os: osx -# language: c -# compiler: clang -# env: DESC="mac/clang build" -# git: -# - depth: 1 -# addons: -# homebrew: -# - cmake -# -# script: -# - cd ${TRAVIS_BUILD_DIR} -# - mkdir debug -# - cd debug -# - cmake .. > /dev/null -# - make > /dev/null + - os: osx + osx_image: xcode11.4 + language: c + compiler: clang + env: DESC="mac/clang build" + git: + - depth: 1 + addons: + homebrew: + - cmake + + script: + - cd ${TRAVIS_BUILD_DIR} + - mkdir debug + - cd debug + - cmake .. > /dev/null + - make > /dev/null From 6942ddcf4fab863c5d7367b944ec55d78a8c6d66 Mon Sep 17 00:00:00 2001 From: zyyang Date: Tue, 26 Jan 2021 14:45:02 +0800 Subject: [PATCH 088/147] change --- .../src/test/java/com/taosdata/jdbc/cases/BatchInsertTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchInsertTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchInsertTest.java index 66749d057c..9f54e4b1ba 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchInsertTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchInsertTest.java @@ -1,7 +1,6 @@ package com.taosdata.jdbc.cases; import com.taosdata.jdbc.TSDBDriver; -import com.taosdata.jdbc.lib.TSDBCommon; import org.junit.After; import org.junit.Before; import org.junit.Test; From 542dd5d4954af45d851d082556ff811a8ae06ede Mon Sep 17 00:00:00 2001 From: zyyang Date: Tue, 26 Jan 2021 14:50:59 +0800 Subject: [PATCH 089/147] change --- .../test/java/com/taosdata/jdbc/TSDBDatabaseMetaDataTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDatabaseMetaDataTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDatabaseMetaDataTest.java index e3f0d1b8b2..9bb22ad577 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDatabaseMetaDataTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDatabaseMetaDataTest.java @@ -33,7 +33,7 @@ public class TSDBDatabaseMetaDataTest { @Test public void getURL() throws SQLException { - Assert.assertEquals("jdbc:TAOS://localhost:6030/?user=root&password=taosdata", metaData.getURL()); + Assert.assertEquals("jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata", metaData.getURL()); } @Test From ea4f3c9f1a2d9839907b96a713d2ae64196be53e Mon Sep 17 00:00:00 2001 From: zyyang Date: Tue, 26 Jan 2021 15:06:49 +0800 Subject: [PATCH 090/147] change --- src/connector/jdbc/pom.xml | 2 +- .../com/taosdata/jdbc/BatchInsertTest.java | 115 ------------------ .../jdbc/TSDBDatabaseMetaDataTest.java | 5 +- .../taosdata/jdbc/cases/BatchInsertTest.java | 43 +++---- 4 files changed, 23 insertions(+), 142 deletions(-) delete mode 100644 src/connector/jdbc/src/test/java/com/taosdata/jdbc/BatchInsertTest.java diff --git a/src/connector/jdbc/pom.xml b/src/connector/jdbc/pom.xml index 6be0ca036e..1fb2c69524 100755 --- a/src/connector/jdbc/pom.xml +++ b/src/connector/jdbc/pom.xml @@ -126,7 +126,7 @@ **/*Test.java - **/BatchInsertTest.java + **/AppMemoryLeakTest.java **/FailOverTest.java true diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/BatchInsertTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/BatchInsertTest.java deleted file mode 100644 index 4046e3edf6..0000000000 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/BatchInsertTest.java +++ /dev/null @@ -1,115 +0,0 @@ -package com.taosdata.jdbc; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.sql.*; -import java.util.Properties; -import java.util.Random; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; - -import static org.junit.Assert.assertEquals; - -public class BatchInsertTest { - - private Connection connection; - - private static String dbName = "test"; - private static String stbName = "meters"; - private static String host = "127.0.0.1"; - private static int numOfTables = 30; - private static int numOfRecordsPerTable = 1000; - private static long ts = 1496732686000l; - private static String tablePrefix = "t"; - - @Before - public void createDatabase() throws SQLException { - try { - Class.forName("com.taosdata.jdbc.TSDBDriver"); - } catch (ClassNotFoundException e) { - return; - } - - Properties properties = new Properties(); - properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); - - Statement stmt = connection.createStatement(); - stmt.execute("drop database if exists " + dbName); - stmt.execute("create database if not exists " + dbName); - stmt.execute("use " + dbName); - - String createTableSql = "create table " + stbName + "(ts timestamp, f1 int, f2 int, f3 int) tags(areaid int, loc binary(20))"; - stmt.execute(createTableSql); - - for (int i = 0; i < numOfTables; i++) { - String loc = i % 2 == 0 ? "beijing" : "shanghai"; - String createSubTalbesSql = "create table " + tablePrefix + i + " using " + stbName + " tags(" + i + ", '" + loc + "')"; - stmt.execute(createSubTalbesSql); - } - stmt.close(); - } - - @Test - public void testBatchInsert() throws SQLException { - ExecutorService executorService = Executors.newFixedThreadPool(numOfTables); - for (int i = 0; i < numOfTables; i++) { - final int index = i; - executorService.execute(() -> { - try { - long startTime = System.currentTimeMillis(); - Statement statement = connection.createStatement(); // get statement - StringBuilder sb = new StringBuilder(); - sb.append("INSERT INTO " + tablePrefix + index + " VALUES"); - Random rand = new Random(); - for (int j = 1; j <= numOfRecordsPerTable; j++) { - sb.append("(" + (ts + j) + ", "); - sb.append(rand.nextInt(100) + ", "); - sb.append(rand.nextInt(100) + ", "); - sb.append(rand.nextInt(100) + ")"); - } - statement.addBatch(sb.toString()); - statement.executeBatch(); - long endTime = System.currentTimeMillis(); - System.out.println("Thread " + index + " takes " + (endTime - startTime) + " microseconds"); - connection.commit(); - statement.close(); - } catch (Exception e) { - e.printStackTrace(); - } - }); - } - executorService.shutdown(); - - try { - executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); - } catch (InterruptedException e) { - e.printStackTrace(); - } - - Statement statement = connection.createStatement(); - ResultSet rs = statement.executeQuery("select * from meters"); - int num = 0; - while (rs.next()) { - num++; - } - assertEquals(num, numOfTables * numOfRecordsPerTable); - rs.close(); - } - - @After - public void close() { - try { - if (connection != null) - connection.close(); - } catch (SQLException e) { - e.printStackTrace(); - } - } - -} \ No newline at end of file diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDatabaseMetaDataTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDatabaseMetaDataTest.java index 9bb22ad577..a7657fa3e5 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDatabaseMetaDataTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDatabaseMetaDataTest.java @@ -949,11 +949,10 @@ public class TSDBDatabaseMetaDataTest { } @BeforeClass - public static void before() { + public static void beforeClass() { try { Class.forName("com.taosdata.jdbc.TSDBDriver"); Properties properties = new Properties(); - properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host); properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); @@ -967,7 +966,7 @@ public class TSDBDatabaseMetaDataTest { } @AfterClass - public static void after() { + public static void afterClass() { try { if (connection != null) connection.close(); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchInsertTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchInsertTest.java index 9f54e4b1ba..472da34980 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchInsertTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchInsertTest.java @@ -60,30 +60,27 @@ public class BatchInsertTest { ExecutorService executorService = Executors.newFixedThreadPool(numOfTables); for (int i = 0; i < numOfTables; i++) { final int index = i; - executorService.execute(new Runnable() { - @Override - public void run() { - try { - long startTime = System.currentTimeMillis(); - Statement statement = connection.createStatement(); // get statement - StringBuilder sb = new StringBuilder(); - sb.append("INSERT INTO " + tablePrefix + index + " VALUES"); - Random rand = new Random(); - for (int j = 1; j <= numOfRecordsPerTable; j++) { - sb.append("(" + (ts + j) + ", "); - sb.append(rand.nextInt(100) + ", "); - sb.append(rand.nextInt(100) + ", "); - sb.append(rand.nextInt(100) + ")"); - } - statement.addBatch(sb.toString()); - statement.executeBatch(); - long endTime = System.currentTimeMillis(); - System.out.println("Thread " + index + " takes " + (endTime - startTime) + " microseconds"); - connection.commit(); - statement.close(); - } catch (Exception e) { - e.printStackTrace(); + executorService.execute(() -> { + try { + long startTime = System.currentTimeMillis(); + Statement statement = connection.createStatement(); // get statement + StringBuilder sb = new StringBuilder(); + sb.append("INSERT INTO " + tablePrefix + index + " VALUES"); + Random rand = new Random(); + for (int j = 1; j <= numOfRecordsPerTable; j++) { + sb.append("(" + (ts + j) + ", "); + sb.append(rand.nextInt(100) + ", "); + sb.append(rand.nextInt(100) + ", "); + sb.append(rand.nextInt(100) + ")"); } + statement.addBatch(sb.toString()); + statement.executeBatch(); + long endTime = System.currentTimeMillis(); + System.out.println("Thread " + index + " takes " + (endTime - startTime) + " microseconds"); + connection.commit(); + statement.close(); + } catch (Exception e) { + e.printStackTrace(); } }); } From b76b45277c2e744b260ee55571df145393f4652a Mon Sep 17 00:00:00 2001 From: zyyang Date: Tue, 26 Jan 2021 15:12:44 +0800 Subject: [PATCH 091/147] change --- .../java/com/taosdata/jdbc/StableTest.java | 1 + .../java/com/taosdata/jdbc/StatementTest.java | 126 ++++-------------- 2 files changed, 29 insertions(+), 98 deletions(-) diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StableTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StableTest.java index c86a0b4c6a..0a1c548baa 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StableTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StableTest.java @@ -29,6 +29,7 @@ public class StableTest { properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); Statement statement = connection.createStatement(); + statement.execute("drop database if exists " + dbName); statement.execute("create database if not exists " + dbName); statement.execute("use " + dbName); statement.close(); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StatementTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StatementTest.java index f78f899294..f35c859540 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StatementTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StatementTest.java @@ -1,6 +1,7 @@ package com.taosdata.jdbc; import org.junit.AfterClass; +import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; @@ -67,106 +68,35 @@ public class StatementTest { @Test public void testUnsupport() { - TSDBStatement tsdbStatement = (TSDBStatement) statement; try { - tsdbStatement.unwrap(null); - } catch (SQLException e) { - } - try { - tsdbStatement.isWrapperFor(null); - } catch (SQLException e) { - } - try { - tsdbStatement.getMaxFieldSize(); - } catch (SQLException e) { - } - try { - tsdbStatement.setMaxFieldSize(0); - } catch (SQLException e) { - } - try { - tsdbStatement.setEscapeProcessing(true); - } catch (SQLException e) { - } - try { - tsdbStatement.cancel(); - } catch (SQLException e) { - } - try { - tsdbStatement.getWarnings(); - } catch (SQLException e) { - } - try { - tsdbStatement.clearWarnings(); - } catch (SQLException e) { - } - try { - tsdbStatement.setCursorName(null); - } catch (SQLException e) { - } - try { - tsdbStatement.getMoreResults(); - } catch (SQLException e) { - } - try { - tsdbStatement.setFetchDirection(0); - } catch (SQLException e) { - } - try { - tsdbStatement.getFetchDirection(); - } catch (SQLException e) { - } - try { - tsdbStatement.getResultSetConcurrency(); - } catch (SQLException e) { - } - try { - tsdbStatement.getResultSetType(); - } catch (SQLException e) { - } - try { - tsdbStatement.getConnection(); - } catch (SQLException e) { - } - try { - tsdbStatement.getMoreResults(); - } catch (SQLException e) { - } - try { - tsdbStatement.getGeneratedKeys(); - } catch (SQLException e) { - } - try { - tsdbStatement.executeUpdate(null, 0); - } catch (SQLException e) { - } - try { - tsdbStatement.executeUpdate(null, new int[]{0}); - } catch (SQLException e) { - } - try { - tsdbStatement.executeUpdate(null, new String[]{"str1", "str2"}); - } catch (SQLException e) { - } - try { - tsdbStatement.getResultSetHoldability(); - } catch (SQLException e) { - } - try { - tsdbStatement.setPoolable(true); - } catch (SQLException e) { - } - try { - tsdbStatement.isPoolable(); - } catch (SQLException e) { - } - try { - tsdbStatement.closeOnCompletion(); - } catch (SQLException e) { - } - try { - tsdbStatement.isCloseOnCompletion(); + Assert.assertNotNull(statement.unwrap(TSDBStatement.class)); + Assert.assertTrue(statement.isWrapperFor(TSDBStatement.class)); + + statement.getMaxFieldSize(); + statement.setMaxFieldSize(0); + statement.setEscapeProcessing(true); + statement.cancel(); + statement.getWarnings(); + statement.clearWarnings(); + statement.setCursorName(null); + statement.getMoreResults(); + statement.setFetchDirection(0); + statement.getFetchDirection(); + statement.getResultSetConcurrency(); + statement.getResultSetType(); + statement.getConnection(); + statement.getMoreResults(); + statement.getGeneratedKeys(); + statement.executeUpdate(null, 0); + statement.executeUpdate(null, new int[]{0}); + statement.executeUpdate(null, new String[]{"str1", "str2"}); + statement.getResultSetHoldability(); + statement.setPoolable(true); + statement.isPoolable(); + statement.closeOnCompletion(); + statement.isCloseOnCompletion(); } catch (SQLException e) { + e.printStackTrace(); } } From e3cd70cf8e20d199321f65bffdf7ac7bddf6130b Mon Sep 17 00:00:00 2001 From: zyyang Date: Tue, 26 Jan 2021 15:17:36 +0800 Subject: [PATCH 092/147] change --- .../test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java index 0e4ed67bf9..ebacf6af6b 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java @@ -167,7 +167,7 @@ public class TSDBPreparedStatementTest { public static void beforeClass() { try { Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); - conn = DriverManager.getConnection("jdbc:TAOS://" + host + ":6030/jdbc_test?user=root&password=taosdata"); + conn = DriverManager.getConnection("jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata"); } catch (ClassNotFoundException | SQLException e) { e.printStackTrace(); } From 3c83219fd905f9642949b1a1ec5550490f41e006 Mon Sep 17 00:00:00 2001 From: zyyang Date: Tue, 26 Jan 2021 15:19:50 +0800 Subject: [PATCH 093/147] change --- .../jdbc/src/test/java/com/taosdata/jdbc/StatementTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StatementTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StatementTest.java index f35c859540..4ce29af020 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StatementTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StatementTest.java @@ -66,7 +66,7 @@ public class StatementTest { assertEquals(false, isClosed); } - @Test + @Test(expected = SQLFeatureNotSupportedException.class) public void testUnsupport() { try { Assert.assertNotNull(statement.unwrap(TSDBStatement.class)); From b9da747d2d85c0c825dbc0fe252ca9ee525e7b85 Mon Sep 17 00:00:00 2001 From: zyyang Date: Tue, 26 Jan 2021 15:25:14 +0800 Subject: [PATCH 094/147] change --- .../java/com/taosdata/jdbc/TSDBConstants.java | 1 + .../java/com/taosdata/jdbc/TSDBStatement.java | 5 ++ .../taosdata/jdbc/rs/RestfulStatement.java | 47 +++++++++---------- .../java/com/taosdata/jdbc/StatementTest.java | 2 +- 4 files changed, 30 insertions(+), 25 deletions(-) diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java index 4f4911aad9..da297810fe 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java @@ -19,6 +19,7 @@ import java.util.Map; public abstract class TSDBConstants { + public static final String STATEMENT_CLOSED = "Statement already closed."; public static final String DEFAULT_PORT = "6200"; public static final String UNSUPPORT_METHOD_EXCEPTIONZ_MSG = "this operation is NOT supported currently!"; public static final String INVALID_VARIABLES = "invalid variables"; diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java index 4c900b5fdc..e7317b8e1d 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java @@ -136,10 +136,15 @@ public class TSDBStatement implements Statement { } public void setMaxFieldSize(int max) throws SQLException { + if (isClosed()) + throw new SQLException(TSDBConstants.STATEMENT_CLOSED); + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); } public int getMaxRows() throws SQLException { + if (isClosed()) + throw new SQLException(TSDBConstants.STATEMENT_CLOSED); // always set maxRows to zero, meaning unlimitted rows in a resultSet return 0; } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java index 8b2276fbb0..93712875b3 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java @@ -14,7 +14,6 @@ import java.util.stream.Collectors; public class RestfulStatement implements Statement { - private static final String STATEMENT_CLOSED = "Statement already closed."; private boolean closed; private String database; private final RestfulConnection conn; @@ -131,14 +130,14 @@ public class RestfulStatement implements Statement { @Override public int getMaxFieldSize() throws SQLException { if (isClosed()) - throw new SQLException(STATEMENT_CLOSED); + throw new SQLException(TSDBConstants.STATEMENT_CLOSED); return TSDBConstants.maxFieldSize; } @Override public void setMaxFieldSize(int max) throws SQLException { if (isClosed()) - throw new SQLException(STATEMENT_CLOSED); + throw new SQLException(TSDBConstants.STATEMENT_CLOSED); if (max < 0) throw new SQLException(TSDBConstants.INVALID_VARIABLES); // nothing to do @@ -147,14 +146,14 @@ public class RestfulStatement implements Statement { @Override public int getMaxRows() throws SQLException { if (isClosed()) - throw new SQLException(STATEMENT_CLOSED); + throw new SQLException(TSDBConstants.STATEMENT_CLOSED); return 0; } @Override public void setMaxRows(int max) throws SQLException { if (isClosed()) - throw new SQLException(STATEMENT_CLOSED); + throw new SQLException(TSDBConstants.STATEMENT_CLOSED); if (max < 0) throw new SQLException(TSDBConstants.INVALID_VARIABLES); // nothing to do @@ -163,20 +162,20 @@ public class RestfulStatement implements Statement { @Override public void setEscapeProcessing(boolean enable) throws SQLException { if (isClosed()) - throw new SQLException(RestfulStatement.STATEMENT_CLOSED); + throw new SQLException(TSDBConstants.STATEMENT_CLOSED); } @Override public int getQueryTimeout() throws SQLException { if (isClosed()) - throw new SQLException(STATEMENT_CLOSED); + throw new SQLException(TSDBConstants.STATEMENT_CLOSED); return 0; } @Override public void setQueryTimeout(int seconds) throws SQLException { if (isClosed()) - throw new SQLException(STATEMENT_CLOSED); + throw new SQLException(TSDBConstants.STATEMENT_CLOSED); if (seconds < 0) throw new SQLException(TSDBConstants.INVALID_VARIABLES); } @@ -189,7 +188,7 @@ public class RestfulStatement implements Statement { @Override public SQLWarning getWarnings() throws SQLException { if (isClosed()) - throw new SQLException(STATEMENT_CLOSED); + throw new SQLException(TSDBConstants.STATEMENT_CLOSED); return null; } @@ -197,13 +196,13 @@ public class RestfulStatement implements Statement { public void clearWarnings() throws SQLException { // nothing to do if (isClosed()) - throw new SQLException(STATEMENT_CLOSED); + throw new SQLException(TSDBConstants.STATEMENT_CLOSED); } @Override public void setCursorName(String name) throws SQLException { if (isClosed()) - throw new SQLException(RestfulStatement.STATEMENT_CLOSED); + throw new SQLException(TSDBConstants.STATEMENT_CLOSED); throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); } @@ -243,7 +242,7 @@ public class RestfulStatement implements Statement { @Override public ResultSet getResultSet() throws SQLException { if (isClosed()) - throw new SQLException(STATEMENT_CLOSED); + throw new SQLException(TSDBConstants.STATEMENT_CLOSED); return resultSet; } @@ -275,7 +274,7 @@ public class RestfulStatement implements Statement { @Override public void setFetchSize(int rows) throws SQLException { if (isClosed()) - throw new SQLException(STATEMENT_CLOSED); + throw new SQLException(TSDBConstants.STATEMENT_CLOSED); if (rows < 0) throw new SQLException(TSDBConstants.INVALID_VARIABLES); //nothing to do @@ -284,28 +283,28 @@ public class RestfulStatement implements Statement { @Override public int getFetchSize() throws SQLException { if (isClosed()) - throw new SQLException(STATEMENT_CLOSED); + throw new SQLException(TSDBConstants.STATEMENT_CLOSED); return 0; } @Override public int getResultSetConcurrency() throws SQLException { if (isClosed()) - throw new SQLException(STATEMENT_CLOSED); + throw new SQLException(TSDBConstants.STATEMENT_CLOSED); return this.resultSet.getConcurrency(); } @Override public int getResultSetType() throws SQLException { if (isClosed()) - throw new SQLException(STATEMENT_CLOSED); + throw new SQLException(TSDBConstants.STATEMENT_CLOSED); return this.resultSet.getType(); } @Override public void addBatch(String sql) throws SQLException { if (isClosed()) - throw new SQLException(STATEMENT_CLOSED); + throw new SQLException(TSDBConstants.STATEMENT_CLOSED); //TODO: } @@ -323,14 +322,14 @@ public class RestfulStatement implements Statement { @Override public Connection getConnection() throws SQLException { if (isClosed()) - throw new SQLException(STATEMENT_CLOSED); + throw new SQLException(TSDBConstants.STATEMENT_CLOSED); return this.conn; } @Override public boolean getMoreResults(int current) throws SQLException { if (isClosed()) - throw new SQLException(STATEMENT_CLOSED); + throw new SQLException(TSDBConstants.STATEMENT_CLOSED); if (resultSet == null) return false; @@ -388,7 +387,7 @@ public class RestfulStatement implements Statement { @Override public int getResultSetHoldability() throws SQLException { if (isClosed()) - throw new SQLException(STATEMENT_CLOSED); + throw new SQLException(TSDBConstants.STATEMENT_CLOSED); return this.resultSet.getHoldability(); } @@ -400,28 +399,28 @@ public class RestfulStatement implements Statement { @Override public void setPoolable(boolean poolable) throws SQLException { if (isClosed()) - throw new SQLException(STATEMENT_CLOSED); + throw new SQLException(TSDBConstants.STATEMENT_CLOSED); //nothing to do } @Override public boolean isPoolable() throws SQLException { if (isClosed()) - throw new SQLException(STATEMENT_CLOSED); + throw new SQLException(TSDBConstants.STATEMENT_CLOSED); return false; } @Override public void closeOnCompletion() throws SQLException { if (isClosed()) - throw new SQLException(STATEMENT_CLOSED); + throw new SQLException(TSDBConstants.STATEMENT_CLOSED); this.closeOnCompletion = true; } @Override public boolean isCloseOnCompletion() throws SQLException { if (isClosed()) - throw new SQLException(STATEMENT_CLOSED); + throw new SQLException(TSDBConstants.STATEMENT_CLOSED); return this.closeOnCompletion; } diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StatementTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StatementTest.java index 4ce29af020..7a9bc57dfa 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StatementTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StatementTest.java @@ -66,7 +66,7 @@ public class StatementTest { assertEquals(false, isClosed); } - @Test(expected = SQLFeatureNotSupportedException.class) + @Test(expected = SQLException.class) public void testUnsupport() { try { Assert.assertNotNull(statement.unwrap(TSDBStatement.class)); From ecd22634ef130896fe4e8ff10cc0a6d0b484ebf2 Mon Sep 17 00:00:00 2001 From: zyyang Date: Tue, 26 Jan 2021 15:26:40 +0800 Subject: [PATCH 095/147] change --- .../java/com/taosdata/jdbc/StatementTest.java | 56 +++++++++---------- 1 file changed, 26 insertions(+), 30 deletions(-) diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StatementTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StatementTest.java index 7a9bc57dfa..b09482fb03 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StatementTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StatementTest.java @@ -67,37 +67,33 @@ public class StatementTest { } @Test(expected = SQLException.class) - public void testUnsupport() { - try { - Assert.assertNotNull(statement.unwrap(TSDBStatement.class)); - Assert.assertTrue(statement.isWrapperFor(TSDBStatement.class)); + public void testUnsupport() throws SQLException { + Assert.assertNotNull(statement.unwrap(TSDBStatement.class)); + Assert.assertTrue(statement.isWrapperFor(TSDBStatement.class)); - statement.getMaxFieldSize(); - statement.setMaxFieldSize(0); - statement.setEscapeProcessing(true); - statement.cancel(); - statement.getWarnings(); - statement.clearWarnings(); - statement.setCursorName(null); - statement.getMoreResults(); - statement.setFetchDirection(0); - statement.getFetchDirection(); - statement.getResultSetConcurrency(); - statement.getResultSetType(); - statement.getConnection(); - statement.getMoreResults(); - statement.getGeneratedKeys(); - statement.executeUpdate(null, 0); - statement.executeUpdate(null, new int[]{0}); - statement.executeUpdate(null, new String[]{"str1", "str2"}); - statement.getResultSetHoldability(); - statement.setPoolable(true); - statement.isPoolable(); - statement.closeOnCompletion(); - statement.isCloseOnCompletion(); - } catch (SQLException e) { - e.printStackTrace(); - } + statement.getMaxFieldSize(); + statement.setMaxFieldSize(0); + statement.setEscapeProcessing(true); + statement.cancel(); + statement.getWarnings(); + statement.clearWarnings(); + statement.setCursorName(null); + statement.getMoreResults(); + statement.setFetchDirection(0); + statement.getFetchDirection(); + statement.getResultSetConcurrency(); + statement.getResultSetType(); + statement.getConnection(); + statement.getMoreResults(); + statement.getGeneratedKeys(); + statement.executeUpdate(null, 0); + statement.executeUpdate(null, new int[]{0}); + statement.executeUpdate(null, new String[]{"str1", "str2"}); + statement.getResultSetHoldability(); + statement.setPoolable(true); + statement.isPoolable(); + statement.closeOnCompletion(); + statement.isCloseOnCompletion(); } @AfterClass From 8d9efdd66e6d89af37878a3f2a5005cead637105 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 26 Jan 2021 15:50:03 +0800 Subject: [PATCH 096/147] [TD-2852] : add Mac platform compiling section to the README.md. --- README.md | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index fafcdc8321..ded262b674 100644 --- a/README.md +++ b/README.md @@ -126,7 +126,7 @@ cmake .. -DCPUTYPE=aarch32 && cmake --build . If you use the Visual Studio 2013, please open a command window by executing "cmd.exe". Please specify "x86_amd64" for 64 bits Windows or specify "x86" is for 32 bits Windows when you execute vcvarsall.bat. -``` +```cmd mkdir debug && cd debug "C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat" < x86_amd64 | x86 > cmake .. -G "NMake Makefiles" @@ -137,7 +137,8 @@ If you use the Visual Studio 2019 or 2017: please open a command window by executing "cmd.exe". Please specify "x64" for 64 bits Windows or specify "x86" is for 32 bits Windows when you execute vcvarsall.bat. -``` + +```cmd mkdir debug && cd debug "c:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvarsall.bat" < x64 | x86 > cmake .. -G "NMake Makefiles" @@ -145,27 +146,37 @@ nmake ``` Or, you can simply open a command window by clicking Windows Start -> "Visual Studio < 2019 | 2017 >" folder -> "x64 Native Tools Command Prompt for VS < 2019 | 2017 >" or "x86 Native Tools Command Prompt for VS < 2019 | 2017 >" depends what architecture your Windows is, then execute commands as follows: -``` +```cmd mkdir debug && cd debug cmake .. -G "NMake Makefiles" nmake ``` +### On Mac OS X platform + +Please install XCode command line tools and cmake. Verified with XCode 11.4+ on Catalina and Big Sur. + +```shell +mkdir debug && cd debug +cmake .. && cmake --build . +``` + # Quick Run + # Quick Run To quickly start a TDengine server after building, run the command below in terminal: -```cmd +```bash ./build/bin/taosd -c test/cfg ``` In another terminal, use the TDengine shell to connect the server: -``` +```bash ./build/bin/taos -c test/cfg ``` option "-c test/cfg" specifies the system configuration file directory. # Installing After building successfully, TDengine can be installed by: -```cmd +```bash make install ``` Users can find more information about directories installed on the system in the [directory and files](https://www.taosdata.com/en/documentation/administrator/#Directory-and-Files) section. It should be noted that installing from source code does not configure service management for TDengine. From a9da437d024f91293fbc35ba95093e1a6986f5c7 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Tue, 26 Jan 2021 16:06:16 +0800 Subject: [PATCH 097/147] fix some error in python test frame --- tests/pytest/util/sql.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index 4f7607fc6c..adb49760bd 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -194,11 +194,12 @@ class TDSql: pstate = 0 pl = psutil.pids() for pid in pl: - if psutil.Process(pid).name == 'taosd': + if psutil.Process(pid).name() == 'taosd': pstate = 1 break - if pstate: - tdLog.sleep(5) + if pstate == state :break + if state or pstate: + tdLog.sleep(1) continue pstate = 0 break @@ -219,9 +220,9 @@ class TDSql: tdLog.info("dir: %s is empty, expect: empty" %dir) else: if state : - tdLog.info("dir: %s is empty, expect: not empty" %dir) + tdLog.info("dir: %s is not empty, expect: not empty" %dir) else: - tdLog.exit("dir: %s is empty, expect: empty" %dir) + tdLog.exit("dir: %s is not empty, expect: empty" %dir) else: tdLog.exit("dir: %s doesn't exist" %dir) def createDir(self, dir): From 030d4532e0702f123219ac9a01cd6b387ca6b0e5 Mon Sep 17 00:00:00 2001 From: zyyang Date: Tue, 26 Jan 2021 16:47:34 +0800 Subject: [PATCH 098/147] change --- .../test/java/com/taosdata/jdbc/ResultSetTest.java | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ResultSetTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ResultSetTest.java index b70bcd3c23..3d80ff066c 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ResultSetTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ResultSetTest.java @@ -1,6 +1,7 @@ package com.taosdata.jdbc; import org.junit.AfterClass; +import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; @@ -114,16 +115,8 @@ public class ResultSetTest { public void testUnsupport() throws SQLException { statement.executeQuery("show databases"); resSet = statement.getResultSet(); - try { - resSet.unwrap(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.isWrapperFor(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } + Assert.assertNotNull(resSet.unwrap(TSDBResultSet.class)); + Assert.assertTrue(resSet.isWrapperFor(TSDBResultSet.class)); try { resSet.getAsciiStream(0); } catch (SQLException e) { From 9bb353498b13111eb031446ae94056d14195beda Mon Sep 17 00:00:00 2001 From: zyyang Date: Tue, 26 Jan 2021 16:49:43 +0800 Subject: [PATCH 099/147] change --- .../java/com/taosdata/jdbc/utils/SqlSyntaxValidatorTest.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SqlSyntaxValidatorTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SqlSyntaxValidatorTest.java index c38958c6ce..ccb0941da0 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SqlSyntaxValidatorTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SqlSyntaxValidatorTest.java @@ -16,10 +16,6 @@ public class SqlSyntaxValidatorTest { @Test public void isUseSQL() { Assert.assertTrue(SqlSyntaxValidator.isUseSql("use database test")); - Assert.assertTrue(SqlSyntaxValidator.isUseSql("create database test")); - Assert.assertTrue(SqlSyntaxValidator.isUseSql("create database if not exist test")); - Assert.assertTrue(SqlSyntaxValidator.isUseSql("drop database test")); - Assert.assertTrue(SqlSyntaxValidator.isUseSql("drop database if exist test")); } } \ No newline at end of file From d1ee728376b5e5bb9de88e4278c042e1e04512b6 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Tue, 26 Jan 2021 16:53:17 +0800 Subject: [PATCH 100/147] fix psutil.process error --- tests/pytest/util/sql.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index adb49760bd..16931cca33 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -194,9 +194,13 @@ class TDSql: pstate = 0 pl = psutil.pids() for pid in pl: - if psutil.Process(pid).name() == 'taosd': - pstate = 1 - break + try: + if psutil.Process(pid).name() == 'taosd': + print('have already started') + pstate = 1 + break + except psutil.NoSuchProcess: + pass if pstate == state :break if state or pstate: tdLog.sleep(1) From 9644f13f09ff27e42d58129c89534d1fe5958c45 Mon Sep 17 00:00:00 2001 From: zyyang <69311263+zyyang-taosdata@users.noreply.github.com> Date: Tue, 26 Jan 2021 17:06:13 +0800 Subject: [PATCH 101/147] Update connector-java-ch.md --- .../webdocs/markdowndocs/connector-java-ch.md | 32 +++++++------------ 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/documentation20/webdocs/markdowndocs/connector-java-ch.md b/documentation20/webdocs/markdowndocs/connector-java-ch.md index 3d067ad206..cd88e093aa 100644 --- a/documentation20/webdocs/markdowndocs/connector-java-ch.md +++ b/documentation20/webdocs/markdowndocs/connector-java-ch.md @@ -334,16 +334,16 @@ conn.close(); ```java public static void main(String[] args) throws SQLException { HikariConfig config = new HikariConfig(); + // jdbc properties config.setJdbcUrl("jdbc:TAOS://127.0.0.1:6030/log"); config.setUsername("root"); config.setPassword("taosdata"); - + // connection pool configurations config.setMinimumIdle(3); //minimum number of idle connection config.setMaximumPoolSize(10); //maximum number of connection in the pool - config.setConnectionTimeout(10000); //maximum wait milliseconds for get connection from pool - config.setIdleTimeout(60000); // max idle time for recycle idle connection - config.setConnectionTestQuery("describe log.dn"); //validation query - config.setValidationTimeout(3000); //validation query timeout + config.setConnectionTimeout(30000); //maximum wait milliseconds for get connection from pool + config.setIdleTimeout(0); // max idle time for recycle idle connection + config.setConnectionTestQuery("select server_status()"); //validation query HikariDataSource ds = new HikariDataSource(config); //create datasource @@ -376,25 +376,17 @@ conn.close(); ```java public static void main(String[] args) throws Exception { Properties properties = new Properties(); + // jdbc properties properties.put("driverClassName","com.taosdata.jdbc.TSDBDriver"); properties.put("url","jdbc:TAOS://127.0.0.1:6030/log"); properties.put("username","root"); properties.put("password","taosdata"); - - properties.put("maxActive","10"); //maximum number of connection in the pool - properties.put("initialSize","3");//initial number of connection - properties.put("maxWait","10000");//maximum wait milliseconds for get connection from pool - properties.put("minIdle","3");//minimum number of connection in the pool - - properties.put("timeBetweenEvictionRunsMillis","3000");// the interval milliseconds to test connection - - properties.put("minEvictableIdleTimeMillis","60000");//the minimum milliseconds to keep idle - properties.put("maxEvictableIdleTimeMillis","90000");//the maximum milliseconds to keep idle - - properties.put("validationQuery","describe log.dn"); //validation query - properties.put("testWhileIdle","true"); // test connection while idle - properties.put("testOnBorrow","false"); // don't need while testWhileIdle is true - properties.put("testOnReturn","false"); // don't need while testWhileIdle is true + // pool configurations + properties.put("maxActive","10"); //maximum number of connection in the pool + properties.put("initialSize","3"); //initial number of connection + properties.put("minIdle","3"); //minimum number of connection in the pool + properties.put("maxWait","30000"); //maximum wait milliseconds for get connection from pool + properties.put("validationQuery","select server_status()"); //validation query //create druid datasource DataSource ds = DruidDataSourceFactory.createDataSource(properties); From c5b69e7109b61668f46171b0eb4aa68d7ec7a2ed Mon Sep 17 00:00:00 2001 From: zyyang Date: Tue, 26 Jan 2021 17:34:34 +0800 Subject: [PATCH 102/147] [TD-2597]: fix Druid and HikariCP configurations --- .../com/taosdata/demo/pool/DruidPoolBuilder.java | 16 +++++++--------- .../com/taosdata/demo/pool/HikariCpBuilder.java | 9 +++++++-- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/DruidPoolBuilder.java b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/DruidPoolBuilder.java index e5dc14c6a5..a03e9758b6 100644 --- a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/DruidPoolBuilder.java +++ b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/DruidPoolBuilder.java @@ -10,20 +10,18 @@ public class DruidPoolBuilder { final String url = "jdbc:TAOS://" + host + ":6030"; DruidDataSource dataSource = new DruidDataSource(); + // jdbc properties dataSource.setUrl(url); dataSource.setDriverClassName("com.taosdata.jdbc.TSDBDriver"); dataSource.setUsername("root"); dataSource.setPassword("taosdata"); - //初始连接数，默认0 - dataSource.setInitialSize(poolSize); - //最大连接数，默认8 - dataSource.setMaxActive(poolSize); - //最小闲置数 - dataSource.setMinIdle(poolSize); - //获取连接的最大等待时间，单位毫秒 - dataSource.setMaxWait(2000); - + // pool configurations + dataSource.setInitialSize(poolSize);//初始连接数，默认0 + dataSource.setMinIdle(poolSize);//最小闲置数 + dataSource.setMaxActive(poolSize);//最大连接数，默认8 + dataSource.setMaxWait(30000);//获取连接的最大等待时间，单位毫秒 + dataSource.setValidationQuery("select server_status()"); return dataSource; } diff --git a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/HikariCpBuilder.java b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/HikariCpBuilder.java index 87f1f4ad2c..2e50dcacff 100644 --- a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/HikariCpBuilder.java +++ b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/HikariCpBuilder.java @@ -9,13 +9,18 @@ public class HikariCpBuilder { public static DataSource getDataSource(String host, int poolSize) { HikariConfig config = new HikariConfig(); + // jdbc properties config.setDriverClassName("com.taosdata.jdbc.TSDBDriver"); config.setJdbcUrl("jdbc:TAOS://" + host + ":6030"); config.setUsername("root"); config.setPassword("taosdata"); + // pool configurations + config.setMinimumIdle(3); //minimum number of idle connection + config.setMaximumPoolSize(10); //maximum number of connection in the pool + config.setConnectionTimeout(30000); //maximum wait milliseconds for get connection from pool + config.setIdleTimeout(0); // max idle time for recycle idle connection + config.setConnectionTestQuery("select server_status()"); //validation query - config.setMaximumPoolSize(poolSize); - config.setMinimumIdle(poolSize); HikariDataSource ds = new HikariDataSource(config); return ds; } From 5bf6c816f039c8c1b050d7b57608efd54612ad6a Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Tue, 26 Jan 2021 17:37:59 +0800 Subject: [PATCH 103/147] fix bug --- src/query/src/qExecutor.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 6140b9824c..4bb62d4be7 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -3828,6 +3828,11 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { setQueryStatus(pQuery, QUERY_NOT_COMPLETED); pRuntimeEnv->scanFlag = REPEAT_SCAN; + if (pRuntimeEnv->pTsBuf) { + bool ret = tsBufNextPos(pRuntimeEnv->pTsBuf); + assert(ret); + } + qDebug("QInfo:%p start to repeat scan data blocks due to query func required, qrange:%"PRId64"-%"PRId64, pQInfo, cond.twindow.skey, cond.twindow.ekey); } From 2e52298f906ecb1b9482fce66f5c2d3e87fe2b6c Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Tue, 26 Jan 2021 17:54:28 +0800 Subject: [PATCH 104/147] add some steps on threeLevelMountPoint --- tests/pytest/multilevel/threeLevelMountPoint.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/pytest/multilevel/threeLevelMountPoint.py b/tests/pytest/multilevel/threeLevelMountPoint.py index ccaf95f47d..d97602eb76 100644 --- a/tests/pytest/multilevel/threeLevelMountPoint.py +++ b/tests/pytest/multilevel/threeLevelMountPoint.py @@ -56,6 +56,15 @@ class TDTestCase: tdDnodes.startWithoutSleep(1) tdSql.taosdStatus(1) + tdSql.haveFile('/mnt/data00',1) + tdSql.haveFile('/mnt/data01',1) + tdSql.haveFile('/mnt/data02',1) + tdSql.haveFile('/mnt/data10',1) + tdSql.haveFile('/mnt/data11',1) + tdSql.haveFile('/mnt/data12',1) + tdSql.haveFile('/mnt/data20',1) + tdSql.haveFile('/mnt/data21',1) + tdSql.haveFile('/mnt/data22',1) def stop(self): tdSql.close() From fe31dc0e1f5c291b93d57a8176b00195f3ac2c22 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Tue, 26 Jan 2021 18:18:04 +0800 Subject: [PATCH 105/147] fix same errors --- .../multilevel/fileDistributionSameLevel.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/tests/pytest/multilevel/fileDistributionSameLevel.py b/tests/pytest/multilevel/fileDistributionSameLevel.py index e2880283b5..88a7216b8e 100644 --- a/tests/pytest/multilevel/fileDistributionSameLevel.py +++ b/tests/pytest/multilevel/fileDistributionSameLevel.py @@ -25,19 +25,19 @@ class TDTestCase: tdSql.init(conn.cursor(), logSql) def run(self): - self.ntables = 10 - self.rowsPerTable = 10 + self.ntables = 1000 self.ts = 1520000010000 tdDnodes.stop(1) # Test1 1 dataDir cfg={ '10' : 'maxVgroupsPerDb', + '100' : 'maxTablesPerVnode', '/mnt/data00 0 1' : 'dataDir', '/mnt/data01 0 0' : 'dataDir', '/mnt/data02 0 0' : 'dataDir', - '/mnt/data03 1 0' : 'dataDir', - '/mnt/data04 1 0' : 'dataDir' + '/mnt/data03 0 0' : 'dataDir', + '/mnt/data04 0 0' : 'dataDir' } tdSql.createDir('/mnt/data00') tdSql.createDir('/mnt/data01') @@ -51,16 +51,17 @@ class TDTestCase: tdSql.execute("create database test days 1") tdSql.execute("use test") - tdSql.execute("create table tb(ts timestamp, c int)") + tdSql.execute("create table stb(ts timestamp, c int) tags(t int)") - for i in range(self.rowsPerTable): - tdSql.execute("insert into tb values(%d, 1)" % (self.ts + i * 86400000)) + for i in range(self.ntables): + tdSql.execute("create table tb%d using stb tags(%d)" %(i, i)) + tdSql.execute("insert into tb%d values(%d, 1)" % (self.ts + int (i / 100) * 86400000)) tdDnodes.stop(1) tdDnodes.start(1) - tdSql.query("select * from test.tb") - tdSql.checkRows(10) + tdSql.query("select * from test.stb") + tdSql.checkRows(1000) def stop(self): From 4b0528e7c4a35ccbabd8a2f9236ed51708f02ea3 Mon Sep 17 00:00:00 2001 From: freemine Date: Tue, 26 Jan 2021 20:43:02 +0800 Subject: [PATCH 106/147] sem_timedwait porting issue --- src/dnode/src/dnodeTelemetry.c | 47 ++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/src/dnode/src/dnodeTelemetry.c b/src/dnode/src/dnodeTelemetry.c index 1da1486dda..1fb6f73092 100644 --- a/src/dnode/src/dnodeTelemetry.c +++ b/src/dnode/src/dnodeTelemetry.c @@ -31,12 +31,16 @@ #include "mnodeAcct.h" #include "dnodeTelemetry.h" -static tsem_t tsExitSem; +// sem_timedwait is NOT implemented on MacOSX +// thus, we use pthread_mutex_t/pthread_cond_t to simulate +static pthread_mutex_t tsExitLock = PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t tsExitCond = PTHREAD_COND_INITIALIZER; +static volatile int tsExit = 0; static pthread_t tsTelemetryThread; #define TELEMETRY_SERVER "telemetry.taosdata.com" #define TELEMETRY_PORT 80 -#define REPORT_INTERVAL 86400 +#define REPORT_INTERVAL 86400 static void beginObject(SBufferWriter* bw) { tbufWriteChar(bw, '{'); @@ -172,8 +176,8 @@ static void addMemoryInfo(SBufferWriter* bw) { static void addVersionInfo(SBufferWriter* bw) { addStringField(bw, "version", version); addStringField(bw, "buildInfo", buildinfo); - addStringField(bw, "gitInfo", gitinfo); - addStringField(bw, "email", tsEmail); + addStringField(bw, "gitInfo", gitinfo); + addStringField(bw, "email", tsEmail); } static void addRuntimeInfo(SBufferWriter* bw) { @@ -248,12 +252,14 @@ static void* telemetryThread(void* param) { clock_gettime(CLOCK_REALTIME, &end); end.tv_sec += 300; // wait 5 minutes before send first report - while (1) { - if (sem_timedwait(&tsExitSem, &end) == 0) { - break; - } else if (errno != ETIMEDOUT) { - continue; - } + while (!tsExit) { + int r = 0; + struct timespec ts = end; + pthread_mutex_lock(&tsExitLock); + r = pthread_cond_timedwait(&tsExitCond, &tsExitLock, &ts); + pthread_mutex_unlock(&tsExitLock); + if (r==0) break; + if (r!=ETIMEDOUT) continue; if (sdbIsMaster()) { sendTelemetryReport(); @@ -269,12 +275,12 @@ static void dnodeGetEmail(char* filepath) { if (fd < 0) { return; } - + if (taosRead(fd, (void *)tsEmail, TSDB_FQDN_LEN) < 0) { dError("failed to read %d bytes from file %s since %s", TSDB_FQDN_LEN, filepath, strerror(errno)); - } + } - taosClose(fd); + taosClose(fd); } int32_t dnodeInitTelemetry() { @@ -282,13 +288,7 @@ int32_t dnodeInitTelemetry() { return 0; } - dnodeGetEmail("/usr/local/taos/email"); - - if (tsem_init(&tsExitSem, 0, 0) == -1) { - // just log the error, it is ok for telemetry to fail - dTrace("failed to create semaphore for telemetry, reason:%s", strerror(errno)); - return 0; - } + dnodeGetEmail("/usr/local/taos/email"); pthread_attr_t attr; pthread_attr_init(&attr); @@ -310,8 +310,11 @@ void dnodeCleanupTelemetry() { } if (taosCheckPthreadValid(tsTelemetryThread)) { - tsem_post(&tsExitSem); + pthread_mutex_lock(&tsExitLock); + tsExit = 1; + pthread_cond_signal(&tsExitCond); + pthread_mutex_unlock(&tsExitLock); + pthread_join(tsTelemetryThread, NULL); - tsem_destroy(&tsExitSem); } } From 9dcee4f8e7222b139d6f232765a2bcefb4c6045e Mon Sep 17 00:00:00 2001 From: freemine Date: Tue, 26 Jan 2021 20:51:28 +0800 Subject: [PATCH 107/147] remove __APPLE__ block of codes --- src/dnode/src/dnodeTelemetry.c | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/dnode/src/dnodeTelemetry.c b/src/dnode/src/dnodeTelemetry.c index 1fb6f73092..4caece1661 100644 --- a/src/dnode/src/dnodeTelemetry.c +++ b/src/dnode/src/dnodeTelemetry.c @@ -240,13 +240,6 @@ static void sendTelemetryReport() { taosCloseSocket(fd); } -#ifdef __APPLE__ -static int sem_timedwait(tsem_t *sem, struct timespec *to) { - fprintf(stderr, "%s[%d]%s(): not implemented yet!\n", basename(__FILE__), __LINE__, __func__); - abort(); -} -#endif // __APPLE__ - static void* telemetryThread(void* param) { struct timespec end = {0}; clock_gettime(CLOCK_REALTIME, &end); From 97acf2dd4ffa2eba0bb9b5d796f1329ef88f6c56 Mon Sep 17 00:00:00 2001 From: zyyang <69311263+zyyang-taosdata@users.noreply.github.com> Date: Wed, 27 Jan 2021 11:48:15 +0800 Subject: [PATCH 108/147] Update connector-java-ch.md modified the connection pool configurations --- .../webdocs/markdowndocs/connector-java-ch.md | 31 +++++++++---------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/documentation20/webdocs/markdowndocs/connector-java-ch.md b/documentation20/webdocs/markdowndocs/connector-java-ch.md index cd88e093aa..0654ccd1b9 100644 --- a/documentation20/webdocs/markdowndocs/connector-java-ch.md +++ b/documentation20/webdocs/markdowndocs/connector-java-ch.md @@ -339,9 +339,10 @@ conn.close(); config.setUsername("root"); config.setPassword("taosdata"); // connection pool configurations - config.setMinimumIdle(3); //minimum number of idle connection + config.setMinimumIdle(10); //minimum number of idle connection config.setMaximumPoolSize(10); //maximum number of connection in the pool config.setConnectionTimeout(30000); //maximum wait milliseconds for get connection from pool + config.setMaxLifetime(0); // maximum life time for each connection config.setIdleTimeout(0); // max idle time for recycle idle connection config.setConnectionTestQuery("select server_status()"); //validation query @@ -375,24 +376,22 @@ conn.close(); * 使用示例如下： ```java public static void main(String[] args) throws Exception { - Properties properties = new Properties(); + + DruidDataSource dataSource = new DruidDataSource(); // jdbc properties - properties.put("driverClassName","com.taosdata.jdbc.TSDBDriver"); - properties.put("url","jdbc:TAOS://127.0.0.1:6030/log"); - properties.put("username","root"); - properties.put("password","taosdata"); + dataSource.setDriverClassName("com.taosdata.jdbc.TSDBDriver"); + dataSource.setUrl(url); + dataSource.setUsername("root"); + dataSource.setPassword("taosdata"); // pool configurations - properties.put("maxActive","10"); //maximum number of connection in the pool - properties.put("initialSize","3"); //initial number of connection - properties.put("minIdle","3"); //minimum number of connection in the pool - properties.put("maxWait","30000"); //maximum wait milliseconds for get connection from pool - properties.put("validationQuery","select server_status()"); //validation query - - //create druid datasource - DataSource ds = DruidDataSourceFactory.createDataSource(properties); - Connection connection = ds.getConnection(); // get connection + dataSource.setInitialSize(10); + dataSource.setMinIdle(10); + dataSource.setMaxActive(10); + dataSource.setMaxWait(30000); + dataSource.setValidationQuery("select server_status()"); + + Connection connection = dataSource.getConnection(); // get connection Statement statement = connection.createStatement(); // get statement - //query or insert // ... From 1ceacaafa89f1b909b59bfca3943d41864212f18 Mon Sep 17 00:00:00 2001 From: zyyang Date: Wed, 27 Jan 2021 11:52:03 +0800 Subject: [PATCH 109/147] [TD-2541]: optimized the connection pool demo codes --- .../com/taosdata/demo/ConnectionPoolDemo.java | 6 +----- .../com/taosdata/demo/pool/DruidPoolBuilder.java | 16 ++++++++-------- .../com/taosdata/demo/pool/HikariCpBuilder.java | 1 + 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/ConnectionPoolDemo.java b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/ConnectionPoolDemo.java index 4e33b75bc5..38e0446c4d 100644 --- a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/ConnectionPoolDemo.java +++ b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/ConnectionPoolDemo.java @@ -1,6 +1,5 @@ package com.taosdata.demo; -import com.taosdata.demo.common.InsertTask; import com.taosdata.demo.pool.C3p0Builder; import com.taosdata.demo.pool.DbcpBuilder; import com.taosdata.demo.pool.DruidPoolBuilder; @@ -11,9 +10,6 @@ import javax.sql.DataSource; import java.sql.Connection; import java.sql.SQLException; import java.sql.Statement; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; public class ConnectionPoolDemo { @@ -96,13 +92,13 @@ public class ConnectionPoolDemo { e.printStackTrace(); } - // ExecutorService executor = Executors.newFixedThreadPool(threadCount); // while (true) { // executor.execute(new InsertTask(dataSource, dbName, tableSize, batchSize)); // if (sleep > 0) // TimeUnit.MILLISECONDS.sleep(sleep); // } + } private static void init(DataSource dataSource) { diff --git a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/DruidPoolBuilder.java b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/DruidPoolBuilder.java index a03e9758b6..7bbac0faf8 100644 --- a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/DruidPoolBuilder.java +++ b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/DruidPoolBuilder.java @@ -1,29 +1,29 @@ package com.taosdata.demo.pool; import com.alibaba.druid.pool.DruidDataSource; +import com.alibaba.druid.pool.DruidDataSourceFactory; import javax.sql.DataSource; +import java.util.Properties; public class DruidPoolBuilder { - public static DataSource getDataSource(String host, int poolSize) { + public static DataSource getDataSource(String host, int poolSize) throws Exception { final String url = "jdbc:TAOS://" + host + ":6030"; DruidDataSource dataSource = new DruidDataSource(); // jdbc properties - dataSource.setUrl(url); dataSource.setDriverClassName("com.taosdata.jdbc.TSDBDriver"); + dataSource.setUrl(url); dataSource.setUsername("root"); dataSource.setPassword("taosdata"); - // pool configurations - dataSource.setInitialSize(poolSize);//初始连接数，默认0 - dataSource.setMinIdle(poolSize);//最小闲置数 - dataSource.setMaxActive(poolSize);//最大连接数，默认8 - dataSource.setMaxWait(30000);//获取连接的最大等待时间，单位毫秒 + dataSource.setInitialSize(10); + dataSource.setMinIdle(10); + dataSource.setMaxActive(10); + dataSource.setMaxWait(30000); dataSource.setValidationQuery("select server_status()"); return dataSource; } - } diff --git a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/HikariCpBuilder.java b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/HikariCpBuilder.java index 2e50dcacff..acf18bf999 100644 --- a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/HikariCpBuilder.java +++ b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/HikariCpBuilder.java @@ -18,6 +18,7 @@ public class HikariCpBuilder { config.setMinimumIdle(3); //minimum number of idle connection config.setMaximumPoolSize(10); //maximum number of connection in the pool config.setConnectionTimeout(30000); //maximum wait milliseconds for get connection from pool + config.setMaxLifetime(0); // maximum life time for each connection config.setIdleTimeout(0); // max idle time for recycle idle connection config.setConnectionTestQuery("select server_status()"); //validation query From ee236c208837692b1ebbe304673bf5efba1bd622 Mon Sep 17 00:00:00 2001 From: zyyang Date: Wed, 27 Jan 2021 12:41:56 +0800 Subject: [PATCH 110/147] change --- .../java/com/taosdata/demo/pool/DruidPoolBuilder.java | 10 ++++------ .../java/com/taosdata/demo/pool/HikariCpBuilder.java | 4 ++-- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/DruidPoolBuilder.java b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/DruidPoolBuilder.java index 7bbac0faf8..e9fc496af3 100644 --- a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/DruidPoolBuilder.java +++ b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/DruidPoolBuilder.java @@ -1,14 +1,12 @@ package com.taosdata.demo.pool; import com.alibaba.druid.pool.DruidDataSource; -import com.alibaba.druid.pool.DruidDataSourceFactory; import javax.sql.DataSource; -import java.util.Properties; public class DruidPoolBuilder { - public static DataSource getDataSource(String host, int poolSize) throws Exception { + public static DataSource getDataSource(String host, int poolSize) { final String url = "jdbc:TAOS://" + host + ":6030"; DruidDataSource dataSource = new DruidDataSource(); @@ -18,9 +16,9 @@ public class DruidPoolBuilder { dataSource.setUsername("root"); dataSource.setPassword("taosdata"); // pool configurations - dataSource.setInitialSize(10); - dataSource.setMinIdle(10); - dataSource.setMaxActive(10); + dataSource.setInitialSize(poolSize); + dataSource.setMinIdle(poolSize); + dataSource.setMaxActive(poolSize); dataSource.setMaxWait(30000); dataSource.setValidationQuery("select server_status()"); return dataSource; diff --git a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/HikariCpBuilder.java b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/HikariCpBuilder.java index acf18bf999..5cfe7e6e6a 100644 --- a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/HikariCpBuilder.java +++ b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/HikariCpBuilder.java @@ -15,8 +15,8 @@ public class HikariCpBuilder { config.setUsername("root"); config.setPassword("taosdata"); // pool configurations - config.setMinimumIdle(3); //minimum number of idle connection - config.setMaximumPoolSize(10); //maximum number of connection in the pool + config.setMinimumIdle(poolSize); //minimum number of idle connection + config.setMaximumPoolSize(poolSize); //maximum number of connection in the pool config.setConnectionTimeout(30000); //maximum wait milliseconds for get connection from pool config.setMaxLifetime(0); // maximum life time for each connection config.setIdleTimeout(0); // max idle time for recycle idle connection From df6bc5285635a570fffb5b209e04547a837168b5 Mon Sep 17 00:00:00 2001 From: zyyang Date: Wed, 27 Jan 2021 13:42:46 +0800 Subject: [PATCH 111/147] change --- .../com/taosdata/demo/ConnectionPoolDemo.java | 71 ++++++++----------- .../com/taosdata/demo/common/InsertTask.java | 39 ++-------- 2 files changed, 35 insertions(+), 75 deletions(-) diff --git a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/ConnectionPoolDemo.java b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/ConnectionPoolDemo.java index 38e0446c4d..03b6af1ec2 100644 --- a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/ConnectionPoolDemo.java +++ b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/ConnectionPoolDemo.java @@ -1,5 +1,6 @@ package com.taosdata.demo; +import com.taosdata.demo.common.InsertTask; import com.taosdata.demo.pool.C3p0Builder; import com.taosdata.demo.pool.DbcpBuilder; import com.taosdata.demo.pool.DruidPoolBuilder; @@ -10,12 +11,16 @@ import javax.sql.DataSource; import java.sql.Connection; import java.sql.SQLException; import java.sql.Statement; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; public class ConnectionPoolDemo { private static Logger logger = Logger.getLogger(DruidPoolBuilder.class); private static final String dbName = "pool_test"; + private static long totalSize = 1_000_000l; private static int batchSize = 10; private static int sleep = 1000; private static int poolSize = 50; @@ -24,36 +29,18 @@ public class ConnectionPoolDemo { private static String poolType = "hikari"; - public static void main(String[] args) throws InterruptedException { + public static void main(String[] args) { String host = null; for (int i = 0; i < args.length; i++) { if ("-host".equalsIgnoreCase(args[i]) && i < args.length - 1) { host = args[++i]; } - if ("-batchSize".equalsIgnoreCase(args[i]) && i < args.length - 1) { - batchSize = Integer.parseInt(args[++i]); - } - if ("-sleep".equalsIgnoreCase(args[i]) && i < args.length - 1) { - sleep = Integer.parseInt(args[++i]); - } - if ("-poolSize".equalsIgnoreCase(args[i]) && i < args.length - 1) { - poolSize = Integer.parseInt(args[++i]); - } - if ("-tableSize".equalsIgnoreCase(args[i]) && i < args.length - 1) { - tableSize = Integer.parseInt(args[++i]); - } if ("-poolType".equalsIgnoreCase(args[i]) && i < args.length - 1) { poolType = args[++i]; } } if (host == null) { - System.out.println("Usage: java -jar XXX.jar " + - "-host " + - "-batchSize " + - "-sleep " + - "-poolSize " + - "-tableSize " + - "-poolType "); + System.out.println("Usage: java -jar XXX.jar -host -poolType "); return; } @@ -75,30 +62,34 @@ public class ConnectionPoolDemo { } logger.info(">>>>>>>>>>>>>> connection pool Type: " + poolType); - init(dataSource); - try { - Connection connection = dataSource.getConnection(); - Statement statement = connection.createStatement(); - String sql = "insert into " + dbName + ".t_1 values('2020-01-01 00:00:00.000',12.12,111)"; - int affectRows = statement.executeUpdate(sql); - System.out.println("affectRows >>> " + affectRows); - affectRows = statement.executeUpdate(sql); - System.out.println("affectRows >>> " + affectRows); - statement.close(); - connection.close(); - } catch (SQLException e) { - e.printStackTrace(); - } - -// ExecutorService executor = Executors.newFixedThreadPool(threadCount); -// while (true) { -// executor.execute(new InsertTask(dataSource, dbName, tableSize, batchSize)); -// if (sleep > 0) -// TimeUnit.MILLISECONDS.sleep(sleep); +// try { +// Connection connection = dataSource.getConnection(); +// Statement statement = connection.createStatement(); +// String sql = "insert into " + dbName + ".t_1 values('2020-01-01 00:00:00.000',12.12,111)"; +// int affectRows = statement.executeUpdate(sql); +// System.out.println("affectRows >>> " + affectRows); +// affectRows = statement.executeUpdate(sql); +// System.out.println("affectRows >>> " + affectRows); +// statement.close(); +// connection.close(); +// } catch (SQLException e) { +// e.printStackTrace(); // } + ExecutorService executor = Executors.newFixedThreadPool(threadCount); + for (long i = 0; i < totalSize / batchSize; i++) { + executor.execute(new InsertTask(dataSource, dbName, tableSize, batchSize)); + // sleep few seconds + try { + TimeUnit.MILLISECONDS.sleep(sleep); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + executor.shutdown(); + } private static void init(DataSource dataSource) { diff --git a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/common/InsertTask.java b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/common/InsertTask.java index ed86acd6e9..2396287fec 100644 --- a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/common/InsertTask.java +++ b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/common/InsertTask.java @@ -26,52 +26,21 @@ public class InsertTask implements Runnable { @Override public void run() { - Connection conn = null; - Statement stmt = null; int affectedRows = 0; - long start = System.currentTimeMillis(); - try { - conn = ds.getConnection(); - stmt = conn.createStatement(); - + try (Connection conn = ds.getConnection(); Statement stmt = conn.createStatement()) { for (int tb_index = 1; tb_index <= tableSize; tb_index++) { StringBuilder sb = new StringBuilder(); - sb.append("insert into "); - sb.append(dbName); - sb.append(".t_"); - sb.append(tb_index); - sb.append("(ts, temperature, humidity) values "); + sb.append("insert into ").append(dbName).append(".t_").append(tb_index).append("(ts, temperature, humidity) values "); for (int i = 0; i < batchSize; i++) { - sb.append("("); - sb.append(start + i); - sb.append(", "); - sb.append(random.nextFloat() * 30); - sb.append(", "); - sb.append(random.nextInt(70)); - sb.append(") "); + sb.append("(").append(start + i).append(", ").append(random.nextFloat() * 30).append(", ").append(random.nextInt(70)).append(") "); } logger.info("SQL >>> " + sb.toString()); affectedRows += stmt.executeUpdate(sb.toString()); } } catch (SQLException e) { e.printStackTrace(); - } finally { - if (stmt != null) { - try { - stmt.close(); - } catch (SQLException e) { - e.printStackTrace(); - } - } - if (conn != null) { - try { - conn.close(); - } catch (SQLException e) { - e.printStackTrace(); - } - } - logger.info(">>> affectedRows:" + affectedRows + " TimeCost:" + (System.currentTimeMillis() - start) + " ms"); } + logger.info(">>> affectedRows:" + affectedRows + " TimeCost:" + (System.currentTimeMillis() - start) + " ms"); } } From 45a0bf5c15fde67abb1dab212caa844e6291b637 Mon Sep 17 00:00:00 2001 From: zyyang Date: Wed, 27 Jan 2021 13:46:26 +0800 Subject: [PATCH 112/147] change --- .../src/main/java/com/taosdata/demo/ConnectionPoolDemo.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/ConnectionPoolDemo.java b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/ConnectionPoolDemo.java index 03b6af1ec2..2bc3059cf8 100644 --- a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/ConnectionPoolDemo.java +++ b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/ConnectionPoolDemo.java @@ -79,7 +79,7 @@ public class ConnectionPoolDemo { // } ExecutorService executor = Executors.newFixedThreadPool(threadCount); - for (long i = 0; i < totalSize / batchSize; i++) { + for (long i = 0; i < totalSize / batchSize / tableSize; i++) { executor.execute(new InsertTask(dataSource, dbName, tableSize, batchSize)); // sleep few seconds try { From 353112c01ee870378d65b6c36e23b178c2f83015 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Wed, 27 Jan 2021 13:53:31 +0800 Subject: [PATCH 113/147] fix bug --- src/client/src/tscUtil.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index d045000e24..824fff574e 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -2127,7 +2127,11 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, __async_cb_func_t pTableMetaInfo->tagColList, pTableMetaInfo->pVgroupTables); } else { // transfer the ownership of pTableMeta to the newly create sql object. STableMetaInfo* pPrevInfo = tscGetTableMetaInfoFromCmd(&pPrevSql->cmd, pPrevSql->cmd.clauseIndex, 0); - + if (pPrevInfo->pTableMeta && pPrevInfo->pTableMeta->tableType < 0) { + terrno = TSDB_CODE_TSC_APP_ERROR; + goto _error; + } + STableMeta* pPrevTableMeta = tscTableMetaDup(pPrevInfo->pTableMeta); SVgroupsInfo* pVgroupsInfo = pPrevInfo->vgroupList; pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, &pTableMetaInfo->name, pPrevTableMeta, pVgroupsInfo, pTableMetaInfo->tagColList, From 9ed2f88bbcca3f69c1c07042b1010d196d6b54e9 Mon Sep 17 00:00:00 2001 From: zyyang Date: Wed, 27 Jan 2021 13:54:39 +0800 Subject: [PATCH 114/147] change --- .../com/taosdata/demo/ConnectionPoolDemo.java | 48 +++++++++---------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/ConnectionPoolDemo.java b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/ConnectionPoolDemo.java index 2bc3059cf8..6787e07423 100644 --- a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/ConnectionPoolDemo.java +++ b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/ConnectionPoolDemo.java @@ -64,31 +64,31 @@ public class ConnectionPoolDemo { logger.info(">>>>>>>>>>>>>> connection pool Type: " + poolType); init(dataSource); -// try { -// Connection connection = dataSource.getConnection(); -// Statement statement = connection.createStatement(); -// String sql = "insert into " + dbName + ".t_1 values('2020-01-01 00:00:00.000',12.12,111)"; -// int affectRows = statement.executeUpdate(sql); -// System.out.println("affectRows >>> " + affectRows); -// affectRows = statement.executeUpdate(sql); -// System.out.println("affectRows >>> " + affectRows); -// statement.close(); -// connection.close(); -// } catch (SQLException e) { -// e.printStackTrace(); -// } - - ExecutorService executor = Executors.newFixedThreadPool(threadCount); - for (long i = 0; i < totalSize / batchSize / tableSize; i++) { - executor.execute(new InsertTask(dataSource, dbName, tableSize, batchSize)); - // sleep few seconds - try { - TimeUnit.MILLISECONDS.sleep(sleep); - } catch (InterruptedException e) { - e.printStackTrace(); - } + try { + Connection connection = dataSource.getConnection(); + Statement statement = connection.createStatement(); + String sql = "insert into " + dbName + ".t_1 values('2020-01-01 00:00:00.000',12.12,111)"; + int affectRows = statement.executeUpdate(sql); + System.out.println("affectRows >>> " + affectRows); + affectRows = statement.executeUpdate(sql); + System.out.println("affectRows >>> " + affectRows); + statement.close(); + connection.close(); + } catch (SQLException e) { + e.printStackTrace(); } - executor.shutdown(); + +// ExecutorService executor = Executors.newFixedThreadPool(threadCount); +// for (long i = 0; i < totalSize / batchSize / tableSize; i++) { +// executor.execute(new InsertTask(dataSource, dbName, tableSize, batchSize)); +// // sleep few seconds +// try { +// TimeUnit.MILLISECONDS.sleep(sleep); +// } catch (InterruptedException e) { +// e.printStackTrace(); +// } +// } +// executor.shutdown(); } From e76038cd0f9a7aa7a1082fa2c03098334f4eb14b Mon Sep 17 00:00:00 2001 From: zyyang Date: Wed, 27 Jan 2021 14:02:18 +0800 Subject: [PATCH 115/147] change --- tests/examples/JDBC/JDBCDemo/pom.xml | 2 +- .../JDBC/connectionPools/README-cn.md | 4 +-- tests/examples/JDBC/connectionPools/pom.xml | 33 +++++++++++++++++-- .../{demo => example}/ConnectionPoolDemo.java | 14 +++----- .../{demo => example}/common/InsertTask.java | 2 +- .../{demo => example}/pool/C3p0Builder.java | 3 +- .../{demo => example}/pool/DbcpBuilder.java | 2 +- .../pool/DruidPoolBuilder.java | 2 +- .../pool/HikariCpBuilder.java | 2 +- 9 files changed, 44 insertions(+), 20 deletions(-) rename tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/{demo => example}/ConnectionPoolDemo.java (91%) rename tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/{demo => example}/common/InsertTask.java (97%) rename tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/{demo => example}/pool/C3p0Builder.java (89%) rename tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/{demo => example}/pool/DbcpBuilder.java (93%) rename tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/{demo => example}/pool/DruidPoolBuilder.java (95%) rename tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/{demo => example}/pool/HikariCpBuilder.java (97%) diff --git a/tests/examples/JDBC/JDBCDemo/pom.xml b/tests/examples/JDBC/JDBCDemo/pom.xml index d075fc8f2a..6f0c58a821 100644 --- a/tests/examples/JDBC/JDBCDemo/pom.xml +++ b/tests/examples/JDBC/JDBCDemo/pom.xml @@ -52,7 +52,7 @@ com.taosdata.jdbc taos-jdbcdriver - 2.0.15 + 2.0.18 diff --git a/tests/examples/JDBC/connectionPools/README-cn.md b/tests/examples/JDBC/connectionPools/README-cn.md index 761596dfc5..139a94e36d 100644 --- a/tests/examples/JDBC/connectionPools/README-cn.md +++ b/tests/examples/JDBC/connectionPools/README-cn.md @@ -12,12 +12,12 @@ ConnectionPoolDemo的程序逻辑： ### 如何运行这个例子： ```shell script -# mvn exec:java -Dexec.mainClass="com.taosdata.demo.ConnectionPoolDemo" -Dexec.args="-host localhost" +# mvn exec:java -Dexec.mainClass="com.taosdata.example.ConnectionPoolDemo" -Dexec.args="-host localhost" ``` 使用mvn运行ConnectionPoolDemo的main方法，可以指定参数 ```shell script Usage: -mvn exec:java -Dexec.mainClass="com.taosdata.demo.ConnectionPoolDemo" -Dexec.args="" +mvn exec:java -Dexec.mainClass="com.taosdata.example.ConnectionPoolDemo" -Dexec.args="" -host : hostname -poolType -poolSize diff --git a/tests/examples/JDBC/connectionPools/pom.xml b/tests/examples/JDBC/connectionPools/pom.xml index fbee256067..84467003f9 100644 --- a/tests/examples/JDBC/connectionPools/pom.xml +++ b/tests/examples/JDBC/connectionPools/pom.xml @@ -12,7 +12,7 @@ com.taosdata.jdbc taos-jdbcdriver - 2.0.17 + 2.0.18 @@ -50,6 +50,35 @@ log4j 1.2.17 - + + + + + org.apache.maven.plugins + maven-assembly-plugin + 3.1.0 + + + + com.taosdata.example.ConnectionPoolDemo + + + + jar-with-dependencies + + + + + make-assembly + package + + single + + + + + + + \ No newline at end of file diff --git a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/ConnectionPoolDemo.java b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/ConnectionPoolDemo.java similarity index 91% rename from tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/ConnectionPoolDemo.java rename to tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/ConnectionPoolDemo.java index 6787e07423..7b603ec97f 100644 --- a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/ConnectionPoolDemo.java +++ b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/ConnectionPoolDemo.java @@ -1,19 +1,15 @@ -package com.taosdata.demo; +package com.taosdata.example; -import com.taosdata.demo.common.InsertTask; -import com.taosdata.demo.pool.C3p0Builder; -import com.taosdata.demo.pool.DbcpBuilder; -import com.taosdata.demo.pool.DruidPoolBuilder; -import com.taosdata.demo.pool.HikariCpBuilder; +import com.taosdata.example.pool.C3p0Builder; +import com.taosdata.example.pool.DbcpBuilder; +import com.taosdata.example.pool.DruidPoolBuilder; +import com.taosdata.example.pool.HikariCpBuilder; import org.apache.log4j.Logger; import javax.sql.DataSource; import java.sql.Connection; import java.sql.SQLException; import java.sql.Statement; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; public class ConnectionPoolDemo { diff --git a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/common/InsertTask.java b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/common/InsertTask.java similarity index 97% rename from tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/common/InsertTask.java rename to tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/common/InsertTask.java index 2396287fec..e39a613aa6 100644 --- a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/common/InsertTask.java +++ b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/common/InsertTask.java @@ -1,4 +1,4 @@ -package com.taosdata.demo.common; +package com.taosdata.example.common; import org.apache.log4j.Logger; diff --git a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/C3p0Builder.java b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/pool/C3p0Builder.java similarity index 89% rename from tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/C3p0Builder.java rename to tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/pool/C3p0Builder.java index 587f417410..235db0bb79 100644 --- a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/C3p0Builder.java +++ b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/pool/C3p0Builder.java @@ -1,7 +1,6 @@ -package com.taosdata.demo.pool; +package com.taosdata.example.pool; import com.mchange.v2.c3p0.ComboPooledDataSource; -import org.apache.commons.dbcp.BasicDataSource; import javax.sql.DataSource; import java.beans.PropertyVetoException; diff --git a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/DbcpBuilder.java b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/pool/DbcpBuilder.java similarity index 93% rename from tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/DbcpBuilder.java rename to tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/pool/DbcpBuilder.java index 3c34a32532..3aa9e4ebcf 100644 --- a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/DbcpBuilder.java +++ b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/pool/DbcpBuilder.java @@ -1,4 +1,4 @@ -package com.taosdata.demo.pool; +package com.taosdata.example.pool; import org.apache.commons.dbcp.BasicDataSource; diff --git a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/DruidPoolBuilder.java b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/pool/DruidPoolBuilder.java similarity index 95% rename from tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/DruidPoolBuilder.java rename to tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/pool/DruidPoolBuilder.java index e9fc496af3..500f0e9e97 100644 --- a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/DruidPoolBuilder.java +++ b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/pool/DruidPoolBuilder.java @@ -1,4 +1,4 @@ -package com.taosdata.demo.pool; +package com.taosdata.example.pool; import com.alibaba.druid.pool.DruidDataSource; diff --git a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/HikariCpBuilder.java b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/pool/HikariCpBuilder.java similarity index 97% rename from tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/HikariCpBuilder.java rename to tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/pool/HikariCpBuilder.java index 5cfe7e6e6a..7e151de3e0 100644 --- a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/HikariCpBuilder.java +++ b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/pool/HikariCpBuilder.java @@ -1,4 +1,4 @@ -package com.taosdata.demo.pool; +package com.taosdata.example.pool; import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariDataSource; From 87530b4935ec30479518e5b50944166fd7f4420a Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Wed, 27 Jan 2021 14:24:11 +0800 Subject: [PATCH 116/147] fix bug --- src/client/src/tscSubquery.c | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 67fd3d24bd..d1136ca4de 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -1991,7 +1991,22 @@ static void tscAbortFurtherRetryRetrieval(SRetrieveSupport *trsupport, TAOS_RES * current query failed, and the retry count is less than the available * count, retry query clear previous retrieved data, then launch a new sub query */ -static int32_t tscReissueSubquery(SRetrieveSupport *trsupport, SSqlObj *pSql, int32_t code) { +static int32_t tscReissueSubquery(SRetrieveSupport *oriTrs, SSqlObj *pSql, int32_t code) { + SRetrieveSupport *trsupport = malloc(sizeof(SRetrieveSupport)); + if (trsupport == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + memcpy(trsupport, oriTrs, sizeof(*trsupport)); + + const uint32_t nBufferSize = (1u << 16u); // 64KB + trsupport->localBuffer = (tFilePage *)calloc(1, nBufferSize + sizeof(tFilePage)); + if (trsupport->localBuffer == NULL) { + tscError("%p failed to malloc buffer for local buffer, reason:%s", pSql, strerror(errno)); + tfree(trsupport); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + SSqlObj *pParentSql = trsupport->pParentSql; int32_t subqueryIndex = trsupport->subqueryIndex; From 3ef5dea4a4176296e8e50ca1f9d651bcd88e3b90 Mon Sep 17 00:00:00 2001 From: zyyang Date: Wed, 27 Jan 2021 14:29:13 +0800 Subject: [PATCH 117/147] change --- .../taosdata/example/ConnectionPoolDemo.java | 58 +++++++++++-------- 1 file changed, 33 insertions(+), 25 deletions(-) diff --git a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/ConnectionPoolDemo.java b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/ConnectionPoolDemo.java index 7b603ec97f..3b288b38dc 100644 --- a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/ConnectionPoolDemo.java +++ b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/ConnectionPoolDemo.java @@ -1,5 +1,6 @@ package com.taosdata.example; +import com.taosdata.example.common.InsertTask; import com.taosdata.example.pool.C3p0Builder; import com.taosdata.example.pool.DbcpBuilder; import com.taosdata.example.pool.DruidPoolBuilder; @@ -10,6 +11,9 @@ import javax.sql.DataSource; import java.sql.Connection; import java.sql.SQLException; import java.sql.Statement; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; public class ConnectionPoolDemo { @@ -31,12 +35,16 @@ public class ConnectionPoolDemo { if ("-host".equalsIgnoreCase(args[i]) && i < args.length - 1) { host = args[++i]; } + if ("-recordNumber".equalsIgnoreCase(args[i]) && i < args.length - 1) { + totalSize = Long.parseLong(args[++i]); + } if ("-poolType".equalsIgnoreCase(args[i]) && i < args.length - 1) { poolType = args[++i]; } + } if (host == null) { - System.out.println("Usage: java -jar XXX.jar -host -poolType "); + System.out.println("Usage: java -jar XXX.jar -host -recordNumber -poolType "); return; } @@ -60,31 +68,31 @@ public class ConnectionPoolDemo { logger.info(">>>>>>>>>>>>>> connection pool Type: " + poolType); init(dataSource); - try { - Connection connection = dataSource.getConnection(); - Statement statement = connection.createStatement(); - String sql = "insert into " + dbName + ".t_1 values('2020-01-01 00:00:00.000',12.12,111)"; - int affectRows = statement.executeUpdate(sql); - System.out.println("affectRows >>> " + affectRows); - affectRows = statement.executeUpdate(sql); - System.out.println("affectRows >>> " + affectRows); - statement.close(); - connection.close(); - } catch (SQLException e) { - e.printStackTrace(); - } - -// ExecutorService executor = Executors.newFixedThreadPool(threadCount); -// for (long i = 0; i < totalSize / batchSize / tableSize; i++) { -// executor.execute(new InsertTask(dataSource, dbName, tableSize, batchSize)); -// // sleep few seconds -// try { -// TimeUnit.MILLISECONDS.sleep(sleep); -// } catch (InterruptedException e) { -// e.printStackTrace(); -// } +// try { +// Connection connection = dataSource.getConnection(); +// Statement statement = connection.createStatement(); +// String sql = "insert into " + dbName + ".t_1 values('2020-01-01 00:00:00.000',12.12,111)"; +// int affectRows = statement.executeUpdate(sql); +// System.out.println("affectRows >>> " + affectRows); +// affectRows = statement.executeUpdate(sql); +// System.out.println("affectRows >>> " + affectRows); +// statement.close(); +// connection.close(); +// } catch (SQLException e) { +// e.printStackTrace(); // } -// executor.shutdown(); + + ExecutorService executor = Executors.newFixedThreadPool(threadCount); + for (long i = 0; i < totalSize / batchSize / tableSize; i++) { + executor.execute(new InsertTask(dataSource, dbName, tableSize, batchSize)); + // sleep few seconds + try { + TimeUnit.MILLISECONDS.sleep(sleep); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + executor.shutdown(); } From 99ef14af98e77378b5be239d65036d14e8c3e069 Mon Sep 17 00:00:00 2001 From: zyyang Date: Wed, 27 Jan 2021 15:12:48 +0800 Subject: [PATCH 118/147] change --- .../taosdata/example/ConnectionPoolDemo.java | 31 +++++++++++++------ 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/ConnectionPoolDemo.java b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/ConnectionPoolDemo.java index 3b288b38dc..183f7345ca 100644 --- a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/ConnectionPoolDemo.java +++ b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/ConnectionPoolDemo.java @@ -20,13 +20,14 @@ public class ConnectionPoolDemo { private static Logger logger = Logger.getLogger(DruidPoolBuilder.class); private static final String dbName = "pool_test"; - private static long totalSize = 1_000_000l; - private static int batchSize = 10; - private static int sleep = 1000; - private static int poolSize = 50; - private static int tableSize = 1000; - private static int threadCount = 50; private static String poolType = "hikari"; + private static long totalSize = 1_000_000l; + private static long tableSize = 1; + private static long batchSize = 1; + + private static int poolSize = 50; + private static int threadCount = 50; + private static int sleep = 1000; public static void main(String[] args) { @@ -35,16 +36,26 @@ public class ConnectionPoolDemo { if ("-host".equalsIgnoreCase(args[i]) && i < args.length - 1) { host = args[++i]; } + if ("-poolType".equalsIgnoreCase(args[i]) && i < args.length - 1) { + poolType = args[++i]; + } if ("-recordNumber".equalsIgnoreCase(args[i]) && i < args.length - 1) { totalSize = Long.parseLong(args[++i]); } - if ("-poolType".equalsIgnoreCase(args[i]) && i < args.length - 1) { - poolType = args[++i]; + if ("-tableNumber".equalsIgnoreCase(args[i]) && i < args.length - 1) { + tableSize = Long.parseLong(args[++i]); + } + if ("-batchNumber".equalsIgnoreCase(args[i]) && i < args.length - 1) { + batchSize = Long.parseLong(args[++i]); } } if (host == null) { - System.out.println("Usage: java -jar XXX.jar -host -recordNumber -poolType "); + System.out.println("Usage: java -jar XXX.jar -host " + + "-recordNumber " + + "-tableNumber " + + "-batchNumber " + + "-poolType "); return; } @@ -83,7 +94,7 @@ public class ConnectionPoolDemo { // } ExecutorService executor = Executors.newFixedThreadPool(threadCount); - for (long i = 0; i < totalSize / batchSize / tableSize; i++) { + for (long i = 0; i < totalSize / tableSize / batchSize; i++) { executor.execute(new InsertTask(dataSource, dbName, tableSize, batchSize)); // sleep few seconds try { From 61a1563864ce355c1f03e3a4b80a2df6b6adf90a Mon Sep 17 00:00:00 2001 From: zyyang Date: Wed, 27 Jan 2021 15:17:10 +0800 Subject: [PATCH 119/147] change --- .../main/java/com/taosdata/example/common/InsertTask.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/common/InsertTask.java b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/common/InsertTask.java index e39a613aa6..da7c9a22b5 100644 --- a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/common/InsertTask.java +++ b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/common/InsertTask.java @@ -13,11 +13,11 @@ public class InsertTask implements Runnable { private static final Logger logger = Logger.getLogger(InsertTask.class); private final DataSource ds; - private final int batchSize; private final String dbName; - private final int tableSize; + private final long tableSize; + private final long batchSize; - public InsertTask(DataSource ds, String dbName, int tableSize, int batchSize) { + public InsertTask(DataSource ds, String dbName, long tableSize, long batchSize) { this.ds = ds; this.dbName = dbName; this.tableSize = tableSize; From 224b850b620cc9dd582e1a71df7ede11c6b05f60 Mon Sep 17 00:00:00 2001 From: zyyang Date: Wed, 27 Jan 2021 15:20:35 +0800 Subject: [PATCH 120/147] change --- .../main/java/com/taosdata/example/ConnectionPoolDemo.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/ConnectionPoolDemo.java b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/ConnectionPoolDemo.java index 183f7345ca..629e0195ef 100644 --- a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/ConnectionPoolDemo.java +++ b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/ConnectionPoolDemo.java @@ -27,8 +27,7 @@ public class ConnectionPoolDemo { private static int poolSize = 50; private static int threadCount = 50; - private static int sleep = 1000; - + private static int sleep = 0; public static void main(String[] args) { String host = null; @@ -98,7 +97,8 @@ public class ConnectionPoolDemo { executor.execute(new InsertTask(dataSource, dbName, tableSize, batchSize)); // sleep few seconds try { - TimeUnit.MILLISECONDS.sleep(sleep); + if (sleep > 0) + TimeUnit.MILLISECONDS.sleep(sleep); } catch (InterruptedException e) { e.printStackTrace(); } From 9b67864f56da431712dc67e4892f8760f70d49bf Mon Sep 17 00:00:00 2001 From: zyyang Date: Wed, 27 Jan 2021 15:21:29 +0800 Subject: [PATCH 121/147] change --- .../main/java/com/taosdata/example/ConnectionPoolDemo.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/ConnectionPoolDemo.java b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/ConnectionPoolDemo.java index 629e0195ef..bd57d138b2 100644 --- a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/ConnectionPoolDemo.java +++ b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/ConnectionPoolDemo.java @@ -51,10 +51,12 @@ public class ConnectionPoolDemo { } if (host == null) { System.out.println("Usage: java -jar XXX.jar -host " + + "-poolType " + "-recordNumber " + "-tableNumber " + "-batchNumber " + - "-poolType "); + "-sleep " + ); return; } From 2839d49757a23127f06c7944e5fdb618b46d52ed Mon Sep 17 00:00:00 2001 From: zyyang Date: Wed, 27 Jan 2021 15:27:12 +0800 Subject: [PATCH 122/147] change --- .../examples/JDBC/connectionPools/README-cn.md | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/tests/examples/JDBC/connectionPools/README-cn.md b/tests/examples/JDBC/connectionPools/README-cn.md index 139a94e36d..9b26df3c2e 100644 --- a/tests/examples/JDBC/connectionPools/README-cn.md +++ b/tests/examples/JDBC/connectionPools/README-cn.md @@ -1,23 +1,25 @@ 这个example中，我们适配了java常见的连接池: -* c3p0 -* dbcp +* HikariCP（默认） * druid -* HikariCP +* dbcp +* c3p0 ### 说明 ConnectionPoolDemo的程序逻辑： 1. 创建到host的connection连接池 2. 创建名称为pool_test的database，创建表超级weather，创建tableSize个子表 -3. 不断向所有子表进行插入。 +3. 总共插入totalNumber条数据。 ### 如何运行这个例子： + ```shell script -# mvn exec:java -Dexec.mainClass="com.taosdata.example.ConnectionPoolDemo" -Dexec.args="-host localhost" +mvn clean package assembly:single +java -jar target/connectionPools-1.0-SNAPSHOT-jar-with-dependencies.jar -host 127.0.0.1 ``` 使用mvn运行ConnectionPoolDemo的main方法，可以指定参数 ```shell script Usage: -mvn exec:java -Dexec.mainClass="com.taosdata.example.ConnectionPoolDemo" -Dexec.args="" +java -jar target/connectionPools-1.0-SNAPSHOT-jar-with-dependencies.jar -host : hostname -poolType -poolSize @@ -26,8 +28,5 @@ mvn exec:java -Dexec.mainClass="com.taosdata.example.ConnectionPoolDemo" -Dexec. -sleep : 每次插入任务提交后的 ``` -### 如何停止程序： -ConnectionPoolDemo不会自己停止，会一直执行插入，需要手动Ctrl+C运行。 - ### 日志 使用log4j，将日志和错误分别输出到了debug.log和error.log中 \ No newline at end of file From 03450b00c06f3b9214abf46a7357ad16a23d485e Mon Sep 17 00:00:00 2001 From: zyyang <69311263+zyyang-taosdata@users.noreply.github.com> Date: Wed, 27 Jan 2021 15:56:53 +0800 Subject: [PATCH 123/147] Update connector-java-ch.md --- documentation20/webdocs/markdowndocs/connector-java-ch.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation20/webdocs/markdowndocs/connector-java-ch.md b/documentation20/webdocs/markdowndocs/connector-java-ch.md index 0654ccd1b9..62c1f11bab 100644 --- a/documentation20/webdocs/markdowndocs/connector-java-ch.md +++ b/documentation20/webdocs/markdowndocs/connector-java-ch.md @@ -418,7 +418,7 @@ Query OK, 1 row(s) in set (0.000141s) ## 与框架使用 * Spring JdbcTemplate 中使用 taos-jdbcdriver，可参考 [SpringJdbcTemplate][11] -* Springboot + Mybatis 中使用，可参考 [springbootdemo +* Springboot + Mybatis 中使用，可参考 [springbootdemo][12] From 70bed13745e20dbdb871f45fb54c2e84db79c606 Mon Sep 17 00:00:00 2001 From: zyyang Date: Wed, 27 Jan 2021 16:07:55 +0800 Subject: [PATCH 124/147] change --- tests/examples/JDBC/JDBCDemo/pom.xml | 16 ++--- .../java/com/taosdata/example/JDBCDemo.java | 70 ++++++++----------- .../com/taosdata/example/JdbcRestfulDemo.java | 6 +- .../com/taosdata/example/SubscribeDemo.java | 1 - 4 files changed, 41 insertions(+), 52 deletions(-) diff --git a/tests/examples/JDBC/JDBCDemo/pom.xml b/tests/examples/JDBC/JDBCDemo/pom.xml index 6f0c58a821..27a03911a2 100644 --- a/tests/examples/JDBC/JDBCDemo/pom.xml +++ b/tests/examples/JDBC/JDBCDemo/pom.xml @@ -9,6 +9,14 @@ SNAPSHOT jar + + + com.taosdata.jdbc + taos-jdbcdriver + 2.0.18 + + + @@ -48,12 +56,4 @@ - - - com.taosdata.jdbc - taos-jdbcdriver - 2.0.18 - - - diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JDBCDemo.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JDBCDemo.java index 0fa0059805..9a34b7d6b1 100644 --- a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JDBCDemo.java +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JDBCDemo.java @@ -5,7 +5,6 @@ import java.util.Properties; public class JDBCDemo { private static String host; - private static String driverType = "jni"; private static final String dbName = "test"; private static final String tbName = "weather"; private Connection connection; @@ -14,17 +13,10 @@ public class JDBCDemo { for (int i = 0; i < args.length; i++) { if ("-host".equalsIgnoreCase(args[i]) && i < args.length - 1) host = args[++i]; - if ("-driverType".equalsIgnoreCase(args[i]) && i < args.length - 1) { - driverType = args[++i]; - if (!"jni".equalsIgnoreCase(driverType) && !"restful".equalsIgnoreCase(driverType)) - printHelp(); - } } - if (host == null) { printHelp(); } - JDBCDemo demo = new JDBCDemo(); demo.init(); demo.createDatabase(); @@ -38,15 +30,10 @@ public class JDBCDemo { } private void init() { + final String url = "jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata"; // get connection try { - String url = "jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata"; - if (driverType.equals("restful")) { - Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); - url = "jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata"; - } else { - Class.forName("com.taosdata.jdbc.TSDBDriver"); - } + Class.forName("com.taosdata.jdbc.TSDBDriver"); Properties properties = new Properties(); properties.setProperty("charset", "UTF-8"); properties.setProperty("locale", "en_US.UTF-8"); @@ -70,11 +57,39 @@ public class JDBCDemo { exuete(sql); } + private void dropTable() { + final String sql = "drop table if exists " + dbName + "." + tbName + ""; + exuete(sql); + } + + private void createTable() { + final String sql = "create table if not exists " + dbName + "." + tbName + " (ts timestamp, temperature float, humidity int)"; + exuete(sql); + } + + private void insert() { + final String sql = "insert into test.weather (ts, temperature, humidity) values(now, 20.5, 34)"; + exuete(sql); + } + private void select() { final String sql = "select * from test.weather"; executeQuery(sql); } + private void close() { + try { + if (connection != null) { + this.connection.close(); + System.out.println("connection closed."); + } + } catch (SQLException e) { + e.printStackTrace(); + } + } + + /************************************************************************/ + private void executeQuery(String sql) { try (Statement statement = connection.createStatement()) { long start = System.currentTimeMillis(); @@ -99,15 +114,6 @@ public class JDBCDemo { } } - private void insert() { - final String sql = "insert into test.weather (ts, temperature, humidity) values(now, 20.5, 34)"; - exuete(sql); - } - - private void createTable() { - final String sql = "create table if not exists " + dbName + "." + tbName + " (ts timestamp, temperature float, humidity int)"; - exuete(sql); - } private void printSql(String sql, boolean succeed, long cost) { System.out.println("[ " + (succeed ? "OK" : "ERROR!") + " ] time cost: " + cost + " ms, execute statement ====> " + sql); @@ -125,22 +131,6 @@ public class JDBCDemo { } } - private void close() { - try { - if (connection != null) { - this.connection.close(); - System.out.println("connection closed."); - } - } catch (SQLException e) { - e.printStackTrace(); - } - } - - private void dropTable() { - final String sql = "drop table if exists " + dbName + "." + tbName + ""; - exuete(sql); - } - private static void printHelp() { System.out.println("Usage: java -jar JdbcDemo.jar -host -driverType "); System.exit(0); diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JdbcRestfulDemo.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JdbcRestfulDemo.java index 91639e85f3..eedb0ba166 100644 --- a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JdbcRestfulDemo.java +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JdbcRestfulDemo.java @@ -14,9 +14,9 @@ public class JdbcRestfulDemo { String url = "jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata"; Properties properties = new Properties(); -// properties.setProperty("charset", "UTF-8"); -// properties.setProperty("locale", "en_US.UTF-8"); -// properties.setProperty("timezone", "UTC-8"); + properties.setProperty("charset", "UTF-8"); + properties.setProperty("locale", "en_US.UTF-8"); + properties.setProperty("timezone", "UTC-8"); Connection conn = DriverManager.getConnection(url, properties); Statement stmt = conn.createStatement(); diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/SubscribeDemo.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/SubscribeDemo.java index bb3712a302..2c3a263729 100644 --- a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/SubscribeDemo.java +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/SubscribeDemo.java @@ -13,7 +13,6 @@ public class SubscribeDemo { public static TSDBConnection getConnection(String host, String database) throws Exception { Class.forName("com.taosdata.jdbc.TSDBDriver"); Properties properties = new Properties(); - properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host); properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); From 4cd9f95150f4ab84af11cdde16e1b68ff3c9a512 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Wed, 27 Jan 2021 16:10:14 +0800 Subject: [PATCH 125/147] [TD-2864]improve filedistribution test --- .../multilevel/fileDistributionSameLevel.py | 44 +++++++++++++++++-- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/tests/pytest/multilevel/fileDistributionSameLevel.py b/tests/pytest/multilevel/fileDistributionSameLevel.py index 88a7216b8e..83ce856717 100644 --- a/tests/pytest/multilevel/fileDistributionSameLevel.py +++ b/tests/pytest/multilevel/fileDistributionSameLevel.py @@ -18,18 +18,36 @@ from util.cases import * from util.sql import * from util.dnodes import * - +dataDir = ['data00','data01','data02','data03','data04'] +dataDict = {'data00':0,'data01':0,'data02':0,'data03':0,'data04':0} class TDTestCase: def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) + def getfiles(self,ospath): + try: + files = os.listdir(ospath) + for f in files: + path = os.path.join(ospath, f) + if os.path.isfile(path): + if path.endswith('.data'): + for i in dataDir: + if i in path: + dataDict[i] = dataDict[i] + 1 + print(path) + if os.path.isdir(path): + self.getfiles(path) + except Exception as e : + print(str(e)) + def run(self): self.ntables = 1000 self.ts = 1520000010000 tdDnodes.stop(1) # Test1 1 dataDir + tdLog.info("================= step1") cfg={ '10' : 'maxVgroupsPerDb', '100' : 'maxTablesPerVnode', @@ -55,15 +73,33 @@ class TDTestCase: for i in range(self.ntables): tdSql.execute("create table tb%d using stb tags(%d)" %(i, i)) - tdSql.execute("insert into tb%d values(%d, 1)" % (self.ts + int (i / 100) * 86400000)) - + tdSql.execute("insert into tb%d values(%d, 1)" % (i,self.ts + int (i / 100) * 86400000)) + + tdLog.info("================= step2") tdDnodes.stop(1) tdDnodes.start(1) tdSql.query("select * from test.stb") tdSql.checkRows(1000) - + tdLog.info("================= step3") + tdSql.execute('drop database test') + for i in range(50): + tdSql.execute("create database test%d days 1" %(i)) + tdSql.execute("use test%d" %(i)) + tdSql.execute("create table tb (ts timestamp,i int)") + for j in range(10): + tdSql.execute("insert into tb values(%d, 1)" % (self.ts + int (i / 100) * 86400000)) + tdDnodes.stop(1) + tdDnodes.start(1) + flag = True + for i in range(4): + if dataDict[dataDir[i]] == dataDict[dataDir[i+1]]: + flag = flag & True + else: + flag = flag & False + break + if not flag : tdLog.exit("%s failed, expect not occured" % (sys.argv[0])) def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) From 1b41b4e0e6edf6159411d72adf3aca870de23b14 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Wed, 27 Jan 2021 16:11:15 +0800 Subject: [PATCH 126/147] [TD-2864]improve filedistribution test --- .../multilevel/fileDistributionSameLevel.py | 61 +++++++++++++++---- 1 file changed, 49 insertions(+), 12 deletions(-) diff --git a/tests/pytest/multilevel/fileDistributionSameLevel.py b/tests/pytest/multilevel/fileDistributionSameLevel.py index e2880283b5..83ce856717 100644 --- a/tests/pytest/multilevel/fileDistributionSameLevel.py +++ b/tests/pytest/multilevel/fileDistributionSameLevel.py @@ -18,26 +18,44 @@ from util.cases import * from util.sql import * from util.dnodes import * - +dataDir = ['data00','data01','data02','data03','data04'] +dataDict = {'data00':0,'data01':0,'data02':0,'data03':0,'data04':0} class TDTestCase: def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) + def getfiles(self,ospath): + try: + files = os.listdir(ospath) + for f in files: + path = os.path.join(ospath, f) + if os.path.isfile(path): + if path.endswith('.data'): + for i in dataDir: + if i in path: + dataDict[i] = dataDict[i] + 1 + print(path) + if os.path.isdir(path): + self.getfiles(path) + except Exception as e : + print(str(e)) + def run(self): - self.ntables = 10 - self.rowsPerTable = 10 + self.ntables = 1000 self.ts = 1520000010000 tdDnodes.stop(1) # Test1 1 dataDir + tdLog.info("================= step1") cfg={ '10' : 'maxVgroupsPerDb', + '100' : 'maxTablesPerVnode', '/mnt/data00 0 1' : 'dataDir', '/mnt/data01 0 0' : 'dataDir', '/mnt/data02 0 0' : 'dataDir', - '/mnt/data03 1 0' : 'dataDir', - '/mnt/data04 1 0' : 'dataDir' + '/mnt/data03 0 0' : 'dataDir', + '/mnt/data04 0 0' : 'dataDir' } tdSql.createDir('/mnt/data00') tdSql.createDir('/mnt/data01') @@ -51,18 +69,37 @@ class TDTestCase: tdSql.execute("create database test days 1") tdSql.execute("use test") - tdSql.execute("create table tb(ts timestamp, c int)") + tdSql.execute("create table stb(ts timestamp, c int) tags(t int)") - for i in range(self.rowsPerTable): - tdSql.execute("insert into tb values(%d, 1)" % (self.ts + i * 86400000)) - + for i in range(self.ntables): + tdSql.execute("create table tb%d using stb tags(%d)" %(i, i)) + tdSql.execute("insert into tb%d values(%d, 1)" % (i,self.ts + int (i / 100) * 86400000)) + + tdLog.info("================= step2") tdDnodes.stop(1) tdDnodes.start(1) - tdSql.query("select * from test.tb") - tdSql.checkRows(10) - + tdSql.query("select * from test.stb") + tdSql.checkRows(1000) + tdLog.info("================= step3") + tdSql.execute('drop database test') + for i in range(50): + tdSql.execute("create database test%d days 1" %(i)) + tdSql.execute("use test%d" %(i)) + tdSql.execute("create table tb (ts timestamp,i int)") + for j in range(10): + tdSql.execute("insert into tb values(%d, 1)" % (self.ts + int (i / 100) * 86400000)) + tdDnodes.stop(1) + tdDnodes.start(1) + flag = True + for i in range(4): + if dataDict[dataDir[i]] == dataDict[dataDir[i+1]]: + flag = flag & True + else: + flag = flag & False + break + if not flag : tdLog.exit("%s failed, expect not occured" % (sys.argv[0])) def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) From 3e588cffd068428219201fc7e6d06ab3df3e98fc Mon Sep 17 00:00:00 2001 From: zyyang Date: Wed, 27 Jan 2021 16:17:23 +0800 Subject: [PATCH 127/147] change --- tests/examples/JDBC/JDBCDemo/readme.md | 4 ++-- .../JDBCDemo/src/main/java/com/taosdata/example/JDBCDemo.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/examples/JDBC/JDBCDemo/readme.md b/tests/examples/JDBC/JDBCDemo/readme.md index da638a0bcc..9484442085 100644 --- a/tests/examples/JDBC/JDBCDemo/readme.md +++ b/tests/examples/JDBC/JDBCDemo/readme.md @@ -11,12 +11,12 @@ Download the tdengine package on our website: ``https://www.taosdata.com/cn/all- ## Run jdbcDemo using mvn plugin run command: ``` -mvn clean compile exec:java -Dexec.mainClass="com.taosdata.example.JdbcDemo" +mvn clean compile exec:java -Dexec.mainClass="com.taosdata.example.JDBCDemo" ``` and run with your customed args ``` -mvn clean compile exec:java -Dexec.mainClass="com.taosdata.example.JdbcDemo" -Dexec.args="-host [HOSTNAME]" +mvn clean compile exec:java -Dexec.mainClass="com.taosdata.example.JDBCDemo" -Dexec.args="-host [HOSTNAME]" ``` ## Compile the Demo Code and Run It diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JDBCDemo.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JDBCDemo.java index 9a34b7d6b1..07a21e79d1 100644 --- a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JDBCDemo.java +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JDBCDemo.java @@ -132,7 +132,7 @@ public class JDBCDemo { } private static void printHelp() { - System.out.println("Usage: java -jar JdbcDemo.jar -host -driverType "); + System.out.println("Usage: java -jar JDBCDemo.jar -host "); System.exit(0); } From f83aad38c614a0b2f2c8eaa9c5ccb313c6c52125 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Wed, 27 Jan 2021 16:20:36 +0800 Subject: [PATCH 128/147] [TD-2714] add test case for stream --- tests/pytest/stream/stream3.py | 108 +++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 tests/pytest/stream/stream3.py diff --git a/tests/pytest/stream/stream3.py b/tests/pytest/stream/stream3.py new file mode 100644 index 0000000000..9a5c6c9aec --- /dev/null +++ b/tests/pytest/stream/stream3.py @@ -0,0 +1,108 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import time +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + ts = 1500000000000 + tbNum = 10 + rowNum = 20 + + tdSql.prepare() + + tdLog.info("===== step1 =====") + tdSql.execute( + "create table stb0(ts timestamp, col1 binary(20), col2 nchar(20)) tags(tgcol int)") + for i in range(tbNum): + tdSql.execute("create table tb%d using stb0 tags(%d)" % (i, i)) + for j in range(rowNum): + tdSql.execute( + "insert into tb%d values (%d, 'binary%d', 'nchar%d')" % + (i, ts + 60000 * j, j, j)) + tdSql.execute("insert into tb0 values(%d, null, null)" % (ts + 10000000)) + time.sleep(0.1) + + tdLog.info("===== step2 =====") + tdSql.query( + "select count(*), count(col1), count(col2) from stb0 interval(1d)") + tdSql.checkData(0, 1, rowNum * tbNum + 1) + tdSql.checkData(0, 2, rowNum * tbNum) + tdSql.checkData(0, 3, rowNum * tbNum) + + tdSql.query("show tables") + tdSql.checkRows(tbNum) + tdSql.execute( + "create table s0 as select count(*), count(col1), count(col2) from stb0 interval(1d)") + tdSql.query("show tables") + tdSql.checkRows(tbNum + 1) + + tdLog.info("===== step3 =====") + tdSql.waitedQuery("select * from s0", 1, 120) + try: + tdSql.checkData(0, 1, rowNum * tbNum + 1) + tdSql.checkData(0, 2, rowNum * tbNum) + tdSql.checkData(0, 3, rowNum * tbNum) + except Exception as e: + tdLog.info(repr(e)) + + tdLog.info("===== step4 =====") + tdSql.execute("drop table s0") + tdSql.query("show tables") + tdSql.checkRows(tbNum) + + tdLog.info("===== step5 =====") + tdSql.error("select * from s0") + + tdLog.info("===== step6 =====") + time.sleep(0.1) + tdSql.execute( + "create table s0 as select count(*), count(col1), count(col2) from tb0 interval(1d)") + tdSql.query("show tables") + tdSql.checkRows(tbNum + 1) + + tdLog.info("===== step7 =====") + tdSql.waitedQuery("select * from s0", 1, 120) + try: + tdSql.checkData(0, 1, rowNum + 1) + tdSql.checkData(0, 2, rowNum) + tdSql.checkData(0, 3, rowNum) + except Exception as e: + tdLog.info(repr(e)) + + tdLog.info("===== step8 =====") + tdSql.query( + "select count(*), count(col1), count(col2) from stb0 interval(1d)") + tdSql.checkData(0, 1, rowNum * tbNum + 1) + tdSql.checkData(0, 2, rowNum * tbNum) + tdSql.checkData(0, 3, rowNum * tbNum) + tdSql.query("show tables") + tdSql.checkRows(tbNum + 1) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From a763cb48fe074f055c4021808de442e8548c9747 Mon Sep 17 00:00:00 2001 From: zyyang Date: Wed, 27 Jan 2021 16:20:43 +0800 Subject: [PATCH 129/147] change --- tests/examples/JDBC/calciteDemo/pom.xml | 53 --------------- .../taosdata/example/calcite/CalciteDemo.java | 67 ------------------- .../src/main/resources/log4j.properties | 6 -- 3 files changed, 126 deletions(-) delete mode 100644 tests/examples/JDBC/calciteDemo/pom.xml delete mode 100644 tests/examples/JDBC/calciteDemo/src/main/java/com/taosdata/example/calcite/CalciteDemo.java delete mode 100644 tests/examples/JDBC/calciteDemo/src/main/resources/log4j.properties diff --git a/tests/examples/JDBC/calciteDemo/pom.xml b/tests/examples/JDBC/calciteDemo/pom.xml deleted file mode 100644 index a7d47837d9..0000000000 --- a/tests/examples/JDBC/calciteDemo/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - 4.0.0 - - com.taosdata.example.calcite - calciteDemo - 1.0-SNAPSHOT - - - - - org.slf4j - slf4j-simple - 1.7.25 - compile - - - - org.apache.calcite - calcite-core - 1.23.0 - - - org.apache.commons - commons-dbcp2 - 2.7.0 - - - org.apache.calcite.avatica - avatica-core - 1.17.0 - - - - - mysql - mysql-connector-java - 5.1.47 - - - - - com.taosdata.jdbc - taos-jdbcdriver - 2.0.8 - - - - - - \ No newline at end of file diff --git a/tests/examples/JDBC/calciteDemo/src/main/java/com/taosdata/example/calcite/CalciteDemo.java b/tests/examples/JDBC/calciteDemo/src/main/java/com/taosdata/example/calcite/CalciteDemo.java deleted file mode 100644 index 7e97956b78..0000000000 --- a/tests/examples/JDBC/calciteDemo/src/main/java/com/taosdata/example/calcite/CalciteDemo.java +++ /dev/null @@ -1,67 +0,0 @@ -package com.taosdata.example.calcite; - -import org.apache.calcite.adapter.jdbc.JdbcSchema; -import org.apache.calcite.jdbc.CalciteConnection; -import org.apache.calcite.schema.Schema; -import org.apache.calcite.schema.SchemaPlus; -import org.apache.calcite.sql.parser.SqlParseException; -import org.apache.commons.dbcp2.BasicDataSource; - -import java.sql.*; -import java.util.Properties; - -public class CalciteDemo { - - private static String url_taos = "jdbc:TAOS://192.168.236.135:6030/test"; - private static String url_mysql = "jdbc:mysql://master:3306/test?useSSL=false&useUnicode=true&characterEncoding=UTF-8"; - - public static void main(String[] args) throws SqlParseException, ClassNotFoundException, SQLException { - Class.forName("org.apache.calcite.jdbc.Driver"); - Properties info = new Properties(); - info.setProperty("caseSensitive", "false"); - - Connection connection = DriverManager.getConnection("jdbc:calcite:", info); - CalciteConnection calciteConnection = connection.unwrap(CalciteConnection.class); - - SchemaPlus rootSchema = calciteConnection.getRootSchema(); - - //这里hdb是在tdengine中创建的数据库名 - Schema schema = mysqlTest(rootSchema); -// Schema schema = tdengineTest(rootSchema); - - //创建新的schema自动映射到原来的hdb数据库 - rootSchema.add("test", schema); - - Statement stmt = calciteConnection.createStatement(); - //查询schema test中的表，表名是tdengine中的表 - ResultSet rs = stmt.executeQuery("select * from test.t"); - ResultSetMetaData metaData = rs.getMetaData(); - while (rs.next()) { - for (int i = 1; i <= metaData.getColumnCount(); i++) { - System.out.println(metaData.getColumnLabel(i) + " : " + rs.getString(i)); - } - } - } - - - private static Schema tdengineTest(SchemaPlus rootSchema) throws ClassNotFoundException { - Class.forName("com.taosdata.jdbc.TSDBDriver"); - BasicDataSource dataSource = new BasicDataSource(); - dataSource.setUrl(url_taos); - dataSource.setUsername("root"); - dataSource.setPassword("taosdata"); - - return JdbcSchema.create(rootSchema, "test", dataSource, "hdb", null); - } - - private static Schema mysqlTest(SchemaPlus rootSchema) throws ClassNotFoundException { - Class.forName("com.mysql.jdbc.Driver"); - BasicDataSource dataSource = new BasicDataSource(); - dataSource.setUrl(url_mysql); - dataSource.setUsername("root"); - dataSource.setPassword("123456"); - - //Schema schema = JdbcSchema.create(rootSchema, "test", dataSource, "hdb", null); - return JdbcSchema.create(rootSchema, "test", dataSource, "test", null); - } -} diff --git a/tests/examples/JDBC/calciteDemo/src/main/resources/log4j.properties b/tests/examples/JDBC/calciteDemo/src/main/resources/log4j.properties deleted file mode 100644 index 1a77ec520c..0000000000 --- a/tests/examples/JDBC/calciteDemo/src/main/resources/log4j.properties +++ /dev/null @@ -1,6 +0,0 @@ -log4j.rootLogger=info,stdout - -#console -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern= [%d{yyyy-MM-dd HH:mm:ss a}]:%p %l%m%n \ No newline at end of file From c8a901c9db9a84087e8d8e855e87384979e7ac2d Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Wed, 27 Jan 2021 16:24:49 +0800 Subject: [PATCH 130/147] [TD-2718] add sim case into full test --- tests/script/jenkins/basic.txt | 1 + tests/script/jenkins/basic_1.txt | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index b5f608f3d3..4f42d043d9 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -196,6 +196,7 @@ cd ../../../debug; make ./test.sh -f general/table/table.sim ./test.sh -f general/table/tinyint.sim ./test.sh -f general/table/vgroup.sim +./test.sh -f general/table/createmulti.sim ./test.sh -f general/tag/3.sim ./test.sh -f general/tag/4.sim diff --git a/tests/script/jenkins/basic_1.txt b/tests/script/jenkins/basic_1.txt index c124b60f80..7f7ef18368 100644 --- a/tests/script/jenkins/basic_1.txt +++ b/tests/script/jenkins/basic_1.txt @@ -157,4 +157,4 @@ ./test.sh -f general/table/table.sim ./test.sh -f general/table/tinyint.sim ./test.sh -f general/table/vgroup.sim - +./test.sh -f general/table/createmulti.sim \ No newline at end of file From 95b466360401ff5e197c44cc2582033efa7f61cc Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 27 Jan 2021 08:36:16 +0000 Subject: [PATCH 131/147] TD-2573 --- src/os/src/detail/osSysinfo.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/os/src/detail/osSysinfo.c b/src/os/src/detail/osSysinfo.c index 2ba0dd6917..360e99bb8f 100644 --- a/src/os/src/detail/osSysinfo.c +++ b/src/os/src/detail/osSysinfo.c @@ -248,7 +248,7 @@ static void taosGetSystemLocale() { // get and set default locale if (cfg_locale && cfg_locale->cfgStatus < TAOS_CFG_CSTATUS_DEFAULT) { locale = setlocale(LC_CTYPE, ""); if (locale == NULL) { - uError("can't get locale from system, set it to en_US.UTF-8"); + uError("can't get locale from system, set it to en_US.UTF-8 since error:%d:%s", errno, strerror(errno)); strcpy(tsLocale, "en_US.UTF-8"); } else { tstrncpy(tsLocale, locale, TSDB_LOCALE_LEN); From 0ae92fd5c5fcfa548ae6629d57d238165ef6c4fd Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Wed, 27 Jan 2021 16:41:16 +0800 Subject: [PATCH 132/147] retest general/tag/bool_int.sim --- tests/script/jenkins/basic_2.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/jenkins/basic_2.txt b/tests/script/jenkins/basic_2.txt index f35a0cd31d..9b3b3f9b18 100644 --- a/tests/script/jenkins/basic_2.txt +++ b/tests/script/jenkins/basic_2.txt @@ -10,7 +10,7 @@ cd ../../../debug; make ./test.sh -f general/tag/binary_binary.sim ./test.sh -f general/tag/binary.sim ./test.sh -f general/tag/bool_binary.sim -#./test.sh -f general/tag/bool_int.sim +./test.sh -f general/tag/bool_int.sim ./test.sh -f general/tag/bool.sim ./test.sh -f general/tag/change.sim ./test.sh -f general/tag/column.sim From b346f51889115b7044d8302c13b337b41eeaa0a5 Mon Sep 17 00:00:00 2001 From: zyyang Date: Wed, 27 Jan 2021 17:10:08 +0800 Subject: [PATCH 133/147] change --- .../com/taosdata/example/SubscribeDemo.java | 64 ++++++++----------- 1 file changed, 27 insertions(+), 37 deletions(-) diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/SubscribeDemo.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/SubscribeDemo.java index 2c3a263729..0fdfcc51a1 100644 --- a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/SubscribeDemo.java +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/SubscribeDemo.java @@ -7,52 +7,45 @@ import com.taosdata.jdbc.TSDBSubscribe; import java.sql.DriverManager; import java.util.Properties; +import java.util.concurrent.TimeUnit; public class SubscribeDemo { + private static final String usage = "java -jar SubscribeDemo.jar -host -database -topic -sql "; - public static TSDBConnection getConnection(String host, String database) throws Exception { - Class.forName("com.taosdata.jdbc.TSDBDriver"); - Properties properties = new Properties(); - properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - - String cs = String.format("jdbc:TAOS://%s:0/%s", host, database); - return (TSDBConnection) DriverManager.getConnection(cs, properties); - } - - public static void main(String[] args) throws Exception { - String usage = "java -Djava.ext.dirs=../ TestTSDBSubscribe [-host host] <-db database> <-topic topic> <-sql sql>"; - if (args.length < 2) { - System.err.println(usage); - return; - } - - String host = "localhost", database = "", topic = "", sql = ""; + public static void main(String[] args) { + // parse args from command line + String host = "", database = "", topic = "", sql = ""; for (int i = 0; i < args.length; i++) { - if ("-db".equalsIgnoreCase(args[i]) && i < args.length - 1) { + if ("-host".equalsIgnoreCase(args[i]) && i < args.length - 1) { + host = args[++i]; + } + if ("-database".equalsIgnoreCase(args[i]) && i < args.length - 1) { database = args[++i]; } if ("-topic".equalsIgnoreCase(args[i]) && i < args.length - 1) { topic = args[++i]; } - if ("-host".equalsIgnoreCase(args[i]) && i < args.length - 1) { - host = args[++i]; - } if ("-sql".equalsIgnoreCase(args[i]) && i < args.length - 1) { sql = args[++i]; } } - if (database.isEmpty() || topic.isEmpty() || sql.isEmpty()) { - System.err.println(usage); + if (host.isEmpty() || database.isEmpty() || topic.isEmpty() || sql.isEmpty()) { + System.out.println(usage); return; } + /*********************************************************************************************/ - TSDBConnection connection = null; - TSDBSubscribe sub = null; try { - connection = getConnection(host, database); - sub = ((TSDBConnection) connection).subscribe(topic, sql, false); + Class.forName("com.taosdata.jdbc.TSDBDriver"); + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + final String url = "jdbc:TAOS://" + host + ":6030/" + database + "?user=root&password=taosdata"; + // get TSDBConnection + TSDBConnection connection = DriverManager.getConnection(url, properties).unwrap(TSDBConnection.class); + // create TSDBSubscribe + TSDBSubscribe sub = connection.subscribe(topic, sql, false); int total = 0; while (true) { @@ -63,17 +56,14 @@ public class SubscribeDemo { } total += count; System.out.printf("%d rows consumed, total %d\n", count, total); - Thread.sleep(900); + if (total >= 10) + break; + TimeUnit.SECONDS.sleep(1); } + sub.close(false); + connection.close(); } catch (Exception e) { e.printStackTrace(); - } finally { - if (null != sub) { - sub.close(true); - } - if (null != connection) { - connection.close(); - } } } } From a95fc2a3023a1088e2798672e8b22bd1fe75c10a Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Wed, 27 Jan 2021 17:11:06 +0800 Subject: [PATCH 134/147] [TD-2648] update script for docker cluster --- tests/pytest/cluster/clusterEnvSetup/basic.py | 26 +++++++++++++++++++ .../clusterEnvSetup/docker-compose.yml | 6 ++--- 2 files changed, 29 insertions(+), 3 deletions(-) create mode 100644 tests/pytest/cluster/clusterEnvSetup/basic.py diff --git a/tests/pytest/cluster/clusterEnvSetup/basic.py b/tests/pytest/cluster/clusterEnvSetup/basic.py new file mode 100644 index 0000000000..10ba91ab06 --- /dev/null +++ b/tests/pytest/cluster/clusterEnvSetup/basic.py @@ -0,0 +1,26 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import os +import random + +class ClusterTestcase: + + + def run(self): + os.system("./buildClusterEnv.sh -n 3 -v 2.0.14.1") + os.system("yes|taosdemo -h 172.27.0.7 -n 100 -t 100 -x") + os.system("python3 ../../concurrent_inquiry.py -H 172.27.0.7 -T 4 -t 4 -l 10") + +clusterTest = ClusterTestcase() +clusterTest.run() \ No newline at end of file diff --git a/tests/pytest/cluster/clusterEnvSetup/docker-compose.yml b/tests/pytest/cluster/clusterEnvSetup/docker-compose.yml index 33deff8d1e..c45a09582b 100644 --- a/tests/pytest/cluster/clusterEnvSetup/docker-compose.yml +++ b/tests/pytest/cluster/clusterEnvSetup/docker-compose.yml @@ -39,7 +39,7 @@ services: networks: taos_update_net: ipv4_address: 172.27.0.7 - command: bash -c "taosd && tail -f /dev/null" + command: taosd td2.0-node2: build: @@ -78,7 +78,7 @@ services: networks: taos_update_net: ipv4_address: 172.27.0.8 - command: bash -c "taosd && tail -f /dev/null" + command: taosd td2.0-node3: build: @@ -117,7 +117,7 @@ services: networks: taos_update_net: ipv4_address: 172.27.0.9 - command: bash -c "taosd && tail -f /dev/null" + command: taosd networks: taos_update_net: From ee85996beb5e760d93b7ded1cbb7f220eabab830 Mon Sep 17 00:00:00 2001 From: zyyang Date: Wed, 27 Jan 2021 17:51:38 +0800 Subject: [PATCH 135/147] change --- .../src/main/java/com/taosdata/example/SubscribeDemo.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/SubscribeDemo.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/SubscribeDemo.java index 0fdfcc51a1..6d036201b3 100644 --- a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/SubscribeDemo.java +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/SubscribeDemo.java @@ -34,7 +34,6 @@ public class SubscribeDemo { return; } /*********************************************************************************************/ - try { Class.forName("com.taosdata.jdbc.TSDBDriver"); Properties properties = new Properties(); @@ -43,7 +42,7 @@ public class SubscribeDemo { properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); final String url = "jdbc:TAOS://" + host + ":6030/" + database + "?user=root&password=taosdata"; // get TSDBConnection - TSDBConnection connection = DriverManager.getConnection(url, properties).unwrap(TSDBConnection.class); + TSDBConnection connection = (TSDBConnection) DriverManager.getConnection(url, properties); // create TSDBSubscribe TSDBSubscribe sub = connection.subscribe(topic, sql, false); @@ -66,4 +65,4 @@ public class SubscribeDemo { e.printStackTrace(); } } -} +} \ No newline at end of file From 40f4534f2a86913bfe6f87d528973c7177f2e206 Mon Sep 17 00:00:00 2001 From: zyyang Date: Wed, 27 Jan 2021 17:55:54 +0800 Subject: [PATCH 136/147] change --- .../src/main/java/com/taosdata/example/SubscribeDemo.java | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/SubscribeDemo.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/SubscribeDemo.java index 6d036201b3..a5b8bc8194 100644 --- a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/SubscribeDemo.java +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/SubscribeDemo.java @@ -62,6 +62,7 @@ public class SubscribeDemo { sub.close(false); connection.close(); } catch (Exception e) { + System.out.println("host: " + host + ", database: " + database + ", topic: " + topic + ", sql: " + sql); e.printStackTrace(); } } From a792e6c3d032a701b38493dc2485b9e23b66fce1 Mon Sep 17 00:00:00 2001 From: zyyang Date: Wed, 27 Jan 2021 18:51:58 +0800 Subject: [PATCH 137/147] change --- .../src/main/java/com/taosdata/example/SubscribeDemo.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/SubscribeDemo.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/SubscribeDemo.java index a5b8bc8194..def4c64902 100644 --- a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/SubscribeDemo.java +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/SubscribeDemo.java @@ -6,6 +6,7 @@ import com.taosdata.jdbc.TSDBResultSet; import com.taosdata.jdbc.TSDBSubscribe; import java.sql.DriverManager; +import java.sql.ResultSetMetaData; import java.util.Properties; import java.util.concurrent.TimeUnit; @@ -50,11 +51,16 @@ public class SubscribeDemo { while (true) { TSDBResultSet rs = sub.consume(); int count = 0; + ResultSetMetaData meta = rs.getMetaData(); while (rs.next()) { + for (int i = 1; i <= meta.getColumnCount(); i++) { + System.out.print(meta.getColumnLabel(i) + ": " + rs.getString(i) + "\t"); + } + System.out.println(); count++; } total += count; - System.out.printf("%d rows consumed, total %d\n", count, total); +// System.out.printf("%d rows consumed, total %d\n", count, total); if (total >= 10) break; TimeUnit.SECONDS.sleep(1); From c076d6aeb225da13e0002c58fb9b67ad0f922b46 Mon Sep 17 00:00:00 2001 From: zyyang Date: Wed, 27 Jan 2021 18:57:04 +0800 Subject: [PATCH 138/147] change --- tests/examples/JDBC/taosdemo/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/examples/JDBC/taosdemo/pom.xml b/tests/examples/JDBC/taosdemo/pom.xml index 15f868a117..a6cbe4615e 100644 --- a/tests/examples/JDBC/taosdemo/pom.xml +++ b/tests/examples/JDBC/taosdemo/pom.xml @@ -67,7 +67,7 @@ com.taosdata.jdbc taos-jdbcdriver - 2.0.17 + 2.0.18 From 6fecb25da815875e8d13a3ebe0a0e96d2fa661bf Mon Sep 17 00:00:00 2001 From: zyyang Date: Wed, 27 Jan 2021 19:25:59 +0800 Subject: [PATCH 139/147] change --- tests/examples/JDBC/springbootdemo/pom.xml | 6 +- tests/examples/JDBC/springbootdemo/readme.md | 6 +- .../SpringbootdemoApplication.java | 8 +-- .../controller/WeatherController.java | 6 +- .../springbootdemo/dao/WeatherMapper.java | 4 +- .../springbootdemo/dao/WeatherMapper.xml | 6 +- .../springbootdemo/domain/Weather.java | 2 +- .../service/WeatherService.java | 6 +- .../controller/RainStationController.java | 28 -------- .../springbootdemo/dao/DatabaseMapper.java | 15 ---- .../springbootdemo/dao/DatabaseMapper.xml | 44 ------------ .../springbootdemo/dao/RainfallMapper.java | 9 --- .../springbootdemo/dao/RainfallMapper.xml | 11 --- .../jdbc/springbootdemo/dao/TableMapper.java | 8 --- .../jdbc/springbootdemo/dao/TableMapper.xml | 21 ------ .../springbootdemo/domain/FieldMetadata.java | 28 -------- .../jdbc/springbootdemo/domain/Rainfall.java | 64 ----------------- .../springbootdemo/domain/TableMetadata.java | 43 ----------- .../springbootdemo/domain/TagMetadata.java | 27 ------- .../service/RainStationService.java | 72 ------------------- .../src/main/resources/application.properties | 20 ++---- .../SpringbootdemoApplicationTests.java | 13 ---- 22 files changed, 27 insertions(+), 420 deletions(-) rename tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/{jdbc => example}/springbootdemo/SpringbootdemoApplication.java (66%) rename tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/{jdbc => example}/springbootdemo/controller/WeatherController.java (87%) rename tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/{jdbc => example}/springbootdemo/dao/WeatherMapper.java (74%) rename tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/{jdbc => example}/springbootdemo/dao/WeatherMapper.xml (85%) rename tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/{jdbc => example}/springbootdemo/domain/Weather.java (93%) rename tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/{jdbc => example}/springbootdemo/service/WeatherService.java (82%) delete mode 100644 tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/controller/RainStationController.java delete mode 100644 tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/DatabaseMapper.java delete mode 100644 tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/DatabaseMapper.xml delete mode 100644 tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/RainfallMapper.java delete mode 100644 tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/RainfallMapper.xml delete mode 100644 tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/TableMapper.java delete mode 100644 tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/TableMapper.xml delete mode 100644 tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/domain/FieldMetadata.java delete mode 100644 tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/domain/Rainfall.java delete mode 100644 tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/domain/TableMetadata.java delete mode 100644 tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/domain/TagMetadata.java delete mode 100644 tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/service/RainStationService.java delete mode 100644 tests/examples/JDBC/springbootdemo/src/test/java/com/taosdata/jdbc/springbootdemo/SpringbootdemoApplicationTests.java diff --git a/tests/examples/JDBC/springbootdemo/pom.xml b/tests/examples/JDBC/springbootdemo/pom.xml index 881ea0d6bf..52fb8caa90 100644 --- a/tests/examples/JDBC/springbootdemo/pom.xml +++ b/tests/examples/JDBC/springbootdemo/pom.xml @@ -8,7 +8,7 @@ 2.2.1.RELEASE - com.taosdata.jdbc + com.taosdata.example springbootdemo 0.0.1-SNAPSHOT springbootdemo @@ -63,7 +63,7 @@ com.taosdata.jdbc taos-jdbcdriver - 2.0.4 + 2.0.18 @@ -71,8 +71,6 @@ druid-spring-boot-starter 1.1.17 - - diff --git a/tests/examples/JDBC/springbootdemo/readme.md b/tests/examples/JDBC/springbootdemo/readme.md index ac3bb44ef9..67a28947d2 100644 --- a/tests/examples/JDBC/springbootdemo/readme.md +++ b/tests/examples/JDBC/springbootdemo/readme.md @@ -47,7 +47,7 @@ logging.level.com.taosdata.jdbc.springbootdemo.dao=debug * 插入单条记录 ```xml - + insert into test.weather (ts, temperature, humidity) values (now, #{temperature,jdbcType=INTEGER}, #{humidity,jdbcType=FLOAT}) ``` @@ -67,9 +67,9 @@ logging.level.com.taosdata.jdbc.springbootdemo.dao=debug - + - + diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/SpringbootdemoApplication.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/SpringbootdemoApplication.java similarity index 66% rename from tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/SpringbootdemoApplication.java rename to tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/SpringbootdemoApplication.java index f693214567..8066126d62 100644 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/SpringbootdemoApplication.java +++ b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/SpringbootdemoApplication.java @@ -1,15 +1,13 @@ -package com.taosdata.jdbc.springbootdemo; +package com.taosdata.example.springbootdemo; import org.mybatis.spring.annotation.MapperScan; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; -@MapperScan(basePackages = {"com.taosdata.jdbc.springbootdemo.dao"}) +@MapperScan(basePackages = {"com.taosdata.example.springbootdemo.dao"}) @SpringBootApplication -public class cd { - +public class SpringbootdemoApplication { public static void main(String[] args) { SpringApplication.run(SpringbootdemoApplication.class, args); } - } diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/controller/WeatherController.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/controller/WeatherController.java similarity index 87% rename from tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/controller/WeatherController.java rename to tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/controller/WeatherController.java index 56a58fbb4d..0e13f9284a 100644 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/controller/WeatherController.java +++ b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/controller/WeatherController.java @@ -1,7 +1,7 @@ -package com.taosdata.jdbc.springbootdemo.controller; +package com.taosdata.example.springbootdemo.controller; -import com.taosdata.jdbc.springbootdemo.domain.Weather; -import com.taosdata.jdbc.springbootdemo.service.WeatherService; +import com.taosdata.example.springbootdemo.domain.Weather; +import com.taosdata.example.springbootdemo.service.WeatherService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.*; diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/WeatherMapper.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.java similarity index 74% rename from tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/WeatherMapper.java rename to tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.java index 1e3db1f491..cae1a1aec0 100644 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/WeatherMapper.java +++ b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.java @@ -1,6 +1,6 @@ -package com.taosdata.jdbc.springbootdemo.dao; +package com.taosdata.example.springbootdemo.dao; -import com.taosdata.jdbc.springbootdemo.domain.Weather; +import com.taosdata.example.springbootdemo.domain.Weather; import org.apache.ibatis.annotations.Param; import java.util.List; diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/WeatherMapper.xml b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.xml similarity index 85% rename from tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/WeatherMapper.xml rename to tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.xml index e894f9a658..a9bcda0b00 100644 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/WeatherMapper.xml +++ b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.xml @@ -1,9 +1,9 @@ - + - + @@ -34,7 +34,7 @@ - + insert into test.weather (ts, temperature, humidity) values (now, #{temperature,jdbcType=INTEGER}, #{humidity,jdbcType=FLOAT}) diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/domain/Weather.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/domain/Weather.java similarity index 93% rename from tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/domain/Weather.java rename to tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/domain/Weather.java index cd7de447ea..60565448ad 100644 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/domain/Weather.java +++ b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/domain/Weather.java @@ -1,4 +1,4 @@ -package com.taosdata.jdbc.springbootdemo.domain; +package com.taosdata.example.springbootdemo.domain; import com.fasterxml.jackson.annotation.JsonFormat; diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/service/WeatherService.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/service/WeatherService.java similarity index 82% rename from tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/service/WeatherService.java rename to tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/service/WeatherService.java index b950a9a4e4..31ce8f1dd9 100644 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/service/WeatherService.java +++ b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/service/WeatherService.java @@ -1,7 +1,7 @@ -package com.taosdata.jdbc.springbootdemo.service; +package com.taosdata.example.springbootdemo.service; -import com.taosdata.jdbc.springbootdemo.dao.WeatherMapper; -import com.taosdata.jdbc.springbootdemo.domain.Weather; +import com.taosdata.example.springbootdemo.dao.WeatherMapper; +import com.taosdata.example.springbootdemo.domain.Weather; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/controller/RainStationController.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/controller/RainStationController.java deleted file mode 100644 index 844ac21bb8..0000000000 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/controller/RainStationController.java +++ /dev/null @@ -1,28 +0,0 @@ -package com.taosdata.jdbc.springbootdemo.controller; - - -import com.taosdata.jdbc.springbootdemo.domain.Rainfall; -import com.taosdata.jdbc.springbootdemo.service.RainStationService; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.*; - -@RestController -@RequestMapping("/rainstation") -public class RainStationController { - - @Autowired - private RainStationService service; - - @GetMapping("/init") - public boolean init() { - service.init(); - service.createTable(); - return true; - } - - @PostMapping("/insert") - public int insert(@RequestBody Rainfall rainfall){ - return service.insert(rainfall); - } - -} diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/DatabaseMapper.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/DatabaseMapper.java deleted file mode 100644 index a9266acb30..0000000000 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/DatabaseMapper.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.taosdata.jdbc.springbootdemo.dao; - -import java.util.Map; - -public interface DatabaseMapper { - - int createDatabase(String dbname); - - int dropDatabase(String dbname); - - int creatDatabaseWithParameters(Map map); - - int useDatabase(String dbname); - -} diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/DatabaseMapper.xml b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/DatabaseMapper.xml deleted file mode 100644 index 329f75b582..0000000000 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/DatabaseMapper.xml +++ /dev/null @@ -1,44 +0,0 @@ - - - - - - - create database if not exists ${dbname} - - - - DROP database if exists ${dbname} - - - - - CREATE database if not EXISTS ${dbname} - - KEEP ${keep} - - - DAYS ${days} - - - REPLICA ${replica} - - - cache ${cache} - - - blocks ${blocks} - - - minrows ${minrows} - - - maxrows ${maxrows} - - - - - use ${dbname} - - - \ No newline at end of file diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/RainfallMapper.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/RainfallMapper.java deleted file mode 100644 index f0efbf40ba..0000000000 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/RainfallMapper.java +++ /dev/null @@ -1,9 +0,0 @@ -package com.taosdata.jdbc.springbootdemo.dao; - -import java.util.Map; - -public interface RainfallMapper { - - - int save(Map map); -} diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/RainfallMapper.xml b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/RainfallMapper.xml deleted file mode 100644 index 319b4f3974..0000000000 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/RainfallMapper.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - - INSERT INTO ${table} using ${dbname}.${stable} tags(#{values.station_code}, #{values.station_name}) (ts, name, code, rainfall) values (#{values.ts}, #{values.name}, #{values.code}, #{values.rainfall}) - - - - \ No newline at end of file diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/TableMapper.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/TableMapper.java deleted file mode 100644 index 7601bf974c..0000000000 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/TableMapper.java +++ /dev/null @@ -1,8 +0,0 @@ -package com.taosdata.jdbc.springbootdemo.dao; - -import com.taosdata.jdbc.springbootdemo.domain.TableMetadata; - -public interface TableMapper { - - boolean createSTable(TableMetadata tableMetadata); -} \ No newline at end of file diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/TableMapper.xml b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/TableMapper.xml deleted file mode 100644 index 5a272eadb4..0000000000 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/TableMapper.xml +++ /dev/null @@ -1,21 +0,0 @@ - - - - - - - create table if not exists ${dbname}.${tablename} - - ${field.name} ${field.type} - - TAGS - - ${tag.name} ${tag.type} - - - - - drop ${tablename} - - - \ No newline at end of file diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/domain/FieldMetadata.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/domain/FieldMetadata.java deleted file mode 100644 index 619b5a303d..0000000000 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/domain/FieldMetadata.java +++ /dev/null @@ -1,28 +0,0 @@ -package com.taosdata.jdbc.springbootdemo.domain; - -public class FieldMetadata { - - private String name; - private String type; - - public FieldMetadata(String name, String type) { - this.name = name; - this.type = type; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type; - } -} diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/domain/Rainfall.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/domain/Rainfall.java deleted file mode 100644 index 93e199d7e6..0000000000 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/domain/Rainfall.java +++ /dev/null @@ -1,64 +0,0 @@ -package com.taosdata.jdbc.springbootdemo.domain; - -import com.fasterxml.jackson.annotation.JsonFormat; - -import java.sql.Timestamp; - -public class Rainfall { - - @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss.SSS",timezone = "GMT+8") - private Timestamp ts; - private String name; - private String code; - private float rainfall; - private String station_code; - private String station_name; - - public Timestamp getTs() { - return ts; - } - - public void setTs(Timestamp ts) { - this.ts = ts; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getCode() { - return code; - } - - public void setCode(String code) { - this.code = code; - } - - public float getRainfall() { - return rainfall; - } - - public void setRainfall(float rainfall) { - this.rainfall = rainfall; - } - - public String getStation_code() { - return station_code; - } - - public void setStation_code(String station_code) { - this.station_code = station_code; - } - - public String getStation_name() { - return station_name; - } - - public void setStation_name(String station_name) { - this.station_name = station_name; - } -} diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/domain/TableMetadata.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/domain/TableMetadata.java deleted file mode 100644 index 74bb434f08..0000000000 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/domain/TableMetadata.java +++ /dev/null @@ -1,43 +0,0 @@ -package com.taosdata.jdbc.springbootdemo.domain; - -import java.util.List; - -public class TableMetadata { - - private String dbname; - private String tablename; - private List fields; - private List tags; - - public String getDbname() { - return dbname; - } - - public void setDbname(String dbname) { - this.dbname = dbname; - } - - public String getTablename() { - return tablename; - } - - public void setTablename(String tablename) { - this.tablename = tablename; - } - - public List getFields() { - return fields; - } - - public void setFields(List fields) { - this.fields = fields; - } - - public List getTags() { - return tags; - } - - public void setTags(List tags) { - this.tags = tags; - } -} diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/domain/TagMetadata.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/domain/TagMetadata.java deleted file mode 100644 index 755ecc0075..0000000000 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/domain/TagMetadata.java +++ /dev/null @@ -1,27 +0,0 @@ -package com.taosdata.jdbc.springbootdemo.domain; - -public class TagMetadata { - private String name; - private String type; - - public TagMetadata(String name, String type) { - this.name = name; - this.type = type; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type; - } -} diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/service/RainStationService.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/service/RainStationService.java deleted file mode 100644 index 3ea63c1760..0000000000 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/service/RainStationService.java +++ /dev/null @@ -1,72 +0,0 @@ -package com.taosdata.jdbc.springbootdemo.service; - -import com.taosdata.jdbc.springbootdemo.dao.DatabaseMapper; -import com.taosdata.jdbc.springbootdemo.dao.RainfallMapper; -import com.taosdata.jdbc.springbootdemo.dao.TableMapper; -import com.taosdata.jdbc.springbootdemo.domain.FieldMetadata; -import com.taosdata.jdbc.springbootdemo.domain.Rainfall; -import com.taosdata.jdbc.springbootdemo.domain.TableMetadata; -import com.taosdata.jdbc.springbootdemo.domain.TagMetadata; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -@Service -public class RainStationService { - - @Autowired - private DatabaseMapper databaseMapper; - @Autowired - private TableMapper tableMapper; - @Autowired - private RainfallMapper rainfallMapper; - - public boolean init() { - databaseMapper.dropDatabase("rainstation"); - - Map map = new HashMap<>(); - map.put("dbname", "rainstation"); - map.put("keep", "36500"); - map.put("days", "30"); - map.put("blocks", "4"); - databaseMapper.creatDatabaseWithParameters(map); - - databaseMapper.useDatabase("rainstation"); - return true; - } - - public boolean createTable() { - TableMetadata tableMetadata = new TableMetadata(); - tableMetadata.setDbname("rainstation"); - tableMetadata.setTablename("monitoring"); - - List fields = new ArrayList<>(); - fields.add(new FieldMetadata("ts", "timestamp")); - fields.add(new FieldMetadata("name", "NCHAR(10)")); - fields.add(new FieldMetadata("code", " BINARY(8)")); - fields.add(new FieldMetadata("rainfall", "float")); - tableMetadata.setFields(fields); - - List tags = new ArrayList<>(); - tags.add(new TagMetadata("station_code", "BINARY(8)")); - tags.add(new TagMetadata("station_name", "NCHAR(10)")); - tableMetadata.setTags(tags); - - tableMapper.createSTable(tableMetadata); - return true; - } - - - public int insert(Rainfall rainfall) { - Map map = new HashMap<>(); - map.put("dbname", "rainstation"); - map.put("table", "S_53646"); - map.put("stable", "monitoring"); - map.put("values", rainfall); - return rainfallMapper.save(map); - } -} \ No newline at end of file diff --git a/tests/examples/JDBC/springbootdemo/src/main/resources/application.properties b/tests/examples/JDBC/springbootdemo/src/main/resources/application.properties index 683884fcdd..4fb68758c4 100644 --- a/tests/examples/JDBC/springbootdemo/src/main/resources/application.properties +++ b/tests/examples/JDBC/springbootdemo/src/main/resources/application.properties @@ -1,24 +1,18 @@ -# datasource config +# datasource config - JDBC-JNI spring.datasource.driver-class-name=com.taosdata.jdbc.TSDBDriver -spring.datasource.url=jdbc:TAOS://localhost:6030/log +spring.datasource.url=jdbc:TAOS://127.0.0.1:6030/test?timezone=UTC-8&charset=UTF-8&locale=en_US.UTF-8 spring.datasource.username=root spring.datasource.password=taosdata +# datasource config - JDBC-RESTful +#spring.datasource.driver-class-name=com.taosdata.jdbc.rs.RestfulDriver +#spring.datasource.url=jdbc:TAOS-RS://master:6041/test?user=root&password=taosdata + spring.datasource.druid.initial-size=5 spring.datasource.druid.min-idle=5 spring.datasource.druid.max-active=5 -# max wait time for get connection, ms -spring.datasource.druid.max-wait=60000 - +spring.datasource.druid.max-wait=30000 spring.datasource.druid.validation-query=select server_status(); -spring.datasource.druid.validation-query-timeout=5000 -spring.datasource.druid.test-on-borrow=false -spring.datasource.druid.test-on-return=false -spring.datasource.druid.test-while-idle=true -spring.datasource.druid.time-between-eviction-runs-millis=60000 -spring.datasource.druid.min-evictable-idle-time-millis=600000 -spring.datasource.druid.max-evictable-idle-time-millis=900000 - #mybatis mybatis.mapper-locations=classpath:mapper/*.xml diff --git a/tests/examples/JDBC/springbootdemo/src/test/java/com/taosdata/jdbc/springbootdemo/SpringbootdemoApplicationTests.java b/tests/examples/JDBC/springbootdemo/src/test/java/com/taosdata/jdbc/springbootdemo/SpringbootdemoApplicationTests.java deleted file mode 100644 index 23a7420dab..0000000000 --- a/tests/examples/JDBC/springbootdemo/src/test/java/com/taosdata/jdbc/springbootdemo/SpringbootdemoApplicationTests.java +++ /dev/null @@ -1,13 +0,0 @@ -package com.taosdata.jdbc.springbootdemo; - -import org.junit.jupiter.api.Test; -import org.springframework.boot.test.context.SpringBootTest; - -@SpringBootTest -class SpringbootdemoApplicationTests { - - @Test - void contextLoads() { - } - -} From 43d154a6903ddc10591d56304c5fec457377d6ac Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 27 Jan 2021 11:30:15 +0000 Subject: [PATCH 140/147] TD-2861 --- src/sync/inc/syncMsg.h | 4 +++- src/sync/src/syncMain.c | 19 ++++++++++++++++ src/sync/src/syncMsg.c | 1 + src/util/CMakeLists.txt | 1 + src/util/src/tnettest.c | 48 ++++++++++++++++++++++++++++++++++++++--- 5 files changed, 69 insertions(+), 4 deletions(-) diff --git a/src/sync/inc/syncMsg.h b/src/sync/inc/syncMsg.h index 73f4223c88..f589379aa2 100644 --- a/src/sync/inc/syncMsg.h +++ b/src/sync/inc/syncMsg.h @@ -37,7 +37,8 @@ typedef enum { TAOS_SMSG_SETUP_RSP = 12, TAOS_SMSG_SYNC_FILE = 13, TAOS_SMSG_SYNC_FILE_RSP = 14, - TAOS_SMSG_END = 15, + TAOS_SMSG_TEST = 15, + TAOS_SMSG_END = 16 } ESyncMsgType; typedef enum { @@ -132,6 +133,7 @@ void syncBuildSyncReqMsg(SSyncMsg *pMsg, int32_t vgId); void syncBuildSyncDataMsg(SSyncMsg *pMsg, int32_t vgId); void syncBuildSyncSetupMsg(SSyncMsg *pMsg, int32_t vgId); void syncBuildPeersStatus(SPeersStatus *pMsg, int32_t vgId); +void syncBuildSyncTestMsg(SSyncMsg *pMsg, int32_t vgId); void syncBuildFileAck(SFileAck *pMsg, int32_t vgId); void syncBuildFileInfo(SFileInfo *pMsg, int32_t vgId); diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index d698432176..6e87039612 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -1179,6 +1179,20 @@ static void syncCreateRestoreDataThread(SSyncPeer *pPeer) { } } +static void syncProcessTestMsg(SSyncMsg *pMsg, SOCKET connFd) { + sInfo("recv sync test msg"); + + SSyncMsg rsp; + syncBuildSyncTestMsg(&rsp, -1); + if (taosWriteMsg(connFd, &rsp, sizeof(SSyncMsg)) != sizeof(SSyncMsg)) { + sInfo("failed to send sync test rsp since %s", strerror(errno)); + } + + sInfo("send sync test rsp"); + taosMsleep(1000); + taosCloseSocket(connFd); +} + static void syncProcessIncommingConnection(SOCKET connFd, uint32_t sourceIp) { char ipstr[24]; int32_t i; @@ -1200,6 +1214,11 @@ static void syncProcessIncommingConnection(SOCKET connFd, uint32_t sourceIp) { return; } + if (msg.head.type == TAOS_SMSG_TEST) { + syncProcessTestMsg(&msg, connFd); + return; + } + int32_t vgId = msg.head.vgId; SSyncNode **ppNode = taosHashGet(tsVgIdHash, &vgId, sizeof(int32_t)); if (ppNode == NULL || *ppNode == NULL) { diff --git a/src/sync/src/syncMsg.c b/src/sync/src/syncMsg.c index 034f9a98a7..9718a3414e 100644 --- a/src/sync/src/syncMsg.c +++ b/src/sync/src/syncMsg.c @@ -86,6 +86,7 @@ static void syncBuildMsg(SSyncMsg *pMsg, int32_t vgId, ESyncMsgType type) { void syncBuildSyncReqMsg(SSyncMsg *pMsg, int32_t vgId) { syncBuildMsg(pMsg, vgId, TAOS_SMSG_SYNC_REQ); } void syncBuildSyncDataMsg(SSyncMsg *pMsg, int32_t vgId) { syncBuildMsg(pMsg, vgId, TAOS_SMSG_SYNC_DATA); } void syncBuildSyncSetupMsg(SSyncMsg *pMsg, int32_t vgId) { syncBuildMsg(pMsg, vgId, TAOS_SMSG_SETUP); } +void syncBuildSyncTestMsg(SSyncMsg *pMsg, int32_t vgId) { syncBuildMsg(pMsg, vgId, TAOS_SMSG_TEST); } void syncBuildPeersStatus(SPeersStatus *pMsg, int32_t vgId) { pMsg->head.type = TAOS_SMSG_STATUS; diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index 3606aea76b..92e030ad81 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -2,6 +2,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 3.5) PROJECT(TDengine) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/rpc/inc) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/sync/inc) AUX_SOURCE_DIRECTORY(src SRC) ADD_LIBRARY(tutil ${SRC}) TARGET_LINK_LIBRARIES(tutil pthread osdetail lz4 z) diff --git a/src/util/src/tnettest.c b/src/util/src/tnettest.c index fe6dfb493d..7e39715cd9 100644 --- a/src/util/src/tnettest.c +++ b/src/util/src/tnettest.c @@ -23,6 +23,8 @@ #include "tsocket.h" #include "trpc.h" #include "rpcHead.h" +#include "tchecksum.h" +#include "syncMsg.h" #define MAX_PKG_LEN (64 * 1000) #define BUFFER_SIZE (MAX_PKG_LEN + 1024) @@ -408,13 +410,51 @@ static void taosNetTestStartup(char *host, int32_t port) { free(pStep); } +static void taosNetCheckSync(char *host, int32_t port) { + uint32_t ip = taosGetIpv4FromFqdn(host); + if (ip == 0xffffffff) { + uError("failed to get IP address from %s since %s", host, strerror(errno)); + return; + } + + SOCKET connFd = taosOpenTcpClientSocket(ip, (uint16_t)port, 0); + if (connFd < 0) { + uError("failed to create socket while test port:%d since %s", port, strerror(errno)); + return; + } + + SSyncMsg msg; + memset(&msg, 0, sizeof(SSyncMsg)); + SSyncHead *pHead = &msg.head; + pHead->type = TAOS_SMSG_TEST; + pHead->protocol = SYNC_PROTOCOL_VERSION; + pHead->signature = SYNC_SIGNATURE; + pHead->code = 0; + pHead->cId = 0; + pHead->vgId = -1; + pHead->len = sizeof(SSyncMsg) - sizeof(SSyncHead); + taosCalcChecksumAppend(0, (uint8_t *)pHead, sizeof(SSyncHead)); + + if (taosWriteMsg(connFd, &msg, sizeof(SSyncMsg)) != sizeof(SSyncMsg)) { + uError("failed to test port:%d while send msg since %s", port, strerror(errno)); + return; + } + + if (taosReadMsg(connFd, &msg, sizeof(SSyncMsg)) != sizeof(SSyncMsg)) { + uError("failed to test port:%d while recv msg since %s", port, strerror(errno)); + } + + uInfo("successed to test TCP port:%d", port); + taosCloseSocket(connFd); +} + static void taosNetTestRpc(char *host, int32_t startPort, int32_t pkgLen) { - int32_t endPort = startPort + 9; + int32_t endPort = startPort + TSDB_PORT_SYNC; char spi = 0; uInfo("check rpc, host:%s startPort:%d endPort:%d pkgLen:%d\n", host, startPort, endPort, pkgLen); - - for (uint16_t port = startPort; port <= endPort; port++) { + + for (uint16_t port = startPort; port < endPort; port++) { int32_t sendpkgLen; if (pkgLen <= tsRpcMaxUdpSize) { sendpkgLen = tsRpcMaxUdpSize + 1000; @@ -442,6 +482,8 @@ static void taosNetTestRpc(char *host, int32_t startPort, int32_t pkgLen) { uInfo("successed to test UDP port:%d", port); } } + + taosNetCheckSync(host, endPort); } static void taosNetTestClient(char *host, int32_t startPort, int32_t pkgLen) { From 6a73886ed2caff23db6add0455fece4581dc08b4 Mon Sep 17 00:00:00 2001 From: zyyang Date: Wed, 27 Jan 2021 19:35:51 +0800 Subject: [PATCH 141/147] change --- .../example/springbootdemo/controller/WeatherController.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/controller/WeatherController.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/controller/WeatherController.java index 0e13f9284a..4a4109dcf3 100644 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/controller/WeatherController.java +++ b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/controller/WeatherController.java @@ -45,7 +45,6 @@ public class WeatherController { */ @PostMapping("/{temperature}/{humidity}") public int saveWeather(@PathVariable int temperature, @PathVariable float humidity) { - return weatherService.save(temperature, humidity); } @@ -57,7 +56,6 @@ public class WeatherController { */ @PostMapping("/batch") public int batchSaveWeather(@RequestBody List weatherList) { - return weatherService.save(weatherList); } From be4fd9831a4c653af46ec67a26763210c229ad6e Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 27 Jan 2021 11:51:27 +0000 Subject: [PATCH 142/147] [TD-2785]: Allow to judge the status of sync and arbitrator ports --- src/kit/shell/src/shellLinux.c | 2 +- src/kit/shell/src/shellWindows.c | 2 +- src/sync/src/syncArbitrator.c | 6 ++++++ src/sync/src/syncMain.c | 2 +- src/util/src/tnettest.c | 5 ++++- 5 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/kit/shell/src/shellLinux.c b/src/kit/shell/src/shellLinux.c index 9eb30ccdcc..a8f457377d 100644 --- a/src/kit/shell/src/shellLinux.c +++ b/src/kit/shell/src/shellLinux.c @@ -47,7 +47,7 @@ static struct argp_option options[] = { {"thread", 'T', "THREADNUM", 0, "Number of threads when using multi-thread to import data."}, {"database", 'd', "DATABASE", 0, "Database to use when connecting to the server."}, {"timezone", 't', "TIMEZONE", 0, "Time zone of the shell, default is local."}, - {"netrole", 'n', "NETROLE", 0, "Net role when network connectivity test, default is startup, options: client|server|rpc|startup."}, + {"netrole", 'n', "NETROLE", 0, "Net role when network connectivity test, default is startup, options: client|server|rpc|startup|sync."}, {"pktlen", 'l', "PKTLEN", 0, "Packet length used for net test, default is 1000 bytes."}, {0}}; diff --git a/src/kit/shell/src/shellWindows.c b/src/kit/shell/src/shellWindows.c index 73216f9ca3..56021d9c34 100644 --- a/src/kit/shell/src/shellWindows.c +++ b/src/kit/shell/src/shellWindows.c @@ -52,7 +52,7 @@ void printHelp() { printf("%s%s\n", indent, "-t"); printf("%s%s%s\n", indent, indent, "Time zone of the shell, default is local."); printf("%s%s\n", indent, "-n"); - printf("%s%s%s\n", indent, indent, "Net role when network connectivity test, default is startup, options: client|server|rpc|startup."); + printf("%s%s%s\n", indent, indent, "Net role when network connectivity test, default is startup, options: client|server|rpc|startup|sync."); printf("%s%s\n", indent, "-l"); printf("%s%s%s\n", indent, indent, "Packet length used for net test, default is 1000 bytes."); printf("%s%s\n", indent, "-V"); diff --git a/src/sync/src/syncArbitrator.c b/src/sync/src/syncArbitrator.c index 9fb6b0ddb7..fdbef4c9f8 100644 --- a/src/sync/src/syncArbitrator.c +++ b/src/sync/src/syncArbitrator.c @@ -27,6 +27,7 @@ #include "syncInt.h" #include "syncTcp.h" +extern void syncProcessTestMsg(SSyncMsg *pMsg, SOCKET connFd); static void arbSignalHandler(int32_t signum, void *sigInfo, void *context); static void arbProcessIncommingConnection(SOCKET connFd, uint32_t sourceIp); static void arbProcessBrokenLink(int64_t rid); @@ -118,6 +119,11 @@ static void arbProcessIncommingConnection(SOCKET connFd, uint32_t sourceIp) { return; } + if (msg.head.type == TAOS_SMSG_TEST) { + syncProcessTestMsg(&msg, connFd); + return; + } + SNodeConn *pNode = calloc(sizeof(SNodeConn), 1); if (pNode == NULL) { sError("failed to allocate memory since %s", strerror(errno)); diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index 6e87039612..8dac89544b 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -1179,7 +1179,7 @@ static void syncCreateRestoreDataThread(SSyncPeer *pPeer) { } } -static void syncProcessTestMsg(SSyncMsg *pMsg, SOCKET connFd) { +void syncProcessTestMsg(SSyncMsg *pMsg, SOCKET connFd) { sInfo("recv sync test msg"); SSyncMsg rsp; diff --git a/src/util/src/tnettest.c b/src/util/src/tnettest.c index 7e39715cd9..28abad356c 100644 --- a/src/util/src/tnettest.c +++ b/src/util/src/tnettest.c @@ -483,7 +483,8 @@ static void taosNetTestRpc(char *host, int32_t startPort, int32_t pkgLen) { } } - taosNetCheckSync(host, endPort); + taosNetCheckSync(host, startPort + TSDB_PORT_SYNC); + taosNetCheckSync(host, startPort + TSDB_PORT_ARBITRATOR); } static void taosNetTestClient(char *host, int32_t startPort, int32_t pkgLen) { @@ -550,6 +551,8 @@ void taosNetTest(char *role, char *host, int32_t port, int32_t pkgLen) { taosNetTestServer(host, port, pkgLen); } else if (0 == strcmp("rpc", role)) { taosNetTestRpc(host, port, pkgLen); + } else if (0 == strcmp("sync", role)) { + taosNetCheckSync(host, port); } else if (0 == strcmp("startup", role)) { taosNetTestStartup(host, port); } else { From 0bcb46afb2291241866d212e63968000561ee553 Mon Sep 17 00:00:00 2001 From: zyyang Date: Wed, 27 Jan 2021 22:42:01 +0800 Subject: [PATCH 143/147] change --- tests/examples/JDBC/mybatisplus-demo/pom.xml | 8 +------- .../mybatisplus-demo/src/main/resources/application.yml | 6 +----- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/tests/examples/JDBC/mybatisplus-demo/pom.xml b/tests/examples/JDBC/mybatisplus-demo/pom.xml index e59b915d2c..a83d0a00e6 100644 --- a/tests/examples/JDBC/mybatisplus-demo/pom.xml +++ b/tests/examples/JDBC/mybatisplus-demo/pom.xml @@ -47,15 +47,9 @@ com.taosdata.jdbc taos-jdbcdriver - 2.0.14 + 2.0.18 - - - mysql - mysql-connector-java - 5.1.47 - org.springframework.boot spring-boot-starter-web diff --git a/tests/examples/JDBC/mybatisplus-demo/src/main/resources/application.yml b/tests/examples/JDBC/mybatisplus-demo/src/main/resources/application.yml index 71e518602e..38180c6d75 100644 --- a/tests/examples/JDBC/mybatisplus-demo/src/main/resources/application.yml +++ b/tests/examples/JDBC/mybatisplus-demo/src/main/resources/application.yml @@ -1,19 +1,15 @@ spring: datasource: driver-class-name: com.taosdata.jdbc.TSDBDriver - url: jdbc:TAOS://localhost:6030/mp_test + url: jdbc:TAOS://localhost:6030/mp_test?charset=UTF-8&locale=en_US.UTF-8&timezone=UTC-8 user: root password: taosdata - charset: UTF-8 - locale: en_US.UTF-8 - timezone: UTC-8 druid: initial-size: 5 min-idle: 5 max-active: 5 - mybatis-plus: configuration: map-underscore-to-camel-case: false From 4299403e8db8c1ba99c6a9cded0865678df55b17 Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Thu, 28 Jan 2021 10:10:03 +0800 Subject: [PATCH 144/147] [TD-2571] : add new keyword DISTINCT to select different values of tag column in super table. --- .../webdocs/markdowndocs/TAOS SQL-ch.md | 11 +++ .../webdocs/markdowndocs/administrator-ch.md | 72 +++++++++---------- 2 files changed, 47 insertions(+), 36 deletions(-) diff --git a/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md b/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md index 7428b3f41d..0da2883605 100644 --- a/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md +++ b/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md @@ -470,6 +470,17 @@ Query OK, 2 row(s) in set (0.003112s) 注意：普通表的通配符 * 中并不包含 _标签列_。 +##### 获取标签列的去重取值 + +从 2.0.15 版本开始，支持在超级表查询标签列时，指定 distinct 关键字，这样将返回指定标签列的所有不重复取值。 +```mysql +SELECT DISTINCT tag_name FROM stb_name; +``` + +注意：目前 distinct 关键字只支持对超级表的标签列进行去重，而不能用于普通列。 + + + #### 结果集列名 ```SELECT```子句中，如果不指定返回结果集合的列名，结果集列名称默认使用```SELECT```子句中的表达式名称作为列名称。此外，用户可使用```AS```来重命名返回结果集合中列的名称。例如： diff --git a/documentation20/webdocs/markdowndocs/administrator-ch.md b/documentation20/webdocs/markdowndocs/administrator-ch.md index 160bab4d08..7c8344d9ca 100644 --- a/documentation20/webdocs/markdowndocs/administrator-ch.md +++ b/documentation20/webdocs/markdowndocs/administrator-ch.md @@ -452,39 +452,39 @@ TDengine的所有可执行文件默认存放在 _/usr/local/taos/bin_ 目录下 | 关键字列表 | | | | | | ---------- | ----------- | ------------ | ---------- | --------- | -| ABLOCKS | CONNECTION | GT | MNODES | SLIDING | -| ABORT | CONNECTIONS | ID | MODULES | SMALLINT | -| ACCOUNT | COPY | IF | NCHAR | SPREAD | -| ACCOUNTS | COUNT | IGNORE | NE | STABLE | -| ADD | CREATE | IMMEDIATE | NONE | STABLES | -| AFTER | CTIME | IMPORT | NOT | STAR | -| ALL | DATABASE | IN | NOTNULL | STATEMENT | -| ALTER | DATABASES | INITIALLY | NOW | STDDEV | -| AND | DAYS | INSERT | OF | STREAM | -| AS | DEFERRED | INSTEAD | OFFSET | STREAMS | -| ASC | DELIMITERS | INTEGER | OR | STRING | -| ATTACH | DESC | INTERVAL | ORDER | SUM | -| AVG | DESCRIBE | INTO | PASS | TABLE | -| BEFORE | DETACH | IP | PERCENTILE | TABLES | -| BEGIN | DIFF | IS | PLUS | TAG | -| BETWEEN | DIVIDE | ISNULL | PRAGMA | TAGS | -| BIGINT | DNODE | JOIN | PREV | TBLOCKS | -| BINARY | DNODES | KEEP | PRIVILEGE | TBNAME | -| BITAND | DOT | KEY | QUERIES | TIMES | -| BITNOT | DOUBLE | KILL | QUERY | TIMESTAMP | -| BITOR | DROP | LAST | RAISE | TINYINT | -| BOOL | EACH | LE | REM | TOP | -| BOTTOM | END | LEASTSQUARES | REPLACE | TRIGGER | -| BY | EQ | LIKE | REPLICA | UMINUS | -| CACHE | EXISTS | LIMIT | RESET | UPLUS | -| CASCADE | EXPLAIN | LINEAR | RESTRICT | USE | -| CHANGE | FAIL | LOCAL | ROW | USER | -| CLOG | FILL | LP | ROWS | USERS | -| CLUSTER | FIRST | LSHIFT | RP | USING | -| COLON | FLOAT | LT | RSHIFT | VALUES | -| COLUMN | FOR | MATCH | SCORES | VARIABLE | -| COMMA | FROM | MAX | SELECT | VGROUPS | -| COMP | GE | METRIC | SEMI | VIEW | -| CONCAT | GLOB | METRICS | SET | WAVG | -| CONFIGS | GRANTS | MIN | SHOW | WHERE | -| CONFLICT | GROUP | MINUS | SLASH | | +| ABLOCKS | CONNECTION | GROUP | MINUS | SLASH | +| ABORT | CONNECTIONS | GT | MNODES | SLIDING | +| ACCOUNT | COPY | ID | MODULES | SMALLINT | +| ACCOUNTS | COUNT | IF | NCHAR | SPREAD | +| ADD | CREATE | IGNORE | NE | STABLE | +| AFTER | CTIME | IMMEDIATE | NONE | STABLES | +| ALL | DATABASE | IMPORT | NOT | STAR | +| ALTER | DATABASES | IN | NOTNULL | STATEMENT | +| AND | DAYS | INITIALLY | NOW | STDDEV | +| AS | DEFERRED | INSERT | OF | STREAM | +| ASC | DELIMITERS | INSTEAD | OFFSET | STREAMS | +| ATTACH | DESC | INTEGER | OR | STRING | +| AVG | DESCRIBE | INTERVAL | ORDER | SUM | +| BEFORE | DETACH | INTO | PASS | TABLE | +| BEGIN | DIFF | IP | PERCENTILE | TABLES | +| BETWEEN | DISTINCT | IS | PLUS | TAG | +| BIGINT | DIVIDE | ISNULL | PRAGMA | TAGS | +| BINARY | DNODE | JOIN | PREV | TBLOCKS | +| BITAND | DNODES | KEEP | PRIVILEGE | TBNAME | +| BITNOT | DOT | KEY | QUERIES | TIMES | +| BITOR | DOUBLE | KILL | QUERY | TIMESTAMP | +| BOOL | DROP | LAST | RAISE | TINYINT | +| BOTTOM | EACH | LE | REM | TOP | +| BY | END | LEASTSQUARES | REPLACE | TRIGGER | +| CACHE | EQ | LIKE | REPLICA | UMINUS | +| CASCADE | EXISTS | LIMIT | RESET | UPLUS | +| CHANGE | EXPLAIN | LINEAR | RESTRICT | USE | +| CLOG | FAIL | LOCAL | ROW | USER | +| CLUSTER | FILL | LP | ROWS | USERS | +| COLON | FIRST | LSHIFT | RP | USING | +| COLUMN | FLOAT | LT | RSHIFT | VALUES | +| COMMA | FOR | MATCH | SCORES | VARIABLE | +| COMP | FROM | MAX | SELECT | VGROUPS | +| CONCAT | GE | METRIC | SEMI | VIEW | +| CONFIGS | GLOB | METRICS | SET | WAVG | +| CONFLICT | GRANTS | MIN | SHOW | WHERE | From a2b5d9de0142a45d5e6139c08f270af59551fdf7 Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Thu, 28 Jan 2021 10:57:27 +0800 Subject: [PATCH 145/147] [TD-1413] : enterprise edition support multi-layer storage now. --- documentation20/webdocs/markdowndocs/architecture-ch.md | 1 - 1 file changed, 1 deletion(-) diff --git a/documentation20/webdocs/markdowndocs/architecture-ch.md b/documentation20/webdocs/markdowndocs/architecture-ch.md index 773d8196f2..8921633c8d 100644 --- a/documentation20/webdocs/markdowndocs/architecture-ch.md +++ b/documentation20/webdocs/markdowndocs/architecture-ch.md @@ -344,7 +344,6 @@ dataDir /mnt/disk6/taos 2 挂载的盘也可以是非本地的网络盘，只要系统能访问即可。 注：多级存储功能仅企业版支持 -**提示：该功能暂未提供** ## 数据查询 TDengine提供了多种多样针对表和超级表的查询处理功能，除了常规的聚合查询之外，还提供针对时序数据的窗口查询、统计聚合等功能。TDengine的查询处理需要客户端、vnode, mnode节点协同完成。 From 8bcc7117ab3ef6c523b60ded25ecf8ede971dbf9 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Thu, 28 Jan 2021 10:59:56 +0800 Subject: [PATCH 146/147] [roll] --- packaging/deb/makedeb.sh | 1 - packaging/rpm/tdengine.spec | 1 - 2 files changed, 2 deletions(-) diff --git a/packaging/deb/makedeb.sh b/packaging/deb/makedeb.sh index a2f58619df..431093be95 100755 --- a/packaging/deb/makedeb.sh +++ b/packaging/deb/makedeb.sh @@ -43,7 +43,6 @@ mkdir -p ${pkg_dir}${install_home_path}/include mkdir -p ${pkg_dir}${install_home_path}/init.d mkdir -p ${pkg_dir}${install_home_path}/script -echo "" > ${pkg_dir}${install_home_path}/email cp ${compile_dir}/../packaging/cfg/taos.cfg ${pkg_dir}${install_home_path}/cfg cp ${compile_dir}/../packaging/deb/taosd ${pkg_dir}${install_home_path}/init.d cp ${compile_dir}/../packaging/tools/post.sh ${pkg_dir}${install_home_path}/script diff --git a/packaging/rpm/tdengine.spec b/packaging/rpm/tdengine.spec index a143d0afdb..6f012aa80e 100644 --- a/packaging/rpm/tdengine.spec +++ b/packaging/rpm/tdengine.spec @@ -51,7 +51,6 @@ mkdir -p %{buildroot}%{homepath}/include mkdir -p %{buildroot}%{homepath}/init.d mkdir -p %{buildroot}%{homepath}/script -echo "" > %{buildroot}%{homepath}/email cp %{_compiledir}/../packaging/cfg/taos.cfg %{buildroot}%{homepath}/cfg cp %{_compiledir}/../packaging/rpm/taosd %{buildroot}%{homepath}/init.d cp %{_compiledir}/../packaging/tools/post.sh %{buildroot}%{homepath}/script From c244f389ff7d2b73f7c78bd57c1ca42af91ea51d Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 28 Jan 2021 11:43:48 +0800 Subject: [PATCH 147/147] [TD-2873] : fix JdbcRestfulDemo.java compile issue. --- tests/examples/JDBC/JDBCDemo/pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/examples/JDBC/JDBCDemo/pom.xml b/tests/examples/JDBC/JDBCDemo/pom.xml index 27a03911a2..61cf50ed4d 100644 --- a/tests/examples/JDBC/JDBCDemo/pom.xml +++ b/tests/examples/JDBC/JDBCDemo/pom.xml @@ -50,6 +50,7 @@ 8 8 + UTF-8