From e5c2b6aa10ab3fe032567afb1e3ba027b2a55b21 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 12 Jan 2021 22:27:43 +0800 Subject: [PATCH 01/62] TD-1207 --- src/common/inc/tdataformat.h | 4 +- src/common/inc/tglobal.h | 4 +- src/common/src/tglobal.c | 8 ++-- src/kit/shell/src/shellMain.c | 2 +- src/mnode/CMakeLists.txt | 2 +- src/mnode/src/mnodeAcct.c | 8 ++-- src/mnode/src/mnodeCluster.c | 10 ++--- src/mnode/src/mnodeDb.c | 8 ++-- src/mnode/src/mnodeDnode.c | 14 +++---- src/mnode/src/mnodeMnode.c | 6 +-- src/mnode/src/mnodeSdb.c | 22 +++++----- src/mnode/src/mnodeShow.c | 2 +- src/mnode/src/mnodeTable.c | 64 ++++++++++++++--------------- src/mnode/src/mnodeUser.c | 12 +++--- src/mnode/src/mnodeVgroup.c | 8 ++-- src/os/inc/osFile.h | 2 +- src/os/inc/osSocket.h | 1 + src/os/src/detail/osFile.c | 2 +- src/os/src/detail/osSocket.c | 10 +++++ src/os/src/windows/wFile.c | 2 +- src/os/src/windows/wSocket.c | 1 + src/plugins/http/CMakeLists.txt | 2 +- src/plugins/http/inc/httpInt.h | 4 +- src/plugins/http/src/httpGcJson.c | 2 +- src/plugins/http/src/httpGzip.c | 4 +- src/plugins/http/src/httpJson.c | 12 +++--- src/plugins/http/src/httpParser.c | 14 +++---- src/plugins/http/src/httpQueue.c | 4 +- src/plugins/http/src/httpServer.c | 22 +++------- src/plugins/http/src/httpTgHandle.c | 2 +- src/plugins/http/src/httpUtil.c | 4 +- src/plugins/monitor/CMakeLists.txt | 2 +- src/plugins/monitor/src/monMain.c | 8 ++-- src/rpc/src/rpcTcp.c | 2 +- src/sync/CMakeLists.txt | 2 +- src/sync/inc/syncInt.h | 4 +- src/sync/inc/syncTcp.h | 4 +- src/sync/src/syncArbitrator.c | 25 +++++++---- src/sync/src/syncMain.c | 14 +++---- src/sync/src/syncRestore.c | 6 +-- src/sync/src/syncRetrieve.c | 11 +++-- src/sync/src/syncTcp.c | 30 +++++++++----- src/util/inc/tsocket.h | 8 ++++ src/vnode/src/vnodeCfg.c | 26 ++++++------ src/wal/CMakeLists.txt | 2 +- src/wal/src/walMgmt.c | 2 +- src/wal/src/walWrite.c | 4 +- 47 files changed, 218 insertions(+), 194 deletions(-) diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index 8d4949d9b4..f1bf89aed4 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -29,7 +29,7 @@ extern "C" { #define STR_TO_VARSTR(x, str) \ do { \ - VarDataLenT __len = strlen(str); \ + VarDataLenT __len = (int32_t)strlen(str); \ *(VarDataLenT *)(x) = __len; \ memcpy(varDataVal(x), (str), __len); \ } while (0); @@ -42,7 +42,7 @@ extern "C" { #define STR_WITH_SIZE_TO_VARSTR(x, str, _size) \ do { \ - *(VarDataLenT *)(x) = (_size); \ + *(VarDataLenT *)(x) = (int32_t)(_size); \ memcpy(varDataVal(x), (str), (_size)); \ } while (0); diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index f6a08f8198..bf0d4e61df 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -89,8 +89,8 @@ extern int32_t tsMinRowsInFileBlock; extern int32_t tsMaxRowsInFileBlock; extern int16_t tsCommitTime; // seconds extern int32_t tsTimePrecision; -extern int16_t tsCompression; -extern int16_t tsWAL; +extern int8_t tsCompression; +extern int8_t tsWAL; extern int32_t tsFsyncPeriod; extern int32_t tsReplications; extern int32_t tsQuorum; diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 551fb96195..a7826f91ad 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -122,8 +122,8 @@ int32_t tsMinRowsInFileBlock = TSDB_DEFAULT_MIN_ROW_FBLOCK; int32_t tsMaxRowsInFileBlock = TSDB_DEFAULT_MAX_ROW_FBLOCK; int16_t tsCommitTime = TSDB_DEFAULT_COMMIT_TIME; // seconds int32_t tsTimePrecision = TSDB_DEFAULT_PRECISION; -int16_t tsCompression = TSDB_DEFAULT_COMP_LEVEL; -int16_t tsWAL = TSDB_DEFAULT_WAL_LEVEL; +int8_t tsCompression = TSDB_DEFAULT_COMP_LEVEL; +int8_t tsWAL = TSDB_DEFAULT_WAL_LEVEL; int32_t tsFsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD; int32_t tsReplications = TSDB_DEFAULT_DB_REPLICA_OPTION; int32_t tsQuorum = TSDB_DEFAULT_DB_QUORUM_OPTION; @@ -769,7 +769,7 @@ static void doInitGlobalConfig(void) { cfg.option = "comp"; cfg.ptr = &tsCompression; - cfg.valType = TAOS_CFG_VTYPE_INT16; + cfg.valType = TAOS_CFG_VTYPE_INT8; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; cfg.minValue = TSDB_MIN_COMP_LEVEL; cfg.maxValue = TSDB_MAX_COMP_LEVEL; @@ -779,7 +779,7 @@ static void doInitGlobalConfig(void) { cfg.option = "walLevel"; cfg.ptr = &tsWAL; - cfg.valType = TAOS_CFG_VTYPE_INT16; + cfg.valType = TAOS_CFG_VTYPE_INT8; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; cfg.minValue = TSDB_MIN_WAL_LEVEL; cfg.maxValue = TSDB_MAX_WAL_LEVEL; diff --git a/src/kit/shell/src/shellMain.c b/src/kit/shell/src/shellMain.c index 041ad71ccb..496cef41ba 100644 --- a/src/kit/shell/src/shellMain.c +++ b/src/kit/shell/src/shellMain.c @@ -21,7 +21,7 @@ pthread_t pid; static tsem_t cancelSem; -void shellQueryInterruptHandler(int signum) { +void shellQueryInterruptHandler(int32_t signum) { tsem_post(&cancelSem); } diff --git a/src/mnode/CMakeLists.txt b/src/mnode/CMakeLists.txt index ff5c9335b6..6e6d515de4 100644 --- a/src/mnode/CMakeLists.txt +++ b/src/mnode/CMakeLists.txt @@ -1,7 +1,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) -IF (TD_LINUX) +IF (TD_LINUX OR TD_WINDOWS) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/dnode/inc) diff --git a/src/mnode/src/mnodeAcct.c b/src/mnode/src/mnodeAcct.c index 6fba05674f..afe474df6b 100644 --- a/src/mnode/src/mnodeAcct.c +++ b/src/mnode/src/mnodeAcct.c @@ -81,7 +81,7 @@ static int32_t mnodeAcctActionDecode(SSdbRow *pRow) { } static int32_t mnodeAcctActionRestored() { - int32_t numOfRows = sdbGetNumOfRows(tsAcctSdb); + int64_t numOfRows = sdbGetNumOfRows(tsAcctSdb); if (numOfRows <= 0 && dnodeIsFirstDeploy()) { mInfo("dnode first deploy, create root acct"); int32_t code = mnodeCreateRootAcct(); @@ -97,14 +97,14 @@ static int32_t mnodeAcctActionRestored() { int32_t mnodeInitAccts() { SAcctObj tObj; - tsAcctUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; + tsAcctUpdateSize = (int32_t)((int8_t *)tObj.updateEnd - (int8_t *)&tObj); SSdbTableDesc desc = { .id = SDB_TABLE_ACCOUNT, .name = "accounts", .hashSessions = TSDB_DEFAULT_ACCOUNTS_HASH_SIZE, .maxRowSize = tsAcctUpdateSize, - .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, + .refCountPos = (int32_t)((int8_t *)(&tObj.refCount) - (int8_t *)&tObj), .keyType = SDB_KEY_STRING, .fpInsert = mnodeAcctActionInsert, .fpDelete = mnodeAcctActionDelete, @@ -206,7 +206,7 @@ void mnodeDropUserFromAcct(SAcctObj *pAcct, SUserObj *pUser) { } static int32_t mnodeCreateRootAcct() { - int32_t numOfAccts = sdbGetNumOfRows(tsAcctSdb); + int64_t numOfAccts = sdbGetNumOfRows(tsAcctSdb); if (numOfAccts != 0) return TSDB_CODE_SUCCESS; SAcctObj *pAcct = malloc(sizeof(SAcctObj)); diff --git a/src/mnode/src/mnodeCluster.c b/src/mnode/src/mnodeCluster.c index a35e304810..0931f31012 100644 --- a/src/mnode/src/mnodeCluster.c +++ b/src/mnode/src/mnodeCluster.c @@ -68,7 +68,7 @@ static int32_t mnodeClusterActionDecode(SSdbRow *pRow) { } static int32_t mnodeClusterActionRestored() { - int32_t numOfRows = sdbGetNumOfRows(tsClusterSdb); + int64_t numOfRows = sdbGetNumOfRows(tsClusterSdb); if (numOfRows <= 0 && dnodeIsFirstDeploy()) { mInfo("dnode first deploy, create cluster"); int32_t code = mnodeCreateCluster(); @@ -84,14 +84,14 @@ static int32_t mnodeClusterActionRestored() { int32_t mnodeInitCluster() { SClusterObj tObj; - tsClusterUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; + tsClusterUpdateSize = (int32_t)((int8_t *)tObj.updateEnd - (int8_t *)&tObj); SSdbTableDesc desc = { .id = SDB_TABLE_CLUSTER, .name = "cluster", .hashSessions = TSDB_DEFAULT_CLUSTER_HASH_SIZE, .maxRowSize = tsClusterUpdateSize, - .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, + .refCountPos = (int32_t)((int8_t *)(&tObj.refCount) - (int8_t *)&tObj), .keyType = SDB_KEY_STRING, .fpInsert = mnodeClusterActionInsert, .fpDelete = mnodeClusterActionDelete, @@ -139,7 +139,7 @@ void mnodeDecClusterRef(SClusterObj *pCluster) { } static int32_t mnodeCreateCluster() { - int32_t numOfClusters = sdbGetNumOfRows(tsClusterSdb); + int64_t numOfClusters = sdbGetNumOfRows(tsClusterSdb); if (numOfClusters != 0) return TSDB_CODE_SUCCESS; SClusterObj *pCluster = malloc(sizeof(SClusterObj)); @@ -226,7 +226,7 @@ static int32_t mnodeRetrieveClusters(SShowObj *pShow, char *data, int32_t rows, cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int32_t *) pWrite = pCluster->createdTime; + *(int64_t *) pWrite = pCluster->createdTime; cols++; mnodeDecClusterRef(pCluster); diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index 8a03b1cd0e..aa402b257a 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -143,14 +143,14 @@ static int32_t mnodeDbActionRestored() { int32_t mnodeInitDbs() { SDbObj tObj; - tsDbUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; + tsDbUpdateSize = (int32_t)((int8_t *)tObj.updateEnd - (int8_t *)&tObj); SSdbTableDesc desc = { .id = SDB_TABLE_DB, .name = "dbs", .hashSessions = TSDB_DEFAULT_DBS_HASH_SIZE, .maxRowSize = tsDbUpdateSize, - .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, + .refCountPos = (int32_t)((int8_t *)(&tObj.refCount) - (int8_t *)&tObj), .keyType = SDB_KEY_STRING, .fpInsert = mnodeDbActionInsert, .fpDelete = mnodeDbActionDelete, @@ -192,11 +192,11 @@ SDbObj *mnodeGetDb(char *db) { } void mnodeIncDbRef(SDbObj *pDb) { - return sdbIncRef(tsDbSdb, pDb); + sdbIncRef(tsDbSdb, pDb); } void mnodeDecDbRef(SDbObj *pDb) { - return sdbDecRef(tsDbSdb, pDb); + sdbDecRef(tsDbSdb, pDb); } SDbObj *mnodeGetDbByTableId(char *tableId) { diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index 792e41dd5b..01034b170f 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -148,7 +148,7 @@ static int32_t mnodeDnodeActionDecode(SSdbRow *pRow) { } static int32_t mnodeDnodeActionRestored() { - int32_t numOfRows = sdbGetNumOfRows(tsDnodeSdb); + int64_t numOfRows = sdbGetNumOfRows(tsDnodeSdb); if (numOfRows <= 0 && dnodeIsFirstDeploy()) { mInfo("dnode first deploy, create dnode:%s", tsLocalEp); mnodeCreateDnode(tsLocalEp, NULL); @@ -165,7 +165,7 @@ static int32_t mnodeDnodeActionRestored() { int32_t mnodeInitDnodes() { SDnodeObj tObj; - tsDnodeUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; + tsDnodeUpdateSize = (int32_t)((int8_t *)tObj.updateEnd - (int8_t *)&tObj); pthread_mutex_init(&tsDnodeEpsMutex, NULL); SSdbTableDesc desc = { @@ -173,7 +173,7 @@ int32_t mnodeInitDnodes() { .name = "dnodes", .hashSessions = TSDB_DEFAULT_DNODES_HASH_SIZE, .maxRowSize = tsDnodeUpdateSize, - .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, + .refCountPos = (int32_t)((int8_t *)(&tObj.refCount) - (int8_t *)&tObj), .keyType = SDB_KEY_AUTO, .fpInsert = mnodeDnodeActionInsert, .fpDelete = mnodeDnodeActionDelete, @@ -227,7 +227,7 @@ void mnodeCancelGetNextDnode(void *pIter) { } int32_t mnodeGetDnodesNum() { - return sdbGetNumOfRows(tsDnodeSdb); + return (int32_t)sdbGetNumOfRows(tsDnodeSdb); } int32_t mnodeGetOnlinDnodesCpuCoreNum() { @@ -407,7 +407,7 @@ static int32_t mnodeCheckClusterCfgPara(const SClusterCfg *clusterCfg) { int64_t checkTime = 0; char timestr[32] = "1970-01-01 00:00:00.00"; - (void)taosParseTime(timestr, &checkTime, strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); + (void)taosParseTime(timestr, &checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); if ((0 != strncasecmp(clusterCfg->timezone, tsTimezone, strlen(tsTimezone))) && (checkTime != clusterCfg->checkTime)) { mError("\"timezone\"[%s - %s] [%" PRId64 " - %" PRId64 "] cfg parameters inconsistent", clusterCfg->timezone, @@ -638,9 +638,9 @@ static int32_t mnodeCreateDnode(char *ep, SMnodeMsg *pMsg) { char *temp = strchr(dnodeEp, ':'); if (!temp) { - int len = strlen(dnodeEp); + int32_t len = (int32_t)strlen(dnodeEp); if (dnodeEp[len - 1] == ';') dnodeEp[len - 1] = 0; - len = strlen(dnodeEp); + len = (int32_t)strlen(dnodeEp); snprintf(dnodeEp + len, TSDB_EP_LEN - len, ":%d", tsServerPort); } ep = dnodeEp; diff --git a/src/mnode/src/mnodeMnode.c b/src/mnode/src/mnodeMnode.c index 3ea41c41c6..49473d3e06 100644 --- a/src/mnode/src/mnodeMnode.c +++ b/src/mnode/src/mnodeMnode.c @@ -136,14 +136,14 @@ int32_t mnodeInitMnodes() { mnodeMnodeInitLock(); SMnodeObj tObj; - tsMnodeUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; + tsMnodeUpdateSize = (int32_t)((int8_t *)tObj.updateEnd - (int8_t *)&tObj); SSdbTableDesc desc = { .id = SDB_TABLE_MNODE, .name = "mnodes", .hashSessions = TSDB_DEFAULT_MNODES_HASH_SIZE, .maxRowSize = tsMnodeUpdateSize, - .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, + .refCountPos = (int32_t)((int8_t *)(&tObj.refCount) - (int8_t *)&tObj), .keyType = SDB_KEY_INT, .fpInsert = mnodeMnodeActionInsert, .fpDelete = mnodeMnodeActionDelete, @@ -176,7 +176,7 @@ void mnodeCleanupMnodes() { } int32_t mnodeGetMnodesNum() { - return sdbGetNumOfRows(tsMnodeSdb); + return (int32_t)sdbGetNumOfRows(tsMnodeSdb); } void *mnodeGetMnode(int32_t mnodeId) { diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index ae495108b3..17cfd3e9d4 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -207,7 +207,7 @@ static void sdbRestoreTables() { (*pTable->fpRestored)(); } - totalRows += pTable->numOfRows; + totalRows += (int32_t)pTable->numOfRows; numOfTables++; sdbInfo("vgId:1, sdb:%s is checked, rows:%" PRId64, pTable->name, pTable->numOfRows); } @@ -475,7 +475,7 @@ void sdbIncRef(void *tparam, void *pRow) { if (pRow == NULL || tparam == NULL) return; SSdbTable *pTable = tparam; - int32_t * pRefCount = (int32_t *)(pRow + pTable->refCountPos); + int32_t * pRefCount = (int32_t *)((char *)pRow + pTable->refCountPos); int32_t refCount = atomic_add_fetch_32(pRefCount, 1); sdbTrace("vgId:1, sdb:%s, inc ref to row:%p:%s:%d", pTable->name, pRow, sdbGetRowStr(pTable, pRow), refCount); } @@ -484,11 +484,11 @@ void sdbDecRef(void *tparam, void *pRow) { if (pRow == NULL || tparam == NULL) return; SSdbTable *pTable = tparam; - int32_t * pRefCount = (int32_t *)(pRow + pTable->refCountPos); + int32_t * pRefCount = (int32_t *)((char *)pRow + pTable->refCountPos); int32_t refCount = atomic_sub_fetch_32(pRefCount, 1); sdbTrace("vgId:1, sdb:%s, dec ref to row:%p:%s:%d", pTable->name, pRow, sdbGetRowStr(pTable, pRow), refCount); - int32_t *updateEnd = pRow + pTable->refCountPos - 4; + int32_t *updateEnd = (int32_t *)((char *)pRow + pTable->refCountPos - 4); if (refCount <= 0 && *updateEnd) { sdbTrace("vgId:1, sdb:%s, row:%p:%s:%d destroyed", pTable->name, pRow, sdbGetRowStr(pTable, pRow), refCount); SSdbRow row = {.pObj = pRow}; @@ -501,7 +501,7 @@ static void *sdbGetRowMeta(SSdbTable *pTable, void *key) { int32_t keySize = sizeof(int32_t); if (pTable->keyType == SDB_KEY_STRING || pTable->keyType == SDB_KEY_VAR_STRING) { - keySize = strlen((char *)key); + keySize = (int32_t)strlen((char *)key); } void **ppRow = (void **)taosHashGet(pTable->iHandle, key, keySize); @@ -534,7 +534,7 @@ static int32_t sdbInsertHash(SSdbTable *pTable, SSdbRow *pRow) { int32_t keySize = sizeof(int32_t); if (pTable->keyType == SDB_KEY_STRING || pTable->keyType == SDB_KEY_VAR_STRING) { - keySize = strlen((char *)key); + keySize = (int32_t)strlen((char *)key); } pthread_mutex_lock(&pTable->mutex); @@ -564,7 +564,7 @@ static int32_t sdbInsertHash(SSdbTable *pTable, SSdbRow *pRow) { } static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbRow *pRow) { - int32_t *updateEnd = pRow->pObj + pTable->refCountPos - 4; + int32_t *updateEnd = (int32_t *)((char*)pRow->pObj + pTable->refCountPos - 4); bool set = atomic_val_compare_exchange_32(updateEnd, 0, 1) == 0; if (!set) { sdbError("vgId:1, sdb:%s, failed to delete key:%s from hash, for it already removed", pTable->name, @@ -577,7 +577,7 @@ static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbRow *pRow) { void * key = sdbGetObjKey(pTable, pRow->pObj); int32_t keySize = sizeof(int32_t); if (pTable->keyType == SDB_KEY_STRING || pTable->keyType == SDB_KEY_VAR_STRING) { - keySize = strlen((char *)key); + keySize = (int32_t)strlen((char *)key); } pthread_mutex_lock(&pTable->mutex); @@ -764,7 +764,7 @@ bool sdbCheckRowDeleted(void *tparam, void *pRow) { SSdbTable *pTable = tparam; if (pTable == NULL) return false; - int32_t *updateEnd = pRow + pTable->refCountPos - 4; + int32_t *updateEnd = (int32_t *)((char*)pRow + pTable->refCountPos - 4); return atomic_val_compare_exchange_32(updateEnd, 1, 1) == 1; } @@ -942,14 +942,14 @@ static int32_t sdbInitWorker() { static void sdbCleanupWorker() { for (int32_t i = 0; i < tsSdbPool.num; ++i) { SSdbWorker *pWorker = tsSdbPool.worker + i; - if (pWorker->thread) { + if (taosCheckPthreadValid(pWorker->thread)) { taosQsetThreadResume(tsSdbWQset); } } for (int32_t i = 0; i < tsSdbPool.num; ++i) { SSdbWorker *pWorker = tsSdbPool.worker + i; - if (pWorker->thread) { + if (taosCheckPthreadValid(pWorker->thread)) { pthread_join(pWorker->thread, NULL); } } diff --git a/src/mnode/src/mnodeShow.c b/src/mnode/src/mnodeShow.c index 6b9f0e26a7..0377df97fd 100644 --- a/src/mnode/src/mnodeShow.c +++ b/src/mnode/src/mnodeShow.c @@ -218,7 +218,7 @@ static int32_t mnodeProcessRetrieveMsg(SMnodeMsg *pMsg) { } pRsp->numOfRows = htonl(rowsRead); - pRsp->precision = htonl(TSDB_TIME_PRECISION_MILLI); // millisecond time precision + pRsp->precision = (int16_t)htonl(TSDB_TIME_PRECISION_MILLI); // millisecond time precision pMsg->rpcRsp.rsp = pRsp; pMsg->rpcRsp.len = size; diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index de37b09345..a0c8d88c51 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -222,23 +222,23 @@ static int32_t mnodeChildTableActionEncode(SSdbRow *pRow) { SCTableObj *pTable = pRow->pObj; assert(pTable != NULL && pRow->rowData != NULL); - int32_t len = strlen(pTable->info.tableId); + int32_t len = (int32_t)strlen(pTable->info.tableId); if (len >= TSDB_TABLE_FNAME_LEN) return TSDB_CODE_MND_INVALID_TABLE_ID; memcpy(pRow->rowData, pTable->info.tableId, len); - memset(pRow->rowData + len, 0, 1); + memset((char *)pRow->rowData + len, 0, 1); len++; - memcpy(pRow->rowData + len, (char*)pTable + sizeof(char *), tsChildTableUpdateSize); + memcpy((char *)pRow->rowData + len, (char *)pTable + sizeof(char *), tsChildTableUpdateSize); len += tsChildTableUpdateSize; if (pTable->info.type != TSDB_CHILD_TABLE) { int32_t schemaSize = pTable->numOfColumns * sizeof(SSchema); - memcpy(pRow->rowData + len, pTable->schema, schemaSize); + memcpy((char *)pRow->rowData + len, pTable->schema, schemaSize); len += schemaSize; if (pTable->sqlLen != 0) { - memcpy(pRow->rowData + len, pTable->sql, pTable->sqlLen); + memcpy((char *)pRow->rowData + len, pTable->sql, pTable->sqlLen); len += pTable->sqlLen; } } @@ -253,7 +253,7 @@ static int32_t mnodeChildTableActionDecode(SSdbRow *pRow) { SCTableObj *pTable = calloc(1, sizeof(SCTableObj)); if (pTable == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; - int32_t len = strlen(pRow->rowData); + int32_t len = (int32_t)strlen(pRow->rowData); if (len >= TSDB_TABLE_FNAME_LEN) { free(pTable); return TSDB_CODE_MND_INVALID_TABLE_ID; @@ -261,7 +261,7 @@ static int32_t mnodeChildTableActionDecode(SSdbRow *pRow) { pTable->info.tableId = strdup(pRow->rowData); len++; - memcpy((char*)pTable + sizeof(char *), pRow->rowData + len, tsChildTableUpdateSize); + memcpy((char *)pTable + sizeof(char *), (char *)pRow->rowData + len, tsChildTableUpdateSize); len += tsChildTableUpdateSize; if (pTable->info.type != TSDB_CHILD_TABLE) { @@ -271,7 +271,7 @@ static int32_t mnodeChildTableActionDecode(SSdbRow *pRow) { mnodeDestroyChildTable(pTable); return TSDB_CODE_MND_INVALID_TABLE_TYPE; } - memcpy(pTable->schema, pRow->rowData + len, schemaSize); + memcpy(pTable->schema, (char *)pRow->rowData + len, schemaSize); len += schemaSize; if (pTable->sqlLen != 0) { @@ -280,7 +280,7 @@ static int32_t mnodeChildTableActionDecode(SSdbRow *pRow) { mnodeDestroyChildTable(pTable); return TSDB_CODE_MND_OUT_OF_MEMORY; } - memcpy(pTable->sql, pRow->rowData + len, pTable->sqlLen); + memcpy(pTable->sql, (char *)pRow->rowData + len, pTable->sqlLen); } } @@ -352,14 +352,14 @@ static int32_t mnodeChildTableActionRestored() { static int32_t mnodeInitChildTables() { SCTableObj tObj; - tsChildTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj.info.type; + tsChildTableUpdateSize = (int32_t)((int8_t *)tObj.updateEnd - (int8_t *)&tObj.info.type); SSdbTableDesc desc = { .id = SDB_TABLE_CTABLE, .name = "ctables", .hashSessions = TSDB_DEFAULT_CTABLES_HASH_SIZE, .maxRowSize = sizeof(SCTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16) + TSDB_TABLE_FNAME_LEN + TSDB_CQ_SQL_SIZE, - .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, + .refCountPos = (int32_t)((int8_t *)(&tObj.refCount) - (int8_t *)&tObj), .keyType = SDB_KEY_VAR_STRING, .fpInsert = mnodeChildTableActionInsert, .fpDelete = mnodeChildTableActionDelete, @@ -501,18 +501,18 @@ static int32_t mnodeSuperTableActionEncode(SSdbRow *pRow) { SSTableObj *pStable = pRow->pObj; assert(pRow->pObj != NULL && pRow->rowData != NULL); - int32_t len = strlen(pStable->info.tableId); + int32_t len = (int32_t)strlen(pStable->info.tableId); if (len >= TSDB_TABLE_FNAME_LEN) len = TSDB_CODE_MND_INVALID_TABLE_ID; memcpy(pRow->rowData, pStable->info.tableId, len); - memset(pRow->rowData + len, 0, 1); + memset((char *)pRow->rowData + len, 0, 1); len++; - memcpy(pRow->rowData + len, (char*)pStable + sizeof(char *), tsSuperTableUpdateSize); + memcpy((char *)pRow->rowData + len, (char *)pStable + sizeof(char *), tsSuperTableUpdateSize); len += tsSuperTableUpdateSize; int32_t schemaSize = sizeof(SSchema) * (pStable->numOfColumns + pStable->numOfTags); - memcpy(pRow->rowData + len, pStable->schema, schemaSize); + memcpy((char *)pRow->rowData + len, pStable->schema, schemaSize); len += schemaSize; pRow->rowSize = len; @@ -525,7 +525,7 @@ static int32_t mnodeSuperTableActionDecode(SSdbRow *pRow) { SSTableObj *pStable = (SSTableObj *) calloc(1, sizeof(SSTableObj)); if (pStable == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; - int32_t len = strlen(pRow->rowData); + int32_t len = (int32_t)strlen(pRow->rowData); if (len >= TSDB_TABLE_FNAME_LEN){ free(pStable); return TSDB_CODE_MND_INVALID_TABLE_ID; @@ -533,7 +533,7 @@ static int32_t mnodeSuperTableActionDecode(SSdbRow *pRow) { pStable->info.tableId = strdup(pRow->rowData); len++; - memcpy((char*)pStable + sizeof(char *), pRow->rowData + len, tsSuperTableUpdateSize); + memcpy((char *)pStable + sizeof(char *), (char *)pRow->rowData + len, tsSuperTableUpdateSize); len += tsSuperTableUpdateSize; int32_t schemaSize = sizeof(SSchema) * (pStable->numOfColumns + pStable->numOfTags); @@ -543,7 +543,7 @@ static int32_t mnodeSuperTableActionDecode(SSdbRow *pRow) { return TSDB_CODE_MND_NOT_SUPER_TABLE; } - memcpy(pStable->schema, pRow->rowData + len, schemaSize); + memcpy(pStable->schema, (char *)pRow->rowData + len, schemaSize); pRow->pObj = pStable; @@ -556,14 +556,14 @@ static int32_t mnodeSuperTableActionRestored() { static int32_t mnodeInitSuperTables() { SSTableObj tObj; - tsSuperTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj.info.type; + tsSuperTableUpdateSize = (int32_t)((int8_t *)tObj.updateEnd - (int8_t *)&tObj.info.type); SSdbTableDesc desc = { .id = SDB_TABLE_STABLE, .name = "stables", .hashSessions = TSDB_DEFAULT_STABLES_HASH_SIZE, .maxRowSize = sizeof(SSTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16) + TSDB_TABLE_FNAME_LEN, - .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, + .refCountPos = (int32_t)((int8_t *)(&tObj.refCount) - (int8_t *)&tObj), .keyType = SDB_KEY_VAR_STRING, .fpInsert = mnodeSuperTableActionInsert, .fpDelete = mnodeSuperTableActionDelete, @@ -1266,7 +1266,7 @@ static int32_t mnodeModifySuperTableTagName(SMnodeMsg *pMsg, char *oldTagName, c } // int32_t rowSize = 0; - uint32_t len = strlen(newTagName); + uint32_t len = (int32_t)strlen(newTagName); if (len >= TSDB_COL_NAME_LEN) { return TSDB_CODE_MND_COL_NAME_TOO_LONG; } @@ -1429,7 +1429,7 @@ static int32_t mnodeChangeSuperTableColumn(SMnodeMsg *pMsg, char *oldName, char } // int32_t rowSize = 0; - uint32_t len = strlen(newName); + uint32_t len = (uint32_t)strlen(newName); if (len >= TSDB_COL_NAME_LEN) { return TSDB_CODE_MND_COL_NAME_TOO_LONG; } @@ -1534,7 +1534,7 @@ int32_t mnodeRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, tstrncpy(prefix, pDb->name, 64); strcat(prefix, TS_PATH_DELIMITER); - prefixLen = strlen(prefix); + prefixLen = (int32_t)strlen(prefix); SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; char stableName[TSDB_TABLE_NAME_LEN] = {0}; @@ -1559,7 +1559,7 @@ int32_t mnodeRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - int16_t len = strnlen(stableName, TSDB_TABLE_NAME_LEN - 1); + int16_t len = (int16_t)strnlen(stableName, TSDB_TABLE_NAME_LEN - 1); *(int16_t*) pWrite = len; pWrite += sizeof(int16_t); // todo refactor @@ -1602,7 +1602,7 @@ void mnodeDropAllSuperTables(SDbObj *pDropDb) { char prefix[64] = {0}; tstrncpy(prefix, pDropDb->name, 64); strcat(prefix, TS_PATH_DELIMITER); - int32_t prefixLen = strlen(prefix); + int32_t prefixLen = (int32_t)strlen(prefix); mInfo("db:%s, all super tables will be dropped from sdb", pDropDb->name); @@ -1755,9 +1755,9 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) { rpcFreeCont(pRsp); return TSDB_CODE_MND_INVALID_TABLE_NAME; } else { - pRsp->numOfTables = htonl(pRsp->numOfTables); + pRsp->numOfTables = (int32_t)htonl(pRsp->numOfTables); pMsg->rpcRsp.rsp = pRsp; - pMsg->rpcRsp.len = msg - (char *)pRsp; + pMsg->rpcRsp.len = (int32_t)((char *)msg - (char *)pRsp); return TSDB_CODE_SUCCESS; } @@ -2030,7 +2030,7 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) { static int32_t mnodeProcessCreateChildTableMsg(SMnodeMsg *pMsg) { //SCMCreateTableMsg* p1 = pMsg->rpcMsg.pCont; // there are several tables here. - SCreateTableMsg* pCreate = (SCreateTableMsg*)(pMsg->rpcMsg.pCont + sizeof(SCMCreateTableMsg)); + SCreateTableMsg* pCreate = (SCreateTableMsg*)((char *)pMsg->rpcMsg.pCont + sizeof(SCMCreateTableMsg)); int32_t code = grantCheck(TSDB_GRANT_TIMESERIES); if (code != TSDB_CODE_SUCCESS) { @@ -2287,7 +2287,7 @@ static int32_t mnodeChangeNormalTableColumn(SMnodeMsg *pMsg, char *oldName, char } // int32_t rowSize = 0; - uint32_t len = strlen(newName); + uint32_t len = (uint32_t)strlen(newName); if (len >= TSDB_COL_NAME_LEN) { return TSDB_CODE_MND_COL_NAME_TOO_LONG; } @@ -2491,7 +2491,7 @@ void mnodeDropAllChildTables(SDbObj *pDropDb) { char prefix[64] = {0}; tstrncpy(prefix, pDropDb->name, 64); strcat(prefix, TS_PATH_DELIMITER); - int32_t prefixLen = strlen(prefix); + int32_t prefixLen = (int32_t)strlen(prefix); mInfo("db:%s, all child tables will be dropped from sdb", pDropDb->name); @@ -2907,7 +2907,7 @@ static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; char prefix[64] = {0}; - int32_t prefixLen = tableIdPrefix(pDb->name, prefix, 64); + int32_t prefixLen = (int32_t)tableIdPrefix(pDb->name, prefix, 64); char* pattern = NULL; if (pShow->payloadLen > 0) { @@ -3143,7 +3143,7 @@ static int32_t mnodeRetrieveStreamTables(SShowObj *pShow, char *data, int32_t ro char prefix[64] = {0}; tstrncpy(prefix, pDb->name, 64); strcat(prefix, TS_PATH_DELIMITER); - int32_t prefixLen = strlen(prefix); + int32_t prefixLen = (int32_t)strlen(prefix); while (numOfRows < rows) { pShow->pIter = mnodeGetNextChildTable(pShow->pIter, &pTable); diff --git a/src/mnode/src/mnodeUser.c b/src/mnode/src/mnodeUser.c index fb26086d04..3a699757d9 100644 --- a/src/mnode/src/mnodeUser.c +++ b/src/mnode/src/mnodeUser.c @@ -128,7 +128,7 @@ static void mnodePrintUserAuth() { } static int32_t mnodeUserActionRestored() { - int32_t numOfRows = sdbGetNumOfRows(tsUserSdb); + int64_t numOfRows = sdbGetNumOfRows(tsUserSdb); if (numOfRows <= 0 && dnodeIsFirstDeploy()) { mInfo("dnode first deploy, create root user"); SAcctObj *pAcct = mnodeGetAcct(TSDB_DEFAULT_USER); @@ -148,14 +148,14 @@ static int32_t mnodeUserActionRestored() { int32_t mnodeInitUsers() { SUserObj tObj; - tsUserUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; + tsUserUpdateSize = (int32_t)((int8_t *)tObj.updateEnd - (int8_t *)&tObj); SSdbTableDesc desc = { .id = SDB_TABLE_USER, .name = "users", .hashSessions = TSDB_DEFAULT_USERS_HASH_SIZE, .maxRowSize = tsUserUpdateSize, - .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, + .refCountPos = (int32_t)((int8_t *)(&tObj.refCount) - (int8_t *)&tObj), .keyType = SDB_KEY_STRING, .fpInsert = mnodeUserActionInsert, .fpDelete = mnodeUserActionDelete, @@ -204,11 +204,11 @@ void mnodeCancelGetNextUser(void *pIter) { } void mnodeIncUserRef(SUserObj *pUser) { - return sdbIncRef(tsUserSdb, pUser); + sdbIncRef(tsUserSdb, pUser); } void mnodeDecUserRef(SUserObj *pUser) { - return sdbDecRef(tsUserSdb, pUser); + sdbDecRef(tsUserSdb, pUser); } static int32_t mnodeUpdateUser(SUserObj *pUser, void *pMsg) { @@ -561,7 +561,7 @@ static int32_t mnodeProcessDropUserMsg(SMnodeMsg *pMsg) { void mnodeDropAllUsers(SAcctObj *pAcct) { void * pIter = NULL; int32_t numOfUsers = 0; - int32_t acctNameLen = strlen(pAcct->user); + int32_t acctNameLen = (int32_t)strlen(pAcct->user); SUserObj *pUser = NULL; while (1) { diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index 827be0687d..bd89f2a267 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -206,14 +206,14 @@ static int32_t mnodeVgroupActionRestored() { int32_t mnodeInitVgroups() { SVgObj tObj; - tsVgUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; + tsVgUpdateSize = (int32_t)((int8_t *)tObj.updateEnd - (int8_t *)&tObj); SSdbTableDesc desc = { .id = SDB_TABLE_VGROUP, .name = "vgroups", .hashSessions = TSDB_DEFAULT_VGROUPS_HASH_SIZE, .maxRowSize = tsVgUpdateSize, - .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, + .refCountPos = (int32_t)((int8_t *)(&tObj.refCount) - (int8_t *)&tObj), .keyType = SDB_KEY_AUTO, .fpInsert = mnodeVgroupActionInsert, .fpDelete = mnodeVgroupActionDelete, @@ -245,11 +245,11 @@ int32_t mnodeInitVgroups() { } void mnodeIncVgroupRef(SVgObj *pVgroup) { - return sdbIncRef(tsVgroupSdb, pVgroup); + sdbIncRef(tsVgroupSdb, pVgroup); } void mnodeDecVgroupRef(SVgObj *pVgroup) { - return sdbDecRef(tsVgroupSdb, pVgroup); + sdbDecRef(tsVgroupSdb, pVgroup); } SVgObj *mnodeGetVgroup(int32_t vgId) { diff --git a/src/os/inc/osFile.h b/src/os/inc/osFile.h index c9b3b9cd76..91ac5c9958 100644 --- a/src/os/inc/osFile.h +++ b/src/os/inc/osFile.h @@ -37,7 +37,7 @@ int32_t taosRenameFile(char *fullPath, char *suffix, char delimiter, char **dstP } // TAOS_OS_FUNC_FILE_SENDIFLE -int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t *offset, int64_t size); +int64_t taosSendFile(SOCKET dfd, int32_t sfd, int64_t *offset, int64_t size); int64_t taosFSendFile(FILE *outfile, FILE *infile, int64_t *offset, int64_t size); #ifdef TAOS_RANDOM_FILE_FAIL diff --git a/src/os/inc/osSocket.h b/src/os/inc/osSocket.h index 13d3fa4079..4e2671ce2b 100644 --- a/src/os/inc/osSocket.h +++ b/src/os/inc/osSocket.h @@ -61,6 +61,7 @@ extern "C" { int32_t taosSetNonblocking(SOCKET sock, int32_t on); void taosIgnSIGPIPE(); void taosBlockSIGPIPE(); +void taosSetMaskSIGPIPE(); // TAOS_OS_FUNC_SOCKET_SETSOCKETOPT int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen); diff --git a/src/os/src/detail/osFile.c b/src/os/src/detail/osFile.c index 2e6886aa21..6ba38635f4 100644 --- a/src/os/src/detail/osFile.c +++ b/src/os/src/detail/osFile.c @@ -121,7 +121,7 @@ int64_t taosLSeekImp(int32_t fd, int64_t offset, int32_t whence) { #ifndef TAOS_OS_FUNC_FILE_SENDIFLE -int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t *offset, int64_t size) { +int64_t taosSendFile(SOCKET dfd, int32_t sfd, int64_t *offset, int64_t size) { int64_t leftbytes = size; int64_t sentbytes; diff --git a/src/os/src/detail/osSocket.c b/src/os/src/detail/osSocket.c index 729471247f..d03e8086bf 100644 --- a/src/os/src/detail/osSocket.c +++ b/src/os/src/detail/osSocket.c @@ -53,6 +53,16 @@ void taosBlockSIGPIPE() { } } +void taosSetMaskSIGPIPE() { + sigset_t signal_mask; + sigemptyset(&signal_mask); + sigaddset(&signal_mask, SIGPIPE); + int32_t rc = pthread_sigmask(SIG_SETMASK, &signal_mask, NULL); + if (rc != 0) { + uError("failed to setmask SIGPIPE"); + } +} + #endif #ifndef TAOS_OS_FUNC_SOCKET_SETSOCKETOPT diff --git a/src/os/src/windows/wFile.c b/src/os/src/windows/wFile.c index 2204135ae6..a127c17ce7 100644 --- a/src/os/src/windows/wFile.c +++ b/src/os/src/windows/wFile.c @@ -78,7 +78,7 @@ int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t co return writeLen; } -int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t* offset, int64_t size) { +int64_t taosSendFile(SOCKET dfd, int32_t sfd, int64_t* offset, int64_t size) { uError("taosSendFile no implemented yet"); return 0; } diff --git a/src/os/src/windows/wSocket.c b/src/os/src/windows/wSocket.c index 9697c5e65f..4e6ee15e77 100644 --- a/src/os/src/windows/wSocket.c +++ b/src/os/src/windows/wSocket.c @@ -48,6 +48,7 @@ int32_t taosSetNonblocking(SOCKET sock, int32_t on) { void taosIgnSIGPIPE() {} void taosBlockSIGPIPE() {} +void taosSetMaskSIGPIPE() {} int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen) { if (level == SOL_SOCKET && optname == TCP_KEEPCNT) { diff --git a/src/plugins/http/CMakeLists.txt b/src/plugins/http/CMakeLists.txt index 5d8b52986a..86d857876d 100644 --- a/src/plugins/http/CMakeLists.txt +++ b/src/plugins/http/CMakeLists.txt @@ -9,7 +9,7 @@ INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) INCLUDE_DIRECTORIES(inc) AUX_SOURCE_DIRECTORY(src SRC) -IF (TD_LINUX) +IF (TD_LINUX OR TD_WINDOWS) ADD_LIBRARY(http ${SRC}) TARGET_LINK_LIBRARIES(http z) diff --git a/src/plugins/http/inc/httpInt.h b/src/plugins/http/inc/httpInt.h index ebdfabf310..ada67efbfd 100644 --- a/src/plugins/http/inc/httpInt.h +++ b/src/plugins/http/inc/httpInt.h @@ -166,7 +166,7 @@ typedef struct HttpThread { HttpContext * pHead; pthread_mutex_t threadMutex; bool stop; - int32_t pollFd; + SOCKET pollFd; int32_t numOfContexts; int32_t threadId; char label[HTTP_LABEL_SIZE]; @@ -177,7 +177,7 @@ typedef struct HttpServer { char label[HTTP_LABEL_SIZE]; uint32_t serverIp; uint16_t serverPort; - int32_t fd; + SOCKET fd; int32_t numOfThreads; int32_t methodScannerLen; int32_t requestNum; diff --git a/src/plugins/http/src/httpGcJson.c b/src/plugins/http/src/httpGcJson.c index 2c9eca11de..3a053afd87 100644 --- a/src/plugins/http/src/httpGcJson.c +++ b/src/plugins/http/src/httpGcJson.c @@ -158,7 +158,7 @@ bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, if (row[i]!= NULL){ len += snprintf(target + len, HTTP_GC_TARGET_SIZE - len, "%s:", fields[i].name); memcpy(target + len, (char *) row[i], length[i]); - len = strlen(target); + len = (int32_t)strlen(target); } break; default: diff --git a/src/plugins/http/src/httpGzip.c b/src/plugins/http/src/httpGzip.c index 54f900c755..d94e3b1235 100644 --- a/src/plugins/http/src/httpGzip.c +++ b/src/plugins/http/src/httpGzip.c @@ -133,7 +133,7 @@ int32_t ehttp_gzip_write(ehttp_gzip_t *gzip, const char *buf, int32_t len) { if (ret!=Z_STREAM_END) continue; } - int32_t len = gzip->gzip->next_out - (z_const Bytef*)gzip->chunk; + int32_t len = (int32_t)(gzip->gzip->next_out - (z_const Bytef*)gzip->chunk); gzip->gzip->next_out[0] = '\0'; gzip->callbacks.on_data(gzip, gzip->arg, gzip->chunk, len); @@ -155,7 +155,7 @@ int32_t ehttp_gzip_finish(ehttp_gzip_t *gzip) { if (ret != Z_STREAM_END) return -1; - int32_t len = gzip->gzip->next_out - (z_const Bytef*)gzip->chunk; + int32_t len = (int32_t)(gzip->gzip->next_out - (z_const Bytef*)gzip->chunk); gzip->gzip->next_out[0] = '\0'; gzip->callbacks.on_data(gzip, gzip->arg, gzip->chunk, len); diff --git a/src/plugins/http/src/httpJson.c b/src/plugins/http/src/httpJson.c index 1aa6cfac4b..132553ffe1 100644 --- a/src/plugins/http/src/httpJson.c +++ b/src/plugins/http/src/httpJson.c @@ -93,7 +93,7 @@ int32_t httpWriteBufNoTrace(struct HttpContext *pContext, const char *buf, int32 int32_t httpWriteJsonBufBody(JsonBuf* buf, bool isTheLast) { int32_t remain = 0; char sLen[24]; - uint64_t srcLen = (uint64_t) (buf->lst - buf->buf); + int32_t srcLen = (int32_t) (buf->lst - buf->buf); if (buf->pContext->fd <= 0) { httpTrace("context:%p, fd:%d, write json body error", buf->pContext, buf->pContext->fd); @@ -113,11 +113,11 @@ int32_t httpWriteJsonBufBody(JsonBuf* buf, bool isTheLast) { httpTrace("context:%p, fd:%d, no data need dump", buf->pContext, buf->pContext->fd); return 0; // there is no data to dump. } else { - int32_t len = sprintf(sLen, "%" PRIx64 "\r\n", srcLen); - httpTrace("context:%p, fd:%d, write body, chunkSize:%" PRIu64 ", response:\n%s", buf->pContext, buf->pContext->fd, + int32_t len = sprintf(sLen, "%d\r\n", srcLen); + httpTrace("context:%p, fd:%d, write body, chunkSize:%d, response:\n%s", buf->pContext, buf->pContext->fd, srcLen, buf->buf); httpWriteBufNoTrace(buf->pContext, sLen, len); - remain = httpWriteBufNoTrace(buf->pContext, buf->buf, (int32_t)srcLen); + remain = httpWriteBufNoTrace(buf->pContext, buf->buf, srcLen); } } else { char compressBuf[JSON_BUFFER_SIZE] = {0}; @@ -126,7 +126,7 @@ int32_t httpWriteJsonBufBody(JsonBuf* buf, bool isTheLast) { if (ret == 0) { if (compressBufLen > 0) { int32_t len = sprintf(sLen, "%x\r\n", compressBufLen); - httpTrace("context:%p, fd:%d, write body, chunkSize:%" PRIu64 ", compressSize:%d, last:%d, response:\n%s", + httpTrace("context:%p, fd:%d, write body, chunkSize:%d, compressSize:%d, last:%d, response:\n%s", buf->pContext, buf->pContext->fd, srcLen, compressBufLen, isTheLast, buf->buf); httpWriteBufNoTrace(buf->pContext, sLen, len); remain = httpWriteBufNoTrace(buf->pContext, (const char*)compressBuf, compressBufLen); @@ -136,7 +136,7 @@ int32_t httpWriteJsonBufBody(JsonBuf* buf, bool isTheLast) { remain = 0; // there is no data to dump. } } else { - httpError("context:%p, fd:%d, failed to compress data, chunkSize:%" PRIu64 ", last:%d, error:%d, response:\n%s", + httpError("context:%p, fd:%d, failed to compress data, chunkSize:%d, last:%d, error:%d, response:\n%s", buf->pContext, buf->pContext->fd, srcLen, isTheLast, ret, buf->buf); remain = 0; } diff --git a/src/plugins/http/src/httpParser.c b/src/plugins/http/src/httpParser.c index b844834537..331ca079fa 100644 --- a/src/plugins/http/src/httpParser.c +++ b/src/plugins/http/src/httpParser.c @@ -153,7 +153,7 @@ static int32_t httpOnRequestLine(HttpParser *pParser, char *method, char *target for (int32_t i = 0; i < HTTP_MAX_URL; i++) { char *pSeek = strchr(pStart, '/'); if (pSeek == NULL) { - (void)httpAppendString(pParser->path + i, pStart, strlen(pStart)); + (void)httpAppendString(pParser->path + i, pStart, (int32_t)strlen(pStart)); break; } else { (void)httpAppendString(pParser->path + i, pStart, (int32_t)(pSeek - pStart)); @@ -285,7 +285,7 @@ static int32_t httpOnParseHeaderField(HttpParser *parser, const char *key, const free(t); free(s); httpTrace("context:%p, fd:%d, basic auth:%s", pContext, pContext->fd, parser->authContent); - int32_t ok = httpParseBasicAuthToken(pContext, parser->authContent, strlen(parser->authContent)); + int32_t ok = httpParseBasicAuthToken(pContext, parser->authContent, (int32_t)strlen(parser->authContent)); if (ok != 0) { httpOnError(parser, 0, TSDB_CODE_HTTP_INVALID_BASIC_AUTH); return -1; @@ -299,7 +299,7 @@ static int32_t httpOnParseHeaderField(HttpParser *parser, const char *key, const free(t); free(s); httpTrace("context:%p, fd:%d, taosd auth:%s", pContext, pContext->fd, parser->authContent); - int32_t ok = httpParseTaosdAuthToken(pContext, parser->authContent, strlen(parser->authContent)); + int32_t ok = httpParseTaosdAuthToken(pContext, parser->authContent, (int32_t)strlen(parser->authContent)); if (ok != 0) { httpOnError(parser, 0, TSDB_CODE_HTTP_INVALID_TAOSD_AUTH); return -1; @@ -524,14 +524,14 @@ char *httpDecodeUrl(const char *enc) { int32_t hex, cnt; int32_t n = sscanf(p+1, "%2x%n", &hex, &cnt); if (n!=1 && cnt !=2) { ok = 0; break; } - if (httpAppendString(&str, enc, p-enc)) { ok = 0; break; } + if (httpAppendString(&str, enc, (int32_t)(p-enc))) { ok = 0; break; } char c = (char)hex; if (httpAppendString(&str, &c, 1)) { ok = 0; break; } enc = p+3; } char *dec = NULL; if (ok && *enc) { - if (httpAppendString(&str, enc, strlen(enc))) { ok = 0; } + if (httpAppendString(&str, enc, (int32_t)strlen(enc))) { ok = 0; } } if (ok) { dec = str.str; @@ -667,7 +667,7 @@ static int32_t httpParserOnVersion(HttpParser *parser, HTTP_PARSER_STATE state, int32_t ok = 0; do { const char *prefix = "HTTP/1."; - int32_t len = strlen(prefix); + int32_t len = (int32_t)strlen(prefix); if (parser->str.pos < len) { if (prefix[parser->str.pos] != c) { httpError("context:%p, fd:%d, parser state:%d, unexpected char:[%c]%02x", pContext, pContext->fd, state, c, c); @@ -811,7 +811,7 @@ static int32_t httpParserOnCrlf(HttpParser *parser, HTTP_PARSER_STATE state, con int32_t ok = 0; do { const char *s = "\r\n"; - int32_t len = strlen(s); + int32_t len = (int32_t)strlen(s); if (s[parser->str.pos] != c) { httpError("context:%p, fd:%d, parser state:%d, unexpected char:[%c]%02x", pContext, pContext->fd, state, c, c); ok = -1; diff --git a/src/plugins/http/src/httpQueue.c b/src/plugins/http/src/httpQueue.c index 1c039abb4d..7ae54717f3 100644 --- a/src/plugins/http/src/httpQueue.c +++ b/src/plugins/http/src/httpQueue.c @@ -134,14 +134,14 @@ void httpCleanupResultQueue() { for (int32_t i = 0; i < tsHttpPool.num; ++i) { SHttpWorker *pWorker = tsHttpPool.httpWorker + i; - if (pWorker->thread) { + if (taosCheckPthreadValid(pWorker->thread)) { taosQsetThreadResume(tsHttpQset); } } for (int32_t i = 0; i < tsHttpPool.num; ++i) { SHttpWorker *pWorker = tsHttpPool.httpWorker + i; - if (pWorker->thread) { + if (taosCheckPthreadValid(pWorker->thread)) { pthread_join(pWorker->thread, NULL); } } diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index 4896d50c6c..ee6addd6fa 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -25,10 +25,6 @@ #include "httpResp.h" #include "httpUtil.h" -#ifndef EPOLLWAKEUP - #define EPOLLWAKEUP (1u << 29) -#endif - static bool httpReadData(HttpContext *pContext); static void httpStopThread(HttpThread* pThread) { @@ -49,10 +45,10 @@ static void httpStopThread(HttpThread* pThread) { pthread_join(pThread->thread, NULL); if (fd != -1) { - close(fd); + taosCloseSocket(fd); } - close(pThread->pollFd); + taosCloseSocket(pThread->pollFd); pthread_mutex_destroy(&(pThread->threadMutex)); } @@ -77,10 +73,7 @@ static void httpProcessHttpData(void *param) { HttpContext *pContext; int32_t fdNum; - sigset_t set; - sigemptyset(&set); - sigaddset(&set, SIGPIPE); - pthread_sigmask(SIG_SETMASK, &set, NULL); + taosSetMaskSIGPIPE(); while (1) { struct epoll_event events[HTTP_MAX_EVENTS]; @@ -162,10 +155,7 @@ static void *httpAcceptHttpConnection(void *arg) { HttpContext * pContext = NULL; int32_t totalFds = 0; - sigset_t set; - sigemptyset(&set); - sigaddset(&set, SIGPIPE); - pthread_sigmask(SIG_SETMASK, &set, NULL); + taosSetMaskSIGPIPE(); pServer->fd = taosOpenTcpServerSocket(pServer->serverIp, pServer->serverPort); @@ -242,7 +232,7 @@ static void *httpAcceptHttpConnection(void *arg) { threadId = threadId % pServer->numOfThreads; } - close(pServer->fd); + taosCloseSocket(pServer->fd); return NULL; } @@ -265,7 +255,7 @@ bool httpInitConnect() { return false; } - pThread->pollFd = epoll_create(HTTP_MAX_EVENTS); // size does not matter + pThread->pollFd = (SOCKET)epoll_create(HTTP_MAX_EVENTS); // size does not matter if (pThread->pollFd < 0) { httpError("http thread:%s, failed to create HTTP epoll", pThread->label); pthread_mutex_destroy(&(pThread->threadMutex)); diff --git a/src/plugins/http/src/httpTgHandle.c b/src/plugins/http/src/httpTgHandle.c index e2b57b87bb..f9c8860afe 100644 --- a/src/plugins/http/src/httpTgHandle.c +++ b/src/plugins/http/src/httpTgHandle.c @@ -276,7 +276,7 @@ int32_t tgReadSchema(char *fileName) { rewind(fp); char * content = (char *)calloc(contentSize + 1, 1); - int32_t result = fread(content, 1, contentSize, fp); + int32_t result = (int32_t)fread(content, 1, contentSize, fp); if (result != contentSize) { httpError("failed to read telegraf schema file:%s", fileName); diff --git a/src/plugins/http/src/httpUtil.c b/src/plugins/http/src/httpUtil.c index 39168ee96d..51333d2118 100644 --- a/src/plugins/http/src/httpUtil.c +++ b/src/plugins/http/src/httpUtil.c @@ -388,7 +388,7 @@ int32_t httpGzipDeCompress(char *srcData, int32_t nSrcData, char *destData, int3 if (inflateEnd(&gzipStream) != Z_OK) { return -4; } - *nDestData = gzipStream.total_out; + *nDestData = (int32_t)gzipStream.total_out; return 0; } @@ -417,7 +417,7 @@ int32_t httpGzipCompress(HttpContext *pContext, char *srcData, int32_t nSrcData, return -1; } - int32_t cacheLen = pContext->gzipStream.total_out - lastTotalLen; + int32_t cacheLen = (int32_t)(pContext->gzipStream.total_out - lastTotalLen); if (cacheLen >= *nDestData) { return -2; } diff --git a/src/plugins/monitor/CMakeLists.txt b/src/plugins/monitor/CMakeLists.txt index edea2187ea..c9314fe1cb 100644 --- a/src/plugins/monitor/CMakeLists.txt +++ b/src/plugins/monitor/CMakeLists.txt @@ -6,7 +6,7 @@ INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) AUX_SOURCE_DIRECTORY(./src SRC) -IF (TD_LINUX) +IF (TD_LINUX OR TD_WINDOWS) ADD_LIBRARY(monitor ${SRC}) IF (TD_SOMODE_STATIC) diff --git a/src/plugins/monitor/src/monMain.c b/src/plugins/monitor/src/monMain.c index 9443b1ce12..85d28bf0a5 100644 --- a/src/plugins/monitor/src/monMain.c +++ b/src/plugins/monitor/src/monMain.c @@ -79,8 +79,8 @@ int32_t monInitSystem() { strcpy(tsMonitor.ep, tsLocalEp); } - int len = strlen(tsMonitor.ep); - for (int i = 0; i < len; ++i) { + int32_t len = (int32_t)strlen(tsMonitor.ep); + for (int32_t i = 0; i < len; ++i) { if (tsMonitor.ep[i] == ':' || tsMonitor.ep[i] == '-' || tsMonitor.ep[i] == '.') { tsMonitor.ep[i] = '_'; } @@ -148,7 +148,7 @@ static void *monThreadFunc(void *param) { } if (tsMonitor.state == MON_STATE_NOT_INIT) { - int code = 0; + int32_t code = 0; for (; tsMonitor.cmdIndex < MON_CMD_MAX; ++tsMonitor.cmdIndex) { monBuildMonitorSql(tsMonitor.sql, tsMonitor.cmdIndex); @@ -330,7 +330,7 @@ static void monSaveSystemInfo() { pos += monBuildReqSql(sql + pos); void *res = taos_query(tsMonitor.conn, tsMonitor.sql); - int code = taos_errno(res); + int32_t code = taos_errno(res); taos_free_result(res); if (code != 0) { diff --git a/src/rpc/src/rpcTcp.c b/src/rpc/src/rpcTcp.c index 178b96c423..d11de3bc58 100644 --- a/src/rpc/src/rpcTcp.c +++ b/src/rpc/src/rpcTcp.c @@ -132,7 +132,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread break; } - pThreadObj->pollFd = (int64_t)epoll_create(10); // size does not matter + pThreadObj->pollFd = (SOCKET)epoll_create(10); // size does not matter if (pThreadObj->pollFd < 0) { tError("%s failed to create TCP epoll", label); code = -1; diff --git a/src/sync/CMakeLists.txt b/src/sync/CMakeLists.txt index aa38a56f38..7a0ea85867 100644 --- a/src/sync/CMakeLists.txt +++ b/src/sync/CMakeLists.txt @@ -4,7 +4,7 @@ PROJECT(TDengine) INCLUDE_DIRECTORIES(inc) AUX_SOURCE_DIRECTORY(src SRC) -IF (TD_LINUX) +IF (TD_LINUX OR TD_WINDOWS) LIST(REMOVE_ITEM SRC src/syncArbitrator.c) ADD_LIBRARY(sync ${SRC}) TARGET_LINK_LIBRARIES(sync tutil pthread common) diff --git a/src/sync/inc/syncInt.h b/src/sync/inc/syncInt.h index eef687d647..e43140d4e6 100644 --- a/src/sync/inc/syncInt.h +++ b/src/sync/inc/syncInt.h @@ -82,8 +82,8 @@ typedef struct SsyncPeer { uint64_t sversion; // track the peer version in retrieve process uint64_t lastFileVer; // track the file version while retrieve uint64_t lastWalVer; // track the wal version while retrieve - int32_t syncFd; - int32_t peerFd; // forward FD + SOCKET syncFd; + SOCKET peerFd; // forward FD int32_t numOfRetrieves; // number of retrieves tried int32_t fileChanged; // a flag to indicate file is changed during retrieving process int32_t refCount; diff --git a/src/sync/inc/syncTcp.h b/src/sync/inc/syncTcp.h index d4674fee6b..b322c3440c 100644 --- a/src/sync/inc/syncTcp.h +++ b/src/sync/inc/syncTcp.h @@ -27,12 +27,12 @@ typedef struct { int32_t bufferSize; void (*processBrokenLink)(int64_t handleId); int32_t (*processIncomingMsg)(int64_t handleId, void *buffer); - void (*processIncomingConn)(int32_t fd, uint32_t ip); + void (*processIncomingConn)(SOCKET fd, uint32_t ip); } SPoolInfo; void *syncOpenTcpThreadPool(SPoolInfo *pInfo); void syncCloseTcpThreadPool(void *); -void *syncAllocateTcpConn(void *, int64_t rid, int32_t connFd); +void *syncAllocateTcpConn(void *, int64_t rid, SOCKET connFd); void syncFreeTcpConn(void *); #ifdef __cplusplus diff --git a/src/sync/src/syncArbitrator.c b/src/sync/src/syncArbitrator.c index fed0774346..187e7a3b46 100644 --- a/src/sync/src/syncArbitrator.c +++ b/src/sync/src/syncArbitrator.c @@ -27,8 +27,12 @@ #include "syncInt.h" #include "syncTcp.h" -static void arbSignalHandler(int32_t signum, siginfo_t *sigInfo, void *context); -static void arbProcessIncommingConnection(int32_t connFd, uint32_t sourceIp); +#ifndef SIGHUP + #define SIGHUP SIGTERM +#endif + +static void arbSignalHandler(int32_t signum); +static void arbProcessIncommingConnection(SOCKET connFd, uint32_t sourceIp); static void arbProcessBrokenLink(int64_t rid); static int32_t arbProcessPeerMsg(int64_t rid, void *buffer); static tsem_t tsArbSem; @@ -36,7 +40,7 @@ static void * tsArbTcpPool; typedef struct { char id[TSDB_EP_LEN + 24]; - int32_t nodeFd; + SOCKET nodeFd; void * pConn; } SNodeConn; @@ -70,8 +74,9 @@ int32_t main(int32_t argc, char *argv[]) { /* Set termination handler. */ struct sigaction act = {{0}}; - act.sa_flags = SA_SIGINFO; - act.sa_sigaction = arbSignalHandler; + memset(&act, 0, sizeof(struct sigaction)); + + act.sa_handler = arbSignalHandler; sigaction(SIGTERM, &act, NULL); sigaction(SIGHUP, &act, NULL); sigaction(SIGINT, &act, NULL); @@ -103,12 +108,11 @@ int32_t main(int32_t argc, char *argv[]) { syncCloseTcpThreadPool(tsArbTcpPool); sInfo("TAOS arbitrator is shut down"); - closelog(); return 0; } -static void arbProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { +static void arbProcessIncommingConnection(SOCKET connFd, uint32_t sourceIp) { char ipstr[24]; tinet_ntoa(ipstr, sourceIp); sDebug("peer TCP connection from ip:%s", ipstr); @@ -172,15 +176,18 @@ static int32_t arbProcessPeerMsg(int64_t rid, void *buffer) { return 0; } -static void arbSignalHandler(int32_t signum, siginfo_t *sigInfo, void *context) { +static void arbSignalHandler(int32_t signum) { struct sigaction act = {{0}}; act.sa_handler = SIG_IGN; sigaction(SIGTERM, &act, NULL); sigaction(SIGHUP, &act, NULL); sigaction(SIGINT, &act, NULL); +#ifndef WINDOWS sInfo("shut down signal is %d, sender PID:%d", signum, sigInfo->si_pid); - +#else + sInfo("shut down signal is %d", signum); +#endif // inform main thread to exit tsem_post(&tsArbSem); } diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index 98100fbdd8..b43f5824be 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -45,7 +45,7 @@ static void syncCheckPeerConnection(void *param, void *tmrId); static int32_t syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack, int8_t type, uint16_t tranId); static void syncProcessBrokenLink(int64_t rid); static int32_t syncProcessPeerMsg(int64_t rid, void *buffer); -static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp); +static void syncProcessIncommingConnection(SOCKET connFd, uint32_t sourceIp); static void syncRemovePeer(SSyncPeer *pPeer); static void syncAddArbitrator(SSyncNode *pNode); static void syncFreeNode(void *); @@ -544,7 +544,7 @@ static void syncClosePeerConn(SSyncPeer *pPeer) { sDebug("%s, pfd:%d sfd:%d will be closed", pPeer->id, pPeer->peerFd, pPeer->syncFd); taosTmrStopA(&pPeer->timer); - taosClose(pPeer->syncFd); + taosCloseSocket(pPeer->syncFd); if (pPeer->peerFd >= 0) { pPeer->peerFd = -1; void *pConn = pPeer->pConn; @@ -869,7 +869,7 @@ static void syncProcessSyncRequest(char *msg, SSyncPeer *pPeer) { if (nodeRole != TAOS_SYNC_ROLE_MASTER) { sError("%s, I am not master anymore", pPeer->id); - taosClose(pPeer->syncFd); + taosCloseSocket(pPeer->syncFd); return; } @@ -1114,7 +1114,7 @@ static void syncSetupPeerConnection(SSyncPeer *pPeer) { return; } - int32_t connFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0); + SOCKET connFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0); if (connFd < 0) { sDebug("%s, failed to open tcp socket since %s", pPeer->id, strerror(errno)); taosTmrReset(syncCheckPeerConnection, SYNC_CHECK_INTERVAL, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer); @@ -1132,7 +1132,7 @@ static void syncSetupPeerConnection(SSyncPeer *pPeer) { if (pPeer->isArb) tsArbOnline = 1; } else { sDebug("%s, failed to setup peer connection to server since %s, try later", pPeer->id, strerror(errno)); - taosClose(connFd); + taosCloseSocket(connFd); taosTmrReset(syncCheckPeerConnection, SYNC_CHECK_INTERVAL, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer); } } @@ -1171,7 +1171,7 @@ static void syncCreateRestoreDataThread(SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; nodeSStatus = TAOS_SYNC_STATUS_INIT; sError("%s, failed to create sync restore thread, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); - taosClose(pPeer->syncFd); + taosCloseSocket(pPeer->syncFd); syncReleasePeer(pPeer); } else { sInfo("%s, sync restore thread:0x%08" PRIx64 " create successfully, rid:%" PRId64, pPeer->id, @@ -1179,7 +1179,7 @@ static void syncCreateRestoreDataThread(SSyncPeer *pPeer) { } } -static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { +static void syncProcessIncommingConnection(SOCKET connFd, uint32_t sourceIp) { char ipstr[24]; int32_t i; diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c index 78520c6608..bd4f9bd3e1 100644 --- a/src/sync/src/syncRestore.c +++ b/src/sync/src/syncRestore.c @@ -365,7 +365,7 @@ void *syncRestoreData(void *param) { SSyncNode *pNode = pPeer->pSyncNode; taosBlockSIGPIPE(); - __sync_fetch_and_add(&tsSyncNum, 1); + atomic_add_fetch_32(&tsSyncNum, 1); sInfo("%s, start to restore data, sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); (*pNode->notifyRole)(pNode->vgId, TAOS_SYNC_ROLE_SYNCING); @@ -390,9 +390,9 @@ void *syncRestoreData(void *param) { nodeSStatus = TAOS_SYNC_STATUS_INIT; sInfo("%s, restore data over, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); - taosClose(pPeer->syncFd); + taosCloseSocket(pPeer->syncFd); syncCloseRecvBuffer(pNode); - __sync_fetch_and_sub(&tsSyncNum, 1); + atomic_sub_fetch_32(&tsSyncNum, 1); // The ref is obtained in both the create thread and the current thread, so it is released twice syncReleasePeer(pPeer); diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c index 153886102e..9ff7955351 100644 --- a/src/sync/src/syncRetrieve.c +++ b/src/sync/src/syncRetrieve.c @@ -14,7 +14,6 @@ */ #define _DEFAULT_SOURCE -#include #include "os.h" #include "taoserror.h" #include "tlog.h" @@ -160,7 +159,7 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) { break; } - ret = taosSendFile(pPeer->syncFd, sfd, NULL, fileInfo.size); + ret = (int32_t)taosSendFile(pPeer->syncFd, sfd, NULL, fileInfo.size); close(sfd); if (ret < 0) { code = -1; @@ -228,7 +227,7 @@ static int32_t syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversi return -1; } - int32_t code = taosLSeek(sfd, offset, SEEK_SET); + int32_t code = (int32_t)taosLSeek(sfd, offset, SEEK_SET); if (code < 0) { sError("%s, failed to seek %" PRId64 " in wal:%s for retrieve since:%s", pPeer->id, offset, name, tstrerror(errno)); close(sfd); @@ -322,7 +321,7 @@ static int32_t syncProcessLastWal(SSyncPeer *pPeer, char *wname, int64_t index) // if all data are read out, and no update if (bytes == 0 && !walModified) { // wal not closed, it means some data not flushed to disk, wait for a while - usleep(10000); + taosMsleep(10); } // if bytes > 0, file is updated, or fversion is not reached but file still open, read again @@ -384,7 +383,7 @@ static int32_t syncRetrieveWal(SSyncPeer *pPeer) { break; } - code = taosSendFile(pPeer->syncFd, sfd, NULL, size); + code = (int32_t)taosSendFile(pPeer->syncFd, sfd, NULL, size); close(sfd); if (code < 0) { sError("%s, failed to send wal:%s for retrieve since %s, code:0x%x", pPeer->id, fname, strerror(errno), code); @@ -501,7 +500,7 @@ void *syncRetrieveData(void *param) { } pPeer->fileChanged = 0; - taosClose(pPeer->syncFd); + taosCloseSocket(pPeer->syncFd); // The ref is obtained in both the create thread and the current thread, so it is released twice sInfo("%s, sync retrieve data over, sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); diff --git a/src/sync/src/syncTcp.c b/src/sync/src/syncTcp.c index 4744666737..829c9ceec6 100644 --- a/src/sync/src/syncTcp.c +++ b/src/sync/src/syncTcp.c @@ -24,10 +24,18 @@ #include "syncInt.h" #include "syncTcp.h" +#ifdef WINDOWS +#include "wepoll.h" +#endif + +#ifndef EPOLLWAKEUP + #define EPOLLWAKEUP (1u << 29) +#endif + typedef struct SThreadObj { pthread_t thread; bool stop; - int32_t pollFd; + SOCKET pollFd; int32_t numOfFds; struct SPoolObj *pPool; } SThreadObj; @@ -37,13 +45,13 @@ typedef struct SPoolObj { SThreadObj **pThread; pthread_t thread; int32_t nextId; - int32_t acceptFd; // FD for accept new connection + SOCKET acceptFd; // FD for accept new connection } SPoolObj; typedef struct { SThreadObj *pThread; int64_t handleId; - int32_t fd; + SOCKET fd; int32_t closedByApp; } SConnObj; @@ -82,7 +90,7 @@ void *syncOpenTcpThreadPool(SPoolInfo *pInfo) { pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); if (pthread_create(&(pPool->thread), &thattr, (void *)syncAcceptPeerTcpConnection, pPool) != 0) { sError("failed to create accept thread for TCP server since %s", strerror(errno)); - close(pPool->acceptFd); + taosCloseSocket(pPool->acceptFd); tfree(pPool->pThread); tfree(pPool); return NULL; @@ -112,7 +120,7 @@ void syncCloseTcpThreadPool(void *param) { tfree(pPool); } -void *syncAllocateTcpConn(void *param, int64_t rid, int32_t connFd) { +void *syncAllocateTcpConn(void *param, int64_t rid, SOCKET connFd) { struct epoll_event event; SPoolObj *pPool = param; @@ -169,7 +177,7 @@ static void taosProcessBrokenLink(SConnObj *pConn) { pThread->numOfFds--; epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, pConn->fd, NULL); sDebug("%p fd:%d is removed from epoll thread, num:%d", pThread, pConn->fd, pThread->numOfFds); - taosClose(pConn->fd); + taosCloseSocket(pConn->fd); tfree(pConn); } @@ -233,7 +241,7 @@ static void *syncProcessTcpData(void *param) { sDebug("%p TCP epoll thread exits", pThread); - close(pThread->pollFd); + taosCloseSocket(pThread->pollFd); tfree(pThread); tfree(buffer); return NULL; @@ -248,7 +256,7 @@ static void *syncAcceptPeerTcpConnection(void *argv) { while (1) { struct sockaddr_in clientAddr; socklen_t addrlen = sizeof(clientAddr); - int32_t connFd = accept(pPool->acceptFd, (struct sockaddr *)&clientAddr, &addrlen); + SOCKET connFd = accept(pPool->acceptFd, (struct sockaddr *)&clientAddr, &addrlen); if (connFd < 0) { if (errno == EINVAL) { sDebug("%p TCP server accept is exiting...", pPool); @@ -264,7 +272,7 @@ static void *syncAcceptPeerTcpConnection(void *argv) { (*pInfo->processIncomingConn)(connFd, clientAddr.sin_addr.s_addr); } - taosClose(pPool->acceptFd); + taosCloseSocket(pPool->acceptFd); return NULL; } @@ -277,7 +285,7 @@ static SThreadObj *syncGetTcpThread(SPoolObj *pPool) { if (pThread == NULL) return NULL; pThread->pPool = pPool; - pThread->pollFd = epoll_create(10); // size does not matter + pThread->pollFd = (SOCKET)epoll_create(10); // size does not matter if (pThread->pollFd < 0) { tfree(pThread); return NULL; @@ -290,7 +298,7 @@ static SThreadObj *syncGetTcpThread(SPoolObj *pPool) { pthread_attr_destroy(&thattr); if (ret != 0) { - close(pThread->pollFd); + taosCloseSocket(pThread->pollFd); tfree(pThread); return NULL; } diff --git a/src/util/inc/tsocket.h b/src/util/inc/tsocket.h index a339955cc0..6b449d6bc9 100644 --- a/src/util/inc/tsocket.h +++ b/src/util/inc/tsocket.h @@ -20,6 +20,14 @@ extern "C" { #endif +#ifdef WINDOWS +#include "wepoll.h" +#endif + +#ifndef EPOLLWAKEUP + #define EPOLLWAKEUP (1u << 29) +#endif + int32_t taosReadn(SOCKET sock, char *buffer, int32_t len); int32_t taosWriteMsg(SOCKET fd, void *ptr, int32_t nbytes); int32_t taosReadMsg(SOCKET fd, void *ptr, int32_t nbytes); diff --git a/src/vnode/src/vnodeCfg.c b/src/vnode/src/vnodeCfg.c index 0b32f97939..1ea774afae 100644 --- a/src/vnode/src/vnodeCfg.c +++ b/src/vnode/src/vnodeCfg.c @@ -78,7 +78,7 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) { goto PARSE_VCFG_ERROR; } - len = fread(content, 1, maxLen, fp); + len = (int32_t)fread(content, 1, maxLen, fp); if (len <= 0) { vError("vgId:%d, failed to read %s, content is null", pVnode->vgId, file); goto PARSE_VCFG_ERROR; @@ -103,14 +103,14 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) { vError("vgId:%d, failed to read %s, cfgVersion not found", pVnode->vgId, file); goto PARSE_VCFG_ERROR; } - vnodeMsg.cfg.dbCfgVersion = dbCfgVersion->valueint; + vnodeMsg.cfg.dbCfgVersion = (int32_t)dbCfgVersion->valueint; cJSON *vgCfgVersion = cJSON_GetObjectItem(root, "vgCfgVersion"); if (!vgCfgVersion || vgCfgVersion->type != cJSON_Number) { vError("vgId:%d, failed to read %s, vgCfgVersion not found", pVnode->vgId, file); vnodeMsg.cfg.vgCfgVersion = 0; } else { - vnodeMsg.cfg.vgCfgVersion = vgCfgVersion->valueint; + vnodeMsg.cfg.vgCfgVersion = (int32_t)vgCfgVersion->valueint; } cJSON *cacheBlockSize = cJSON_GetObjectItem(root, "cacheBlockSize"); @@ -118,56 +118,56 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) { vError("vgId:%d, failed to read %s, cacheBlockSize not found", pVnode->vgId, file); goto PARSE_VCFG_ERROR; } - vnodeMsg.cfg.cacheBlockSize = cacheBlockSize->valueint; + vnodeMsg.cfg.cacheBlockSize = (int32_t)cacheBlockSize->valueint; cJSON *totalBlocks = cJSON_GetObjectItem(root, "totalBlocks"); if (!totalBlocks || totalBlocks->type != cJSON_Number) { vError("vgId:%d, failed to read %s, totalBlocks not found", pVnode->vgId, file); goto PARSE_VCFG_ERROR; } - vnodeMsg.cfg.totalBlocks = totalBlocks->valueint; + vnodeMsg.cfg.totalBlocks = (int32_t)totalBlocks->valueint; cJSON *daysPerFile = cJSON_GetObjectItem(root, "daysPerFile"); if (!daysPerFile || daysPerFile->type != cJSON_Number) { vError("vgId:%d, failed to read %s, daysPerFile not found", pVnode->vgId, file); goto PARSE_VCFG_ERROR; } - vnodeMsg.cfg.daysPerFile = daysPerFile->valueint; + vnodeMsg.cfg.daysPerFile = (int32_t)daysPerFile->valueint; cJSON *daysToKeep = cJSON_GetObjectItem(root, "daysToKeep"); if (!daysToKeep || daysToKeep->type != cJSON_Number) { vError("vgId:%d, failed to read %s, daysToKeep not found", pVnode->vgId, file); goto PARSE_VCFG_ERROR; } - vnodeMsg.cfg.daysToKeep = daysToKeep->valueint; + vnodeMsg.cfg.daysToKeep = (int32_t)daysToKeep->valueint; cJSON *daysToKeep1 = cJSON_GetObjectItem(root, "daysToKeep1"); if (!daysToKeep1 || daysToKeep1->type != cJSON_Number) { vError("vgId:%d, failed to read %s, daysToKeep1 not found", pVnode->vgId, file); goto PARSE_VCFG_ERROR; } - vnodeMsg.cfg.daysToKeep1 = daysToKeep1->valueint; + vnodeMsg.cfg.daysToKeep1 = (int32_t)daysToKeep1->valueint; cJSON *daysToKeep2 = cJSON_GetObjectItem(root, "daysToKeep2"); if (!daysToKeep2 || daysToKeep2->type != cJSON_Number) { vError("vgId:%d, failed to read %s, daysToKeep2 not found", pVnode->vgId, file); goto PARSE_VCFG_ERROR; } - vnodeMsg.cfg.daysToKeep2 = daysToKeep2->valueint; + vnodeMsg.cfg.daysToKeep2 = (int32_t)daysToKeep2->valueint; cJSON *minRowsPerFileBlock = cJSON_GetObjectItem(root, "minRowsPerFileBlock"); if (!minRowsPerFileBlock || minRowsPerFileBlock->type != cJSON_Number) { vError("vgId:%d, failed to read %s, minRowsPerFileBlock not found", pVnode->vgId, file); goto PARSE_VCFG_ERROR; } - vnodeMsg.cfg.minRowsPerFileBlock = minRowsPerFileBlock->valueint; + vnodeMsg.cfg.minRowsPerFileBlock = (int32_t)minRowsPerFileBlock->valueint; cJSON *maxRowsPerFileBlock = cJSON_GetObjectItem(root, "maxRowsPerFileBlock"); if (!maxRowsPerFileBlock || maxRowsPerFileBlock->type != cJSON_Number) { vError("vgId:%d, failed to read %s, maxRowsPerFileBlock not found", pVnode->vgId, file); goto PARSE_VCFG_ERROR; } - vnodeMsg.cfg.maxRowsPerFileBlock = maxRowsPerFileBlock->valueint; + vnodeMsg.cfg.maxRowsPerFileBlock = (int32_t)maxRowsPerFileBlock->valueint; cJSON *precision = cJSON_GetObjectItem(root, "precision"); if (!precision || precision->type != cJSON_Number) { @@ -195,7 +195,7 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) { vError("vgId:%d, failed to read %s, fsyncPeriod not found", pVnode->vgId, file); goto PARSE_VCFG_ERROR; } - vnodeMsg.cfg.fsyncPeriod = fsyncPeriod->valueint; + vnodeMsg.cfg.fsyncPeriod = (int32_t)fsyncPeriod->valueint; cJSON *wals = cJSON_GetObjectItem(root, "wals"); if (!wals || wals->type != cJSON_Number) { @@ -258,7 +258,7 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) { vError("vgId:%d, failed to read %s, nodeId not found", pVnode->vgId, file); goto PARSE_VCFG_ERROR; } - node->nodeId = nodeId->valueint; + node->nodeId = (int32_t)nodeId->valueint; cJSON *nodeEp = cJSON_GetObjectItem(nodeInfo, "nodeEp"); if (!nodeEp || nodeEp->type != cJSON_String || nodeEp->valuestring == NULL) { diff --git a/src/wal/CMakeLists.txt b/src/wal/CMakeLists.txt index 359e09287a..2125e4f33c 100644 --- a/src/wal/CMakeLists.txt +++ b/src/wal/CMakeLists.txt @@ -4,7 +4,7 @@ PROJECT(TDengine) INCLUDE_DIRECTORIES(inc) AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR}/src SRC) -IF (TD_LINUX) +IF (TD_LINUX OR TD_WINDOWS) ADD_LIBRARY(twal ${SRC}) TARGET_LINK_LIBRARIES(twal tutil common) ADD_SUBDIRECTORY(test) diff --git a/src/wal/src/walMgmt.c b/src/wal/src/walMgmt.c index 72ea239817..62b066500c 100644 --- a/src/wal/src/walMgmt.c +++ b/src/wal/src/walMgmt.c @@ -210,7 +210,7 @@ static int32_t walCreateThread() { static void walStopThread() { tsWal.stop = 1; - if (tsWal.thread) { + if (taosCheckPthreadValid(tsWal.thread)) { pthread_join(tsWal.thread, NULL); } diff --git a/src/wal/src/walWrite.c b/src/wal/src/walWrite.c index e67127d6e4..0eda6ff786 100644 --- a/src/wal/src/walWrite.c +++ b/src/wal/src/walWrite.c @@ -272,7 +272,7 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch SWalHead *pHead = buffer; while (1) { - int32_t ret = tfRead(tfd, pHead, sizeof(SWalHead)); + int32_t ret = (int32_t)tfRead(tfd, pHead, sizeof(SWalHead)); if (ret == 0) break; if (ret < 0) { @@ -307,7 +307,7 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch } } - ret = tfRead(tfd, pHead->cont, pHead->len); + ret = (int32_t)tfRead(tfd, pHead->cont, pHead->len); if (ret < 0) { wError("vgId:%d, file:%s, failed to read wal body since %s", pWal->vgId, name, strerror(errno)); code = TAOS_SYSTEM_ERROR(errno); From f4539411dd6063d1c66b64451b75d288cbe1e79a Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 14 Jan 2021 17:25:32 +0800 Subject: [PATCH 02/62] compile errors --- src/os/inc/osFile.h | 2 ++ src/sync/src/syncArbitrator.c | 5 +---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/os/inc/osFile.h b/src/os/inc/osFile.h index 91ac5c9958..19cc78472c 100644 --- a/src/os/inc/osFile.h +++ b/src/os/inc/osFile.h @@ -20,6 +20,8 @@ extern "C" { #endif +#include "osSocket.h" + int64_t taosReadImp(int32_t fd, void *buf, int64_t count); int64_t taosWriteImp(int32_t fd, void *buf, int64_t count); int64_t taosLSeekImp(int32_t fd, int64_t offset, int32_t whence); diff --git a/src/sync/src/syncArbitrator.c b/src/sync/src/syncArbitrator.c index 187e7a3b46..8c0d02bedc 100644 --- a/src/sync/src/syncArbitrator.c +++ b/src/sync/src/syncArbitrator.c @@ -183,11 +183,8 @@ static void arbSignalHandler(int32_t signum) { sigaction(SIGHUP, &act, NULL); sigaction(SIGINT, &act, NULL); -#ifndef WINDOWS - sInfo("shut down signal is %d, sender PID:%d", signum, sigInfo->si_pid); -#else sInfo("shut down signal is %d", signum); -#endif + // inform main thread to exit tsem_post(&tsArbSem); } From ac49a78d42aacb612580357f53d5604b66c17b20 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 14 Jan 2021 22:36:15 +0800 Subject: [PATCH 03/62] TD-1207 --- CMakeLists.txt | 7 +++- src/balance/CMakeLists.txt | 2 +- src/balance/inc/bnThread.h | 2 +- src/balance/src/bnMain.c | 2 +- src/balance/src/bnScore.c | 10 ++--- src/balance/src/bnThread.c | 2 +- src/cq/CMakeLists.txt | 2 +- src/cq/src/cqMain.c | 2 +- src/cq/test/CMakeLists.txt | 2 +- src/dnode/CMakeLists.txt | 4 +- src/dnode/src/dnodeCfg.c | 4 +- src/dnode/src/dnodeCheck.c | 40 +++++++++---------- src/dnode/src/dnodeEps.c | 4 +- src/dnode/src/dnodeMInfos.c | 16 ++++---- src/dnode/src/dnodeMPeer.c | 4 +- src/dnode/src/dnodeMRead.c | 6 +-- src/dnode/src/dnodeMWrite.c | 4 +- src/dnode/src/dnodePeer.c | 5 ++- src/dnode/src/dnodeShell.c | 2 +- src/dnode/src/dnodeStep.c | 2 +- src/dnode/src/dnodeSystem.c | 71 ++++++++++++++++++++-------------- src/dnode/src/dnodeTelemetry.c | 23 ++++++----- src/dnode/src/dnodeVWrite.c | 4 +- src/dnode/src/dnodeVnodes.c | 4 +- src/mnode/src/mnodeDnode.c | 2 +- src/mnode/src/mnodeTable.c | 1 + src/os/inc/osWindows.h | 6 +++ src/os/src/windows/wSysLog.c | 19 +++++++++ src/os/src/windows/wSysinfo.c | 7 +++- src/plugins/CMakeLists.txt | 2 +- src/sync/src/syncArbitrator.c | 12 +++--- src/vnode/CMakeLists.txt | 3 +- src/vnode/src/vnodeVersion.c | 2 +- src/vnode/src/vnodeWorker.c | 4 +- src/vnode/src/vnodeWrite.c | 2 +- 35 files changed, 168 insertions(+), 116 deletions(-) create mode 100644 src/os/src/windows/wSysLog.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 0c474a355d..7ac06c165d 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,7 +13,7 @@ ENDIF () SET(TD_ACCOUNT FALSE) SET(TD_ADMIN FALSE) SET(TD_GRANT FALSE) -SET(TD_MQTT TRUE) +SET(TD_MQTT FALSE) SET(TD_TSDB_PLUGINS FALSE) SET(TD_COVER FALSE) @@ -29,6 +29,11 @@ MESSAGE(STATUS "Community directory: " ${TD_COMMUNITY_DIR}) INCLUDE(cmake/input.inc) INCLUDE(cmake/platform.inc) + +IF (TD_WINDOWS) + SET(TD_SOMODE_STATIC TRUE) +ENDIF () + INCLUDE(cmake/define.inc) INCLUDE(cmake/env.inc) INCLUDE(cmake/version.inc) diff --git a/src/balance/CMakeLists.txt b/src/balance/CMakeLists.txt index fdc43ea32f..3c93f63d1a 100644 --- a/src/balance/CMakeLists.txt +++ b/src/balance/CMakeLists.txt @@ -8,6 +8,6 @@ INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/inc) INCLUDE_DIRECTORIES(inc) AUX_SOURCE_DIRECTORY(src SRC) -IF (TD_LINUX) +IF (TD_LINUX OR TD_WINDOWS) ADD_LIBRARY(balance ${SRC}) ENDIF () diff --git a/src/balance/inc/bnThread.h b/src/balance/inc/bnThread.h index 8f54b66028..74a761299d 100644 --- a/src/balance/inc/bnThread.h +++ b/src/balance/inc/bnThread.h @@ -24,7 +24,7 @@ extern "C" { int32_t bnInitThread(); void bnCleanupThread(); void bnNotify(); -void bnStartTimer(int64_t mseconds); +void bnStartTimer(int32_t mseconds); #ifdef __cplusplus } diff --git a/src/balance/src/bnMain.c b/src/balance/src/bnMain.c index 3e1d5eda76..236b22afaf 100644 --- a/src/balance/src/bnMain.c +++ b/src/balance/src/bnMain.c @@ -30,7 +30,7 @@ #include "mnodeVgroup.h" extern int64_t tsDnodeRid; -extern int64_t tsSdbRid; +extern int32_t tsSdbRid; static SBnMgmt tsBnMgmt; static void bnMonitorDnodeModule(); diff --git a/src/balance/src/bnScore.c b/src/balance/src/bnScore.c index cbc2c62184..7d94df1c23 100644 --- a/src/balance/src/bnScore.c +++ b/src/balance/src/bnScore.c @@ -271,23 +271,23 @@ static int32_t bnRetrieveScores(SShowObj *pShow, char *data, int32_t rows, void cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(float *)pWrite = systemScore; + *(float *)pWrite = (float)systemScore; cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(float *)pWrite = pDnode->customScore; + *(float *)pWrite = (float)pDnode->customScore; cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(float *)pWrite = (int32_t)moduleScore; + *(float *)pWrite = (float)moduleScore; cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(float *)pWrite = (int32_t)vnodeScore; + *(float *)pWrite = (float)vnodeScore; cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(float *)pWrite = (int32_t)(vnodeScore + moduleScore + pDnode->customScore + systemScore); + *(float *)pWrite = (float)(vnodeScore + moduleScore + pDnode->customScore + systemScore); cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; diff --git a/src/balance/src/bnThread.c b/src/balance/src/bnThread.c index 84f8694fca..caf33061d1 100644 --- a/src/balance/src/bnThread.c +++ b/src/balance/src/bnThread.c @@ -119,7 +119,7 @@ static void bnProcessTimer(void *handle, void *tmrId) { } } -void bnStartTimer(int64_t mseconds) { +void bnStartTimer(int32_t mseconds) { if (tsBnThread.stop) return; bool updateSoon = (mseconds != -1); diff --git a/src/cq/CMakeLists.txt b/src/cq/CMakeLists.txt index 9da831c9c1..dd84e96ecf 100644 --- a/src/cq/CMakeLists.txt +++ b/src/cq/CMakeLists.txt @@ -6,7 +6,7 @@ INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR}/src SRC) -IF (TD_LINUX) +IF (TD_LINUX OR TD_WINDOWS) ADD_LIBRARY(tcq ${SRC}) IF (TD_SOMODE_STATIC) TARGET_LINK_LIBRARIES(tcq tutil common taos_static) diff --git a/src/cq/src/cqMain.c b/src/cq/src/cqMain.c index 0fe5ea78d4..5f1fecc494 100644 --- a/src/cq/src/cqMain.c +++ b/src/cq/src/cqMain.c @@ -343,7 +343,7 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) { char buf[TSDB_MAX_NCHAR_LEN]; int32_t len = taos_fetch_lengths(tres)[i]; taosMbsToUcs4(val, len, buf, sizeof(buf), &len); - memcpy(val + sizeof(VarDataLenT), buf, len); + memcpy((char *)val + sizeof(VarDataLenT), buf, len); varDataLen(val) = len; } tdAppendColVal(trow, val, c->type, c->bytes, c->offset); diff --git a/src/cq/test/CMakeLists.txt b/src/cq/test/CMakeLists.txt index 87a8705fcd..fc3a1ea93a 100644 --- a/src/cq/test/CMakeLists.txt +++ b/src/cq/test/CMakeLists.txt @@ -3,4 +3,4 @@ PROJECT(TDengine) LIST(APPEND CQTEST_SRC ./cqtest.c) ADD_EXECUTABLE(cqtest ${CQTEST_SRC}) -TARGET_LINK_LIBRARIES(cqtest tcq) +TARGET_LINK_LIBRARIES(cqtest tcq taos_static) diff --git a/src/dnode/CMakeLists.txt b/src/dnode/CMakeLists.txt index 699ca00a25..59a8b2b486 100644 --- a/src/dnode/CMakeLists.txt +++ b/src/dnode/CMakeLists.txt @@ -10,7 +10,7 @@ INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/inc) INCLUDE_DIRECTORIES(inc) AUX_SOURCE_DIRECTORY(src SRC) -IF (TD_LINUX) +IF (TD_LINUX OR TD_WINDOWS) ADD_EXECUTABLE(taosd ${SRC}) TARGET_LINK_LIBRARIES(taosd mnode monitor http tsdb twal vnode cJson lz4 balance sync) @@ -28,7 +28,7 @@ IF (TD_LINUX) TARGET_LINK_LIBRARIES(taosd grant) ENDIF () - IF (TD_MQTT) + IF (TD_LINUX AND TD_MQTT) TARGET_LINK_LIBRARIES(taosd mqtt) ENDIF () diff --git a/src/dnode/src/dnodeCfg.c b/src/dnode/src/dnodeCfg.c index f495dbe285..e008d2fa0d 100644 --- a/src/dnode/src/dnodeCfg.c +++ b/src/dnode/src/dnodeCfg.c @@ -97,7 +97,7 @@ static int32_t dnodeReadCfg() { goto PARSE_CFG_OVER; } - len = fread(content, 1, maxLen, fp); + len = (int32_t)fread(content, 1, maxLen, fp); if (len <= 0) { dError("failed to read %s, content is null", file); goto PARSE_CFG_OVER; @@ -115,7 +115,7 @@ static int32_t dnodeReadCfg() { dError("failed to read %s, dnodeId not found", file); goto PARSE_CFG_OVER; } - cfg.dnodeId = dnodeId->valueint; + cfg.dnodeId = (int32_t)dnodeId->valueint; cJSON *clusterId = cJSON_GetObjectItem(root, "clusterId"); if (!clusterId || clusterId->type != cJSON_String) { diff --git a/src/dnode/src/dnodeCheck.c b/src/dnode/src/dnodeCheck.c index be26bb967b..8955fb5643 100644 --- a/src/dnode/src/dnodeCheck.c +++ b/src/dnode/src/dnodeCheck.c @@ -29,8 +29,8 @@ typedef struct { static SCheckItem tsCheckItem[TSDB_CHECK_ITEM_MAX] = {{0}}; int64_t tsMinFreeMemSizeForStart = 0; -static int bindTcpPort(int port) { - int serverSocket; +static int32_t bindTcpPort(int32_t port) { + SOCKET serverSocket; struct sockaddr_in server_addr; if ((serverSocket = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0) { @@ -45,22 +45,22 @@ static int bindTcpPort(int port) { if (bind(serverSocket, (struct sockaddr *)&server_addr, sizeof(server_addr)) < 0) { dError("port:%d tcp bind() fail: %s", port, strerror(errno)); - close(serverSocket); + taosCloseSocket(serverSocket); return -1; } if (listen(serverSocket, 5) < 0) { dError("port:%d listen() fail: %s", port, strerror(errno)); - close(serverSocket); + taosCloseSocket(serverSocket); return -1; } - close(serverSocket); + taosCloseSocket(serverSocket); return 0; } -static int bindUdpPort(int port) { - int serverSocket; +static int32_t bindUdpPort(int32_t port) { + SOCKET serverSocket; struct sockaddr_in server_addr; if ((serverSocket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0) { @@ -75,19 +75,19 @@ static int bindUdpPort(int port) { if (bind(serverSocket, (struct sockaddr *)&server_addr, sizeof(server_addr)) < 0) { dError("port:%d udp bind() fail: %s", port, strerror(errno)); - close(serverSocket); + taosCloseSocket(serverSocket); return -1; } - close(serverSocket); + taosCloseSocket(serverSocket); return 0; } -static int dnodeCheckNetwork() { - int ret; - int startPort = tsServerPort; +static int32_t dnodeCheckNetwork() { + int32_t ret; + int32_t startPort = tsServerPort; - for (int port = startPort; port < startPort + 12; port++) { + for (int32_t port = startPort; port < startPort + 12; port++) { ret = bindTcpPort(port); if (0 != ret) { dError("failed to tcp bind port %d, quit", port); @@ -103,7 +103,7 @@ static int dnodeCheckNetwork() { return 0; } -static int dnodeCheckMem() { +static int32_t dnodeCheckMem() { float memoryUsedMB; float memoryAvailMB; if (true != taosGetSysMemory(&memoryUsedMB)) { @@ -121,12 +121,12 @@ static int dnodeCheckMem() { return 0; } -static int dnodeCheckCpu() { +static int32_t dnodeCheckCpu() { // TODO: return 0; } -static int dnodeCheckDisk() { +static int32_t dnodeCheckDisk() { taosGetDisk(); if (tsAvailDataDirGB < tsMinimalDataDirGB) { @@ -147,24 +147,24 @@ static int dnodeCheckDisk() { return 0; } -static int dnodeCheckOs() { +static int32_t dnodeCheckOs() { // TODO: return 0; } -static int dnodeCheckAccess() { +static int32_t dnodeCheckAccess() { // TODO: return 0; } -static int dnodeCheckVersion() { +static int32_t dnodeCheckVersion() { // TODO: return 0; } -static int dnodeCheckDatafile() { +static int32_t dnodeCheckDatafile() { // TODO: return 0; diff --git a/src/dnode/src/dnodeEps.c b/src/dnode/src/dnodeEps.c index e7dc7efeb2..1e05c696ce 100644 --- a/src/dnode/src/dnodeEps.c +++ b/src/dnode/src/dnodeEps.c @@ -152,7 +152,7 @@ static int32_t dnodeReadEps() { goto PRASE_EPS_OVER; } - len = fread(content, 1, maxLen, fp); + len = (int32_t)fread(content, 1, maxLen, fp); if (len <= 0) { dError("failed to read %s, content is null", file); goto PRASE_EPS_OVER; @@ -199,7 +199,7 @@ static int32_t dnodeReadEps() { dError("failed to read %s, dnodeId not found", file); goto PRASE_EPS_OVER; } - ep->dnodeId = dnodeId->valueint; + ep->dnodeId = (int32_t)dnodeId->valueint; cJSON *dnodeFqdn = cJSON_GetObjectItem(dnodeInfo, "dnodeFqdn"); if (!dnodeFqdn || dnodeFqdn->type != cJSON_String || dnodeFqdn->valuestring == NULL) { diff --git a/src/dnode/src/dnodeMInfos.c b/src/dnode/src/dnodeMInfos.c index dc89487f8b..884924f113 100644 --- a/src/dnode/src/dnodeMInfos.c +++ b/src/dnode/src/dnodeMInfos.c @@ -80,7 +80,7 @@ void dnodeUpdateEpSetForPeer(SRpcEpSet *ep) { pthread_mutex_lock(&tsMInfosMutex); dInfo("minfos is changed, numOfEps:%d inUse:%d", ep->numOfEps, ep->inUse); - for (int i = 0; i < ep->numOfEps; ++i) { + for (int32_t i = 0; i < ep->numOfEps; ++i) { ep->port[i] -= TSDB_PORT_DNODEDNODE; dInfo("minfo:%d %s:%u", i, ep->fqdn[i], ep->port[i]); } @@ -108,7 +108,7 @@ void dnodeGetMInfos(SMInfos *pMinfos) { void dnodeGetEpSetForPeer(SRpcEpSet *epSet) { pthread_mutex_lock(&tsMInfosMutex); *epSet = tsMEpSet; - for (int i = 0; i < epSet->numOfEps; ++i) { + for (int32_t i = 0; i < epSet->numOfEps; ++i) { epSet->port[i] += TSDB_PORT_DNODEDNODE; } pthread_mutex_unlock(&tsMInfosMutex); @@ -171,7 +171,7 @@ static int32_t dnodeReadMInfos() { goto PARSE_MINFOS_OVER; } - len = fread(content, 1, maxLen, fp); + len = (int32_t)fread(content, 1, maxLen, fp); if (len <= 0) { dError("failed to read %s, content is null", file); goto PARSE_MINFOS_OVER; @@ -189,14 +189,14 @@ static int32_t dnodeReadMInfos() { dError("failed to read mnodeEpSet.json, inUse not found"); goto PARSE_MINFOS_OVER; } - tsMInfos.inUse = inUse->valueint; + tsMInfos.inUse = (int8_t)inUse->valueint; cJSON *nodeNum = cJSON_GetObjectItem(root, "nodeNum"); if (!nodeNum || nodeNum->type != cJSON_Number) { dError("failed to read mnodeEpSet.json, nodeNum not found"); goto PARSE_MINFOS_OVER; } - minfos.mnodeNum = nodeNum->valueint; + minfos.mnodeNum = (int8_t)nodeNum->valueint; cJSON *nodeInfos = cJSON_GetObjectItem(root, "nodeInfos"); if (!nodeInfos || nodeInfos->type != cJSON_Array) { @@ -204,13 +204,13 @@ static int32_t dnodeReadMInfos() { goto PARSE_MINFOS_OVER; } - int size = cJSON_GetArraySize(nodeInfos); + int32_t size = cJSON_GetArraySize(nodeInfos); if (size != minfos.mnodeNum) { dError("failed to read mnodeEpSet.json, nodeInfos size not matched"); goto PARSE_MINFOS_OVER; } - for (int i = 0; i < size; ++i) { + for (int32_t i = 0; i < size; ++i) { cJSON *nodeInfo = cJSON_GetArrayItem(nodeInfos, i); if (nodeInfo == NULL) continue; @@ -227,7 +227,7 @@ static int32_t dnodeReadMInfos() { } SMInfo *pMinfo = &minfos.mnodeInfos[i]; - pMinfo->mnodeId = nodeId->valueint; + pMinfo->mnodeId = (int32_t)nodeId->valueint; tstrncpy(pMinfo->mnodeEp, nodeEp->valuestring, TSDB_EP_LEN); bool changed = dnodeCheckEpChanged(pMinfo->mnodeId, pMinfo->mnodeEp); diff --git a/src/dnode/src/dnodeMPeer.c b/src/dnode/src/dnodeMPeer.c index 0863666f76..e4942c49aa 100644 --- a/src/dnode/src/dnodeMPeer.c +++ b/src/dnode/src/dnodeMPeer.c @@ -60,7 +60,7 @@ int32_t dnodeInitMPeer() { void dnodeCleanupMPeer() { for (int32_t i = 0; i < tsMPeerWP.maxNum; ++i) { SMPeerWorker *pWorker = tsMPeerWP.worker + i; - if (pWorker->thread) { + if (taosCheckPthreadValid(pWorker->thread)) { taosQsetThreadResume(tsMPeerQset); } dDebug("dnode mpeer worker:%d is closed", i); @@ -69,7 +69,7 @@ void dnodeCleanupMPeer() { for (int32_t i = 0; i < tsMPeerWP.maxNum; ++i) { SMPeerWorker *pWorker = tsMPeerWP.worker + i; dDebug("dnode mpeer worker:%d start to join", i); - if (pWorker->thread) { + if (taosCheckPthreadValid(pWorker->thread)) { pthread_join(pWorker->thread, NULL); } dDebug("dnode mpeer worker:%d join success", i); diff --git a/src/dnode/src/dnodeMRead.c b/src/dnode/src/dnodeMRead.c index 9027c346f5..90332e6783 100644 --- a/src/dnode/src/dnodeMRead.c +++ b/src/dnode/src/dnodeMRead.c @@ -40,7 +40,7 @@ static void *dnodeProcessMReadQueue(void *param); int32_t dnodeInitMRead() { tsMReadQset = taosOpenQset(); - tsMReadWP.maxNum = tsNumOfCores * tsNumOfThreadsPerCore / 2; + tsMReadWP.maxNum = (int32_t)(tsNumOfCores * tsNumOfThreadsPerCore / 2); tsMReadWP.maxNum = MAX(2, tsMReadWP.maxNum); tsMReadWP.maxNum = MIN(4, tsMReadWP.maxNum); tsMReadWP.curNum = 0; @@ -60,7 +60,7 @@ int32_t dnodeInitMRead() { void dnodeCleanupMRead() { for (int32_t i = 0; i < tsMReadWP.maxNum; ++i) { SMReadWorker *pWorker = tsMReadWP.worker + i; - if (pWorker->thread) { + if (taosCheckPthreadValid(pWorker->thread)) { taosQsetThreadResume(tsMReadQset); } dDebug("dnode mread worker:%d is closed", i); @@ -69,7 +69,7 @@ void dnodeCleanupMRead() { for (int32_t i = 0; i < tsMReadWP.maxNum; ++i) { SMReadWorker *pWorker = tsMReadWP.worker + i; dDebug("dnode mread worker:%d start to join", i); - if (pWorker->thread) { + if (taosCheckPthreadValid(pWorker->thread)) { pthread_join(pWorker->thread, NULL); } dDebug("dnode mread worker:%d start to join", i); diff --git a/src/dnode/src/dnodeMWrite.c b/src/dnode/src/dnodeMWrite.c index 8c9e22ef4b..218d487473 100644 --- a/src/dnode/src/dnodeMWrite.c +++ b/src/dnode/src/dnodeMWrite.c @@ -60,7 +60,7 @@ int32_t dnodeInitMWrite() { void dnodeCleanupMWrite() { for (int32_t i = 0; i < tsMWriteWP.maxNum; ++i) { SMWriteWorker *pWorker = tsMWriteWP.worker + i; - if (pWorker->thread) { + if (taosCheckPthreadValid(pWorker->thread)) { taosQsetThreadResume(tsMWriteQset); } dDebug("dnode mwrite worker:%d is closed", i); @@ -69,7 +69,7 @@ void dnodeCleanupMWrite() { for (int32_t i = 0; i < tsMWriteWP.maxNum; ++i) { SMWriteWorker *pWorker = tsMWriteWP.worker + i; dDebug("dnode mwrite worker:%d start to join", i); - if (pWorker->thread) { + if (taosCheckPthreadValid(pWorker->thread)) { pthread_join(pWorker->thread, NULL); } dDebug("dnode mwrite worker:%d join success", i); diff --git a/src/dnode/src/dnodePeer.c b/src/dnode/src/dnodePeer.c index 5ee10abc30..79c60874f9 100644 --- a/src/dnode/src/dnodePeer.c +++ b/src/dnode/src/dnodePeer.c @@ -90,7 +90,10 @@ static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { }; if (pMsg->pCont == NULL) return; - if (pMsg->msgType == TSDB_MSG_TYPE_NETWORK_TEST) return dnodeSendStartupStep(pMsg); + if (pMsg->msgType == TSDB_MSG_TYPE_NETWORK_TEST) { + dnodeSendStartupStep(pMsg); + return; + } if (dnodeGetRunStatus() != TSDB_RUN_STATUS_RUNING) { rspMsg.code = TSDB_CODE_APP_NOT_READY; diff --git a/src/dnode/src/dnodeShell.c b/src/dnode/src/dnodeShell.c index 79cc70005b..b80f75e963 100644 --- a/src/dnode/src/dnodeShell.c +++ b/src/dnode/src/dnodeShell.c @@ -70,7 +70,7 @@ int32_t dnodeInitShell() { dnodeProcessShellMsgFp[TSDB_MSG_TYPE_NETWORK_TEST] = dnodeSendStartupStep; - int32_t numOfThreads = (tsNumOfCores * tsNumOfThreadsPerCore) / 2.0; + int32_t numOfThreads = (int32_t)((tsNumOfCores * tsNumOfThreadsPerCore) / 2.0); if (numOfThreads < 1) { numOfThreads = 1; } diff --git a/src/dnode/src/dnodeStep.c b/src/dnode/src/dnodeStep.c index 2354b1d5a3..8878299d94 100644 --- a/src/dnode/src/dnodeStep.c +++ b/src/dnode/src/dnodeStep.c @@ -70,5 +70,5 @@ int32_t dnodeStepInit(SStep *pSteps, int32_t stepSize) { } void dnodeStepCleanup(SStep *pSteps, int32_t stepSize) { - return taosStepCleanupImp(pSteps, stepSize - 1); + taosStepCleanupImp(pSteps, stepSize - 1); } \ No newline at end of file diff --git a/src/dnode/src/dnodeSystem.c b/src/dnode/src/dnodeSystem.c index a4d7e791e6..6c28552e1e 100644 --- a/src/dnode/src/dnodeSystem.c +++ b/src/dnode/src/dnodeSystem.c @@ -19,9 +19,43 @@ #include "tconfig.h" #include "dnodeMain.h" -static void signal_handler(int32_t signum, siginfo_t *sigInfo, void *context); static tsem_t exitSem; +#ifdef WINDOWS +static void signal_handler(int32_t signum) { + dInfo("shut down signal is %d", signum); +#else +static void signal_handler(int32_t signum, siginfo_t *sigInfo, void *context) { + if (signum == SIGUSR1) { + taosCfgDynamicOptions("debugFlag 143"); + return; + } + if (signum == SIGUSR2) { + taosCfgDynamicOptions("resetlog"); + return; + } + dInfo("shut down signal is %d, sender PID:%d cmdline:%s", signum, sigInfo->si_pid, taosGetCmdlineByPID(sigInfo->si_pid)); +#endif + + syslog(LOG_INFO, "Shut down signal is %d", signum); + syslog(LOG_INFO, "Shutting down TDengine service..."); + + // protect the application from receive another signal + struct sigaction act = {{0}}; + act.sa_handler = SIG_IGN; + sigaction(SIGTERM, &act, NULL); + sigaction(SIGINT, &act, NULL); + +#ifndef WINDOWS + sigaction(SIGHUP, &act, NULL); + sigaction(SIGUSR1, &act, NULL); + sigaction(SIGUSR2, &act, NULL); +#endif + + // inform main thread to exit + tsem_post(&exitSem); +} + int32_t main(int32_t argc, char *argv[]) { int dump_config = 0; @@ -113,6 +147,8 @@ int32_t main(int32_t argc, char *argv[]) { /* Set termination handler. */ struct sigaction act = {{0}}; + +#ifndef WINDOWS act.sa_flags = SA_SIGINFO; act.sa_sigaction = signal_handler; sigaction(SIGTERM, &act, NULL); @@ -120,6 +156,11 @@ int32_t main(int32_t argc, char *argv[]) { sigaction(SIGINT, &act, NULL); sigaction(SIGUSR1, &act, NULL); sigaction(SIGUSR2, &act, NULL); +#else + act.sa_handler = signal_handler; + sigaction(SIGTERM, &act, NULL); + sigaction(SIGINT, &act, NULL); +#endif // Open /var/log/syslog file to record information. openlog("TDengine:", LOG_PID | LOG_CONS | LOG_NDELAY, LOG_LOCAL1); @@ -146,31 +187,3 @@ int32_t main(int32_t argc, char *argv[]) { closelog(); return EXIT_SUCCESS; } - -static void signal_handler(int32_t signum, siginfo_t *sigInfo, void *context) { - if (signum == SIGUSR1) { - taosCfgDynamicOptions("debugFlag 143"); - return; - } - if (signum == SIGUSR2) { - taosCfgDynamicOptions("resetlog"); - return; - } - - syslog(LOG_INFO, "Shut down signal is %d", signum); - syslog(LOG_INFO, "Shutting down TDengine service..."); - // clean the system. - dInfo("shut down signal is %d, sender PID:%d cmdline:%s", signum, sigInfo->si_pid, taosGetCmdlineByPID(sigInfo->si_pid)); - - // protect the application from receive another signal - struct sigaction act = {{0}}; - act.sa_handler = SIG_IGN; - sigaction(SIGTERM, &act, NULL); - sigaction(SIGHUP, &act, NULL); - sigaction(SIGINT, &act, NULL); - sigaction(SIGUSR1, &act, NULL); - sigaction(SIGUSR2, &act, NULL); - - // inform main thread to exit - tsem_post(&exitSem); -} diff --git a/src/dnode/src/dnodeTelemetry.c b/src/dnode/src/dnodeTelemetry.c index ff9598ecc5..9291831884 100644 --- a/src/dnode/src/dnodeTelemetry.c +++ b/src/dnode/src/dnodeTelemetry.c @@ -93,14 +93,14 @@ static void addStringField(SBufferWriter* bw, const char* k, const char* v) { static void addCpuInfo(SBufferWriter* bw) { char * line = NULL; size_t size = 0; - int done = 0; + int32_t done = 0; FILE* fp = fopen("/proc/cpuinfo", "r"); if (fp == NULL) { return; } - while (done != 3 && (size = getline(&line, &size, fp)) != -1) { + while (done != 3 && (size = tgetline(&line, &size, fp)) != -1) { line[size - 1] = '\0'; if (((done&1) == 0) && strncmp(line, "model name", 10) == 0) { const char* v = strchr(line, ':') + 2; @@ -129,7 +129,7 @@ static void addOsInfo(SBufferWriter* bw) { return; } - while ((size = getline(&line, &size, fp)) != -1) { + while ((size = tgetline(&line, &size, fp)) != -1) { line[size - 1] = '\0'; if (strncmp(line, "PRETTY_NAME", 11) == 0) { const char* p = strchr(line, '=') + 1; @@ -155,7 +155,7 @@ static void addMemoryInfo(SBufferWriter* bw) { return; } - while ((size = getline(&line, &size, fp)) != -1) { + while ((size = tgetline(&line, &size, fp)) != -1) { line[size - 1] = '\0'; if (strncmp(line, "MemTotal", 8) == 0) { const char* p = strchr(line, ':') + 1; @@ -200,7 +200,7 @@ static void sendTelemetryReport() { dTrace("failed to get IP address of " TELEMETRY_SERVER ", reason:%s", strerror(errno)); return; } - int fd = taosOpenTcpClientSocket(ip, TELEMETRY_PORT, 0); + SOCKET fd = taosOpenTcpClientSocket(ip, TELEMETRY_PORT, 0); if (fd < 0) { dTrace("failed to create socket for telemetry, reason:%s", strerror(errno)); return; @@ -222,10 +222,10 @@ static void sendTelemetryReport() { "Content-Type: application/json\n" "Content-Length: "; - taosWriteSocket(fd, header, strlen(header)); - int contLen = tbufTell(&bw) - 1; + taosWriteSocket(fd, header, (int32_t)strlen(header)); + int32_t contLen = (int32_t)(tbufTell(&bw) - 1); sprintf(buf, "%d\n\n", contLen); - taosWriteSocket(fd, buf, strlen(buf)); + taosWriteSocket(fd, buf, (int32_t)strlen(buf)); taosWriteSocket(fd, tbufGetData(&bw, false), contLen); tbufCloseWriter(&bw); @@ -258,7 +258,7 @@ static void* telemetryThread(void* param) { } static void dnodeGetEmail(char* filepath) { - int fd = open(filepath, O_RDONLY); + int32_t fd = open(filepath, O_RDONLY); if (fd < 0) { return; } @@ -267,10 +267,9 @@ static void dnodeGetEmail(char* filepath) { dError("failed to read %d bytes from file %s since %s", TSDB_FQDN_LEN, filepath, strerror(errno)); } - close(fd); + taosClose(fd); } - int32_t dnodeInitTelemetry() { if (!tsEnableTelemetryReporting) { return 0; @@ -303,7 +302,7 @@ void dnodeCleanupTelemetry() { return; } - if (tsTelemetryThread) { + if (taosCheckPthreadValid(tsTelemetryThread)) { tsem_post(&tsExitSem); pthread_join(tsTelemetryThread, NULL); tsem_destroy(&tsExitSem); diff --git a/src/dnode/src/dnodeVWrite.c b/src/dnode/src/dnodeVWrite.c index ab5a4ccaad..93d1611ebc 100644 --- a/src/dnode/src/dnodeVWrite.c +++ b/src/dnode/src/dnodeVWrite.c @@ -52,14 +52,14 @@ int32_t dnodeInitVWrite() { void dnodeCleanupVWrite() { for (int32_t i = 0; i < tsVWriteWP.max; ++i) { SVWriteWorker *pWorker = tsVWriteWP.worker + i; - if (pWorker->thread) { + if (taosCheckPthreadValid(pWorker->thread)) { taosQsetThreadResume(pWorker->qset); } } for (int32_t i = 0; i < tsVWriteWP.max; ++i) { SVWriteWorker *pWorker = tsVWriteWP.worker + i; - if (pWorker->thread) { + if (taosCheckPthreadValid(pWorker->thread)) { pthread_join(pWorker->thread, NULL); taosFreeQall(pWorker->qall); taosCloseQset(pWorker->qset); diff --git a/src/dnode/src/dnodeVnodes.c b/src/dnode/src/dnodeVnodes.c index c62d5a8207..9f32541612 100644 --- a/src/dnode/src/dnodeVnodes.c +++ b/src/dnode/src/dnodeVnodes.c @@ -157,7 +157,7 @@ int32_t dnodeInitVnodes() { int32_t failedVnodes = 0; for (int32_t t = 0; t < threadNum; ++t) { SOpenVnodeThread *pThread = &threads[t]; - if (pThread->vnodeNum > 0 && pThread->thread) { + if (pThread->vnodeNum > 0 && taosCheckPthreadValid(pThread->thread)) { pthread_join(pThread->thread, NULL); } openVnodes += pThread->opened; @@ -260,7 +260,7 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) { tstrncpy(pStatus->clusterCfg.timezone, tsTimezone, 64); pStatus->clusterCfg.checkTime = 0; char timestr[32] = "1970-01-01 00:00:00.00"; - (void)taosParseTime(timestr, &pStatus->clusterCfg.checkTime, strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); + (void)taosParseTime(timestr, &pStatus->clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); tstrncpy(pStatus->clusterCfg.locale, tsLocale, TSDB_LOCALE_LEN); tstrncpy(pStatus->clusterCfg.charset, tsCharset, TSDB_LOCALE_LEN); diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index 01034b170f..d76ecd9ba0 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -39,7 +39,7 @@ #include "mnodeCluster.h" int32_t tsAccessSquence = 0; -int64_t tsDnodeRid = -1; +int64_t tsDnodeRid = -1; static void * tsDnodeSdb = NULL; static int32_t tsDnodeUpdateSize = 0; extern void * tsMnodeSdb; diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index a0c8d88c51..a520a7d52d 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -841,6 +841,7 @@ static int32_t mnodeProcessBatchCreateTableMsg(SMnodeMsg *pMsg) { return TSDB_CODE_MND_ACTION_IN_PROGRESS; } else { // batch master replay, reprocess the whole batch assert(0); + return TSDB_CODE_MND_MSG_NOT_PROCESSED; } } } diff --git a/src/os/inc/osWindows.h b/src/os/inc/osWindows.h index 5a1e642572..396577a5d7 100644 --- a/src/os/inc/osWindows.h +++ b/src/os/inc/osWindows.h @@ -208,6 +208,12 @@ typedef struct { int wordexp(const char *words, wordexp_t *pwordexp, int flags); void wordfree(wordexp_t *pwordexp); +#define openlog(a, b, c) +#define closelog() +#define LOG_ERR 0 +#define LOG_INFO 1 +void syslog(int unused, const char *format, ...); + #define TAOS_OS_FUNC_ATOMIC #define atomic_load_8(ptr) (*(char volatile*)(ptr)) #define atomic_load_16(ptr) (*(short volatile*)(ptr)) diff --git a/src/os/src/windows/wSysLog.c b/src/os/src/windows/wSysLog.c new file mode 100644 index 0000000000..866cacbaba --- /dev/null +++ b/src/os/src/windows/wSysLog.c @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" + +void syslog(int unused, const char *format, ...) {} \ No newline at end of file diff --git a/src/os/src/windows/wSysinfo.c b/src/os/src/windows/wSysinfo.c index 1bfee25c4a..b3fd4c28b9 100644 --- a/src/os/src/windows/wSysinfo.c +++ b/src/os/src/windows/wSysinfo.c @@ -235,4 +235,9 @@ LONG WINAPI FlCrashDump(PEXCEPTION_POINTERS ep) { return EXCEPTION_CONTINUE_SEARCH; } -void taosSetCoreDump() { SetUnhandledExceptionFilter(&FlCrashDump); } \ No newline at end of file +void taosSetCoreDump() { SetUnhandledExceptionFilter(&FlCrashDump); } + +bool taosGetSystemUid(char *uid) { + sprintf(uid, "uid_not_implemented_yet"); + return true; +} \ No newline at end of file diff --git a/src/plugins/CMakeLists.txt b/src/plugins/CMakeLists.txt index d03717b6e1..e66997dc8e 100644 --- a/src/plugins/CMakeLists.txt +++ b/src/plugins/CMakeLists.txt @@ -3,6 +3,6 @@ PROJECT(TDengine) ADD_SUBDIRECTORY(monitor) ADD_SUBDIRECTORY(http) -IF (TD_MQTT) +IF (TD_LINUX AND TD_MQTT) ADD_SUBDIRECTORY(mqtt) ENDIF () \ No newline at end of file diff --git a/src/sync/src/syncArbitrator.c b/src/sync/src/syncArbitrator.c index 8c0d02bedc..dbcc4b40a6 100644 --- a/src/sync/src/syncArbitrator.c +++ b/src/sync/src/syncArbitrator.c @@ -27,10 +27,6 @@ #include "syncInt.h" #include "syncTcp.h" -#ifndef SIGHUP - #define SIGHUP SIGTERM -#endif - static void arbSignalHandler(int32_t signum); static void arbProcessIncommingConnection(SOCKET connFd, uint32_t sourceIp); static void arbProcessBrokenLink(int64_t rid); @@ -78,8 +74,10 @@ int32_t main(int32_t argc, char *argv[]) { act.sa_handler = arbSignalHandler; sigaction(SIGTERM, &act, NULL); - sigaction(SIGHUP, &act, NULL); sigaction(SIGINT, &act, NULL); +#ifndef WINDOWS + sigaction(SIGHUP, &act, NULL); +#endif tsAsyncLog = 0; strcat(arbLogPath, "/arbitrator.log"); @@ -180,8 +178,10 @@ static void arbSignalHandler(int32_t signum) { struct sigaction act = {{0}}; act.sa_handler = SIG_IGN; sigaction(SIGTERM, &act, NULL); - sigaction(SIGHUP, &act, NULL); sigaction(SIGINT, &act, NULL); +#ifndef WINDOWS + sigaction(SIGHUP, &act, NULL); +#endif sInfo("shut down signal is %d", signum); diff --git a/src/vnode/CMakeLists.txt b/src/vnode/CMakeLists.txt index c953883361..202f11e03e 100644 --- a/src/vnode/CMakeLists.txt +++ b/src/vnode/CMakeLists.txt @@ -1,6 +1,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 3.5) PROJECT(TDengine) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/tsdb/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/dnode/inc) @@ -9,7 +10,7 @@ INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/inc) INCLUDE_DIRECTORIES(inc) AUX_SOURCE_DIRECTORY(src SRC) -IF (TD_LINUX) +IF (TD_LINUX OR TD_WINDOWS) ADD_LIBRARY(vnode ${SRC}) TARGET_LINK_LIBRARIES(vnode tsdb tcq) ENDIF () diff --git a/src/vnode/src/vnodeVersion.c b/src/vnode/src/vnodeVersion.c index fb3b3ebd9e..68fa32b2de 100644 --- a/src/vnode/src/vnodeVersion.c +++ b/src/vnode/src/vnodeVersion.c @@ -41,7 +41,7 @@ int32_t vnodeReadVersion(SVnodeObj *pVnode) { goto PARSE_VER_ERROR; } - len = fread(content, 1, maxLen, fp); + len = (int32_t)fread(content, 1, maxLen, fp); if (len <= 0) { vError("vgId:%d, failed to read %s, content is null", pVnode->vgId, file); goto PARSE_VER_ERROR; diff --git a/src/vnode/src/vnodeWorker.c b/src/vnode/src/vnodeWorker.c index d6053cf18e..6fb79d10fe 100644 --- a/src/vnode/src/vnodeWorker.c +++ b/src/vnode/src/vnodeWorker.c @@ -109,7 +109,7 @@ static void vnodeStopMWorker() { void vnodeCleanupMWorker() { for (int32_t i = 0; i < tsVMWorkerPool.maxNum; ++i) { SVMWorker *pWorker = tsVMWorkerPool.worker + i; - if (pWorker->thread) { + if (taosCheckPthreadValid(pWorker->thread)) { taosQsetThreadResume(tsVMWorkerQset); } vDebug("vmworker:%d is closed", i); @@ -118,7 +118,7 @@ void vnodeCleanupMWorker() { for (int32_t i = 0; i < tsVMWorkerPool.maxNum; ++i) { SVMWorker *pWorker = tsVMWorkerPool.worker + i; vDebug("vmworker:%d start to join", i); - if (pWorker->thread) { + if (taosCheckPthreadValid(pWorker->thread)) { pthread_join(pWorker->thread, NULL); } vDebug("vmworker:%d join success", i); diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index 4b9f59279c..90b1e91979 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -333,7 +333,7 @@ static int32_t vnodePerformFlowCtrl(SVWriteMsg *pWrite) { if (pVnode->queuedWMsg < MAX_QUEUED_MSG_NUM && pVnode->flowctrlLevel <= 0) return 0; if (tsEnableFlowCtrl == 0) { - int32_t ms = pow(2, pVnode->flowctrlLevel + 2); + int32_t ms = (int32_t)pow(2, pVnode->flowctrlLevel + 2); if (ms > 100) ms = 100; vTrace("vgId:%d, msg:%p, app:%p, perform flowctrl for %d ms", pVnode->vgId, pWrite, pWrite->rpcMsg.ahandle, ms); taosMsleep(ms); From 8a9ed8e2efcc751a7efc2e3a4f4d391058b7da13 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 15 Jan 2021 18:14:50 +0800 Subject: [PATCH 04/62] TD-1207 --- src/balance/src/bnThread.c | 4 +- src/os/src/detail/osSysinfo.c | 117 +++++++++++++++++----------------- 2 files changed, 62 insertions(+), 59 deletions(-) diff --git a/src/balance/src/bnThread.c b/src/balance/src/bnThread.c index caf33061d1..c6a16da430 100644 --- a/src/balance/src/bnThread.c +++ b/src/balance/src/bnThread.c @@ -124,8 +124,8 @@ void bnStartTimer(int32_t mseconds) { bool updateSoon = (mseconds != -1); if (updateSoon) { - mTrace("balance function will be called after %" PRId64 " ms", mseconds); - taosTmrReset(bnProcessTimer, mseconds, (void *)mseconds, tsMnodeTmr, &tsBnThread.timer); + mTrace("balance function will be called after %d ms", mseconds); + taosTmrReset(bnProcessTimer, mseconds, (void *)(int64_t)mseconds, tsMnodeTmr, &tsBnThread.timer); } else { taosTmrReset(bnProcessTimer, tsStatusInterval * 1000, NULL, tsMnodeTmr, &tsBnThread.timer); } diff --git a/src/os/src/detail/osSysinfo.c b/src/os/src/detail/osSysinfo.c index b0ca6139ed..2ba0dd6917 100644 --- a/src/os/src/detail/osSysinfo.c +++ b/src/os/src/detail/osSysinfo.c @@ -45,6 +45,21 @@ static char tsProcMemFile[25] = {0}; static char tsProcIOFile[25] = {0}; static float tsPageSizeKB = 0; +static void taosGetProcInfos() { + tsPageSize = sysconf(_SC_PAGESIZE); + tsOpenMax = sysconf(_SC_OPEN_MAX); + tsStreamMax = sysconf(_SC_STREAM_MAX); + + tsProcId = (pid_t)syscall(SYS_gettid); + tsPageSizeKB = (float)(sysconf(_SC_PAGESIZE)) / 1024; + + snprintf(tsProcMemFile, 25, "/proc/%d/status", tsProcId); + snprintf(tsProcCpuFile, 25, "/proc/%d/stat", tsProcId); + snprintf(tsProcIOFile, 25, "/proc/%d/io", tsProcId); +} + +static int32_t taosGetTotalMemory() { return (int32_t)((float)sysconf(_SC_PHYS_PAGES) * tsPageSizeKB / 1024); } + bool taosGetSysMemory(float *memoryUsedMB) { float memoryAvailMB = (float)sysconf(_SC_AVPHYS_PAGES) * tsPageSizeKB / 1024; *memoryUsedMB = (float)tsTotalMemoryMB - memoryAvailMB; @@ -105,7 +120,8 @@ static bool taosGetSysCpuInfo(SysCpuInfo *cpuInfo) { } char cpu[10] = {0}; - sscanf(line, "%s %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64, cpu, &cpuInfo->user, &cpuInfo->nice, &cpuInfo->system, &cpuInfo->idle); + sscanf(line, "%s %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64, cpu, &cpuInfo->user, &cpuInfo->nice, &cpuInfo->system, + &cpuInfo->idle); tfree(line); fclose(fp); @@ -131,7 +147,8 @@ static bool taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { for (int i = 0, blank = 0; line[i] != 0; ++i) { if (line[i] == ' ') blank++; if (blank == PROCESS_ITEM) { - sscanf(line + i + 1, "%" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64, &cpuInfo->utime, &cpuInfo->stime, &cpuInfo->cutime, &cpuInfo->cstime); + sscanf(line + i + 1, "%" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64, &cpuInfo->utime, &cpuInfo->stime, + &cpuInfo->cutime, &cpuInfo->cstime); break; } } @@ -162,12 +179,12 @@ static void taosGetSystemTimezone() { char buf[68] = {0}; if (f != NULL) { int len = fread(buf, 64, 1, f); - if(len < 64 && ferror(f)) { + if (len < 64 && ferror(f)) { fclose(f); uError("read /etc/timezone error, reason:%s", strerror(errno)); return; } - + fclose(f); buf[sizeof(buf) - 1] = 0; @@ -258,6 +275,8 @@ static void taosGetSystemLocale() { // get and set default locale } } +static int32_t taosGetCpuCores() { return (int32_t)sysconf(_SC_NPROCESSORS_ONLN); } + bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) { static uint64_t lastSysUsed = 0; static uint64_t lastSysTotal = 0; @@ -300,11 +319,9 @@ bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) { bool taosGetDisk() { struct statvfs info; const double unit = 1024 * 1024 * 1024; - + if (tscEmbedded) { if (statvfs(tsDataDir, &info)) { - //tsTotalDataDirGB = 0; - //tsAvailDataDirGB = 0; uError("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno)); return false; } else { @@ -314,8 +331,6 @@ bool taosGetDisk() { } if (statvfs(tsLogDir, &info)) { - //tsTotalLogDirGB = 0; - //tsAvailLogDirGB = 0; uError("failed to get disk size, logDir:%s errno:%s", tsLogDir, strerror(errno)); return false; } else { @@ -324,8 +339,6 @@ bool taosGetDisk() { } if (statvfs("/tmp", &info)) { - //tsTotalTmpDirGB = 0; - //tsAvailTmpDirectorySpace = 0; uError("failed to get disk size, tmpDir:/tmp errno:%s", strerror(errno)); return false; } else { @@ -344,13 +357,12 @@ static bool taosGetCardInfo(int64_t *bytes) { return false; } - size_t len = 2048; char * line = calloc(1, len); while (!feof(fp)) { memset(line, 0, len); - + int64_t rbytes = 0; int64_t rpackts = 0; int64_t tbytes = 0; @@ -465,7 +477,7 @@ bool taosGetProcIO(float *readKB, float *writeKB) { static int64_t lastReadbyte = -1; static int64_t lastWritebyte = -1; - int64_t curReadbyte = 0; + int64_t curReadbyte = 0; int64_t curWritebyte = 0; if (!taosReadProcIO(&curReadbyte, &curWritebyte)) { @@ -490,18 +502,10 @@ bool taosGetProcIO(float *readKB, float *writeKB) { } void taosGetSystemInfo() { - tsNumOfCores = (int32_t)sysconf(_SC_NPROCESSORS_ONLN); - tsPageSize = sysconf(_SC_PAGESIZE); - tsOpenMax = sysconf(_SC_OPEN_MAX); - tsStreamMax = sysconf(_SC_STREAM_MAX); + taosGetProcInfos(); - tsProcId = (pid_t)syscall(SYS_gettid); - tsPageSizeKB = (float)(sysconf(_SC_PAGESIZE)) / 1024; - tsTotalMemoryMB = (int32_t)((float)sysconf(_SC_PHYS_PAGES) * tsPageSizeKB / 1024); - - snprintf(tsProcMemFile, 25, "/proc/%d/status", tsProcId); - snprintf(tsProcCpuFile, 25, "/proc/%d/stat", tsProcId); - snprintf(tsProcIOFile, 25, "/proc/%d/io", tsProcId); + tsNumOfCores = taosGetCpuCores(); + tsTotalMemoryMB = taosGetTotalMemory(); float tmp1, tmp2; taosGetSysMemory(&tmp1); @@ -573,16 +577,16 @@ void taosSetCoreDump() { if (0 == tsEnableCoreFile) { return; } - + // 1. set ulimit -c unlimited struct rlimit rlim; struct rlimit rlim_new; if (getrlimit(RLIMIT_CORE, &rlim) == 0) { - #ifndef _ALPINE +#ifndef _ALPINE uInfo("the old unlimited para: rlim_cur=%" PRIu64 ", rlim_max=%" PRIu64, rlim.rlim_cur, rlim.rlim_max); - #else +#else uInfo("the old unlimited para: rlim_cur=%llu, rlim_max=%llu", rlim.rlim_cur, rlim.rlim_max); - #endif +#endif rlim_new.rlim_cur = RLIM_INFINITY; rlim_new.rlim_max = RLIM_INFINITY; if (setrlimit(RLIMIT_CORE, &rlim_new) != 0) { @@ -594,57 +598,56 @@ void taosSetCoreDump() { } if (getrlimit(RLIMIT_CORE, &rlim) == 0) { - #ifndef _ALPINE +#ifndef _ALPINE uInfo("the new unlimited para: rlim_cur=%" PRIu64 ", rlim_max=%" PRIu64, rlim.rlim_cur, rlim.rlim_max); - #else +#else uInfo("the new unlimited para: rlim_cur=%llu, rlim_max=%llu", rlim.rlim_cur, rlim.rlim_max); - #endif +#endif } #ifndef _TD_ARM_ // 2. set the path for saving core file struct __sysctl_args args; - int old_usespid = 0; - size_t old_len = 0; - int new_usespid = 1; - size_t new_len = sizeof(new_usespid); - + + int old_usespid = 0; + size_t old_len = 0; + int new_usespid = 1; + size_t new_len = sizeof(new_usespid); + int name[] = {CTL_KERN, KERN_CORE_USES_PID}; - + memset(&args, 0, sizeof(struct __sysctl_args)); - args.name = name; - args.nlen = sizeof(name)/sizeof(name[0]); - args.oldval = &old_usespid; + args.name = name; + args.nlen = sizeof(name) / sizeof(name[0]); + args.oldval = &old_usespid; args.oldlenp = &old_len; - args.newval = &new_usespid; - args.newlen = new_len; - + args.newval = &new_usespid; + args.newlen = new_len; + old_len = sizeof(old_usespid); - + if (syscall(SYS__sysctl, &args) == -1) { - uInfo("_sysctl(kern_core_uses_pid) set fail: %s", strerror(errno)); + uInfo("_sysctl(kern_core_uses_pid) set fail: %s", strerror(errno)); } - + uInfo("The old core_uses_pid[%" PRIu64 "]: %d", old_len, old_usespid); - old_usespid = 0; - old_len = 0; + old_len = 0; memset(&args, 0, sizeof(struct __sysctl_args)); - args.name = name; - args.nlen = sizeof(name)/sizeof(name[0]); - args.oldval = &old_usespid; + args.name = name; + args.nlen = sizeof(name) / sizeof(name[0]); + args.oldval = &old_usespid; args.oldlenp = &old_len; - + old_len = sizeof(old_usespid); - + if (syscall(SYS__sysctl, &args) == -1) { - uInfo("_sysctl(kern_core_uses_pid) get fail: %s", strerror(errno)); + uInfo("_sysctl(kern_core_uses_pid) get fail: %s", strerror(errno)); } - + uInfo("The new core_uses_pid[%" PRIu64 "]: %d", old_len, old_usespid); #endif - } bool taosGetSystemUid(char *uid) { From de09879d50e9be7eb41c59622c3dc494140826a0 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 15 Jan 2021 18:55:29 +0800 Subject: [PATCH 05/62] TD-1207 --- src/dnode/src/dnodeSystem.c | 73 +++++++--------- src/kit/shell/src/shellMain.c | 2 +- src/os/inc/osWindows.h | 12 ++- src/os/src/linux/linuxEnv.c | 15 ++-- src/os/src/windows/wSysinfo.c | 156 +++++++++++++++++++++------------- src/sync/src/syncArbitrator.c | 28 +++--- 6 files changed, 161 insertions(+), 125 deletions(-) diff --git a/src/dnode/src/dnodeSystem.c b/src/dnode/src/dnodeSystem.c index 6c28552e1e..a16ff826bf 100644 --- a/src/dnode/src/dnodeSystem.c +++ b/src/dnode/src/dnodeSystem.c @@ -19,43 +19,9 @@ #include "tconfig.h" #include "dnodeMain.h" +static void signal_handler(int32_t signum, siginfo_t *sigInfo, void *context); static tsem_t exitSem; -#ifdef WINDOWS -static void signal_handler(int32_t signum) { - dInfo("shut down signal is %d", signum); -#else -static void signal_handler(int32_t signum, siginfo_t *sigInfo, void *context) { - if (signum == SIGUSR1) { - taosCfgDynamicOptions("debugFlag 143"); - return; - } - if (signum == SIGUSR2) { - taosCfgDynamicOptions("resetlog"); - return; - } - dInfo("shut down signal is %d, sender PID:%d cmdline:%s", signum, sigInfo->si_pid, taosGetCmdlineByPID(sigInfo->si_pid)); -#endif - - syslog(LOG_INFO, "Shut down signal is %d", signum); - syslog(LOG_INFO, "Shutting down TDengine service..."); - - // protect the application from receive another signal - struct sigaction act = {{0}}; - act.sa_handler = SIG_IGN; - sigaction(SIGTERM, &act, NULL); - sigaction(SIGINT, &act, NULL); - -#ifndef WINDOWS - sigaction(SIGHUP, &act, NULL); - sigaction(SIGUSR1, &act, NULL); - sigaction(SIGUSR2, &act, NULL); -#endif - - // inform main thread to exit - tsem_post(&exitSem); -} - int32_t main(int32_t argc, char *argv[]) { int dump_config = 0; @@ -147,8 +113,6 @@ int32_t main(int32_t argc, char *argv[]) { /* Set termination handler. */ struct sigaction act = {{0}}; - -#ifndef WINDOWS act.sa_flags = SA_SIGINFO; act.sa_sigaction = signal_handler; sigaction(SIGTERM, &act, NULL); @@ -156,11 +120,6 @@ int32_t main(int32_t argc, char *argv[]) { sigaction(SIGINT, &act, NULL); sigaction(SIGUSR1, &act, NULL); sigaction(SIGUSR2, &act, NULL); -#else - act.sa_handler = signal_handler; - sigaction(SIGTERM, &act, NULL); - sigaction(SIGINT, &act, NULL); -#endif // Open /var/log/syslog file to record information. openlog("TDengine:", LOG_PID | LOG_CONS | LOG_NDELAY, LOG_LOCAL1); @@ -187,3 +146,33 @@ int32_t main(int32_t argc, char *argv[]) { closelog(); return EXIT_SUCCESS; } + +static void signal_handler(int32_t signum, siginfo_t *sigInfo, void *context) { + if (signum == SIGUSR1) { + taosCfgDynamicOptions("debugFlag 143"); + return; + } + if (signum == SIGUSR2) { + taosCfgDynamicOptions("resetlog"); + return; + } + + syslog(LOG_INFO, "Shut down signal is %d", signum); + syslog(LOG_INFO, "Shutting down TDengine service..."); + // clean the system. + dInfo("shut down signal is %d, sender PID:%d cmdline:%s", signum, sigInfo->si_pid, taosGetCmdlineByPID(sigInfo->si_pid)); + + // protect the application from receive another signal + struct sigaction act = {{0}}; +#ifndef WINDOWS + act.sa_handler = SIG_IGN; +#endif + sigaction(SIGTERM, &act, NULL); + sigaction(SIGHUP, &act, NULL); + sigaction(SIGINT, &act, NULL); + sigaction(SIGUSR1, &act, NULL); + sigaction(SIGUSR2, &act, NULL); + + // inform main thread to exit + tsem_post(&exitSem); +} \ No newline at end of file diff --git a/src/kit/shell/src/shellMain.c b/src/kit/shell/src/shellMain.c index 496cef41ba..cd801ef07a 100644 --- a/src/kit/shell/src/shellMain.c +++ b/src/kit/shell/src/shellMain.c @@ -21,7 +21,7 @@ pthread_t pid; static tsem_t cancelSem; -void shellQueryInterruptHandler(int32_t signum) { +void shellQueryInterruptHandler(int32_t signum, siginfo_t *sigInfo, void *context) { tsem_post(&cancelSem); } diff --git a/src/os/inc/osWindows.h b/src/os/inc/osWindows.h index 396577a5d7..15b26d208d 100644 --- a/src/os/inc/osWindows.h +++ b/src/os/inc/osWindows.h @@ -193,9 +193,19 @@ int gettimeofday(struct timeval *ptv, void *pTimeZone); //for signal, not dispose #define SIGALRM 1234 +#define SIGHUP 1234 +#define SIGUSR1 1234 +#define SIGUSR2 1234 +#define SA_SIGINFO 1234 + typedef int sigset_t; +typedef struct siginfo_t { + int si_pid; +} siginfo_t; struct sigaction { - void (*sa_handler)(int); + int sa_flags; + void (*sa_handler)(int32_t signum, siginfo_t *sigInfo, void *context); + void (*sa_sigaction)(int32_t signum, siginfo_t *sigInfo, void *context); }; int sigaction(int, struct sigaction *, void *); diff --git a/src/os/src/linux/linuxEnv.c b/src/os/src/linux/linuxEnv.c index 5772885cb4..e3eadbc94b 100644 --- a/src/os/src/linux/linuxEnv.c +++ b/src/os/src/linux/linuxEnv.c @@ -18,7 +18,6 @@ #include "tglobal.h" void osInit() { - #ifdef _TD_POWER_ if (configDir[0] == 0) { strcpy(configDir, "/etc/power"); @@ -43,16 +42,14 @@ void osInit() { char cmdline[1024]; -char *taosGetCmdlineByPID(int pid) -{ - sprintf(cmdline, "/proc/%d/cmdline",pid); - FILE* f = fopen(cmdline,"r"); - if(f){ +char* taosGetCmdlineByPID(int pid) { + sprintf(cmdline, "/proc/%d/cmdline", pid); + FILE* f = fopen(cmdline, "r"); + if (f) { size_t size; size = fread(cmdline, sizeof(char), 1024, f); - if(size>0){ - if('\n'==cmdline[size-1]) - cmdline[size-1]='\0'; + if (size > 0) { + if ('\n' == cmdline[size - 1]) cmdline[size - 1] = '\0'; } fclose(f); } diff --git a/src/os/src/windows/wSysinfo.c b/src/os/src/windows/wSysinfo.c index b3fd4c28b9..0499173a0b 100644 --- a/src/os/src/windows/wSysinfo.c +++ b/src/os/src/windows/wSysinfo.c @@ -32,10 +32,51 @@ #endif #pragma warning(push) -#pragma warning(disable:4091) +#pragma warning(disable : 4091) #include #pragma warning(pop) +static int32_t taosGetTotalMemory() { + MEMORYSTATUSEX memsStat; + memsStat.dwLength = sizeof(memsStat); + if (!GlobalMemoryStatusEx(&memsStat)) { + return 0; + } + + float nMemTotal = memsStat.ullTotalPhys / (1024.0f * 1024.0f); + return (int32_t)nMemTotal; +} + +bool taosGetSysMemory(float *memoryUsedMB) { + MEMORYSTATUSEX memsStat; + memsStat.dwLength = sizeof(memsStat); + if (!GlobalMemoryStatusEx(&memsStat)) { + return false; + } + + float nMemFree = memsStat.ullAvailPhys / (1024.0f * 1024.0f); + float nMemTotal = memsStat.ullTotalPhys / (1024.0f * 1024.0f); + + *memoryUsedMB = nMemTotal - nMemFree; + return true; +} + +bool taosGetProcMemory(float *memoryUsedMB) { + unsigned bytes_used = 0; + +#if defined(_WIN32) && defined(_MSC_VER) + PROCESS_MEMORY_COUNTERS pmc; + HANDLE cur_proc = GetCurrentProcess(); + + if (GetProcessMemoryInfo(cur_proc, &pmc, sizeof(pmc))) { + bytes_used = (unsigned)(pmc.WorkingSetSize + pmc.PagefileUsage); + } +#endif + + *memoryUsedMB = (float)bytes_used / 1024 / 1024; + return true; +} + static void taosGetSystemTimezone() { // get and set default timezone SGlobalCfg *cfg_timezone = taosGetConfigOption("timezone"); @@ -71,16 +112,16 @@ static void taosGetSystemLocale() { } } -void taosPrintOsInfo() {} - -void taosKillSystem() { - uError("function taosKillSystem, exit!"); - exit(0); +static int32_t taosGetCpuCores() { + SYSTEM_INFO info; + GetSystemInfo(&info); + return (int32_t)info.dwNumberOfProcessors; } -void taosGetSystemInfo() { - taosGetSystemTimezone(); - taosGetSystemLocale(); +bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) { + *sysCpuUsage = 0; + *procCpuUsage = 0; + return true; } bool taosGetDisk() { @@ -89,20 +130,35 @@ bool taosGetDisk() { unsigned _int64 i64FreeBytesToCaller; unsigned _int64 i64TotalBytes; unsigned _int64 i64FreeBytes; - char dir[4] = {'C', ':', '\\', '\0'}; - int drive_type; if (tscEmbedded) { - drive_type = GetDriveTypeA(dir); - if (drive_type == DRIVE_FIXED) { - fResult = GetDiskFreeSpaceExA(dir, (PULARGE_INTEGER)&i64FreeBytesToCaller, (PULARGE_INTEGER)&i64TotalBytes, - (PULARGE_INTEGER)&i64FreeBytes); - if (fResult) { - tsTotalDataDirGB = tsTotalLogDirGB = tsTotalTmpDirGB = (float)(i64TotalBytes / unit); - tsAvailDataDirGB = tsAvailLogDirGB = tsAvailTmpDirectorySpace = (float)(i64FreeBytes / unit); - } + fResult = GetDiskFreeSpaceExA(tsDataDir, (PULARGE_INTEGER)&i64FreeBytesToCaller, (PULARGE_INTEGER)&i64TotalBytes, + (PULARGE_INTEGER)&i64FreeBytes); + if (fResult) { + tsTotalDataDirGB = (float)(i64TotalBytes / unit); + tsAvailDataDirGB = (float)(i64FreeBytes / unit); } } + + fResult = GetDiskFreeSpaceExA(tsLogDir, (PULARGE_INTEGER)&i64FreeBytesToCaller, (PULARGE_INTEGER)&i64TotalBytes, + (PULARGE_INTEGER)&i64FreeBytes); + if (fResult) { + tsTotalLogDirGB = (float)(i64TotalBytes / unit); + tsAvailLogDirGB = (float)(i64FreeBytes / unit); + } + + fResult = GetDiskFreeSpaceExA(tsTempDir, (PULARGE_INTEGER)&i64FreeBytesToCaller, (PULARGE_INTEGER)&i64TotalBytes, + (PULARGE_INTEGER)&i64FreeBytes); + if (fResult) { + tsTotalTmpDirGB = (float)(i64TotalBytes / unit); + tsAvailTmpDirectorySpace = (float)(i64FreeBytes / unit); + } + + return true; +} + +bool taosGetBandSpeed(float *bandSpeedKb) { + *bandSpeedKb = 0; return true; } @@ -144,48 +200,30 @@ bool taosGetProcIO(float *readKB, float *writeKB) { return true; } -bool taosGetBandSpeed(float *bandSpeedKb) { - *bandSpeedKb = 0; - return true; +void taosGetSystemInfo() { + tsNumOfCores = taosGetCpuCores(); + tsTotalMemoryMB = taosGetTotalMemory(); + + float tmp1, tmp2; + taosGetDisk(); + taosGetBandSpeed(&tmp1); + taosGetCpuUsage(&tmp1, &tmp2); + taosGetProcIO(&tmp1, &tmp2); + + taosGetSystemTimezone(); + taosGetSystemLocale(); } -bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) { - *sysCpuUsage = 0; - *procCpuUsage = 0; - return true; +void taosPrintOsInfo() { + uInfo(" os numOfCores: %d", tsNumOfCores); + uInfo(" os totalDisk: %f(GB)", tsTotalDataDirGB); + uInfo(" os totalMemory: %d(MB)", tsTotalMemoryMB); + uInfo("=================================="); } -bool taosGetProcMemory(float *memoryUsedMB) { - unsigned bytes_used = 0; -#if 0 -#if defined(_WIN32) && defined(_MSC_VER) - PROCESS_MEMORY_COUNTERS pmc; - HANDLE cur_proc = GetCurrentProcess(); - - if (GetProcessMemoryInfo(cur_proc, &pmc, sizeof(pmc))) { - bytes_used = (unsigned)(pmc.WorkingSetSize + pmc.PagefileUsage); - } -#endif -#endif - - *memoryUsedMB = (float)bytes_used / 1024 / 1024; - - return true; -} - -bool taosGetSysMemory(float *memoryUsedMB) { - MEMORYSTATUSEX memsStat; - float nMemFree; - float nMemTotal; - - memsStat.dwLength = sizeof(memsStat); - if (!GlobalMemoryStatusEx(&memsStat)) { - return false; - } - nMemFree = memsStat.ullAvailPhys / (1024.0f * 1024.0f); - nMemTotal = memsStat.ullTotalPhys / (1024.0f * 1024.0f); - *memoryUsedMB = nMemTotal - nMemFree; - return true; +void taosKillSystem() { + uError("function taosKillSystem, exit!"); + exit(0); } int taosSystem(const char *cmd) { @@ -240,4 +278,6 @@ void taosSetCoreDump() { SetUnhandledExceptionFilter(&FlCrashDump); } bool taosGetSystemUid(char *uid) { sprintf(uid, "uid_not_implemented_yet"); return true; -} \ No newline at end of file +} + +char *taosGetCmdlineByPID(int pid) { return ""; } diff --git a/src/sync/src/syncArbitrator.c b/src/sync/src/syncArbitrator.c index dbcc4b40a6..06b4a61d6f 100644 --- a/src/sync/src/syncArbitrator.c +++ b/src/sync/src/syncArbitrator.c @@ -27,7 +27,7 @@ #include "syncInt.h" #include "syncTcp.h" -static void arbSignalHandler(int32_t signum); +static void arbSignalHandler(int32_t signum, siginfo_t *sigInfo, void *context); static void arbProcessIncommingConnection(SOCKET connFd, uint32_t sourceIp); static void arbProcessBrokenLink(int64_t rid); static int32_t arbProcessPeerMsg(int64_t rid, void *buffer); @@ -35,9 +35,9 @@ static tsem_t tsArbSem; static void * tsArbTcpPool; typedef struct { - char id[TSDB_EP_LEN + 24]; - SOCKET nodeFd; - void * pConn; + char id[TSDB_EP_LEN + 24]; + SOCKET nodeFd; + void * pConn; } SNodeConn; int32_t main(int32_t argc, char *argv[]) { @@ -70,14 +70,13 @@ int32_t main(int32_t argc, char *argv[]) { /* Set termination handler. */ struct sigaction act = {{0}}; - memset(&act, 0, sizeof(struct sigaction)); + act.sa_flags = SA_SIGINFO; + act.sa_sigaction = arbSignalHandler; act.sa_handler = arbSignalHandler; sigaction(SIGTERM, &act, NULL); - sigaction(SIGINT, &act, NULL); -#ifndef WINDOWS sigaction(SIGHUP, &act, NULL); -#endif + sigaction(SIGINT, &act, NULL); tsAsyncLog = 0; strcat(arbLogPath, "/arbitrator.log"); @@ -107,6 +106,7 @@ int32_t main(int32_t argc, char *argv[]) { syncCloseTcpThreadPool(tsArbTcpPool); sInfo("TAOS arbitrator is shut down"); + closelog(); return 0; } @@ -174,16 +174,16 @@ static int32_t arbProcessPeerMsg(int64_t rid, void *buffer) { return 0; } -static void arbSignalHandler(int32_t signum) { +static void arbSignalHandler(int32_t signum, siginfo_t *sigInfo, void *context) { struct sigaction act = {{0}}; - act.sa_handler = SIG_IGN; - sigaction(SIGTERM, &act, NULL); - sigaction(SIGINT, &act, NULL); #ifndef WINDOWS - sigaction(SIGHUP, &act, NULL); + act.sa_handler = SIG_IGN; #endif + sigaction(SIGTERM, &act, NULL); + sigaction(SIGHUP, &act, NULL); + sigaction(SIGINT, &act, NULL); - sInfo("shut down signal is %d", signum); + sInfo("shut down signal is %d, sender PID:%d", signum, sigInfo->si_pid); // inform main thread to exit tsem_post(&tsArbSem); From a649f55195c37c45c7f66a4d42a3268b7603f48b Mon Sep 17 00:00:00 2001 From: freemine Date: Fri, 15 Jan 2021 22:34:48 +0800 Subject: [PATCH 06/62] eok --- src/CMakeLists.txt | 2 +- src/client/CMakeLists.txt | 4 +- src/client/src/tscSQLParser.c | 2 + src/client/src/tscUtil.c | 4 + src/kit/shell/src/shellDarwin.c | 2 + src/os/inc/eok.h | 74 ++++ src/os/inc/osDarwin.h | 11 +- src/os/src/darwin/darwinEnv.c | 3 + src/os/src/darwin/eok.c | 580 ++++++++++++++++++++++++++++++ src/os/src/detail/osSocket.c | 3 +- src/plugins/http/src/httpServer.c | 4 + src/rpc/src/rpcTcp.c | 22 ++ src/sync/src/syncTcp.c | 14 + src/tsdb/inc/tsdbMain.h | 6 +- src/tsdb/src/tsdbCommit.c | 4 + src/tsdb/src/tsdbMain.c | 16 + src/tsdb/src/tsdbMemTable.c | 11 +- src/util/src/tsocket.c | 4 +- tests/examples/c/CMakeLists.txt | 5 + 19 files changed, 763 insertions(+), 8 deletions(-) create mode 100644 src/os/inc/eok.h create mode 100644 src/os/src/darwin/eok.c diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 931a0a132e..f7304ae72f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -19,5 +19,5 @@ ADD_SUBDIRECTORY(wal) ADD_SUBDIRECTORY(cq) ADD_SUBDIRECTORY(dnode) #ADD_SUBDIRECTORY(connector/odbc) -ADD_SUBDIRECTORY(connector/jdbc) +#ADD_SUBDIRECTORY(connector/jdbc) diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index daf7c5e534..3fd29b474e 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -49,12 +49,12 @@ ELSEIF (TD_DARWIN) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/linux) ADD_LIBRARY(taos_static STATIC ${SRC}) - TARGET_LINK_LIBRARIES(taos_static trpc tutil pthread m) + TARGET_LINK_LIBRARIES(taos_static query trpc tutil pthread m) SET_TARGET_PROPERTIES(taos_static PROPERTIES OUTPUT_NAME "taos_static") # generate dynamic library (*.dylib) ADD_LIBRARY(taos SHARED ${SRC}) - TARGET_LINK_LIBRARIES(taos trpc tutil pthread m) + TARGET_LINK_LIBRARIES(taos query trpc tutil pthread m) SET_TARGET_PROPERTIES(taos PROPERTIES CLEAN_DIRECT_OUTPUT 1) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index a08482f570..4a24f2bfdb 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -13,10 +13,12 @@ * along with this program. If not, see . */ +#ifndef __APPLE__ #define _BSD_SOURCE #define _XOPEN_SOURCE 500 #define _DEFAULT_SOURCE #define _GNU_SOURCE +#endif // __APPLE__ #include "os.h" #include "ttype.h" diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index b44ebb3c98..d6151c1a88 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -2486,7 +2486,11 @@ bool tscSetSqlOwner(SSqlObj* pSql) { SSqlRes* pRes = &pSql->res; // set the sql object owner +#ifdef __APPLE__ + pthread_t threadId = (pthread_t)taosGetSelfPthreadId(); +#else uint64_t threadId = taosGetSelfPthreadId(); +#endif if (atomic_val_compare_exchange_64(&pSql->owner, 0, threadId) != 0) { pRes->code = TSDB_CODE_QRY_IN_EXEC; return false; diff --git a/src/kit/shell/src/shellDarwin.c b/src/kit/shell/src/shellDarwin.c index ddf7b21bef..d6aed4401c 100644 --- a/src/kit/shell/src/shellDarwin.c +++ b/src/kit/shell/src/shellDarwin.c @@ -21,6 +21,8 @@ #include "shellCommand.h" #include "tkey.h" +#include "tscLog.h" + #define OPT_ABORT 1 /* �Cabort */ int indicator = 1; diff --git a/src/os/inc/eok.h b/src/os/inc/eok.h new file mode 100644 index 0000000000..8892e50c35 --- /dev/null +++ b/src/os/inc/eok.h @@ -0,0 +1,74 @@ +#ifndef _eok_h_fd274616_996c_400e_9023_ae70be881fa3_ +#define _eok_h_fd274616_996c_400e_9023_ae70be881fa3_ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +enum EPOLL_EVENTS + { + EPOLLIN = 0x001, +#define EPOLLIN EPOLLIN + EPOLLPRI = 0x002, +#define EPOLLPRI EPOLLPRI + EPOLLOUT = 0x004, +#define EPOLLOUT EPOLLOUT + EPOLLRDNORM = 0x040, +#define EPOLLRDNORM EPOLLRDNORM + EPOLLRDBAND = 0x080, +#define EPOLLRDBAND EPOLLRDBAND + EPOLLWRNORM = 0x100, +#define EPOLLWRNORM EPOLLWRNORM + EPOLLWRBAND = 0x200, +#define EPOLLWRBAND EPOLLWRBAND + EPOLLMSG = 0x400, +#define EPOLLMSG EPOLLMSG + EPOLLERR = 0x008, +#define EPOLLERR EPOLLERR + EPOLLHUP = 0x010, +#define EPOLLHUP EPOLLHUP + EPOLLRDHUP = 0x2000, +#define EPOLLRDHUP EPOLLRDHUP + EPOLLEXCLUSIVE = 1u << 28, +#define EPOLLEXCLUSIVE EPOLLEXCLUSIVE + EPOLLWAKEUP = 1u << 29, +#define EPOLLWAKEUP EPOLLWAKEUP + EPOLLONESHOT = 1u << 30, +#define EPOLLONESHOT EPOLLONESHOT + EPOLLET = 1u << 31 +#define EPOLLET EPOLLET + }; + +/* Valid opcodes ( "op" parameter ) to issue to epoll_ctl(). */ +#define EPOLL_CTL_ADD 1 /* Add a file descriptor to the interface. */ +#define EPOLL_CTL_DEL 2 /* Remove a file descriptor from the interface. */ +#define EPOLL_CTL_MOD 3 /* Change file descriptor epoll_event structure. */ + + +typedef union epoll_data +{ + void *ptr; + int fd; + uint32_t u32; + uint64_t u64; +} epoll_data_t; + +struct epoll_event +{ + uint32_t events; /* Epoll events */ + epoll_data_t data; /* User data variable */ +}; + +int epoll_create(int size); +int epoll_ctl(int epfd, int op, int fd, struct epoll_event *event); +int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout); +int epoll_close(int epfd); + +#ifdef __cplusplus +} +#endif + +#endif // _eok_h_fd274616_996c_400e_9023_ae70be881fa3_ + diff --git a/src/os/inc/osDarwin.h b/src/os/inc/osDarwin.h index 1461ec6d3b..52bb661f89 100644 --- a/src/os/inc/osDarwin.h +++ b/src/os/inc/osDarwin.h @@ -91,7 +91,7 @@ extern "C" { typedef int(*__compar_fn_t)(const void *, const void *); // for send function in tsocket.c -#define MSG_NOSIGNAL 0 +// #define MSG_NOSIGNAL 0 #define SO_NO_CHECK 0x1234 #define SOL_TCP 0x1234 #define TCP_KEEPIDLE 0x1234 @@ -100,6 +100,15 @@ typedef int(*__compar_fn_t)(const void *, const void *); #define PTHREAD_MUTEX_RECURSIVE_NP PTHREAD_MUTEX_RECURSIVE #endif +int64_t tsosStr2int64(char *str); + +#include "eok.h" + + + + + + #ifdef __cplusplus } #endif diff --git a/src/os/src/darwin/darwinEnv.c b/src/os/src/darwin/darwinEnv.c index 6adebabec0..28388f24d2 100644 --- a/src/os/src/darwin/darwinEnv.c +++ b/src/os/src/darwin/darwinEnv.c @@ -17,6 +17,8 @@ #include "os.h" #include "tglobal.h" +#include + void osInit() { if (configDir[0] == 0) { strcpy(configDir, "~/TDengine/cfg"); @@ -30,3 +32,4 @@ void osInit() { strcpy(tsScriptDir, "~/TDengine/cfg"); strcpy(tsOsName, "Darwin"); } + diff --git a/src/os/src/darwin/eok.c b/src/os/src/darwin/eok.c new file mode 100644 index 0000000000..ea67223c25 --- /dev/null +++ b/src/os/src/darwin/eok.c @@ -0,0 +1,580 @@ +#include "eok.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define D(fmt, ...) fprintf(stderr, "%s[%d]%s(): " fmt "\n", basename(__FILE__), __LINE__, __func__, ##__VA_ARGS__) +#define A(statement, fmt, ...) do { \ + if (statement) break; \ + fprintf(stderr, "%s[%d]%s(): assert [%s] failed: %d[%s]: " fmt "\n", \ + basename(__FILE__), __LINE__, __func__, \ + #statement, errno, strerror(errno), \ + ##__VA_ARGS__); \ + abort(); \ +} while (0) + +#define E(fmt, ...) do { \ + fprintf(stderr, "%s[%d]%s(): %d[%s]: " fmt "\n", \ + basename(__FILE__), __LINE__, __func__, \ + errno, strerror(errno), \ + ##__VA_ARGS__); \ +} while (0) + +static int eok_dummy = 0; + +typedef struct ep_over_kq_s ep_over_kq_t; +typedef struct eok_event_s eok_event_t; + +struct ep_over_kq_s { + int kq; + int idx; + ep_over_kq_t *next; + + int sv[2]; // 0 for read, 1 for write + + eok_event_t *evs_head; + eok_event_t *evs_tail; + eok_event_t *evs_free; + + struct kevent64_s *kchanges; + int nchanges; + int ichanges; + + struct kevent64_s *kevslist; + int nevslist; + + pthread_mutex_t lock; + + volatile unsigned int lock_valid:1; + volatile unsigned int waiting:1; + volatile unsigned int changed:1; + volatile unsigned int wakenup:1; + volatile unsigned int stopping:1; +}; + +struct eok_event_s { + int fd; + struct epoll_event epev; + volatile unsigned int changed; // 0:registered;1:add;2:mod;3:del + + eok_event_t *next; + eok_event_t *prev; +}; + +typedef struct eoks_s eoks_t; +struct eoks_s { + pthread_mutex_t lock; + ep_over_kq_t *eoks; + int neoks; + ep_over_kq_t *eoks_free; +}; +static eoks_t eoks = { + .lock = PTHREAD_MUTEX_INITIALIZER, + .eoks = NULL, + .neoks = 0, + .eoks_free = NULL, +}; + +static ep_over_kq_t* eoks_alloc(void); +static void eoks_free(ep_over_kq_t *eok); +static ep_over_kq_t* eoks_find(int epfd); + +static eok_event_t* eok_find_ev(ep_over_kq_t *eok, int fd); +static eok_event_t* eok_calloc_ev(ep_over_kq_t *eok); +static void eok_free_ev(ep_over_kq_t *eok, eok_event_t *ev); +static void eok_wakeup(ep_over_kq_t *eok); + +static int eok_chgs_refresh(ep_over_kq_t *eok, eok_event_t *oev, eok_event_t *ev, struct kevent64_s *krev, struct kevent64_s *kwev); + +static struct kevent64_s* eok_alloc_eventslist(ep_over_kq_t *eok, int maxevents); + +int epoll_create(int size) { + (void)size; + int e = 0; + ep_over_kq_t *eok = eoks_alloc(); + if (!eok) { + errno = ENOMEM; + return -1; + } + + A(eok->kq==-1, "internal logic error"); + A(eok->lock_valid, "internal logic error"); + A(eok->idx>=0 && eok->idxnext==NULL, "internal logic error"); + A(eok->sv[0]==-1, "internal logic error"); + A(eok->sv[1]==-1, "internal logic error"); + + eok->kq = kqueue(); + if (eok->kq==-1) { + e = errno; + eoks_free(eok); + errno = e; + return -1; + } + + if (socketpair(AF_LOCAL, SOCK_STREAM, 0, eok->sv)) { + e = errno; + eoks_free(eok); + errno = e; + return -1; + } + + struct epoll_event ev = {0}; + ev.events = EPOLLIN; + ev.data.ptr = &eok_dummy; + if (epoll_ctl(eok->idx, EPOLL_CTL_ADD, eok->sv[0], &ev)) { + A(0, "internal logic error"); + epoll_close(eok->idx); + return -1; + } + + return eok->idx; +} + +int epoll_ctl(int epfd, int op, int fd, struct epoll_event *event) { + int e = 0; + if (epfd<0 || epfd>=eoks.neoks) { + errno = EBADF; + return -1; + } + if (fd==-1) { + errno = EBADF; + return -1; + } + if (event && !(event->events & (EPOLLIN | EPOLLERR | EPOLLHUP | EPOLLRDHUP | EPOLLOUT))) { + e = ENOTSUP; + return -1; + } + + ep_over_kq_t *eok = eoks_find(epfd); + if (!eok) { + errno = EBADF; + return -1; + } + + A(0==pthread_mutex_lock(&eok->lock), ""); + do { + eok_event_t* oev = eok_find_ev(eok, fd); + if (op==EPOLL_CTL_ADD && oev) { + e = EEXIST; + break; + } + if (op!=EPOLL_CTL_ADD && !oev) { + e = ENOENT; + break; + } + + struct kevent64_s krev = {0}; + struct kevent64_s kwev = {0}; + krev.ident = -1; + kwev.ident = -1; + uint16_t flags = 0; + eok_event_t ev = {0}; + ev.fd = fd; + if (event) ev.epev = *event; + struct epoll_event *pev = event; + switch (op) { + case EPOLL_CTL_ADD: { + if (!event) { + e = EINVAL; + break; + } + flags = EV_ADD; + ev.changed = 1; + } break; + case EPOLL_CTL_MOD: { + if (!event) { + e = EINVAL; + break; + } + flags = EV_ADD; + ev.changed = 2; + } break; + case EPOLL_CTL_DEL: { + // event is ignored + pev = &oev->epev; + flags = EV_DELETE; + ev.changed = 3; + } break; + default: { + e = ENOTSUP; + } break; + } + + if (e) break; + + if (pev->events & (EPOLLIN | EPOLLERR | EPOLLHUP | EPOLLRDHUP)) { + flags |= EV_EOF; + EV_SET64(&krev, ev.fd, EVFILT_READ, flags, 0, 0, -1, 0, 0); + D("...."); + } + if (pev->events & EPOLLOUT) { + EV_SET64(&kwev, ev.fd, EVFILT_WRITE, flags, 0, 0, -1, 0, 0); + D("...."); + } + + if (eok_chgs_refresh(eok, oev, &ev, &krev, &kwev)) { + e = errno; + A(e, "internal logic error"); + break; + } + eok->changed = 1; + eok_wakeup(eok); + } while (0); + A(0==pthread_mutex_unlock(&eok->lock), ""); + + if (e) { + errno = e; + return -1; + } + + return 0; +} + +static struct timespec do_timespec_diff(struct timespec *from, struct timespec *to); + +int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) { + int e = 0; + if (epfd<0 || epfd>=eoks.neoks) { + errno = EBADF; + return -1; + } + if (!events) { + errno = EINVAL; + return -1; + } + if (maxevents<=0) { + errno = EINVAL; + return -1; + } + + int r = 0; + + ep_over_kq_t *eok = eoks_find(epfd); + if (!eok) { + errno = EBADF; + return -1; + } + + struct timespec abstime = {0}; + A(TIME_UTC==timespec_get(&abstime, TIME_UTC), "internal logic error"); + + if (timeout!=-1) { + if (timeout<0) timeout = 0; + int64_t t = abstime.tv_nsec + timeout * 1000000; + abstime.tv_sec += t / 1000000000; + abstime.tv_nsec %= 1000000000; + } + + int cnts = 0; + A(0==pthread_mutex_lock(&eok->lock), ""); + do { + cnts = 0; + A(eok->waiting==0, "internal logic error"); + struct kevent64_s *eventslist = eok_alloc_eventslist(eok, maxevents); + if (!eventslist) { + e = ENOMEM; + break; + } + memset(eventslist, 0, maxevents * sizeof(*eventslist)); + + struct timespec now = {0}; + A(TIME_UTC==timespec_get(&now, TIME_UTC), "internal logic error"); + struct timespec to = do_timespec_diff(&now, &abstime); + struct timespec *pto = NULL; + if (timeout!=-1) { + pto = &to; + } + + eok->changed = 0; + eok->wakenup = 0; + eok->waiting = 1; + + struct kevent64_s *kchanges = eok->kchanges; + int nchanges = eok->nchanges; + int ichanges = eok->ichanges; + eok->kchanges = NULL; + eok->nchanges = 0; + eok->ichanges = 0; + + A(0==pthread_mutex_unlock(&eok->lock), ""); + r = kevent64(eok->kq, kchanges, ichanges, eventslist, maxevents, 0, pto); + e = errno; + A(0==pthread_mutex_lock(&eok->lock), ""); + + eok->waiting = 0; + if (kchanges) { + free(kchanges); + kchanges = NULL; + nchanges = 0; + ichanges = 0; + } + + eok->waiting = 0; + if (r<0) break; + if (r==0) { + A(timeout!=-1, "internal logic error"); + } + for (int i=0; iudata && eok->evs_head && eok->evs_tail, "internal logic error"); + eok_event_t *ev = (eok_event_t*)kev->udata; + A(kev->ident == ev->fd, "internal logic error"); + D("..."); + switch (kev->filter) { + case EVFILT_READ: { + A((ev->epev.events & EPOLLIN), "internal logic errro"); + if (ev->epev.data.ptr==&eok_dummy) { + char c = '\0'; + A(1==recv(kev->ident, &c, 1, 0), "internal logic error"); + A(0==memcmp(&c, "1", 1), "internal logic error"); + D("..............."); + } else { + if (ev->changed==3) { + D("already requested to delete"); + // EV_DELETE? + continue; + } + struct epoll_event pev = {0}; + pev.data.ptr = ev->epev.data.ptr; + pev.events = EPOLLIN; + if (kev->flags & EV_EOF) { + pev.events |= (EPOLLHUP | EPOLLERR | EPOLLRDHUP); + } + pev.events &= ev->epev.events; + events[cnts++] = pev; + } + } break; + case EVFILT_WRITE: { + A(0, "not implemented yet"); + } break; + default: { + A(0, "internal logic error"); + } break; + } + } + } while (cnts==0); + A(0==pthread_mutex_unlock(&eok->lock), ""); + + if (e) { + errno = e; + return -1; + } + + return cnts; +} + +static struct timespec do_timespec_diff(struct timespec *from, struct timespec *to) { + struct timespec delta; + delta.tv_sec = to->tv_sec - from->tv_sec; + delta.tv_nsec = to->tv_nsec - from->tv_nsec; + while (delta.tv_nsec<0) { + delta.tv_sec -= 1; + delta.tv_nsec += 1000000000; + } + return delta; +} + +int epoll_close(int epfd) { + if (epfd<0 || epfd>=eoks.neoks) { + errno = EBADF; + return -1; + } + ep_over_kq_t *eok = eoks_find(epfd); + if (!eok) { + errno = EBADF; + return -1; + } + + A(0==pthread_mutex_lock(&eok->lock), ""); + do { + A(eok->stopping==0, "internal logic error"); + eok->stopping = 1; + A(eok->waiting, "internal logic error"); + + if (eok->kq!=-1) { + close(eok->kq); + eok->kq = -1; + } + } while (0); + A(0==pthread_mutex_unlock(&eok->lock), ""); + eoks_free(eok); + + return 0; +} + +static struct kevent64_s* eok_alloc_eventslist(ep_over_kq_t *eok, int maxevents) { + if (maxevents<=eok->nevslist) return eok->kevslist; + struct kevent64_s *p = (struct kevent64_s*)realloc(eok->kevslist, sizeof(*p)*maxevents); + if (!p) return NULL; + eok->kevslist = p; + eok->nevslist = maxevents; + return p; +} + +static eok_event_t* eok_find_ev(ep_over_kq_t *eok, int fd) { + eok_event_t *p = eok->evs_head; + while (p) { + if (p->fd == fd) return p; + p = p->next; + } + errno = ENOENT; + return NULL; +} + +static eok_event_t* eok_calloc_ev(ep_over_kq_t *eok) { + eok_event_t *p = NULL; + if (eok->evs_free) { + p = eok->evs_free; + eok->evs_free = p->next; + p->next = NULL; + } else { + p = (eok_event_t*)calloc(1, sizeof(*p)); + if (!p) return NULL; + } + // dirty link + p->prev = eok->evs_tail; + if (eok->evs_tail) eok->evs_tail->next = p; + else eok->evs_head = p; + eok->evs_tail = p; + + return p; +} + +static void eok_free_ev(ep_over_kq_t *eok, eok_event_t *ev) { + if (ev->prev) ev->prev->next = ev->next; + else eok->evs_head = ev->next; + ev->prev = NULL; + if (ev->next) ev->next->prev = ev->prev; + else eok->evs_tail = ev->prev; + ev->next = eok->evs_free; + eok->evs_free = ev->next; +} + +static void eok_wakeup(ep_over_kq_t *eok) { + if (!eok->waiting) return; + if (eok->wakenup) return; + eok->wakenup = 1; + send(eok->sv[1], "1", 1, 0); +} + +static int eok_chgs_refresh(ep_over_kq_t *eok, eok_event_t *oev, eok_event_t *ev, struct kevent64_s *krev, struct kevent64_s *kwev) { + if (!oev) oev = eok_calloc_ev(eok); + if (!oev) return -1; + int n = 0; + if (krev->ident==ev->fd) ++n; + if (kwev->ident==ev->fd) ++n; + A(n, "internal logic error"); + if (eok->ichanges+n>eok->nchanges) { + struct kevent64_s *p = (struct kevent64_s*)realloc(eok->kchanges, sizeof(*p) * (eok->nchanges + 10)); + if (!p) { + if (ev->changed==1) { + eok_free_ev(eok, oev); + } + errno = ENOMEM; + return -1; + } + eok->kchanges = p; + eok->nchanges += 10; + } + + oev->fd = ev->fd; + oev->epev = ev->epev; + oev->changed = ev->changed; + + if (krev->ident==ev->fd) { + krev->udata = (uint64_t)oev; + eok->kchanges[eok->ichanges++] = *krev; + } + if (kwev->ident==ev->fd) { + kwev->udata = (uint64_t)oev; + eok->kchanges[eok->ichanges++] = *kwev; + } + return 0; +} + +static ep_over_kq_t* eoks_alloc(void) { + ep_over_kq_t *eok = NULL; + + A(0==pthread_mutex_lock(&eoks.lock), ""); + do { + if (eoks.eoks_free) { + eok = eoks.eoks_free; + eoks.eoks_free = eok->next; + eok->next = NULL; + break; + } + ep_over_kq_t *p = (ep_over_kq_t*)realloc(eoks.eoks, sizeof(*p) * (eoks.neoks+1)); + if (!p) break; + eoks.eoks = p; + eok = eoks.eoks + eoks.neoks; + memset(eok, 0, sizeof(*eok)); + eok->idx = eoks.neoks; + eok->kq = -1; + eok->sv[0] = -1; + eok->sv[1] = -1; + eoks.neoks += 1; + } while (0); + A(0==pthread_mutex_unlock(&eoks.lock), ""); + + if (!eok) return NULL; + if (eok->lock_valid) { + return eok; + } + + if (0==pthread_mutex_init(&eok->lock, NULL)) { + eok->lock_valid = 1; + return eok; + } + + eoks_free(eok); + return NULL; +} + +static void eoks_free(ep_over_kq_t *eok) { + A(0==pthread_mutex_lock(&eoks.lock), ""); + do { + A(eok->next==NULL, "internal logic error"); + A(eok->evs_head==NULL, "internal logic error"); + A(eok->waiting==0, "internal logic error"); + if (eok->sv[0]!=-1) { + close(eok->sv[0]); + eok->sv[0] = -1; + } + if (eok->sv[1]!=-1) { + close(eok->sv[1]); + eok->sv[1] = -1; + } + if (eok->kq!=-1) { + close(eok->kq); + eok->kq = -1; + } + eok->next = eoks.eoks_free; + eoks.eoks_free = eok; + } while (0); + A(0==pthread_mutex_unlock(&eoks.lock), ""); +} + +static ep_over_kq_t* eoks_find(int epfd) { + ep_over_kq_t *eok = NULL; + A(0==pthread_mutex_lock(&eoks.lock), ""); + do { + if (epfd<0 || epfd>=eoks.neoks) { + break; + } + A(eoks.eoks, "internal logic error"); + eok = eoks.eoks + epfd; + A(eok->next==NULL, "internal logic error"); + A(eok->lock_valid, "internal logic error"); + } while (0); + A(0==pthread_mutex_unlock(&eoks.lock), ""); + return eok; +} + diff --git a/src/os/src/detail/osSocket.c b/src/os/src/detail/osSocket.c index 729471247f..c6cbbe6b83 100644 --- a/src/os/src/detail/osSocket.c +++ b/src/os/src/detail/osSocket.c @@ -58,6 +58,7 @@ void taosBlockSIGPIPE() { #ifndef TAOS_OS_FUNC_SOCKET_SETSOCKETOPT int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen) { + fprintf(stderr, "==%s[%d]%s()==\n", basename(__FILE__), __LINE__, __func__); return setsockopt(socketfd, level, optname, optval, (socklen_t)optlen); } @@ -73,4 +74,4 @@ const char *taosInetNtoa(struct in_addr ipInt) { return inet_ntoa(ipInt); } -#endif \ No newline at end of file +#endif diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index 4896d50c6c..1cc73aef06 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -52,7 +52,11 @@ static void httpStopThread(HttpThread* pThread) { close(fd); } +#ifdef __APPLE__ + epoll_close(pThread->pollFd); +#else close(pThread->pollFd); +#endif pthread_mutex_destroy(&(pThread->threadMutex)); } diff --git a/src/rpc/src/rpcTcp.c b/src/rpc/src/rpcTcp.c index 178b96c423..ff2a5882f8 100644 --- a/src/rpc/src/rpcTcp.c +++ b/src/rpc/src/rpcTcp.c @@ -133,6 +133,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread } pThreadObj->pollFd = (int64_t)epoll_create(10); // size does not matter + fprintf(stderr, "==%s[%d]%s()==\n", basename(__FILE__), __LINE__, __func__); if (pThreadObj->pollFd < 0) { tError("%s failed to create TCP epoll", label); code = -1; @@ -293,6 +294,7 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int num, void * } pThreadObj->pollFd = (SOCKET)epoll_create(10); // size does not matter + fprintf(stderr, "==%s[%d]%s()==\n", basename(__FILE__), __LINE__, __func__); if (pThreadObj->pollFd < 0) { tError("%s failed to create TCP client epoll", label); free(pThreadObj); @@ -307,7 +309,11 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int num, void * int code = pthread_create(&(pThreadObj->thread), &thattr, taosProcessTcpData, (void *)(pThreadObj)); pthread_attr_destroy(&thattr); if (code != 0) { +#ifdef __APPLE__ + epoll_close(pThreadObj->pollFd); +#else taosCloseSocket(pThreadObj->pollFd); +#endif free(pThreadObj); terrno = TAOS_SYSTEM_ERROR(errno); tError("%s failed to create TCP read data thread(%s)", label, strerror(errno)); @@ -337,6 +343,10 @@ void taosCleanUpTcpClient(void *chandle) { void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port) { SThreadObj * pThreadObj = shandle; + fprintf(stderr, "==%s[%d]%s()==\n", basename(__FILE__), __LINE__, __func__); + fprintf(stderr, "pThreadObj->ip:%d\n", pThreadObj->ip); + fprintf(stderr, "PF_INET/AF_INET:%d/%d\n", PF_INET, AF_INET); + fprintf(stderr, "ip/port:%x/%d\n", ip, port); SOCKET fd = taosOpenTcpClientSocket(ip, port, pThreadObj->ip); if (fd < 0) return NULL; @@ -348,6 +358,7 @@ void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uin localPort = (uint16_t)ntohs(sin.sin_port); } + fprintf(stderr, "==%s[%d]%s()==\n", basename(__FILE__), __LINE__, __func__); SFdObj *pFdObj = taosMallocFdObj(pThreadObj, fd); if (pFdObj) { @@ -358,6 +369,7 @@ void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uin pThreadObj->label, thandle, ip, port, localPort, pFdObj, pThreadObj->numOfFds); } else { tError("%s failed to malloc client FdObj(%s)", pThreadObj->label, strerror(errno)); + fprintf(stderr, "==%s[%d]%s()==\n", basename(__FILE__), __LINE__, __func__); taosCloseSocket(fd); } @@ -480,27 +492,32 @@ static void *taosProcessTcpData(void *param) { if (fdNum < 0) continue; for (int i = 0; i < fdNum; ++i) { + fprintf(stderr, "==%s[%d]%s()==\n", basename(__FILE__), __LINE__, __func__); pFdObj = events[i].data.ptr; if (events[i].events & EPOLLERR) { tDebug("%s %p FD:%p epoll errors", pThreadObj->label, pFdObj->thandle, pFdObj); + fprintf(stderr, "==%s[%d]%s()==\n", basename(__FILE__), __LINE__, __func__); taosReportBrokenLink(pFdObj); continue; } if (events[i].events & EPOLLRDHUP) { tDebug("%s %p FD:%p RD hang up", pThreadObj->label, pFdObj->thandle, pFdObj); + fprintf(stderr, "==%s[%d]%s()==\n", basename(__FILE__), __LINE__, __func__); taosReportBrokenLink(pFdObj); continue; } if (events[i].events & EPOLLHUP) { tDebug("%s %p FD:%p hang up", pThreadObj->label, pFdObj->thandle, pFdObj); + fprintf(stderr, "==%s[%d]%s()==\n", basename(__FILE__), __LINE__, __func__); taosReportBrokenLink(pFdObj); continue; } if (taosReadTcpData(pFdObj, &recvInfo) < 0) { + fprintf(stderr, "==%s[%d]%s()==\n", basename(__FILE__), __LINE__, __func__); shutdown(pFdObj->fd, SHUT_WR); continue; } @@ -512,7 +529,11 @@ static void *taosProcessTcpData(void *param) { if (pThreadObj->stop) break; } +#ifdef __APPLE__ + if (pThreadObj->pollFd >=0) epoll_close(pThreadObj->pollFd); +#else if (pThreadObj->pollFd >=0) taosCloseSocket(pThreadObj->pollFd); +#endif while (pThreadObj->pHead) { SFdObj *pFdObj = pThreadObj->pHead; @@ -542,6 +563,7 @@ static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, SOCKET fd) { event.events = EPOLLIN | EPOLLRDHUP; event.data.ptr = pFdObj; + fprintf(stderr, "==%s[%d]%s()==\n", basename(__FILE__), __LINE__, __func__); if (epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) { tfree(pFdObj); terrno = TAOS_SYSTEM_ERROR(errno); diff --git a/src/sync/src/syncTcp.c b/src/sync/src/syncTcp.c index 4744666737..be85853819 100644 --- a/src/sync/src/syncTcp.c +++ b/src/sync/src/syncTcp.c @@ -136,12 +136,15 @@ void *syncAllocateTcpConn(void *param, int64_t rid, int32_t connFd) { event.events = EPOLLIN | EPOLLRDHUP; event.data.ptr = pConn; + fprintf(stderr, ">>>>>>>>>>>>>>>>>\n"); if (epoll_ctl(pThread->pollFd, EPOLL_CTL_ADD, connFd, &event) < 0) { + fprintf(stderr, "<<<<<<<<<<<<<<<<<\n"); sError("failed to add fd:%d since %s", connFd, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); tfree(pConn); pConn = NULL; } else { + fprintf(stderr, "<<<<<<<<<<<<<<<<<\n"); pThread->numOfFds++; sDebug("%p fd:%d is added to epoll thread, num:%d", pThread, connFd, pThread->numOfFds); } @@ -167,7 +170,9 @@ static void taosProcessBrokenLink(SConnObj *pConn) { (*pInfo->processBrokenLink)(pConn->handleId); pThread->numOfFds--; + fprintf(stderr, "<<<<<<<<<<<<<<<<<\n"); epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, pConn->fd, NULL); + fprintf(stderr, "<<<<<<<<<<<<<<<<<\n"); sDebug("%p fd:%d is removed from epoll thread, num:%d", pThread, pConn->fd, pThread->numOfFds); taosClose(pConn->fd); tfree(pConn); @@ -233,7 +238,11 @@ static void *syncProcessTcpData(void *param) { sDebug("%p TCP epoll thread exits", pThread); +#ifdef __APPLE__ + epoll_close(pThread->pollFd); +#else close(pThread->pollFd); +#endif tfree(pThread); tfree(buffer); return NULL; @@ -278,6 +287,7 @@ static SThreadObj *syncGetTcpThread(SPoolObj *pPool) { pThread->pPool = pPool; pThread->pollFd = epoll_create(10); // size does not matter + fprintf(stderr, "...............\n"); if (pThread->pollFd < 0) { tfree(pThread); return NULL; @@ -290,7 +300,11 @@ static SThreadObj *syncGetTcpThread(SPoolObj *pPool) { pthread_attr_destroy(&thattr); if (ret != 0) { +#ifdef __APPLE__ + epoll_close(pThread->pollFd); +#else close(pThread->pollFd); +#endif tfree(pThread); return NULL; } diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 5067974903..984839162e 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -233,7 +233,11 @@ typedef struct { SMemTable* mem; SMemTable* imem; STsdbFileH* tsdbFileH; +#ifdef __APPLE__ + sem_t *readyToCommit; +#else sem_t readyToCommit; +#endif pthread_mutex_t mutex; bool repoLocked; int32_t code; // Commit code @@ -616,4 +620,4 @@ int tsdbScheduleCommit(STsdbRepo *pRepo); } #endif -#endif \ No newline at end of file +#endif diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c index 696270d670..9551d1b3a5 100644 --- a/src/tsdb/src/tsdbCommit.c +++ b/src/tsdb/src/tsdbCommit.c @@ -166,7 +166,11 @@ static void tsdbEndCommit(STsdbRepo *pRepo, int eno) { pRepo->imem = NULL; tsdbUnlockRepo(pRepo); tsdbUnRefMemTable(pRepo, pIMem); +#ifdef __APPLE__ + sem_post(pRepo->readyToCommit); +#else sem_post(&(pRepo->readyToCommit)); +#endif } static int tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey) { diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index b34b2fa9e6..602aea70e3 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -146,7 +146,11 @@ int tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit) { if (toCommit) { tsdbAsyncCommit(pRepo); +#ifdef __APPLE__ + sem_wait(pRepo->readyToCommit); +#else sem_wait(&(pRepo->readyToCommit)); +#endif terrno = pRepo->code; } tsdbUnRefMemTable(pRepo, pRepo->mem); @@ -643,11 +647,19 @@ static STsdbRepo *tsdbNewRepo(char *rootDir, STsdbAppH *pAppH, STsdbCfg *pCfg) { goto _err; } +#ifdef __APPLE__ + pRepo->readyToCommit = sem_open(NULL, O_CREAT, 0644, 1); + if (!pRepo->readyToCommit) { + terrno = TAOS_SYSTEM_ERROR(code); + goto _err; + } +#else code = sem_init(&(pRepo->readyToCommit), 0, 1); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(code); goto _err; } +#endif pRepo->repoLocked = false; @@ -693,7 +705,11 @@ static void tsdbFreeRepo(STsdbRepo *pRepo) { // tsdbFreeMemTable(pRepo->mem); // tsdbFreeMemTable(pRepo->imem); tfree(pRepo->rootDir); +#ifdef __APPLE__ + sem_close(pRepo->readyToCommit); +#else sem_destroy(&(pRepo->readyToCommit)); +#endif pthread_mutex_destroy(&pRepo->mutex); free(pRepo); } diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 07f001f68a..202405df00 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -207,7 +207,11 @@ void *tsdbAllocBytes(STsdbRepo *pRepo, int bytes) { int tsdbAsyncCommit(STsdbRepo *pRepo) { if (pRepo->mem == NULL) return 0; +#ifdef __APPLE__ + sem_wait(pRepo->readyToCommit); +#else sem_wait(&(pRepo->readyToCommit)); +#endif ASSERT(pRepo->imem == NULL); @@ -229,8 +233,13 @@ int tsdbSyncCommit(TSDB_REPO_T *repo) { STsdbRepo *pRepo = (STsdbRepo *)repo; tsdbAsyncCommit(pRepo); +#ifdef __APPLE__ + sem_wait(pRepo->readyToCommit); + sem_post(pRepo->readyToCommit); +#else sem_wait(&(pRepo->readyToCommit)); sem_post(&(pRepo->readyToCommit)); +#endif if (pRepo->code != TSDB_CODE_SUCCESS) { terrno = pRepo->code; @@ -927,4 +936,4 @@ static int tsdbUpdateTableLatestInfo(STsdbRepo *pRepo, STable *pTable, SDataRow } return 0; -} \ No newline at end of file +} diff --git a/src/util/src/tsocket.c b/src/util/src/tsocket.c index 1a5c3bd64d..6b15d3677c 100644 --- a/src/util/src/tsocket.c +++ b/src/util/src/tsocket.c @@ -35,7 +35,7 @@ int32_t taosGetFqdn(char *fqdn) { hints.ai_flags = AI_CANONNAME; int32_t ret = getaddrinfo(hostname, NULL, &hints, &result); if (!result) { - uError("failed to get fqdn, code:%d, reason:%s", ret, gai_strerror(ret)); + uError("failed to get fqdn for hostname <%s>, code:%d, reason:%s", hostname, ret, gai_strerror(ret)); return -1; } @@ -341,6 +341,7 @@ int32_t taosKeepTcpAlive(SOCKET sockFd) { return -1; } +#ifndef __APPLE__ int32_t probes = 3; if (taosSetSockOpt(sockFd, SOL_TCP, TCP_KEEPCNT, (void *)&probes, sizeof(probes)) < 0) { uError("fd:%d setsockopt SO_KEEPCNT failed: %d (%s)", sockFd, errno, strerror(errno)); @@ -361,6 +362,7 @@ int32_t taosKeepTcpAlive(SOCKET sockFd) { taosCloseSocket(sockFd); return -1; } +#endif int32_t nodelay = 1; if (taosSetSockOpt(sockFd, IPPROTO_TCP, TCP_NODELAY, (void *)&nodelay, sizeof(nodelay)) < 0) { diff --git a/tests/examples/c/CMakeLists.txt b/tests/examples/c/CMakeLists.txt index 59bcb6eaff..4689ed8868 100644 --- a/tests/examples/c/CMakeLists.txt +++ b/tests/examples/c/CMakeLists.txt @@ -6,3 +6,8 @@ IF (TD_LINUX) ADD_EXECUTABLE(demo demo.c) TARGET_LINK_LIBRARIES(demo taos_static trpc tutil pthread ) ENDIF () +IF (TD_DARWIN) + INCLUDE_DIRECTORIES(. ${TD_COMMUNITY_DIR}/src/inc ${TD_COMMUNITY_DIR}/src/client/inc ${TD_COMMUNITY_DIR}/inc) + ADD_EXECUTABLE(epoll epoll.c) + TARGET_LINK_LIBRARIES(epoll taos_static trpc tutil pthread ) +ENDIF () From c46f6ac022309951f2af325018dacf04d5ccb064 Mon Sep 17 00:00:00 2001 From: freemine Date: Fri, 15 Jan 2021 22:39:56 +0800 Subject: [PATCH 07/62] add epoll.c tests --- tests/examples/c/epoll.c | 183 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 183 insertions(+) create mode 100644 tests/examples/c/epoll.c diff --git a/tests/examples/c/epoll.c b/tests/examples/c/epoll.c new file mode 100644 index 0000000000..e0d63bae32 --- /dev/null +++ b/tests/examples/c/epoll.c @@ -0,0 +1,183 @@ +#ifdef __APPLE__ +#include "eok.h" +#else +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define D(fmt, ...) fprintf(stderr, "%s[%d]%s(): " fmt "\n", basename(__FILE__), __LINE__, __func__, ##__VA_ARGS__) +#define A(statement, fmt, ...) do { \ + if (statement) break; \ + fprintf(stderr, "%s[%d]%s(): assert [%s] failed: %d[%s]: " fmt "\n", \ + basename(__FILE__), __LINE__, __func__, \ + #statement, errno, strerror(errno), \ + ##__VA_ARGS__); \ + abort(); \ +} while (0) + +#define E(fmt, ...) do { \ + fprintf(stderr, "%s[%d]%s(): %d[%s]: " fmt "\n", \ + basename(__FILE__), __LINE__, __func__, \ + errno, strerror(errno), \ + ##__VA_ARGS__); \ +} while (0) + +typedef struct ep_s ep_t; +struct ep_s { + int ep; + + pthread_mutex_t lock; + int sv[2]; // 0 for read, 1 for write; + pthread_t thread; + + volatile unsigned int stopping:1; + volatile unsigned int waiting:1; + volatile unsigned int wakenup:1; +}; + +static int ep_dummy = 0; + +static ep_t* ep_create(void); +static void ep_destroy(ep_t *ep); +static void* routine(void* arg); +static int open_connect(unsigned short port); + +int main(int argc, char *argv[]) { + ep_t* ep = ep_create(); + A(ep, "failed"); + int skt = open_connect(6789); + if (skt!=-1) { + struct epoll_event ev = {0}; + ev.events = EPOLLIN | EPOLLERR | EPOLLHUP | EPOLLRDHUP; + ev.data.ptr = &skt; + A(0==epoll_ctl(ep->ep, EPOLL_CTL_ADD, skt, &ev), ""); + } + getchar(); + ep_destroy(ep); + D(""); + return 0; +} + +ep_t* ep_create(void) { + ep_t *ep = (ep_t*)calloc(1, sizeof(*ep)); + A(ep, "out of memory"); + A(-1!=(ep->ep = epoll_create(1)), ""); + ep->sv[0] = -1; + ep->sv[1] = -1; + A(0==socketpair(AF_LOCAL, SOCK_STREAM, 0, ep->sv), ""); + A(0==pthread_mutex_init(&ep->lock, NULL), ""); + A(0==pthread_mutex_lock(&ep->lock), ""); + struct epoll_event ev = {0}; + ev.events = EPOLLIN; + ev.data.ptr = &ep_dummy; + A(0==epoll_ctl(ep->ep, EPOLL_CTL_ADD, ep->sv[0], &ev), ""); + A(0==pthread_create(&ep->thread, NULL, routine, ep), ""); + A(0==pthread_mutex_unlock(&ep->lock), ""); + return ep; +} + +static void ep_destroy(ep_t *ep) { + A(ep, "invalid argument"); + ep->stopping = 1; + A(1==send(ep->sv[1], "1", 1, 0), ""); + A(0==pthread_join(ep->thread, NULL), ""); + A(0==pthread_mutex_destroy(&ep->lock), ""); + A(0==close(ep->sv[0]), ""); + A(0==close(ep->sv[1]), ""); + A(0==close(ep->ep), ""); + free(ep); +} + +static void* routine(void* arg) { + A(arg, "invalid argument"); + ep_t *ep = (ep_t*)arg; + + while (!ep->stopping) { + struct epoll_event evs[10] = {0}; + + A(0==pthread_mutex_lock(&ep->lock), ""); + A(ep->waiting==0, "internal logic error"); + ep->waiting = 1; + A(0==pthread_mutex_unlock(&ep->lock), ""); + + int r = epoll_wait(ep->ep, evs, sizeof(evs)/sizeof(evs[0]), -1); + A(r>0, "indefinite epoll_wait shall not timeout"); + + A(0==pthread_mutex_lock(&ep->lock), ""); + A(ep->waiting==1, "internal logic error"); + ep->waiting = 0; + A(0==pthread_mutex_unlock(&ep->lock), ""); + + for (int i=0; idata.ptr == &ep_dummy) { + char c = '\0'; + A(1==recv(ep->sv[0], &c, 1, 0), "internal logic error"); + A(0==pthread_mutex_lock(&ep->lock), ""); + ep->wakenup = 0; + A(0==pthread_mutex_unlock(&ep->lock), ""); + D("........"); + continue; + } + A(ev->data.ptr, "internal logic error"); + int skt = *(int*)ev->data.ptr; + if (ev->events & EPOLLIN) { + char buf[4]; + int n = recv(skt, buf, sizeof(buf)-1, 0); + A(n>=0 && nevents, buf); + } + if (ev->events & EPOLLRDHUP) { + A(0==epoll_ctl(ep->ep, EPOLL_CTL_DEL, skt, NULL), ""); + } + continue; + } + } + return NULL; +} + +static int open_connect(unsigned short port) { + int r = 0; + int skt = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + if (skt==-1) { + E("socket() failed"); + return -1; + } + do { + struct sockaddr_in si = {0}; + si.sin_family = AF_INET; + si.sin_addr.s_addr = inet_addr("127.0.0.1"); + si.sin_port = htons(port); + r = connect(skt, (struct sockaddr*)&si, sizeof(si)); + if (r) { + E("connect(%u) failed", port); + break; + } + memset(&si, 0, sizeof(si)); + socklen_t len = sizeof(si); + r = getsockname(skt, (struct sockaddr *)&si, &len); + if (r) { + E("getsockname() failed"); + } + A(len==sizeof(si), "internal logic error"); + D("connected: %d", ntohs(si.sin_port)); + return skt; + } while (0); + close(skt); + return -1; +} + From a47e778b5cc30388110abc7e6e227bf2ea65dc74 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 15 Jan 2021 22:45:31 +0800 Subject: [PATCH 08/62] TD-1207 --- src/os/src/darwin/darwinFile.c | 38 ++++++++-- src/os/src/detail/osFile.c | 4 -- src/os/src/windows/wFile.c | 125 +++++++++++++++++++++++++++------ src/os/src/windows/wSysinfo.c | 17 ++++- 4 files changed, 153 insertions(+), 31 deletions(-) diff --git a/src/os/src/darwin/darwinFile.c b/src/os/src/darwin/darwinFile.c index dacf4db741..1e77cd68d8 100644 --- a/src/os/src/darwin/darwinFile.c +++ b/src/os/src/darwin/darwinFile.c @@ -51,7 +51,37 @@ int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t co return writeLen; } -int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t* offset, int64_t size) { - uError("taosSendFile not implemented yet"); - return -1; -} \ No newline at end of file +int64_t taosSendFile(SOCKET dfd, int32_t sfd, int64_t* offset, int64_t count) { + lseek(sfd, (int32_t)(*offset), 0); + int64_t writeLen = 0; + uint8_t buffer[_SEND_FILE_STEP_] = { 0 }; + + for (int64_t len = 0; len < (count - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) { + int32_t rlen = (int32_t)read(sfd, buffer, _SEND_FILE_STEP_); + if (rlen <= 0) { + return writeLen; + } + else if (rlen < _SEND_FILE_STEP_) { + taosWriteSocket(dfd, buffer, rlen); + return (int64_t)(writeLen + rlen); + } + else { + taosWriteSocket(dfd, buffer, _SEND_FILE_STEP_); + writeLen += _SEND_FILE_STEP_; + } + } + + int64_t remain = count - writeLen; + if (remain > 0) { + int32_t rlen = read(sfd, buffer, (int32_t)remain); + if (rlen <= 0) { + return writeLen; + } + else { + taosWriteSocket(sfd, buffer, (int32_t)remain); + writeLen += remain; + } + } + + return writeLen; +} diff --git a/src/os/src/detail/osFile.c b/src/os/src/detail/osFile.c index 6ba38635f4..bb68622731 100644 --- a/src/os/src/detail/osFile.c +++ b/src/os/src/detail/osFile.c @@ -126,10 +126,6 @@ int64_t taosSendFile(SOCKET dfd, int32_t sfd, int64_t *offset, int64_t size) { int64_t sentbytes; while (leftbytes > 0) { - /* - * TODO : Think to check if file is larger than 1GB - */ - // if (leftbytes > 1000000000) leftbytes = 1000000000; sentbytes = sendfile(dfd, sfd, offset, leftbytes); if (sentbytes == -1) { if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) { diff --git a/src/os/src/windows/wFile.c b/src/os/src/windows/wFile.c index a127c17ce7..e61218cac3 100644 --- a/src/os/src/windows/wFile.c +++ b/src/os/src/windows/wFile.c @@ -15,18 +15,19 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "tulog.h" +#include "osSocket.h" #include "tglobal.h" +#include "tulog.h" void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath) { - const char* tdengineTmpFileNamePrefix = "tdengine-"; - char tmpPath[PATH_MAX]; + const char *tdengineTmpFileNamePrefix = "tdengine-"; + char tmpPath[PATH_MAX]; int32_t len = (int32_t)strlen(tsTempDir); memcpy(tmpPath, tsTempDir, len); if (tmpPath[len - 1] != '/' && tmpPath[len - 1] != '\\') { - tmpPath[len++] = '\\'; + tmpPath[len++] = '\\'; } strcpy(tmpPath + len, tdengineTmpFileNamePrefix); @@ -35,7 +36,7 @@ void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath) { strcat(tmpPath, fileNamePrefix); strcat(tmpPath, "-%d-%s"); } - + char rand[8] = {0}; taosRandStr(rand, tListLen(rand) - 1); snprintf(dstPath, PATH_MAX, tmpPath, getpid(), rand); @@ -46,18 +47,16 @@ void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath) { int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t count) { fseek(in_file, (int32_t)(*offset), 0); int64_t writeLen = 0; - uint8_t buffer[_SEND_FILE_STEP_] = { 0 }; - + uint8_t buffer[_SEND_FILE_STEP_] = {0}; + for (int64_t len = 0; len < (count - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) { size_t rlen = fread(buffer, 1, _SEND_FILE_STEP_, in_file); if (rlen <= 0) { return writeLen; - } - else if (rlen < _SEND_FILE_STEP_) { + } else if (rlen < _SEND_FILE_STEP_) { fwrite(buffer, 1, rlen, out_file); return (int64_t)(writeLen + rlen); - } - else { + } else { fwrite(buffer, 1, _SEND_FILE_STEP_, in_file); writeLen += _SEND_FILE_STEP_; } @@ -65,12 +64,11 @@ int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t co int64_t remain = count - writeLen; if (remain > 0) { - size_t rlen = fread(buffer, 1, (size_t) remain, in_file); + size_t rlen = fread(buffer, 1, (size_t)remain, in_file); if (rlen <= 0) { return writeLen; - } - else { - fwrite(buffer, 1, (size_t) remain, out_file); + } else { + fwrite(buffer, 1, (size_t)remain, out_file); writeLen += remain; } } @@ -78,12 +76,99 @@ int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t co return writeLen; } -int64_t taosSendFile(SOCKET dfd, int32_t sfd, int64_t* offset, int64_t size) { - uError("taosSendFile no implemented yet"); +int64_t taosSendFile(SOCKET dfd, int32_t sfd, int64_t *offset, int64_t count) { + lseek(sfd, (int32_t)(*offset), 0); + int64_t writeLen = 0; + uint8_t buffer[_SEND_FILE_STEP_] = {0}; + + for (int64_t len = 0; len < (count - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) { + int32_t rlen = (int32_t)read(sfd, buffer, _SEND_FILE_STEP_); + if (rlen <= 0) { + return writeLen; + } else if (rlen < _SEND_FILE_STEP_) { + taosWriteSocket(dfd, buffer, rlen); + return (int64_t)(writeLen + rlen); + } else { + taosWriteSocket(dfd, buffer, _SEND_FILE_STEP_); + writeLen += _SEND_FILE_STEP_; + } + } + + int64_t remain = count - writeLen; + if (remain > 0) { + int32_t rlen = read(sfd, buffer, (int32_t)remain); + if (rlen <= 0) { + return writeLen; + } else { + taosWriteSocket(sfd, buffer, (int32_t)remain); + writeLen += remain; + } + } + + return writeLen; +} + +int32_t taosFtruncate(int32_t fd, int64_t l_size) { + if (fd < 0) { + errno = EBADF; + uError("%s\n", "fd arg was negative"); + return -1; + } + + HANDLE h = (HANDLE)_get_osfhandle(fd); + + LARGE_INTEGER li_0; + li_0.QuadPart = (int64_t)0; + BOOL cur = SetFilePointerEx(h, li_0, NULL, FILE_CURRENT); + if (!cur) { + uError("SetFilePointerEx Error getting current position in file.\n"); + return -1; + } + + LARGE_INTEGER li_size; + li_size.QuadPart = l_size; + BOOL cur2 = SetFilePointerEx(h, li_size, NULL, FILE_BEGIN); + if (cur2 == 0) { + int error = GetLastError(); + uError("SetFilePointerEx GetLastError is: %d\n", error); + switch (error) { + case ERROR_INVALID_HANDLE: + errno = EBADF; + break; + default: + errno = EIO; + break; + } + return -1; + } + + if (!SetEndOfFile(h)) { + int error = GetLastError(); + uError("SetEndOfFile GetLastError is:%d", error); + switch (error) { + case ERROR_INVALID_HANDLE: + errno = EBADF; + break; + default: + errno = EIO; + break; + } + return -1; + } + return 0; } -int32_t taosFtruncate(int32_t fd, int64_t length) { - uError("taosFtruncate no implemented yet"); + +int fsync(int filedes) { + if (filedes < 0) { + errno = EBADF; + uError("%s\n", "fd arg was negative"); + return -1; + } + + HANDLE h = (HANDLE)_get_osfhandle(filedes); + FlushFileBuffers(h); + return 0; -} \ No newline at end of file + } diff --git a/src/os/src/windows/wSysinfo.c b/src/os/src/windows/wSysinfo.c index 0499173a0b..c024521219 100644 --- a/src/os/src/windows/wSysinfo.c +++ b/src/os/src/windows/wSysinfo.c @@ -31,6 +31,8 @@ #pragma comment(lib, "Mswsock.lib ") #endif +#include + #pragma warning(push) #pragma warning(disable : 4091) #include @@ -233,8 +235,6 @@ int taosSystem(const char *cmd) { int flock(int fd, int option) { return 0; } -int fsync(int filedes) { return 0; } - int sigaction(int sig, struct sigaction *d, void *p) { return 0; } LONG WINAPI FlCrashDump(PEXCEPTION_POINTERS ep) { @@ -276,7 +276,18 @@ LONG WINAPI FlCrashDump(PEXCEPTION_POINTERS ep) { void taosSetCoreDump() { SetUnhandledExceptionFilter(&FlCrashDump); } bool taosGetSystemUid(char *uid) { - sprintf(uid, "uid_not_implemented_yet"); + GUID guid; + CoCreateGuid(&guid); + + sprintf( + uid, + "%08X-%04X-%04X-%02X%02X-%02X%02X%02X%02X%02X%02X", + guid.Data1, guid.Data2, guid.Data3, + guid.Data4[0], guid.Data4[1], + guid.Data4[2], guid.Data4[3], + guid.Data4[4], guid.Data4[5], + guid.Data4[6], guid.Data4[7]); + return true; } From 499644d5a7afc6c12d72d682b31915c9fc284b07 Mon Sep 17 00:00:00 2001 From: freemine Date: Fri, 15 Jan 2021 23:11:24 +0800 Subject: [PATCH 09/62] epoll.c test in CMakeLists.txt --- tests/examples/c/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/examples/c/CMakeLists.txt b/tests/examples/c/CMakeLists.txt index 4689ed8868..7cacefda57 100644 --- a/tests/examples/c/CMakeLists.txt +++ b/tests/examples/c/CMakeLists.txt @@ -5,6 +5,8 @@ IF (TD_LINUX) AUX_SOURCE_DIRECTORY(. SRC) ADD_EXECUTABLE(demo demo.c) TARGET_LINK_LIBRARIES(demo taos_static trpc tutil pthread ) + ADD_EXECUTABLE(epoll epoll.c) + TARGET_LINK_LIBRARIES(epoll taos_static trpc tutil pthread ) ENDIF () IF (TD_DARWIN) INCLUDE_DIRECTORIES(. ${TD_COMMUNITY_DIR}/src/inc ${TD_COMMUNITY_DIR}/src/client/inc ${TD_COMMUNITY_DIR}/inc) From 65a28796cdc267c12ee8706adc288a6ec5f3adcf Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 16 Jan 2021 12:59:40 +0800 Subject: [PATCH 10/62] TD-1207 --- src/dnode/src/dnodeSystem.c | 69 ++++++++++++++++------------------- src/kit/shell/src/shellMain.c | 12 +++--- src/os/inc/os.h | 1 + src/os/inc/osSignal.h | 56 ++++++++++++++++++++++++++++ src/os/inc/osWindows.h | 18 --------- src/os/src/detail/osSignal.c | 38 +++++++++++++++++++ src/os/src/linux/linuxEnv.c | 4 ++ src/os/src/windows/wSysinfo.c | 2 - src/sync/src/syncArbitrator.c | 29 ++++++--------- tests/tsim/src/simExe.c | 4 +- tests/tsim/src/simMain.c | 2 +- 11 files changed, 148 insertions(+), 87 deletions(-) create mode 100644 src/os/inc/osSignal.h create mode 100644 src/os/src/detail/osSignal.c diff --git a/src/dnode/src/dnodeSystem.c b/src/dnode/src/dnodeSystem.c index a16ff826bf..bdaecf2e0f 100644 --- a/src/dnode/src/dnodeSystem.c +++ b/src/dnode/src/dnodeSystem.c @@ -19,8 +19,10 @@ #include "tconfig.h" #include "dnodeMain.h" -static void signal_handler(int32_t signum, siginfo_t *sigInfo, void *context); static tsem_t exitSem; +static void siguser1Handler(int32_t signum); +static void siguser2Handler(int32_t signum); +static void sigintHandler(int32_t signum); int32_t main(int32_t argc, char *argv[]) { int dump_config = 0; @@ -28,7 +30,7 @@ int32_t main(int32_t argc, char *argv[]) { // Set global configuration file for (int32_t i = 1; i < argc; ++i) { if (strcmp(argv[i], "-c") == 0) { - if (i < argc - 1) { + if (i < argc - 1) { if (strlen(argv[++i]) >= TSDB_FILENAME_LEN) { printf("config file path overflow"); exit(EXIT_FAILURE); @@ -80,8 +82,8 @@ int32_t main(int32_t argc, char *argv[]) { taosSetRandomFileFailOutput(NULL); } } else if (strcmp(argv[i], "--random-file-fail-factor") == 0) { - if ( (i+1) < argc ) { - int factor = atoi(argv[i+1]); + if ((i + 1) < argc) { + int factor = atoi(argv[i + 1]); printf("The factor of random failure is %d\n", factor); taosSetRandomFileFailFactor(factor); } else { @@ -93,7 +95,7 @@ int32_t main(int32_t argc, char *argv[]) { } if (0 != dump_config) { - tscEmbedded = 1; + tscEmbedded = 1; taosInitGlobalCfg(); taosReadGlobalLogCfg(); @@ -112,14 +114,12 @@ int32_t main(int32_t argc, char *argv[]) { } /* Set termination handler. */ - struct sigaction act = {{0}}; - act.sa_flags = SA_SIGINFO; - act.sa_sigaction = signal_handler; - sigaction(SIGTERM, &act, NULL); - sigaction(SIGHUP, &act, NULL); - sigaction(SIGINT, &act, NULL); - sigaction(SIGUSR1, &act, NULL); - sigaction(SIGUSR2, &act, NULL); + taosSetSignal(SIGUSR1, siguser1Handler); + taosSetSignal(SIGUSR2, siguser2Handler); + taosSetSignal(SIGTERM, sigintHandler); + taosSetSignal(SIGHUP, sigintHandler); + taosSetSignal(SIGINT, sigintHandler); + taosSetSignal(SIGABRT, sigintHandler); // Open /var/log/syslog file to record information. openlog("TDengine:", LOG_PID | LOG_CONS | LOG_NDELAY, LOG_LOCAL1); @@ -147,32 +147,25 @@ int32_t main(int32_t argc, char *argv[]) { return EXIT_SUCCESS; } -static void signal_handler(int32_t signum, siginfo_t *sigInfo, void *context) { - if (signum == SIGUSR1) { - taosCfgDynamicOptions("debugFlag 143"); - return; - } - if (signum == SIGUSR2) { - taosCfgDynamicOptions("resetlog"); - return; - } +static void siguser1Handler(int32_t signum) { taosCfgDynamicOptions("debugFlag 143"); } - syslog(LOG_INFO, "Shut down signal is %d", signum); - syslog(LOG_INFO, "Shutting down TDengine service..."); - // clean the system. - dInfo("shut down signal is %d, sender PID:%d cmdline:%s", signum, sigInfo->si_pid, taosGetCmdlineByPID(sigInfo->si_pid)); +static void siguser2Handler(int32_t signum) { taosCfgDynamicOptions("resetlog"); } - // protect the application from receive another signal - struct sigaction act = {{0}}; -#ifndef WINDOWS - act.sa_handler = SIG_IGN; -#endif - sigaction(SIGTERM, &act, NULL); - sigaction(SIGHUP, &act, NULL); - sigaction(SIGINT, &act, NULL); - sigaction(SIGUSR1, &act, NULL); - sigaction(SIGUSR2, &act, NULL); +static void sigintHandler(int32_t signum) { + // clean the system. + dInfo("shut down signal is %d", signum); - // inform main thread to exit - tsem_post(&exitSem); + syslog(LOG_INFO, "Shut down signal is %d", signum); + syslog(LOG_INFO, "Shutting down TDengine service..."); + + // protect the application from receive another signal + taosIgnSignal(SIGUSR1); + taosIgnSignal(SIGUSR2); + taosIgnSignal(SIGTERM); + taosIgnSignal(SIGHUP); + taosIgnSignal(SIGINT); + taosIgnSignal(SIGABRT); + + // inform main thread to exit + tsem_post(&exitSem); } \ No newline at end of file diff --git a/src/kit/shell/src/shellMain.c b/src/kit/shell/src/shellMain.c index cd801ef07a..c7a0dab2dd 100644 --- a/src/kit/shell/src/shellMain.c +++ b/src/kit/shell/src/shellMain.c @@ -21,7 +21,7 @@ pthread_t pid; static tsem_t cancelSem; -void shellQueryInterruptHandler(int32_t signum, siginfo_t *sigInfo, void *context) { +void shellQueryInterruptHandler(int32_t signum) { tsem_post(&cancelSem); } @@ -130,12 +130,10 @@ int main(int argc, char* argv[]) { pthread_create(&spid, NULL, cancelHandler, NULL); /* Interrupt handler. */ - struct sigaction act; - memset(&act, 0, sizeof(struct sigaction)); - - act.sa_handler = shellQueryInterruptHandler; - sigaction(SIGTERM, &act, NULL); - sigaction(SIGINT, &act, NULL); + taosSetSignal(SIGTERM, shellQueryInterruptHandler); + taosSetSignal(SIGINT, shellQueryInterruptHandler); + taosSetSignal(SIGHUP, shellQueryInterruptHandler); + taosSetSignal(SIGABRT, shellQueryInterruptHandler); /* Get grant information */ shellGetGrantInfo(con); diff --git a/src/os/inc/os.h b/src/os/inc/os.h index 9383ae48dc..8312b74a50 100644 --- a/src/os/inc/os.h +++ b/src/os/inc/os.h @@ -62,6 +62,7 @@ extern "C" { #include "osMemory.h" #include "osRand.h" #include "osSemphone.h" +#include "osSignal.h" #include "osSocket.h" #include "osString.h" #include "osSysinfo.h" diff --git a/src/os/inc/osSignal.h b/src/os/inc/osSignal.h new file mode 100644 index 0000000000..8b047b9e25 --- /dev/null +++ b/src/os/inc/osSignal.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_OS_SIGNAL_H +#define TDENGINE_OS_SIGNAL_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "os.h" +#include "taosdef.h" +#include + +#ifndef SIGALRM + #define SIGALRM 1234 +#endif + +#ifndef SIGHUP + #define SIGHUP 1234 +#endif + +#ifndef SIGCHLD + #define SIGCHLD 1234 +#endif + +#ifndef SIGUSR1 + #define SIGUSR1 1234 +#endif + +#ifndef SIGUSR2 + #define SIGUSR2 1234 +#endif + +typedef void (*FSignalHandler)(int32_t signum); +void taosSetSignal(int32_t signum, FSignalHandler sigfp); +void taosIgnSignal(int32_t signum); +void taosDflSignal(int32_t signum); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_TTIME_H diff --git a/src/os/inc/osWindows.h b/src/os/inc/osWindows.h index 15b26d208d..1b28cfa90f 100644 --- a/src/os/inc/osWindows.h +++ b/src/os/inc/osWindows.h @@ -191,24 +191,6 @@ int gettimeofday(struct timeval *ptv, void *pTimeZone); #define PATH_MAX 256 #endif -//for signal, not dispose -#define SIGALRM 1234 -#define SIGHUP 1234 -#define SIGUSR1 1234 -#define SIGUSR2 1234 -#define SA_SIGINFO 1234 - -typedef int sigset_t; -typedef struct siginfo_t { - int si_pid; -} siginfo_t; -struct sigaction { - int sa_flags; - void (*sa_handler)(int32_t signum, siginfo_t *sigInfo, void *context); - void (*sa_sigaction)(int32_t signum, siginfo_t *sigInfo, void *context); -}; -int sigaction(int, struct sigaction *, void *); - typedef struct { int we_wordc; char **we_wordv; diff --git a/src/os/src/detail/osSignal.c b/src/os/src/detail/osSignal.c new file mode 100644 index 0000000000..c97a3343de --- /dev/null +++ b/src/os/src/detail/osSignal.c @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "tconfig.h" +#include "tglobal.h" +#include "tulog.h" + +#ifndef TAOS_OS_FUNC_SIGNAL + +void taosSetSignal(int32_t signum, FSignalHandler sigfp) { + struct sigaction act = {{0}}; + act.sa_handler = sigfp; + sigaction(signum, &act, NULL); +} + +void taosIgnSignal(int32_t signum) { + signal(signum, SIG_IGN); +} + +void taosDflSignal(int32_t signum) { + signal(signum, SIG_DFL); +} + +#endif diff --git a/src/os/src/linux/linuxEnv.c b/src/os/src/linux/linuxEnv.c index e3eadbc94b..abcdabcfd5 100644 --- a/src/os/src/linux/linuxEnv.c +++ b/src/os/src/linux/linuxEnv.c @@ -43,6 +43,7 @@ void osInit() { char cmdline[1024]; char* taosGetCmdlineByPID(int pid) { +#if 0 sprintf(cmdline, "/proc/%d/cmdline", pid); FILE* f = fopen(cmdline, "r"); if (f) { @@ -54,4 +55,7 @@ char* taosGetCmdlineByPID(int pid) { fclose(f); } return cmdline; +#else + return ""; +#endif } diff --git a/src/os/src/windows/wSysinfo.c b/src/os/src/windows/wSysinfo.c index c024521219..9a79f5a6bf 100644 --- a/src/os/src/windows/wSysinfo.c +++ b/src/os/src/windows/wSysinfo.c @@ -235,8 +235,6 @@ int taosSystem(const char *cmd) { int flock(int fd, int option) { return 0; } -int sigaction(int sig, struct sigaction *d, void *p) { return 0; } - LONG WINAPI FlCrashDump(PEXCEPTION_POINTERS ep) { typedef BOOL(WINAPI * FxMiniDumpWriteDump)(IN HANDLE hProcess, IN DWORD ProcessId, IN HANDLE hFile, IN MINIDUMP_TYPE DumpType, diff --git a/src/sync/src/syncArbitrator.c b/src/sync/src/syncArbitrator.c index 06b4a61d6f..4fc5e2196c 100644 --- a/src/sync/src/syncArbitrator.c +++ b/src/sync/src/syncArbitrator.c @@ -27,7 +27,7 @@ #include "syncInt.h" #include "syncTcp.h" -static void arbSignalHandler(int32_t signum, siginfo_t *sigInfo, void *context); +static void arbSignalHandler(int32_t signum); static void arbProcessIncommingConnection(SOCKET connFd, uint32_t sourceIp); static void arbProcessBrokenLink(int64_t rid); static int32_t arbProcessPeerMsg(int64_t rid, void *buffer); @@ -69,14 +69,10 @@ int32_t main(int32_t argc, char *argv[]) { } /* Set termination handler. */ - struct sigaction act = {{0}}; - act.sa_flags = SA_SIGINFO; - act.sa_sigaction = arbSignalHandler; - - act.sa_handler = arbSignalHandler; - sigaction(SIGTERM, &act, NULL); - sigaction(SIGHUP, &act, NULL); - sigaction(SIGINT, &act, NULL); + taosSetSignal(SIGTERM, arbSignalHandler); + taosSetSignal(SIGINT, arbSignalHandler); + taosSetSignal(SIGHUP, arbSignalHandler); + taosSetSignal(SIGABRT, arbSignalHandler); tsAsyncLog = 0; strcat(arbLogPath, "/arbitrator.log"); @@ -174,16 +170,13 @@ static int32_t arbProcessPeerMsg(int64_t rid, void *buffer) { return 0; } -static void arbSignalHandler(int32_t signum, siginfo_t *sigInfo, void *context) { - struct sigaction act = {{0}}; -#ifndef WINDOWS - act.sa_handler = SIG_IGN; -#endif - sigaction(SIGTERM, &act, NULL); - sigaction(SIGHUP, &act, NULL); - sigaction(SIGINT, &act, NULL); +static void arbSignalHandler(int32_t signum) { + taosIgnSignal(SIGTERM); + taosIgnSignal(SIGINT); + taosIgnSignal(SIGABRT); + taosIgnSignal(SIGHUP); - sInfo("shut down signal is %d, sender PID:%d", signum, sigInfo->si_pid); + sInfo("shut down signal is %d", signum); // inform main thread to exit tsem_post(&tsArbSem); diff --git a/tests/tsim/src/simExe.c b/tests/tsim/src/simExe.c index a23dff13be..10e87eefe6 100644 --- a/tests/tsim/src/simExe.c +++ b/tests/tsim/src/simExe.c @@ -314,9 +314,7 @@ bool simExecuteSystemCmd(SScript *script, char *option) { simError("script:%s, failed to execute %s , code %d, errno:%d %s, repeatTimes:%d", script->fileName, buf, code, errno, strerror(errno), repeatTimes); taosMsleep(1000); -#ifdef LINUX - signal(SIGCHLD, SIG_DFL); -#endif + taosDflSignal(SIGCHLD); if (repeatTimes++ >= 10) { exit(0); } diff --git a/tests/tsim/src/simMain.c b/tests/tsim/src/simMain.c index 8f13254f68..de560901aa 100644 --- a/tests/tsim/src/simMain.c +++ b/tests/tsim/src/simMain.c @@ -51,7 +51,7 @@ int32_t main(int32_t argc, char *argv[]) { } simInfo("simulator is running ..."); - signal(SIGINT, simHandleSignal); + taosSetSignal(SIGINT, simHandleSignal); SScript *script = simParseScript(scriptFile); if (script == NULL) { From 1e7037e8e17210a0e64bf9f130e11b4734e2675f Mon Sep 17 00:00:00 2001 From: freemine Date: Sat, 16 Jan 2021 13:11:33 +0800 Subject: [PATCH 11/62] epoll and kqueue --- src/os/src/darwin/eok.c | 199 +++++++++++++++++++++++++++++++++++++-- tests/examples/c/epoll.c | 155 ++++++++++++++++++++++++++---- 2 files changed, 329 insertions(+), 25 deletions(-) diff --git a/src/os/src/darwin/eok.c b/src/os/src/darwin/eok.c index ea67223c25..f83846b734 100644 --- a/src/os/src/darwin/eok.c +++ b/src/os/src/darwin/eok.c @@ -39,6 +39,7 @@ struct ep_over_kq_s { int sv[2]; // 0 for read, 1 for write + int evs_count; eok_event_t *evs_head; eok_event_t *evs_tail; eok_event_t *evs_free; @@ -82,6 +83,139 @@ static eoks_t eoks = { .eoks_free = NULL, }; +static const char* op_str(int op) { + switch (op) { + case EPOLL_CTL_ADD: return "EPOLL_CTL_ADD"; + case EPOLL_CTL_MOD: return "EPOLL_CTL_MOD"; + case EPOLL_CTL_DEL: return "EPOLL_CTL_DEL"; + default: return "UNKNOWN"; + } +} + +static __thread char buf_slots[10][1024] = {0}; +static __thread int buf_slots_linelen = sizeof(buf_slots[0])/sizeof(buf_slots[0][0]); +static __thread int buf_slots_count = sizeof(buf_slots)/(sizeof(buf_slots[0])/sizeof(buf_slots[0][0])); +static const char* events_str(uint32_t events, int slots) { + A(slots>=0 && slots0 && (events & (ev))==(ev)) { \ + n = snprintf(p, len, "%s%s", p!=buf ? "|" : "", #ev); \ + p += n; \ + len -= n; \ + } + CHK_EV(EPOLLIN); + CHK_EV(EPOLLPRI); + CHK_EV(EPOLLOUT); + CHK_EV(EPOLLRDNORM); + CHK_EV(EPOLLRDBAND); + CHK_EV(EPOLLWRNORM); + CHK_EV(EPOLLWRBAND); + CHK_EV(EPOLLMSG); + CHK_EV(EPOLLERR); + CHK_EV(EPOLLHUP); + CHK_EV(EPOLLRDHUP); + CHK_EV(EPOLLEXCLUSIVE); + CHK_EV(EPOLLWAKEUP); + CHK_EV(EPOLLONESHOT); + CHK_EV(EPOLLET); +#undef CHK_EV + return buf; +} + +static const char* kev_flags_str(uint16_t flags, int slots) { + A(slots>=0 && slots0 && (flags & (ev))==(ev)) { \ + n = snprintf(p, len, "%s%s", p!=buf ? "|" : "", #ev); \ + p += n; \ + len -= n; \ + } + CHK_EV(EV_ADD); + CHK_EV(EV_DELETE); + CHK_EV(EV_ENABLE); + CHK_EV(EV_DISABLE); + CHK_EV(EV_ONESHOT); + CHK_EV(EV_CLEAR); + CHK_EV(EV_RECEIPT); + CHK_EV(EV_DISPATCH); + CHK_EV(EV_UDATA_SPECIFIC); + CHK_EV(EV_DISPATCH2); + CHK_EV(EV_VANISHED); + CHK_EV(EV_SYSFLAGS); + CHK_EV(EV_FLAG0); + CHK_EV(EV_FLAG1); + CHK_EV(EV_EOF); + CHK_EV(EV_ERROR); +#undef CHK_EV + return buf; +} + static ep_over_kq_t* eoks_alloc(void); static void eoks_free(ep_over_kq_t *eok); static ep_over_kq_t* eoks_find(int epfd); @@ -135,10 +269,15 @@ int epoll_create(int size) { return -1; } + D("epoll_create epfd:[%d]", eok->idx); return eok->idx; } int epoll_ctl(int epfd, int op, int fd, struct epoll_event *event) { + D("epoll_ctling epfd:[%d], op:[%s], fd:[%d], events:[%04x:%s]", + epfd, op_str(op), fd, + event ? event->events : 0, + event ? events_str(event->events, 0) : "NULL"); int e = 0; if (epfd<0 || epfd>=eoks.neoks) { errno = EBADF; @@ -198,6 +337,7 @@ int epoll_ctl(int epfd, int op, int fd, struct epoll_event *event) { ev.changed = 2; } break; case EPOLL_CTL_DEL: { + D("epoll_ctl adding..."); // event is ignored pev = &oev->epev; flags = EV_DELETE; @@ -213,11 +353,9 @@ int epoll_ctl(int epfd, int op, int fd, struct epoll_event *event) { if (pev->events & (EPOLLIN | EPOLLERR | EPOLLHUP | EPOLLRDHUP)) { flags |= EV_EOF; EV_SET64(&krev, ev.fd, EVFILT_READ, flags, 0, 0, -1, 0, 0); - D("...."); } if (pev->events & EPOLLOUT) { EV_SET64(&kwev, ev.fd, EVFILT_WRITE, flags, 0, 0, -1, 0, 0); - D("...."); } if (eok_chgs_refresh(eok, oev, &ev, &krev, &kwev)) { @@ -244,14 +382,17 @@ int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) int e = 0; if (epfd<0 || epfd>=eoks.neoks) { errno = EBADF; + E("epoll_waiting epfd:[%d], maxevents:[%d], timeout:[%d] failed", epfd, maxevents, timeout); return -1; } if (!events) { errno = EINVAL; + E("epoll_waiting epfd:[%d], maxevents:[%d], timeout:[%d] failed", epfd, maxevents, timeout); return -1; } if (maxevents<=0) { errno = EINVAL; + E("epoll_waiting epfd:[%d], maxevents:[%d], timeout:[%d] failed", epfd, maxevents, timeout); return -1; } @@ -260,6 +401,7 @@ int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) ep_over_kq_t *eok = eoks_find(epfd); if (!eok) { errno = EBADF; + E("epoll_waiting epfd:[%d], maxevents:[%d], timeout:[%d] failed", epfd, maxevents, timeout); return -1; } @@ -281,6 +423,7 @@ int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) struct kevent64_s *eventslist = eok_alloc_eventslist(eok, maxevents); if (!eventslist) { e = ENOMEM; + E("epoll_waiting epfd:[%d], maxevents:[%d], timeout:[%d] failed", epfd, maxevents, timeout); break; } memset(eventslist, 0, maxevents * sizeof(*eventslist)); @@ -305,8 +448,15 @@ int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) eok->ichanges = 0; A(0==pthread_mutex_unlock(&eok->lock), ""); + if (ichanges>0) { + D("kevent64 changing [%d] changes and waiting...", ichanges); + } + errno = 0; r = kevent64(eok->kq, kchanges, ichanges, eventslist, maxevents, 0, pto); e = errno; + if (e) { + E("kevent64 waiting done, with r[%d]", r); + } A(0==pthread_mutex_lock(&eok->lock), ""); eok->waiting = 0; @@ -318,7 +468,6 @@ int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) } eok->waiting = 0; - if (r<0) break; if (r==0) { A(timeout!=-1, "internal logic error"); } @@ -327,7 +476,10 @@ int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) A(kev->udata && eok->evs_head && eok->evs_tail, "internal logic error"); eok_event_t *ev = (eok_event_t*)kev->udata; A(kev->ident == ev->fd, "internal logic error"); - D("..."); + if (kev->flags & EV_ERROR) { + D("error when processing change list for fd[%d], error[%s], kev_flags:[%04x:%s]", + ev->fd, strerror(kev->data), kev->flags, kev_flags_str(kev->flags, 0)); + } switch (kev->filter) { case EVFILT_READ: { A((ev->epev.events & EPOLLIN), "internal logic errro"); @@ -335,10 +487,10 @@ int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) char c = '\0'; A(1==recv(kev->ident, &c, 1, 0), "internal logic error"); A(0==memcmp(&c, "1", 1), "internal logic error"); - D("..............."); + D("wokenup"); } else { if (ev->changed==3) { - D("already requested to delete"); + D("already requested to delete for fd[%d]", ev->fd); // EV_DELETE? continue; } @@ -347,8 +499,13 @@ int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) pev.events = EPOLLIN; if (kev->flags & EV_EOF) { pev.events |= (EPOLLHUP | EPOLLERR | EPOLLRDHUP); + D(".........."); } - pev.events &= ev->epev.events; + pev.events = pev.events & ev->epev.events; + D("events found for fd[%d]: [%04x:%s], which was registered: [%04x:%s], kev_flags: [%04x:%s]", + ev->fd, pev.events, events_str(pev.events, 0), + ev->epev.events, events_str(ev->epev.events, 1), + kev->flags, kev_flags_str(kev->flags, 2)); events[cnts++] = pev; } } break; @@ -360,11 +517,26 @@ int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) } break; } } + if (r>=0) { + eok_event_t *p = eok->evs_head; + while (p) { + eok_event_t *next = p->next; + if (p->changed==3) { + D("removing registered event for fd[%d]: [%04x:%s]", p->fd, p->epev.events, events_str(p->epev.events, 0)); + eok_free_ev(eok, p); + } + p = next; + } + } } while (cnts==0); + if (cnts>0) { + D("kevent64 waiting done with [%d] events", cnts); + } A(0==pthread_mutex_unlock(&eok->lock), ""); if (e) { errno = e; + E("epoll_wait failed"); return -1; } @@ -383,6 +555,7 @@ static struct timespec do_timespec_diff(struct timespec *from, struct timespec * } int epoll_close(int epfd) { + D("epoll_closing epfd: [%d]", epfd); if (epfd<0 || epfd>=eoks.neoks) { errno = EBADF; return -1; @@ -445,6 +618,8 @@ static eok_event_t* eok_calloc_ev(ep_over_kq_t *eok) { else eok->evs_head = p; eok->evs_tail = p; + eok->evs_count += 1; + return p; } @@ -456,6 +631,8 @@ static void eok_free_ev(ep_over_kq_t *eok, eok_event_t *ev) { else eok->evs_tail = ev->prev; ev->next = eok->evs_free; eok->evs_free = ev->next; + + eok->evs_count -= 1; } static void eok_wakeup(ep_over_kq_t *eok) { @@ -486,17 +663,23 @@ static int eok_chgs_refresh(ep_over_kq_t *eok, eok_event_t *oev, eok_event_t *ev } oev->fd = ev->fd; - oev->epev = ev->epev; + if (ev->changed!=3) { + oev->epev = ev->epev; + } oev->changed = ev->changed; + n = 0; if (krev->ident==ev->fd) { krev->udata = (uint64_t)oev; eok->kchanges[eok->ichanges++] = *krev; + ++n; } if (kwev->ident==ev->fd) { kwev->udata = (uint64_t)oev; eok->kchanges[eok->ichanges++] = *kwev; + ++n; } + D("add changes[%d] for fd[%d], and changes/registers [%d/%d]", n, ev->fd, eok->ichanges, eok->evs_count); return 0; } diff --git a/tests/examples/c/epoll.c b/tests/examples/c/epoll.c index e0d63bae32..6047c9e4ea 100644 --- a/tests/examples/c/epoll.c +++ b/tests/examples/c/epoll.c @@ -54,18 +54,74 @@ static ep_t* ep_create(void); static void ep_destroy(ep_t *ep); static void* routine(void* arg); static int open_connect(unsigned short port); +static int open_listen(unsigned short port); + +typedef struct client_s client_t; +struct client_s { + int skt; + void (*on_event)(ep_t *ep, struct epoll_event *events, client_t *client); + volatile unsigned int state; // 1: listenning; 2: connected +}; + +static void echo_event(ep_t *ep, struct epoll_event *ev, client_t *client); int main(int argc, char *argv[]) { ep_t* ep = ep_create(); A(ep, "failed"); int skt = open_connect(6789); if (skt!=-1) { - struct epoll_event ev = {0}; - ev.events = EPOLLIN | EPOLLERR | EPOLLHUP | EPOLLRDHUP; - ev.data.ptr = &skt; - A(0==epoll_ctl(ep->ep, EPOLL_CTL_ADD, skt, &ev), ""); + client_t *client = (client_t*)calloc(1, sizeof(*client)); + if (client) { + client->skt = skt; + client->on_event = echo_event; + client->state = 2; + struct epoll_event ev = {0}; + ev.events = EPOLLIN | EPOLLERR | EPOLLHUP | EPOLLRDHUP; + ev.data.ptr = client; + A(0==epoll_ctl(ep->ep, EPOLL_CTL_ADD, skt, &ev), ""); + } + } + skt = open_listen(0); + if (skt!=-1) { + client_t *client = (client_t*)calloc(1, sizeof(*client)); + if (client) { + client->skt = skt; + client->on_event = echo_event; + client->state = 1; + struct epoll_event ev = {0}; + ev.events = EPOLLIN | EPOLLERR | EPOLLHUP | EPOLLRDHUP; + ev.data.ptr = client; + A(0==epoll_ctl(ep->ep, EPOLL_CTL_ADD, skt, &ev), ""); + } + } + // char c = '\0'; + // while ((c=getchar())!=EOF) { + // switch (c) { + // case 'q': break; + // default: continue; + // } + // } + // getchar(); + char *line = NULL; + size_t linecap = 0; + ssize_t linelen; + while ((linelen = getline(&line, &linecap, stdin)) > 0) { + line[strlen(line)-1] = '\0'; + if (0==strcmp(line, "exit")) break; + if (0==strcmp(line, "quit")) break; + if (line==strstr(line, "close")) { + int fd = 0; + sscanf(line, "close %d", &fd); + if (fd<=2) { + fprintf(stderr, "fd [%d] invalid\n", fd); + continue; + } + A(0==epoll_ctl(ep->ep, EPOLL_CTL_DEL, fd, NULL), ""); + continue; + } + if (strlen(line)==0) continue; + fprintf(stderr, "unknown cmd:[%s]\n", line); } - getchar(); ep_destroy(ep); D(""); return 0; @@ -114,7 +170,7 @@ static void* routine(void* arg) { A(0==pthread_mutex_unlock(&ep->lock), ""); int r = epoll_wait(ep->ep, evs, sizeof(evs)/sizeof(evs[0]), -1); - A(r>0, "indefinite epoll_wait shall not timeout"); + A(r>0, "indefinite epoll_wait shall not timeout:[%d]", r); A(0==pthread_mutex_lock(&ep->lock), ""); A(ep->waiting==1, "internal logic error"); @@ -133,23 +189,50 @@ static void* routine(void* arg) { continue; } A(ev->data.ptr, "internal logic error"); - int skt = *(int*)ev->data.ptr; - if (ev->events & EPOLLIN) { - char buf[4]; - int n = recv(skt, buf, sizeof(buf)-1, 0); - A(n>=0 && nevents, buf); - } - if (ev->events & EPOLLRDHUP) { - A(0==epoll_ctl(ep->ep, EPOLL_CTL_DEL, skt, NULL), ""); - } + client_t *client = (client_t*)ev->data.ptr; + client->on_event(ep, ev, client); continue; } } return NULL; } +static int open_listen(unsigned short port) { + int r = 0; + int skt = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + if (skt==-1) { + E("socket() failed"); + return -1; + } + do { + struct sockaddr_in si = {0}; + si.sin_family = AF_INET; + si.sin_addr.s_addr = inet_addr("127.0.0.1"); + si.sin_port = htons(port); + r = bind(skt, (struct sockaddr*)&si, sizeof(si)); + if (r) { + E("bind(%u) failed", port); + break; + } + r = listen(skt, 100); + if (r) { + E("listen() failed"); + break; + } + memset(&si, 0, sizeof(si)); + socklen_t len = sizeof(si); + r = getsockname(skt, (struct sockaddr *)&si, &len); + if (r) { + E("getsockname() failed"); + } + A(len==sizeof(si), "internal logic error"); + D("listenning at: %d", ntohs(si.sin_port)); + return skt; + } while (0); + close(skt); + return -1; +} + static int open_connect(unsigned short port) { int r = 0; int skt = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); @@ -181,3 +264,41 @@ static int open_connect(unsigned short port) { return -1; } +static void echo_event(ep_t *ep, struct epoll_event *ev, client_t *client) { + if (ev->events & EPOLLIN) { + if (client->state==1) { + struct sockaddr_in si = {0}; + socklen_t silen = sizeof(si); + int skt = accept(client->skt, (struct sockaddr*)&si, &silen); + if (skt!=-1) { + client_t *server = (client_t*)calloc(1, sizeof(*server)); + if (server) { + server->skt = skt; + server->on_event = echo_event; + server->state = 2; + struct epoll_event ev = {0}; + ev.events = EPOLLIN | EPOLLERR | EPOLLHUP | EPOLLRDHUP; + ev.data.ptr = server; + A(0==epoll_ctl(ep->ep, EPOLL_CTL_ADD, skt, &ev), ""); + } + } + } + if (client->state==2) { + char buf[4]; + int n = recv(client->skt, buf, sizeof(buf)-1, 0); + A(n>=0 && nevents, buf); + } + } + if (ev->events & (EPOLLERR | EPOLLHUP | EPOLLRDHUP)) { + A(0==pthread_mutex_lock(&ep->lock), ""); + A(0==epoll_ctl(ep->ep, EPOLL_CTL_DEL, client->skt, NULL), ""); + A(0==pthread_mutex_unlock(&ep->lock), ""); + close(client->skt); + client->skt = -1; + client->on_event = NULL; + free(client); + } +} + From 3998f0895e2a1a384bd93b4c1fd100ed4ee4f643 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 16 Jan 2021 23:16:17 +0800 Subject: [PATCH 12/62] TD-1207 --- src/os/inc/osWindows.h | 2 ++ src/os/src/windows/wFile.c | 10 +++++----- src/os/src/windows/wSignal.c | 31 +++++++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 5 deletions(-) create mode 100644 src/os/src/windows/wSignal.c diff --git a/src/os/inc/osWindows.h b/src/os/inc/osWindows.h index 1b28cfa90f..a984a5ba88 100644 --- a/src/os/inc/osWindows.h +++ b/src/os/inc/osWindows.h @@ -191,6 +191,8 @@ int gettimeofday(struct timeval *ptv, void *pTimeZone); #define PATH_MAX 256 #endif +#define TAOS_OS_FUNC_SIGNAL + typedef struct { int we_wordc; char **we_wordv; diff --git a/src/os/src/windows/wFile.c b/src/os/src/windows/wFile.c index e61218cac3..4ad195dc6f 100644 --- a/src/os/src/windows/wFile.c +++ b/src/os/src/windows/wFile.c @@ -77,7 +77,8 @@ int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t co } int64_t taosSendFile(SOCKET dfd, int32_t sfd, int64_t *offset, int64_t count) { - lseek(sfd, (int32_t)(*offset), 0); + if (offset != NULL) lseek(sfd, (int32_t)(*offset), 0); + int64_t writeLen = 0; uint8_t buffer[_SEND_FILE_STEP_] = {0}; @@ -168,7 +169,6 @@ int fsync(int filedes) { } HANDLE h = (HANDLE)_get_osfhandle(filedes); - FlushFileBuffers(h); - - return 0; - } + + return FlushFileBuffers(h); +} diff --git a/src/os/src/windows/wSignal.c b/src/os/src/windows/wSignal.c new file mode 100644 index 0000000000..6767076fc4 --- /dev/null +++ b/src/os/src/windows/wSignal.c @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include + +void taosSetSignal(int32_t signum, FSignalHandler sigfp) { + if (signum == SIGUSR1) return; + signal(signum, sigfp); +} + +void taosIgnSignal(int32_t signum) { + signal(signum, SIG_IGN); +} + +void taosDflSignal(int32_t signum) { + signal(signum, SIG_DFL); +} From 9fbe6e471eb9b9e4f35d883915702ed9c67fe59c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 16 Jan 2021 23:58:45 +0800 Subject: [PATCH 13/62] TD-1207 --- deps/zlib-1.2.11/src/gzlib.c | 4 +- src/client/src/tscParseInsert.c | 2 +- src/client/src/tscSub.c | 4 +- src/client/src/tscSubquery.c | 6 +- src/os/src/detail/osDir.c | 5 +- src/os/src/windows/wSignal.c | 2 + src/query/src/qTsbuf.c | 4 +- src/sync/src/syncRestore.c | 2 +- src/sync/src/syncRetrieve.c | 6 +- src/tsdb/src/tsdbCommit.c | 4 +- src/tsdb/src/tsdbFile.c | 4 +- src/tsdb/src/tsdbMain.c | 4 +- src/util/src/tfile.c | 4 +- src/util/src/tkvstore.c | 12 +-- tests/script/sh/deploy.bat | 155 ++++++++++++++++++++++++++++++++ tests/script/sh/exec.bat | 115 ++++++++++++++++++++++++ tests/script/sh/stop_dnodes.bat | 24 +++++ tests/script/wtest.bat | 37 ++++---- 18 files changed, 344 insertions(+), 50 deletions(-) create mode 100644 tests/script/sh/deploy.bat create mode 100644 tests/script/sh/exec.bat create mode 100644 tests/script/sh/stop_dnodes.bat diff --git a/deps/zlib-1.2.11/src/gzlib.c b/deps/zlib-1.2.11/src/gzlib.c index 381cdc2c7d..f3bd685be0 100644 --- a/deps/zlib-1.2.11/src/gzlib.c +++ b/deps/zlib-1.2.11/src/gzlib.c @@ -240,9 +240,9 @@ local gzFile gz_open(path, fd, mode) /* open the file with the appropriate flags (or just use fd) */ state->fd = fd > -1 ? fd : ( #ifdef WIDECHAR - fd == -2 ? _wopen(path, oflag, 0666) : + fd == -2 ? _wopen(path, oflag | O_BINARY, 0666) : #endif - open((const char *)path, oflag, 0666)); + open((const char *)path, oflag | O_BINARY, 0666)); if (state->fd == -1) { free(state->path); free(state); diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 7151c33393..27abb0d2da 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -1545,7 +1545,7 @@ void tscProcessMultiVnodesImportFromFile(SSqlObj *pSql) { SSqlObj *pNew = createSubqueryObj(pSql, 0, parseFileSendDataBlock, pSupporter, TSDB_SQL_INSERT, NULL); pCmd->count = 1; - FILE *fp = fopen(pCmd->payload, "r"); + FILE *fp = fopen(pCmd->payload, "rb"); if (fp == NULL) { pSql->res.code = TAOS_SYSTEM_ERROR(errno); tscError("%p failed to open file %s to load data from file, code:%s", pSql, pCmd->payload, tstrerror(pSql->res.code)); diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c index 7f0b174ad3..527531b31a 100644 --- a/src/client/src/tscSub.c +++ b/src/client/src/tscSub.c @@ -313,7 +313,7 @@ static int tscLoadSubscriptionProgress(SSub* pSub) { char buf[TSDB_MAX_SQL_LEN]; sprintf(buf, "%s/subscribe/%s", tsDataDir, pSub->topic); - FILE* fp = fopen(buf, "r"); + FILE* fp = fopen(buf, "rb"); if (fp == NULL) { tscDebug("subscription progress file does not exist: %s", pSub->topic); return 1; @@ -368,7 +368,7 @@ void tscSaveSubscriptionProgress(void* sub) { } sprintf(path, "%s/subscribe/%s", tsDataDir, pSub->topic); - FILE* fp = fopen(path, "w+"); + FILE* fp = fopen(path, "wb+"); if (fp == NULL) { tscError("failed to create progress file for subscription: %s", pSub->topic); return; diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 2622246111..c207e1a6cf 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -930,7 +930,7 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow if (numOfRows > 0) { // write the compressed timestamp to disk file if(pSupporter->f == NULL) { - pSupporter->f = fopen(pSupporter->path, "w"); + pSupporter->f = fopen(pSupporter->path, "wb"); if (pSupporter->f == NULL) { tscError("%p failed to create tmp file:%s, reason:%s", pSql, pSupporter->path, strerror(errno)); @@ -974,7 +974,7 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow // continue to retrieve ts-comp data from vnode if (!pRes->completed) { taosGetTmpfilePath("ts-join", pSupporter->path); - pSupporter->f = fopen(pSupporter->path, "w"); + pSupporter->f = fopen(pSupporter->path, "wb"); pRes->row = pRes->numOfRows; taos_fetch_rows_a(tres, tsCompRetrieveCallback, param); @@ -1000,7 +1000,7 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow taosGetTmpfilePath("ts-join", pSupporter->path); // TODO check for failure - pSupporter->f = fopen(pSupporter->path, "w"); + pSupporter->f = fopen(pSupporter->path, "wb"); pRes->row = pRes->numOfRows; // set the callback function diff --git a/src/os/src/detail/osDir.c b/src/os/src/detail/osDir.c index d3f0fda1a5..4f2985548c 100644 --- a/src/os/src/detail/osDir.c +++ b/src/os/src/detail/osDir.c @@ -54,6 +54,9 @@ int taosMkDir(const char *path, mode_t mode) { void taosRename(char* oldName, char *newName) { // if newName in not empty, rename return fail. // the newName must be empty or does not exist +#ifdef WINDOWS + remove(newName); +#endif if (rename(oldName, newName)) { uError("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno)); } else { @@ -117,7 +120,7 @@ int32_t taosCompressFile(char *srcFileName, char *destFileName) { goto cmp_end; } - int32_t fd = open(destFileName, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO); + int32_t fd = open(destFileName, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, S_IRWXU | S_IRWXG | S_IRWXO); if (fd < 0) { ret = -2; goto cmp_end; diff --git a/src/os/src/windows/wSignal.c b/src/os/src/windows/wSignal.c index 6767076fc4..8a3c2cfbd4 100644 --- a/src/os/src/windows/wSignal.c +++ b/src/os/src/windows/wSignal.c @@ -23,9 +23,11 @@ void taosSetSignal(int32_t signum, FSignalHandler sigfp) { } void taosIgnSignal(int32_t signum) { + if (signum == SIGUSR1) return; signal(signum, SIG_IGN); } void taosDflSignal(int32_t signum) { + if (signum == SIGUSR1) return; signal(signum, SIG_DFL); } diff --git a/src/query/src/qTsbuf.c b/src/query/src/qTsbuf.c index a53f8935ee..1f43c5b33c 100644 --- a/src/query/src/qTsbuf.c +++ b/src/query/src/qTsbuf.c @@ -23,7 +23,7 @@ STSBuf* tsBufCreate(bool autoDelete, int32_t order) { pTSBuf->autoDelete = autoDelete; taosGetTmpfilePath("join", pTSBuf->path); - pTSBuf->f = fopen(pTSBuf->path, "w+"); + pTSBuf->f = fopen(pTSBuf->path, "wb+"); if (pTSBuf->f == NULL) { free(pTSBuf); return NULL; @@ -59,7 +59,7 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { tstrncpy(pTSBuf->path, path, sizeof(pTSBuf->path)); - pTSBuf->f = fopen(pTSBuf->path, "r+"); + pTSBuf->f = fopen(pTSBuf->path, "rb+"); if (pTSBuf->f == NULL) { free(pTSBuf); return NULL; diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c index bd4f9bd3e1..99f4ce1c17 100644 --- a/src/sync/src/syncRestore.c +++ b/src/sync/src/syncRestore.c @@ -128,7 +128,7 @@ static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { minfo.name[sizeof(minfo.name) - 1] = 0; snprintf(name, sizeof(name), "%s/%s", pNode->path, minfo.name); - int32_t dfd = open(name, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO); + int32_t dfd = open(name, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, S_IRWXU | S_IRWXG | S_IRWXO); if (dfd < 0) { sError("%s, failed to open file:%s while restore file since %s", pPeer->id, minfo.name, strerror(errno)); break; diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c index c7017c1af8..e748898e6e 100644 --- a/src/sync/src/syncRetrieve.c +++ b/src/sync/src/syncRetrieve.c @@ -152,7 +152,7 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) { snprintf(name, sizeof(name), "%s/%s", pNode->path, fileInfo.name); // send the file to peer - int32_t sfd = open(name, O_RDONLY); + int32_t sfd = open(name, O_RDONLY | O_BINARY); if (sfd < 0) { code = -1; sError("%s, failed to open file:%s while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno)); @@ -221,7 +221,7 @@ static int32_t syncReadOneWalRecord(int32_t sfd, SWalHead *pHead) { } static int32_t syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion, int64_t offset) { - int32_t sfd = open(name, O_RDONLY); + int32_t sfd = open(name, O_RDONLY | O_BINARY); if (sfd < 0) { sError("%s, failed to open wal:%s for retrieve since:%s", pPeer->id, name, tstrerror(errno)); return -1; @@ -376,7 +376,7 @@ static int32_t syncRetrieveWal(SSyncPeer *pPeer) { size = fstat.st_size; sDebug("%s, retrieve wal:%s size:%d", pPeer->id, fname, size); - int32_t sfd = open(fname, O_RDONLY); + int32_t sfd = open(fname, O_RDONLY | O_BINARY); if (sfd < 0) { code = -1; sError("%s, failed to open wal:%s for retrieve since %s, code:0x%x", pPeer->id, fname, strerror(errno), code); diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c index 696270d670..9e7b0cf108 100644 --- a/src/tsdb/src/tsdbCommit.c +++ b/src/tsdb/src/tsdbCommit.c @@ -269,11 +269,11 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe pthread_rwlock_wrlock(&(pFileH->fhlock)); - (void)rename(helperNewHeadF(pHelper)->fname, helperHeadF(pHelper)->fname); + (void)taosRename(helperNewHeadF(pHelper)->fname, helperHeadF(pHelper)->fname); pGroup->files[TSDB_FILE_TYPE_HEAD].info = helperNewHeadF(pHelper)->info; if (newLast) { - (void)rename(helperNewLastF(pHelper)->fname, helperLastF(pHelper)->fname); + (void)taosRename(helperNewLastF(pHelper)->fname, helperLastF(pHelper)->fname); pGroup->files[TSDB_FILE_TYPE_LAST].info = helperNewLastF(pHelper)->info; } else { pGroup->files[TSDB_FILE_TYPE_LAST].info = helperLastF(pHelper)->info; diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 5d8933d141..7a8622b110 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -175,7 +175,7 @@ int tsdbOpenFileH(STsdbRepo *pRepo) { } sprintf(fname2, "%s/%s_back_%" PRId64, tDataDir, dp->d_name, taosGetTimestamp(TSDB_TIME_PRECISION_MILLI)); - (void)rename(fname1, fname2); + (void)taosRename(fname1, fname2); tsdbDebug("vgId:%d file %s exists, backup it as %s", REPO_ID(pRepo), fname1, fname2); @@ -339,7 +339,7 @@ SFileGroup *tsdbGetFileGroupNext(SFileGroupIter *pIter) { int tsdbOpenFile(SFile *pFile, int oflag) { ASSERT(!TSDB_IS_FILE_OPENED(pFile)); - pFile->fd = open(pFile->fname, oflag, 0755); + pFile->fd = open(pFile->fname, oflag | O_BINARY, 0755); if (pFile->fd < 0) { tsdbError("failed to open file %s since %s", pFile->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index b34b2fa9e6..3903f1dedc 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -537,7 +537,7 @@ static int32_t tsdbSaveConfig(char *rootDir, STsdbCfg *pCfg) { goto _err; } - fd = open(fname, O_WRONLY | O_CREAT, 0755); + fd = open(fname, O_WRONLY | O_CREAT | O_BINARY, 0755); if (fd < 0) { tsdbError("vgId:%d failed to open file %s since %s", pCfg->tsdbId, fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); @@ -583,7 +583,7 @@ static int tsdbLoadConfig(char *rootDir, STsdbCfg *pCfg) { goto _err; } - fd = open(fname, O_RDONLY); + fd = open(fname, O_RDONLY | O_BINARY); if (fd < 0) { tsdbError("failed to open file %s since %s", fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); diff --git a/src/util/src/tfile.c b/src/util/src/tfile.c index 64fea2843b..dd621b3199 100644 --- a/src/util/src/tfile.c +++ b/src/util/src/tfile.c @@ -54,12 +54,12 @@ static int64_t tfOpenImp(int32_t fd) { } int64_t tfOpen(const char *pathname, int32_t flags) { - int32_t fd = open(pathname, flags); + int32_t fd = open(pathname, flags | O_BINARY); return tfOpenImp(fd); } int64_t tfOpenM(const char *pathname, int32_t flags, mode_t mode) { - int32_t fd = open(pathname, flags, mode); + int32_t fd = open(pathname, flags | O_BINARY, mode); return tfOpenImp(fd); } diff --git a/src/util/src/tkvstore.c b/src/util/src/tkvstore.c index 2b1d13c78b..0abba410b0 100644 --- a/src/util/src/tkvstore.c +++ b/src/util/src/tkvstore.c @@ -50,7 +50,7 @@ static void * tdDecodeKVRecord(void *buf, SKVRecord *pRecord); static int tdRestoreKVStore(SKVStore *pStore); int tdCreateKVStore(char *fname) { - int fd = open(fname, O_RDWR | O_CREAT, 0755); + int fd = open(fname, O_RDWR | O_CREAT | O_BINARY, 0755); if (fd < 0) { uError("failed to open file %s since %s", fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); @@ -96,14 +96,14 @@ SKVStore *tdOpenKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void *appH SKVStore *pStore = tdNewKVStore(fname, iFunc, aFunc, appH); if (pStore == NULL) return NULL; - pStore->fd = open(pStore->fname, O_RDWR); + pStore->fd = open(pStore->fname, O_RDWR | O_BINARY); if (pStore->fd < 0) { uError("failed to open file %s since %s", pStore->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); goto _err; } - pStore->sfd = open(pStore->fsnap, O_RDONLY); + pStore->sfd = open(pStore->fsnap, O_RDONLY | O_BINARY); if (pStore->sfd < 0) { if (errno != ENOENT) { uError("failed to open file %s since %s", pStore->fsnap, strerror(errno)); @@ -172,14 +172,14 @@ void tdCloseKVStore(SKVStore *pStore) { tdFreeKVStore(pStore); } int tdKVStoreStartCommit(SKVStore *pStore) { ASSERT(pStore->fd < 0); - pStore->fd = open(pStore->fname, O_RDWR); + pStore->fd = open(pStore->fname, O_RDWR | O_BINARY); if (pStore->fd < 0) { uError("failed to open file %s since %s", pStore->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); goto _err; } - pStore->sfd = open(pStore->fsnap, O_WRONLY | O_CREAT, 0755); + pStore->sfd = open(pStore->fsnap, O_WRONLY | O_CREAT | O_BINARY, 0755); if (pStore->sfd < 0) { uError("failed to open file %s since %s", pStore->fsnap, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); @@ -336,7 +336,7 @@ void tsdbGetStoreInfo(char *fname, uint32_t *magic, int64_t *size) { char buf[TD_KVSTORE_HEADER_SIZE] = "\0"; SStoreInfo info = {0}; - int fd = open(fname, O_RDONLY); + int fd = open(fname, O_RDONLY | O_BINARY); if (fd < 0) goto _err; if (taosRead(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) goto _err; diff --git a/tests/script/sh/deploy.bat b/tests/script/sh/deploy.bat new file mode 100644 index 0000000000..9b61a33d45 --- /dev/null +++ b/tests/script/sh/deploy.bat @@ -0,0 +1,155 @@ +#!/bin/bash + +echo "Executing deploy.sh" + +if [ $# != 4 ]; then + echo "argument list need input : " + echo " -n nodeName" + echo " -i nodePort" + exit 1 +fi + +NODE_NAME= +NODE= +while getopts "n:i:" arg +do + case $arg in + n) + NODE_NAME=$OPTARG + ;; + i) + NODE=$OPTARG + ;; + ?) + echo "unkonw argument" + ;; + esac +done + +SCRIPT_DIR=`dirname $0` +cd $SCRIPT_DIR/../ +SCRIPT_DIR=`pwd` +echo "SCRIPT_DIR: $SCRIPT_DIR" + +IN_TDINTERNAL="community" +if [[ "$SCRIPT_DIR" == *"$IN_TDINTERNAL"* ]]; then + cd ../../.. +else + cd ../../ +fi + +TAOS_DIR=`pwd` +TAOSD_DIR=`find . -name "taosd"|grep bin|head -n1` + +if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then + BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2,3` +else + BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2` +fi + +BUILD_DIR=$TAOS_DIR/$BIN_DIR/build + +SIM_DIR=$TAOS_DIR/sim + +NODE_DIR=$SIM_DIR/$NODE_NAME +EXE_DIR=$BUILD_DIR/bin +CFG_DIR=$NODE_DIR/cfg +LOG_DIR=$NODE_DIR/log +DATA_DIR=$NODE_DIR/data + +rm -rf $NODE_DIR + +mkdir -p $SIM_DIR +mkdir -p $NODE_DIR +mkdir -p $LOG_DIR +mkdir -p $DATA_DIR + +#cp -rf $TAOS_DIR/cfg $NODE_DIR/ +mkdir -p $CFG_DIR + +#allow normal user to read/write log +chmod -R 777 $NODE_DIR + +TAOS_CFG=$NODE_DIR/cfg/taos.cfg +touch -f $TAOS_CFG + +TAOS_FLAG=$SIM_DIR/tsim/flag +if [ -f "$TAOS_FLAG" ] ; then + TAOS_CFG=/etc/taos/taos.cfg + DATA_DIR=/var/lib/taos + LOG_DIR=/var/log/taos + sudo rm -f /etc/taos/*.cfg + sudo cp -rf $TAOS_DIR/cfg/*.cfg /etc/taos + sudo rm -rf $DATA_DIR + sudo rm -rf $LOG_DIR +fi + +HOSTNAME=`hostname -f` + +if [ $NODE -eq 1 ]; then + NODE=7100 +elif [ $NODE -eq 2 ]; then + NODE=7200 +elif [ $NODE -eq 3 ]; then + NODE=7300 +elif [ $NODE -eq 4 ]; then + NODE=7400 +elif [ $NODE -eq 5 ]; then + NODE=7500 +elif [ $NODE -eq 6 ]; then + NODE=7600 +elif [ $NODE -eq 7 ]; then + NODE=7700 +elif [ $NODE -eq 8 ]; then + NODE=7800 +fi + +echo " " >> $TAOS_CFG +echo "firstEp ${HOSTNAME}:7100" >> $TAOS_CFG +echo "secondEp ${HOSTNAME}:7200" >> $TAOS_CFG +echo "serverPort ${NODE}" >> $TAOS_CFG +echo "dataDir $DATA_DIR" >> $TAOS_CFG +echo "logDir $LOG_DIR" >> $TAOS_CFG +echo "debugFlag 0" >> $TAOS_CFG +echo "mDebugFlag 143" >> $TAOS_CFG +echo "sdbDebugFlag 143" >> $TAOS_CFG +echo "dDebugFlag 143" >> $TAOS_CFG +echo "vDebugFlag 143" >> $TAOS_CFG +echo "tsdbDebugFlag 143" >> $TAOS_CFG +echo "cDebugFlag 143" >> $TAOS_CFG +echo "jnidebugFlag 143" >> $TAOS_CFG +echo "odbcdebugFlag 143" >> $TAOS_CFG +echo "httpDebugFlag 143" >> $TAOS_CFG +echo "monDebugFlag 143" >> $TAOS_CFG +echo "mqttDebugFlag 143" >> $TAOS_CFG +echo "qdebugFlag 143" >> $TAOS_CFG +echo "rpcDebugFlag 143" >> $TAOS_CFG +echo "tmrDebugFlag 131" >> $TAOS_CFG +echo "udebugFlag 143" >> $TAOS_CFG +echo "sdebugFlag 143" >> $TAOS_CFG +echo "wdebugFlag 143" >> $TAOS_CFG +echo "cqdebugFlag 143" >> $TAOS_CFG +echo "monitor 0" >> $TAOS_CFG +echo "monitorInterval 1" >> $TAOS_CFG +echo "http 0" >> $TAOS_CFG +echo "slaveQuery 0" >> $TAOS_CFG +echo "numOfThreadsPerCore 2.0" >> $TAOS_CFG +echo "defaultPass taosdata" >> $TAOS_CFG +echo "numOfLogLines 20000000" >> $TAOS_CFG +echo "mnodeEqualVnodeNum 0" >> $TAOS_CFG +echo "balanceInterval 1" >> $TAOS_CFG +echo "clog 2" >> $TAOS_CFG +#echo "cache 1" >> $TAOS_CFG +echo "days 10" >> $TAOS_CFG +echo "statusInterval 1" >> $TAOS_CFG +echo "maxVgroupsPerDb 4" >> $TAOS_CFG +echo "minTablesPerVnode 4" >> $TAOS_CFG +echo "maxTablesPerVnode 1000" >> $TAOS_CFG +echo "tableIncStepPerVnode 10000" >> $TAOS_CFG +echo "asyncLog 0" >> $TAOS_CFG +echo "numOfMnodes 1" >> $TAOS_CFG +echo "locale en_US.UTF-8" >> $TAOS_CFG +echo "fsync 0" >> $TAOS_CFG +echo "telemetryReporting 0" >> $TAOS_CFG +echo " " >> $TAOS_CFG + diff --git a/tests/script/sh/exec.bat b/tests/script/sh/exec.bat new file mode 100644 index 0000000000..1c84a6b10e --- /dev/null +++ b/tests/script/sh/exec.bat @@ -0,0 +1,115 @@ +#!/bin/bash + +# if [ $# != 4 || $# != 5 ]; then + # echo "argument list need input : " + # echo " -n nodeName" + # echo " -s start/stop" + # echo " -c clear" + # exit 1 +# fi + +NODE_NAME= +EXEC_OPTON= +CLEAR_OPTION="false" +while getopts "n:s:u:x:ct" arg +do + case $arg in + n) + NODE_NAME=$OPTARG + ;; + s) + EXEC_OPTON=$OPTARG + ;; + c) + CLEAR_OPTION="clear" + ;; + t) + SHELL_OPTION="true" + ;; + u) + USERS=$OPTARG + ;; + x) + SIGNAL=$OPTARG + ;; + ?) + echo "unkown argument" + ;; + esac +done + +SCRIPT_DIR=`dirname $0` +cd $SCRIPT_DIR/../ +SCRIPT_DIR=`pwd` + +IN_TDINTERNAL="community" +if [[ "$SCRIPT_DIR" == *"$IN_TDINTERNAL"* ]]; then + cd ../../.. +else + cd ../../ +fi + +TAOS_DIR=`pwd` +TAOSD_DIR=`find . -name "taosd"|grep bin|head -n1` + +if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then + BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2,3` +else + BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2` +fi + +BUILD_DIR=$TAOS_DIR/$BIN_DIR/build + +SIM_DIR=$TAOS_DIR/sim +NODE_DIR=$SIM_DIR/$NODE_NAME +EXE_DIR=$BUILD_DIR/bin +CFG_DIR=$NODE_DIR/cfg +LOG_DIR=$NODE_DIR/log +DATA_DIR=$NODE_DIR/data +MGMT_DIR=$NODE_DIR/data/mgmt +TSDB_DIR=$NODE_DIR/data/tsdb + +TAOS_CFG=$NODE_DIR/cfg/taos.cfg + +echo ------------ $EXEC_OPTON $NODE_NAME + +TAOS_FLAG=$SIM_DIR/tsim/flag +if [ -f "$TAOS_FLAG" ]; then + EXE_DIR=/usr/local/bin/taos +fi + +if [ "$CLEAR_OPTION" = "clear" ]; then + echo rm -rf $MGMT_DIR $TSDB_DIR + rm -rf $TSDB_DIR + rm -rf $MGMT_DIR +fi + +if [ "$EXEC_OPTON" = "start" ]; then + echo "ExcuteCmd:" $EXE_DIR/taosd -c $CFG_DIR + + if [ "$SHELL_OPTION" = "true" ]; then + TT=`date +%s` + mkdir ${LOG_DIR}/${TT} + nohup valgrind --log-file=${LOG_DIR}/${TT}/valgrind.log --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 & + else + nohup $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 & + fi + +else + #relative path + RCFG_DIR=sim/$NODE_NAME/cfg + PID=`ps -ef|grep taosd | grep $RCFG_DIR | grep -v grep | awk '{print $2}'` + while [ -n "$PID" ] + do + if [ "$SIGNAL" = "SIGKILL" ]; then + echo try to kill by signal SIGKILL + kill -9 $PID + else + echo try to kill by signal SIGINT + kill -SIGINT $PID + fi + sleep 1 + PID=`ps -ef|grep taosd | grep $RCFG_DIR | grep -v grep | awk '{print $2}'` + done +fi + diff --git a/tests/script/sh/stop_dnodes.bat b/tests/script/sh/stop_dnodes.bat new file mode 100644 index 0000000000..1a6e7153c3 --- /dev/null +++ b/tests/script/sh/stop_dnodes.bat @@ -0,0 +1,24 @@ +#!/bin/sh + +PID=`ps -ef|grep /usr/bin/taosd | grep -v grep | awk '{print $2}'` +if [ -n "$PID" ]; then + echo systemctl stop taosd + systemctl stop taosd +fi + +PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` +while [ -n "$PID" ]; do + echo kill -9 $PID + pkill -9 taosd + fuser -k -n tcp 6030 + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` +done + +PID=`ps -ef|grep -w tarbitrator | grep -v grep | awk '{print $2}'` +while [ -n "$PID" ]; do + echo kill -9 $PID + pkill -9 tarbitrator + fuser -k -n tcp 6040 + PID=`ps -ef|grep -w tarbitrator | grep -v grep | awk '{print $2}'` +done + diff --git a/tests/script/wtest.bat b/tests/script/wtest.bat index 6cdd63b42d..8ca1f22518 100644 --- a/tests/script/wtest.bat +++ b/tests/script/wtest.bat @@ -6,8 +6,8 @@ echo Start TDengine Testing Case ... set "SCRIPT_DIR=%~dp0" echo SCRIPT_DIR: %SCRIPT_DIR% -set "BUILD_DIR=%~dp0..\..\debug\32\build\bin" -set "TSIM=%~dp0..\..\debug\32\build\bin\tsim" +set "BUILD_DIR=%~dp0..\..\debug\build\bin" +set "TSIM=%~dp0..\..\debug\build\bin\tsim" echo BUILD_DIR: %BUILD_DIR% set "SIM_DIR=%~dp0..\..\sim" @@ -32,29 +32,24 @@ if exist %LOG_DIR% rmdir /s/q %LOG_DIR% if not exist %CFG_DIR% mkdir %CFG_DIR% if not exist %LOG_DIR% mkdir %LOG_DIR% -echo firstEp %FIRSTEP% > %TAOS_CFG% -echo serverPort 6030 >> %TAOS_CFG% -echo wal 2 >> %TAOS_CFG% -echo asyncLog 0 >> %TAOS_CFG% -echo locale en_US.UTF-8 >> %TAOS_CFG% -echo logDir %LOG_DIR% >> %TAOS_CFG% -echo scriptDir %SCRIPT_DIR% >> %TAOS_CFG% -echo numOfLogLines 100000000 >> %TAOS_CFG% -echo tmrDebugFlag 131 >> %TAOS_CFG% -echo rpcDebugFlag 143 >> %TAOS_CFG% -echo cDebugFlag 143 >> %TAOS_CFG% -echo qdebugFlag 143 >> %TAOS_CFG% -echo udebugFlag 143 >> %TAOS_CFG% +echo firstEp localhost > %TAOS_CFG% +echo serverPort 7100 >> %TAOS_CFG% +echo logDir %LOG_DIR% >> %TAOS_CFG% +echo scriptDir %SCRIPT_DIR% >> %TAOS_CFG% +echo numOfLogLines 100000000 >> %TAOS_CFG% +echo rpcDebugFlag 143 >> %TAOS_CFG% +echo tmrDebugFlag 131 >> %TAOS_CFG% +echo cDebugFlag 143 >> %TAOS_CFG% +echo udebugFlag 143 >> %TAOS_CFG% +echo wal 0 >> %TAOS_CFG% +echo asyncLog 0 >> %TAOS_CFG% +echo locale en_US.UTF-8 >> %TAOS_CFG% +echo enableCoreFile 1 >> %TAOS_CFG% -set "FILE_NAME=windows\testSuite.sim" -set "FIRSTEP=192.168.1.182" +set "FILE_NAME=testSuite.sim" if "%1" == "-f" set "FILE_NAME=%2" -if "%1" == "-h" set "FIRSTEP=%2" -if "%3" == "-f" set "FILE_NAME=%4" -if "%3" == "-h" set "FIRSTEP=%4" echo FILE_NAME: %FILE_NAME% -echo FIRSTEP: %FIRSTEP% echo ExcuteCmd: %tsim% -c %CFG_DIR% -f %FILE_NAME% %tsim% -c %CFG_DIR% -f %FILE_NAME% \ No newline at end of file From 985de782080ae45639aaf701df760c2a5ffcea1b Mon Sep 17 00:00:00 2001 From: freemine Date: Sun, 17 Jan 2021 00:47:16 +0800 Subject: [PATCH 14/62] with epoll over kqueue, taos can be built on MacOSX --- src/kit/shell/src/shellDarwin.c | 3 + src/os/inc/eok.h | 2 +- src/os/src/darwin/darwinEnv.c | 2 - src/os/src/darwin/eok.c | 176 +++++++++++++++++++++----------- src/os/src/detail/osSocket.c | 1 - src/rpc/src/rpcTcp.c | 14 --- src/sync/src/syncTcp.c | 6 -- src/tsdb/src/tsdbMain.c | 4 +- src/util/src/tsocket.c | 3 +- 9 files changed, 127 insertions(+), 84 deletions(-) diff --git a/src/kit/shell/src/shellDarwin.c b/src/kit/shell/src/shellDarwin.c index d6aed4401c..dbec3fdb05 100644 --- a/src/kit/shell/src/shellDarwin.c +++ b/src/kit/shell/src/shellDarwin.c @@ -350,6 +350,9 @@ void *shellLoopQuery(void *arg) { reset_terminal_mode(); } while (shellRunCommand(con, command) == 0); + tfree(command); + exitShell(); + pthread_cleanup_pop(1); return NULL; diff --git a/src/os/inc/eok.h b/src/os/inc/eok.h index 8892e50c35..a0fd5b5d8e 100644 --- a/src/os/inc/eok.h +++ b/src/os/inc/eok.h @@ -70,5 +70,5 @@ int epoll_close(int epfd); } #endif -#endif // _eok_h_fd274616_996c_400e_9023_ae70be881fa3_ +#endif // _eok_h_fd274616_996c_400e_9023_ae70be881fa3_ diff --git a/src/os/src/darwin/darwinEnv.c b/src/os/src/darwin/darwinEnv.c index 28388f24d2..da4b32139e 100644 --- a/src/os/src/darwin/darwinEnv.c +++ b/src/os/src/darwin/darwinEnv.c @@ -17,8 +17,6 @@ #include "os.h" #include "tglobal.h" -#include - void osInit() { if (configDir[0] == 0) { strcpy(configDir, "~/TDengine/cfg"); diff --git a/src/os/src/darwin/eok.c b/src/os/src/darwin/eok.c index f83846b734..3f7137f6a1 100644 --- a/src/os/src/darwin/eok.c +++ b/src/os/src/darwin/eok.c @@ -34,20 +34,31 @@ typedef struct eok_event_s eok_event_t; struct ep_over_kq_s { int kq; + + // !!! + // idx in the eoks list + // used as pseudo-file-desciptor + // must be 'closed' with epoll_close int idx; + ep_over_kq_t *next; int sv[2]; // 0 for read, 1 for write + // all registered 'epoll events, key by fd' int evs_count; eok_event_t *evs_head; eok_event_t *evs_tail; eok_event_t *evs_free; + // all kev changes list pending to be processed by kevent64 + // key by tuple (ident,filter), ident === fd in this case struct kevent64_s *kchanges; int nchanges; int ichanges; + // kev eventslist for kevent64 to store active events + // they remain alive among kevent64 calls struct kevent64_s *kevslist; int nevslist; @@ -76,6 +87,7 @@ struct eoks_s { int neoks; ep_over_kq_t *eoks_free; }; + static eoks_t eoks = { .lock = PTHREAD_MUTEX_INITIALIZER, .eoks = NULL, @@ -93,8 +105,9 @@ static const char* op_str(int op) { } static __thread char buf_slots[10][1024] = {0}; -static __thread int buf_slots_linelen = sizeof(buf_slots[0])/sizeof(buf_slots[0][0]); -static __thread int buf_slots_count = sizeof(buf_slots)/(sizeof(buf_slots[0])/sizeof(buf_slots[0][0])); +static __thread int buf_slots_linelen = sizeof(buf_slots[0])/sizeof(buf_slots[0][0]); +static __thread int buf_slots_count = sizeof(buf_slots)/(sizeof(buf_slots[0])/sizeof(buf_slots[0][0])); + static const char* events_str(uint32_t events, int slots) { A(slots>=0 && slots on linux // EPOLLIN = 0x001, // #define EPOLLIN EPOLLIN // EPOLLPRI = 0x002, @@ -164,6 +178,7 @@ static const char* kev_flags_str(uint16_t flags, int slots) { size_t len = buf_slots_linelen; int n = 0; buf[0] = '\0'; + // copied to // #define EV_ADD 0x0001 /* add event to kq (implies enable) */ // #define EV_DELETE 0x0002 /* delete event from kq */ // #define EV_ENABLE 0x0004 /* enable event */ @@ -233,10 +248,7 @@ int epoll_create(int size) { (void)size; int e = 0; ep_over_kq_t *eok = eoks_alloc(); - if (!eok) { - errno = ENOMEM; - return -1; - } + if (!eok) return -1; A(eok->kq==-1, "internal logic error"); A(eok->lock_valid, "internal logic error"); @@ -263,13 +275,14 @@ int epoll_create(int size) { struct epoll_event ev = {0}; ev.events = EPOLLIN; ev.data.ptr = &eok_dummy; + D("epoll_create epfd:[%d]", eok->idx); if (epoll_ctl(eok->idx, EPOLL_CTL_ADD, eok->sv[0], &ev)) { - A(0, "internal logic error"); + e = errno; epoll_close(eok->idx); + errno = e; return -1; } - D("epoll_create epfd:[%d]", eok->idx); return eok->idx; } @@ -309,36 +322,34 @@ int epoll_ctl(int epfd, int op, int fd, struct epoll_event *event) { e = ENOENT; break; } + if (op!=EPOLL_CTL_DEL && !event) { + e = EINVAL; + break; + } + // prepare krev/kwev struct kevent64_s krev = {0}; struct kevent64_s kwev = {0}; krev.ident = -1; kwev.ident = -1; uint16_t flags = 0; + // prepare internal eok event eok_event_t ev = {0}; ev.fd = fd; if (event) ev.epev = *event; struct epoll_event *pev = event; switch (op) { case EPOLL_CTL_ADD: { - if (!event) { - e = EINVAL; - break; - } flags = EV_ADD; ev.changed = 1; } break; case EPOLL_CTL_MOD: { - if (!event) { - e = EINVAL; - break; - } flags = EV_ADD; ev.changed = 2; } break; case EPOLL_CTL_DEL: { - D("epoll_ctl adding..."); // event is ignored + // pev points to registered epoll_event pev = &oev->epev; flags = EV_DELETE; ev.changed = 3; @@ -350,6 +361,7 @@ int epoll_ctl(int epfd, int op, int fd, struct epoll_event *event) { if (e) break; + // udata will be delayed to be set if (pev->events & (EPOLLIN | EPOLLERR | EPOLLHUP | EPOLLRDHUP)) { flags |= EV_EOF; EV_SET64(&krev, ev.fd, EVFILT_READ, flags, 0, 0, -1, 0, 0); @@ -358,6 +370,7 @@ int epoll_ctl(int epfd, int op, int fd, struct epoll_event *event) { EV_SET64(&kwev, ev.fd, EVFILT_WRITE, flags, 0, 0, -1, 0, 0); } + // refresh registered evlist and changelist in a transaction way if (eok_chgs_refresh(eok, oev, &ev, &krev, &kwev)) { e = errno; A(e, "internal logic error"); @@ -380,11 +393,6 @@ static struct timespec do_timespec_diff(struct timespec *from, struct timespec * int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) { int e = 0; - if (epfd<0 || epfd>=eoks.neoks) { - errno = EBADF; - E("epoll_waiting epfd:[%d], maxevents:[%d], timeout:[%d] failed", epfd, maxevents, timeout); - return -1; - } if (!events) { errno = EINVAL; E("epoll_waiting epfd:[%d], maxevents:[%d], timeout:[%d] failed", epfd, maxevents, timeout); @@ -396,15 +404,6 @@ int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) return -1; } - int r = 0; - - ep_over_kq_t *eok = eoks_find(epfd); - if (!eok) { - errno = EBADF; - E("epoll_waiting epfd:[%d], maxevents:[%d], timeout:[%d] failed", epfd, maxevents, timeout); - return -1; - } - struct timespec abstime = {0}; A(TIME_UTC==timespec_get(&abstime, TIME_UTC), "internal logic error"); @@ -412,7 +411,17 @@ int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) if (timeout<0) timeout = 0; int64_t t = abstime.tv_nsec + timeout * 1000000; abstime.tv_sec += t / 1000000000; - abstime.tv_nsec %= 1000000000; + abstime.tv_nsec = t % 1000000000; + } + + int r = 0; + + ep_over_kq_t *eok = eoks_find(epfd); + if (!eok) { + errno = EBADF; + E("epoll_waiting epfd:[%d], maxevents:[%d], timeout:[%d] failed", epfd, maxevents, timeout); + errno = EBADF; + return -1; } int cnts = 0; @@ -431,43 +440,52 @@ int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) struct timespec now = {0}; A(TIME_UTC==timespec_get(&now, TIME_UTC), "internal logic error"); struct timespec to = do_timespec_diff(&now, &abstime); - struct timespec *pto = NULL; - if (timeout!=-1) { - pto = &to; + struct timespec *pto = &to; + if (timeout==-1) { + pto = NULL; } - eok->changed = 0; - eok->wakenup = 0; - eok->waiting = 1; - + // taking the changelist struct kevent64_s *kchanges = eok->kchanges; int nchanges = eok->nchanges; int ichanges = eok->ichanges; + // let outside world to add changes eok->kchanges = NULL; eok->nchanges = 0; eok->ichanges = 0; + eok->changed = 0; + eok->wakenup = 0; + eok->waiting = 1; + A(0==pthread_mutex_unlock(&eok->lock), ""); if (ichanges>0) { - D("kevent64 changing [%d] changes and waiting...", ichanges); + D("kevent64 epfd[%d] changing [%d] changes and waiting...", eok->idx, ichanges); } errno = 0; r = kevent64(eok->kq, kchanges, ichanges, eventslist, maxevents, 0, pto); e = errno; if (e) { - E("kevent64 waiting done, with r[%d]", r); + E("kevent64 epfd[%d] waiting done, with r[%d]", eok->idx, r); } A(0==pthread_mutex_lock(&eok->lock), ""); eok->waiting = 0; + if (kchanges) { - free(kchanges); - kchanges = NULL; + if (eok->kchanges==NULL) { + // reuse + A(eok->nchanges==0 && eok->ichanges==0, "internal logic error"); + eok->kchanges = kchanges; + eok->nchanges = nchanges; + } else { + free(kchanges); + kchanges = NULL; + } nchanges = 0; ichanges = 0; } - eok->waiting = 0; if (r==0) { A(timeout!=-1, "internal logic error"); } @@ -484,28 +502,37 @@ int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) case EVFILT_READ: { A((ev->epev.events & EPOLLIN), "internal logic errro"); if (ev->epev.data.ptr==&eok_dummy) { + // it's coming from wakeup socket pair char c = '\0'; A(1==recv(kev->ident, &c, 1, 0), "internal logic error"); A(0==memcmp(&c, "1", 1), "internal logic error"); D("wokenup"); + continue; } else { if (ev->changed==3) { D("already requested to delete for fd[%d]", ev->fd); + // TODO: write a unit test for this case // EV_DELETE? continue; } + // converting to epoll_event + // we shall collect all kevents for the uniq fd into one epoll_evnt + // but currently, taos never use EPOLLOUT + // just let it this way for the moment struct epoll_event pev = {0}; pev.data.ptr = ev->epev.data.ptr; pev.events = EPOLLIN; if (kev->flags & EV_EOF) { + // take all these as EOF for the moment pev.events |= (EPOLLHUP | EPOLLERR | EPOLLRDHUP); - D(".........."); } + // rounded to what user care pev.events = pev.events & ev->epev.events; D("events found for fd[%d]: [%04x:%s], which was registered: [%04x:%s], kev_flags: [%04x:%s]", ev->fd, pev.events, events_str(pev.events, 0), ev->epev.events, events_str(ev->epev.events, 1), kev->flags, kev_flags_str(kev->flags, 2)); + // now we get ev and store it events[cnts++] = pev; } } break; @@ -518,6 +545,8 @@ int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) } } if (r>=0) { + // we can safely rule out delete-requested-events from the regitered evlists + // if only changelist are correctly registered eok_event_t *p = eok->evs_head; while (p) { eok_event_t *next = p->next; @@ -528,6 +557,14 @@ int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) p = next; } } + if (cnts==0) { + // if no user-cared-events is up + // we check to see if time is up + A(TIME_UTC==timespec_get(&now, TIME_UTC), "internal logic error"); + to = do_timespec_diff(&now, &abstime); + if (to.tv_sec==0 && to.tv_nsec==0) break; + // time is not up yet, continue loop + } } while (cnts==0); if (cnts>0) { D("kevent64 waiting done with [%d] events", cnts); @@ -540,6 +577,7 @@ int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) return -1; } + // tell user how many events are valid return cnts; } @@ -547,19 +585,20 @@ static struct timespec do_timespec_diff(struct timespec *from, struct timespec * struct timespec delta; delta.tv_sec = to->tv_sec - from->tv_sec; delta.tv_nsec = to->tv_nsec - from->tv_nsec; + // norm and round up while (delta.tv_nsec<0) { delta.tv_sec -= 1; delta.tv_nsec += 1000000000; } + if (delta.tv_sec < 0) { + delta.tv_sec = 0; + delta.tv_nsec = 0; + } return delta; } int epoll_close(int epfd) { D("epoll_closing epfd: [%d]", epfd); - if (epfd<0 || epfd>=eoks.neoks) { - errno = EBADF; - return -1; - } ep_over_kq_t *eok = eoks_find(epfd); if (!eok) { errno = EBADF; @@ -568,9 +607,11 @@ int epoll_close(int epfd) { A(0==pthread_mutex_lock(&eok->lock), ""); do { + // panic if it would be double-closed A(eok->stopping==0, "internal logic error"); eok->stopping = 1; - A(eok->waiting, "internal logic error"); + // panic if epoll_wait is pending + A(eok->waiting==0, "internal logic error"); if (eok->kq!=-1) { close(eok->kq); @@ -608,9 +649,12 @@ static eok_event_t* eok_calloc_ev(ep_over_kq_t *eok) { p = eok->evs_free; eok->evs_free = p->next; p->next = NULL; + A(p->prev==NULL, "internal logic error"); } else { p = (eok_event_t*)calloc(1, sizeof(*p)); if (!p) return NULL; + A(p->next==NULL, "internal logic error"); + A(p->prev==NULL, "internal logic error"); } // dirty link p->prev = eok->evs_tail; @@ -626,9 +670,9 @@ static eok_event_t* eok_calloc_ev(ep_over_kq_t *eok) { static void eok_free_ev(ep_over_kq_t *eok, eok_event_t *ev) { if (ev->prev) ev->prev->next = ev->next; else eok->evs_head = ev->next; - ev->prev = NULL; if (ev->next) ev->next->prev = ev->prev; else eok->evs_tail = ev->prev; + ev->prev = NULL; ev->next = eok->evs_free; eok->evs_free = ev->next; @@ -639,7 +683,7 @@ static void eok_wakeup(ep_over_kq_t *eok) { if (!eok->waiting) return; if (eok->wakenup) return; eok->wakenup = 1; - send(eok->sv[1], "1", 1, 0); + A(1==send(eok->sv[1], "1", 1, 0), ""); } static int eok_chgs_refresh(ep_over_kq_t *eok, eok_event_t *oev, eok_event_t *ev, struct kevent64_s *krev, struct kevent64_s *kwev) { @@ -653,6 +697,8 @@ static int eok_chgs_refresh(ep_over_kq_t *eok, eok_event_t *oev, eok_event_t *ev struct kevent64_s *p = (struct kevent64_s*)realloc(eok->kchanges, sizeof(*p) * (eok->nchanges + 10)); if (!p) { if (ev->changed==1) { + // roll back + A(oev, "internal logic error"); eok_free_ev(eok, oev); } errno = ENOMEM; @@ -662,12 +708,15 @@ static int eok_chgs_refresh(ep_over_kq_t *eok, eok_event_t *oev, eok_event_t *ev eok->nchanges += 10; } + // copy to registered event slot oev->fd = ev->fd; if (ev->changed!=3) { + // if add/mod, copy epoll_event oev->epev = ev->epev; } oev->changed = ev->changed; + // copy to changes list n = 0; if (krev->ident==ev->fd) { krev->udata = (uint64_t)oev; @@ -679,7 +728,7 @@ static int eok_chgs_refresh(ep_over_kq_t *eok, eok_event_t *oev, eok_event_t *ev eok->kchanges[eok->ichanges++] = *kwev; ++n; } - D("add changes[%d] for fd[%d], and changes/registers [%d/%d]", n, ev->fd, eok->ichanges, eok->evs_count); + D("add #changes[%d] for fd[%d], and now #changes/registers [%d/%d]", n, ev->fd, eok->ichanges, eok->evs_count); return 0; } @@ -707,7 +756,10 @@ static ep_over_kq_t* eoks_alloc(void) { } while (0); A(0==pthread_mutex_unlock(&eoks.lock), ""); - if (!eok) return NULL; + if (!eok) { + errno = ENOMEM; + return NULL; + } if (eok->lock_valid) { return eok; } @@ -718,6 +770,7 @@ static ep_over_kq_t* eoks_alloc(void) { } eoks_free(eok); + errno = ENOMEM; return NULL; } @@ -725,16 +778,23 @@ static void eoks_free(ep_over_kq_t *eok) { A(0==pthread_mutex_lock(&eoks.lock), ""); do { A(eok->next==NULL, "internal logic error"); - A(eok->evs_head==NULL, "internal logic error"); + + // leave eok->kchanges as is + A(eok->ichanges==0, "internal logic error"); + A(eok->waiting==0, "internal logic error"); - if (eok->sv[0]!=-1) { + if (eok->evs_count==1) { + A(eok->evs_head && eok->evs_tail && eok->evs_head==eok->evs_tail, "internal logic error"); + A(eok->evs_head->fd==eok->sv[0] && eok->sv[0]!=-1 && eok->sv[1]!=-1, "internal logic error"); + // fd is critical system resource close(eok->sv[0]); eok->sv[0] = -1; - } - if (eok->sv[1]!=-1) { close(eok->sv[1]); eok->sv[1] = -1; + eok_free_ev(eok, eok->evs_head); } + A(eok->evs_head==NULL && eok->evs_tail==NULL && eok->evs_count==0, "internal logic error"); + A(eok->sv[0]==-1 && eok->sv[1]==-1, "internal logic error"); if (eok->kq!=-1) { close(eok->kq); eok->kq = -1; diff --git a/src/os/src/detail/osSocket.c b/src/os/src/detail/osSocket.c index c6cbbe6b83..659c8510dc 100644 --- a/src/os/src/detail/osSocket.c +++ b/src/os/src/detail/osSocket.c @@ -58,7 +58,6 @@ void taosBlockSIGPIPE() { #ifndef TAOS_OS_FUNC_SOCKET_SETSOCKETOPT int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen) { - fprintf(stderr, "==%s[%d]%s()==\n", basename(__FILE__), __LINE__, __func__); return setsockopt(socketfd, level, optname, optval, (socklen_t)optlen); } diff --git a/src/rpc/src/rpcTcp.c b/src/rpc/src/rpcTcp.c index ff2a5882f8..c65c0db66b 100644 --- a/src/rpc/src/rpcTcp.c +++ b/src/rpc/src/rpcTcp.c @@ -133,7 +133,6 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread } pThreadObj->pollFd = (int64_t)epoll_create(10); // size does not matter - fprintf(stderr, "==%s[%d]%s()==\n", basename(__FILE__), __LINE__, __func__); if (pThreadObj->pollFd < 0) { tError("%s failed to create TCP epoll", label); code = -1; @@ -294,7 +293,6 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int num, void * } pThreadObj->pollFd = (SOCKET)epoll_create(10); // size does not matter - fprintf(stderr, "==%s[%d]%s()==\n", basename(__FILE__), __LINE__, __func__); if (pThreadObj->pollFd < 0) { tError("%s failed to create TCP client epoll", label); free(pThreadObj); @@ -343,10 +341,6 @@ void taosCleanUpTcpClient(void *chandle) { void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port) { SThreadObj * pThreadObj = shandle; - fprintf(stderr, "==%s[%d]%s()==\n", basename(__FILE__), __LINE__, __func__); - fprintf(stderr, "pThreadObj->ip:%d\n", pThreadObj->ip); - fprintf(stderr, "PF_INET/AF_INET:%d/%d\n", PF_INET, AF_INET); - fprintf(stderr, "ip/port:%x/%d\n", ip, port); SOCKET fd = taosOpenTcpClientSocket(ip, port, pThreadObj->ip); if (fd < 0) return NULL; @@ -358,7 +352,6 @@ void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uin localPort = (uint16_t)ntohs(sin.sin_port); } - fprintf(stderr, "==%s[%d]%s()==\n", basename(__FILE__), __LINE__, __func__); SFdObj *pFdObj = taosMallocFdObj(pThreadObj, fd); if (pFdObj) { @@ -369,7 +362,6 @@ void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uin pThreadObj->label, thandle, ip, port, localPort, pFdObj, pThreadObj->numOfFds); } else { tError("%s failed to malloc client FdObj(%s)", pThreadObj->label, strerror(errno)); - fprintf(stderr, "==%s[%d]%s()==\n", basename(__FILE__), __LINE__, __func__); taosCloseSocket(fd); } @@ -492,32 +484,27 @@ static void *taosProcessTcpData(void *param) { if (fdNum < 0) continue; for (int i = 0; i < fdNum; ++i) { - fprintf(stderr, "==%s[%d]%s()==\n", basename(__FILE__), __LINE__, __func__); pFdObj = events[i].data.ptr; if (events[i].events & EPOLLERR) { tDebug("%s %p FD:%p epoll errors", pThreadObj->label, pFdObj->thandle, pFdObj); - fprintf(stderr, "==%s[%d]%s()==\n", basename(__FILE__), __LINE__, __func__); taosReportBrokenLink(pFdObj); continue; } if (events[i].events & EPOLLRDHUP) { tDebug("%s %p FD:%p RD hang up", pThreadObj->label, pFdObj->thandle, pFdObj); - fprintf(stderr, "==%s[%d]%s()==\n", basename(__FILE__), __LINE__, __func__); taosReportBrokenLink(pFdObj); continue; } if (events[i].events & EPOLLHUP) { tDebug("%s %p FD:%p hang up", pThreadObj->label, pFdObj->thandle, pFdObj); - fprintf(stderr, "==%s[%d]%s()==\n", basename(__FILE__), __LINE__, __func__); taosReportBrokenLink(pFdObj); continue; } if (taosReadTcpData(pFdObj, &recvInfo) < 0) { - fprintf(stderr, "==%s[%d]%s()==\n", basename(__FILE__), __LINE__, __func__); shutdown(pFdObj->fd, SHUT_WR); continue; } @@ -563,7 +550,6 @@ static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, SOCKET fd) { event.events = EPOLLIN | EPOLLRDHUP; event.data.ptr = pFdObj; - fprintf(stderr, "==%s[%d]%s()==\n", basename(__FILE__), __LINE__, __func__); if (epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) { tfree(pFdObj); terrno = TAOS_SYSTEM_ERROR(errno); diff --git a/src/sync/src/syncTcp.c b/src/sync/src/syncTcp.c index be85853819..1e5761316a 100644 --- a/src/sync/src/syncTcp.c +++ b/src/sync/src/syncTcp.c @@ -136,15 +136,12 @@ void *syncAllocateTcpConn(void *param, int64_t rid, int32_t connFd) { event.events = EPOLLIN | EPOLLRDHUP; event.data.ptr = pConn; - fprintf(stderr, ">>>>>>>>>>>>>>>>>\n"); if (epoll_ctl(pThread->pollFd, EPOLL_CTL_ADD, connFd, &event) < 0) { - fprintf(stderr, "<<<<<<<<<<<<<<<<<\n"); sError("failed to add fd:%d since %s", connFd, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); tfree(pConn); pConn = NULL; } else { - fprintf(stderr, "<<<<<<<<<<<<<<<<<\n"); pThread->numOfFds++; sDebug("%p fd:%d is added to epoll thread, num:%d", pThread, connFd, pThread->numOfFds); } @@ -170,9 +167,7 @@ static void taosProcessBrokenLink(SConnObj *pConn) { (*pInfo->processBrokenLink)(pConn->handleId); pThread->numOfFds--; - fprintf(stderr, "<<<<<<<<<<<<<<<<<\n"); epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, pConn->fd, NULL); - fprintf(stderr, "<<<<<<<<<<<<<<<<<\n"); sDebug("%p fd:%d is removed from epoll thread, num:%d", pThread, pConn->fd, pThread->numOfFds); taosClose(pConn->fd); tfree(pConn); @@ -287,7 +282,6 @@ static SThreadObj *syncGetTcpThread(SPoolObj *pPool) { pThread->pPool = pPool; pThread->pollFd = epoll_create(10); // size does not matter - fprintf(stderr, "...............\n"); if (pThread->pollFd < 0) { tfree(pThread); return NULL; diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 602aea70e3..5493188c09 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -649,13 +649,15 @@ static STsdbRepo *tsdbNewRepo(char *rootDir, STsdbAppH *pAppH, STsdbCfg *pCfg) { #ifdef __APPLE__ pRepo->readyToCommit = sem_open(NULL, O_CREAT, 0644, 1); - if (!pRepo->readyToCommit) { + if (pRepo->readyToCommit==SEM_FAILED) { + code = errno; terrno = TAOS_SYSTEM_ERROR(code); goto _err; } #else code = sem_init(&(pRepo->readyToCommit), 0, 1); if (code != 0) { + code = errno; terrno = TAOS_SYSTEM_ERROR(code); goto _err; } diff --git a/src/util/src/tsocket.c b/src/util/src/tsocket.c index 6b15d3677c..9fc10576ac 100644 --- a/src/util/src/tsocket.c +++ b/src/util/src/tsocket.c @@ -35,7 +35,7 @@ int32_t taosGetFqdn(char *fqdn) { hints.ai_flags = AI_CANONNAME; int32_t ret = getaddrinfo(hostname, NULL, &hints, &result); if (!result) { - uError("failed to get fqdn for hostname <%s>, code:%d, reason:%s", hostname, ret, gai_strerror(ret)); + uError("failed to get fqdn, code:%d, reason:%s", ret, gai_strerror(ret)); return -1; } @@ -342,6 +342,7 @@ int32_t taosKeepTcpAlive(SOCKET sockFd) { } #ifndef __APPLE__ + // all fails on macosx int32_t probes = 3; if (taosSetSockOpt(sockFd, SOL_TCP, TCP_KEEPCNT, (void *)&probes, sizeof(probes)) < 0) { uError("fd:%d setsockopt SO_KEEPCNT failed: %d (%s)", sockFd, errno, strerror(errno)); From 0c19d10929de5f598bdccb35c4295c72fcbdeecb Mon Sep 17 00:00:00 2001 From: freemine Date: Sun, 17 Jan 2021 01:04:09 +0800 Subject: [PATCH 15/62] add ENABLE_LOG macro to swith on/off D/E stuffs --- src/os/src/darwin/eok.c | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/os/src/darwin/eok.c b/src/os/src/darwin/eok.c index 3f7137f6a1..322aa7cf86 100644 --- a/src/os/src/darwin/eok.c +++ b/src/os/src/darwin/eok.c @@ -10,7 +10,19 @@ #include #include +#ifdef ENABLE_LOG #define D(fmt, ...) fprintf(stderr, "%s[%d]%s(): " fmt "\n", basename(__FILE__), __LINE__, __func__, ##__VA_ARGS__) +#define E(fmt, ...) do { \ + fprintf(stderr, "%s[%d]%s(): %d[%s]: " fmt "\n", \ + basename(__FILE__), __LINE__, __func__, \ + errno, strerror(errno), \ + ##__VA_ARGS__); \ +} while (0) +#else // !ENABLE_LOG +#define D(fmt, ...) (void)fmt +#define E(fmt, ...) (void)fmt +#endif // ENABLE_LOG + #define A(statement, fmt, ...) do { \ if (statement) break; \ fprintf(stderr, "%s[%d]%s(): assert [%s] failed: %d[%s]: " fmt "\n", \ @@ -20,13 +32,6 @@ abort(); \ } while (0) -#define E(fmt, ...) do { \ - fprintf(stderr, "%s[%d]%s(): %d[%s]: " fmt "\n", \ - basename(__FILE__), __LINE__, __func__, \ - errno, strerror(errno), \ - ##__VA_ARGS__); \ -} while (0) - static int eok_dummy = 0; typedef struct ep_over_kq_s ep_over_kq_t; @@ -95,6 +100,7 @@ static eoks_t eoks = { .eoks_free = NULL, }; +#ifdef ENABLE_LOG static const char* op_str(int op) { switch (op) { case EPOLL_CTL_ADD: return "EPOLL_CTL_ADD"; @@ -230,6 +236,7 @@ static const char* kev_flags_str(uint16_t flags, int slots) { #undef CHK_EV return buf; } +#endif // ENABLE_LOG static ep_over_kq_t* eoks_alloc(void); static void eoks_free(ep_over_kq_t *eok); From 54af96b6f06e04725eec3ea48ff876d10b76e06f Mon Sep 17 00:00:00 2001 From: freemine Date: Sun, 17 Jan 2021 01:37:58 +0800 Subject: [PATCH 16/62] adding TAOS license header --- src/CMakeLists.txt | 2 +- src/os/inc/eok.h | 15 +++++++++++++++ src/os/src/darwin/eok.c | 15 +++++++++++++++ tests/examples/c/epoll.c | 15 +++++++++++++++ 4 files changed, 46 insertions(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 71ed16afa1..04bc61ed9e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -19,5 +19,5 @@ ADD_SUBDIRECTORY(wal) ADD_SUBDIRECTORY(cq) ADD_SUBDIRECTORY(dnode) #ADD_SUBDIRECTORY(connector/odbc) -#ADD_SUBDIRECTORY(connector/jdbc) +ADD_SUBDIRECTORY(connector/jdbc) diff --git a/src/os/inc/eok.h b/src/os/inc/eok.h index a0fd5b5d8e..4d38e8e632 100644 --- a/src/os/inc/eok.h +++ b/src/os/inc/eok.h @@ -1,3 +1,18 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + #ifndef _eok_h_fd274616_996c_400e_9023_ae70be881fa3_ #define _eok_h_fd274616_996c_400e_9023_ae70be881fa3_ diff --git a/src/os/src/darwin/eok.c b/src/os/src/darwin/eok.c index 322aa7cf86..f9465d29dd 100644 --- a/src/os/src/darwin/eok.c +++ b/src/os/src/darwin/eok.c @@ -1,3 +1,18 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + #include "eok.h" #include diff --git a/tests/examples/c/epoll.c b/tests/examples/c/epoll.c index 6047c9e4ea..1ef9b3eddf 100644 --- a/tests/examples/c/epoll.c +++ b/tests/examples/c/epoll.c @@ -1,3 +1,18 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + #ifdef __APPLE__ #include "eok.h" #else From 60a9725d031d36ff0f0903863080d434182555e4 Mon Sep 17 00:00:00 2001 From: freemine Date: Sun, 17 Jan 2021 02:07:25 +0800 Subject: [PATCH 17/62] replace array initializer with memset --- tests/examples/c/epoll.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/examples/c/epoll.c b/tests/examples/c/epoll.c index 1ef9b3eddf..eb66156c14 100644 --- a/tests/examples/c/epoll.c +++ b/tests/examples/c/epoll.c @@ -177,7 +177,8 @@ static void* routine(void* arg) { ep_t *ep = (ep_t*)arg; while (!ep->stopping) { - struct epoll_event evs[10] = {0}; + struct epoll_event evs[10]; + memset(evs, 0, sizeof(evs)); A(0==pthread_mutex_lock(&ep->lock), ""); A(ep->waiting==0, "internal logic error"); From 6530365e580e034615b4c550a9d0514a2b5f4be8 Mon Sep 17 00:00:00 2001 From: freemine Date: Sun, 17 Jan 2021 08:13:51 +0800 Subject: [PATCH 18/62] bugFix: check timeout as well --- src/os/src/darwin/eok.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/os/src/darwin/eok.c b/src/os/src/darwin/eok.c index f9465d29dd..e2f7c55284 100644 --- a/src/os/src/darwin/eok.c +++ b/src/os/src/darwin/eok.c @@ -25,6 +25,8 @@ #include #include +#define ENABLE_LOG + #ifdef ENABLE_LOG #define D(fmt, ...) fprintf(stderr, "%s[%d]%s(): " fmt "\n", basename(__FILE__), __LINE__, __func__, ##__VA_ARGS__) #define E(fmt, ...) do { \ @@ -582,9 +584,11 @@ int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) if (cnts==0) { // if no user-cared-events is up // we check to see if time is up - A(TIME_UTC==timespec_get(&now, TIME_UTC), "internal logic error"); - to = do_timespec_diff(&now, &abstime); - if (to.tv_sec==0 && to.tv_nsec==0) break; + if (timeout!=-1) { + A(TIME_UTC==timespec_get(&now, TIME_UTC), "internal logic error"); + to = do_timespec_diff(&now, &abstime); + if (to.tv_sec==0 && to.tv_nsec==0) break; + } // time is not up yet, continue loop } } while (cnts==0); From c5dfa590ec3bc5546e23d15357e92dda2156452c Mon Sep 17 00:00:00 2001 From: freemine Date: Sun, 17 Jan 2021 08:15:31 +0800 Subject: [PATCH 19/62] typo: by default, ENABLE_LOG is not defined --- src/os/src/darwin/eok.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/os/src/darwin/eok.c b/src/os/src/darwin/eok.c index e2f7c55284..a6dceb3df3 100644 --- a/src/os/src/darwin/eok.c +++ b/src/os/src/darwin/eok.c @@ -25,8 +25,6 @@ #include #include -#define ENABLE_LOG - #ifdef ENABLE_LOG #define D(fmt, ...) fprintf(stderr, "%s[%d]%s(): " fmt "\n", basename(__FILE__), __LINE__, __func__, ##__VA_ARGS__) #define E(fmt, ...) do { \ From f8ee5fc4133ddbe413c02d52abcb3b3148acd455 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 17 Jan 2021 13:07:37 +0800 Subject: [PATCH 20/62] TD-1207 --- src/plugins/http/inc/httpInt.h | 2 ++ src/plugins/http/src/httpServer.c | 5 +++++ src/plugins/http/src/httpSystem.c | 5 +++++ src/rpc/src/rpcTcp.c | 16 +++++++++++++++- src/sync/src/syncTcp.c | 13 +++++++++++++ 5 files changed, 40 insertions(+), 1 deletion(-) diff --git a/src/plugins/http/inc/httpInt.h b/src/plugins/http/inc/httpInt.h index 31b06714d6..00a7bcf6c5 100644 --- a/src/plugins/http/inc/httpInt.h +++ b/src/plugins/http/inc/httpInt.h @@ -177,6 +177,8 @@ typedef struct HttpServer { char label[HTTP_LABEL_SIZE]; uint32_t serverIp; uint16_t serverPort; + int8_t stop; + int8_t reserve; SOCKET fd; int32_t numOfThreads; int32_t methodScannerLen; diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index ee6addd6fa..5a04a021cd 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -171,6 +171,11 @@ static void *httpAcceptHttpConnection(void *arg) { while (1) { socklen_t addrlen = sizeof(clientAddr); connFd = (int32_t)accept(pServer->fd, (struct sockaddr *)&clientAddr, &addrlen); + if (pServer->stop) { + httpDebug("http server:%s socket stop, exiting...", pServer->label); + break; + } + if (connFd == -1) { if (errno == EINVAL) { httpDebug("http server:%s socket was shutdown, exiting...", pServer->label); diff --git a/src/plugins/http/src/httpSystem.c b/src/plugins/http/src/httpSystem.c index 34a70a658b..df39c080a5 100644 --- a/src/plugins/http/src/httpSystem.c +++ b/src/plugins/http/src/httpSystem.c @@ -89,7 +89,12 @@ int32_t httpStartSystem() { void httpStopSystem() { tsHttpServer.status = HTTP_SERVER_CLOSING; + tsHttpServer.stop = 1; +#ifdef WINDOWS + closesocket(tsHttpServer.fd); +#else shutdown(tsHttpServer.fd, SHUT_RD); +#endif tgCleanupHandle(); } diff --git a/src/rpc/src/rpcTcp.c b/src/rpc/src/rpcTcp.c index 08ae4b5b8b..4cc7530784 100644 --- a/src/rpc/src/rpcTcp.c +++ b/src/rpc/src/rpcTcp.c @@ -59,6 +59,8 @@ typedef struct { SOCKET fd; uint32_t ip; uint16_t port; + int8_t stop; + int8_t reserve; char label[TSDB_LABEL_LEN]; int numOfThreads; void * shandle; @@ -188,8 +190,15 @@ void taosStopTcpServer(void *handle) { SServerObj *pServerObj = handle; if (pServerObj == NULL) return; - if(pServerObj->fd >=0) shutdown(pServerObj->fd, SHUT_RD); + pServerObj->stop = 1; + if (pServerObj->fd >= 0) { +#ifdef WINDOWS + closesocket(pServerObj->fd); +#else + shutdown(pServerObj->fd, SHUT_RD); +#endif + } if (taosCheckPthreadValid(pServerObj->thread)) { if (taosComparePthread(pServerObj->thread, pthread_self())) { pthread_detach(pthread_self()); @@ -230,6 +239,11 @@ static void *taosAcceptTcpConnection(void *arg) { while (1) { socklen_t addrlen = sizeof(caddr); connFd = accept(pServerObj->fd, (struct sockaddr *)&caddr, &addrlen); + if (pServerObj->stop) { + tDebug("%s TCP server stop accepting new connections", pServerObj->label); + break; + } + if (connFd == -1) { if (errno == EINVAL) { tDebug("%s TCP server stop accepting new connections, exiting", pServerObj->label); diff --git a/src/sync/src/syncTcp.c b/src/sync/src/syncTcp.c index 829c9ceec6..72ba8e26b2 100644 --- a/src/sync/src/syncTcp.c +++ b/src/sync/src/syncTcp.c @@ -46,6 +46,7 @@ typedef struct SPoolObj { pthread_t thread; int32_t nextId; SOCKET acceptFd; // FD for accept new connection + int8_t stop; } SPoolObj; typedef struct { @@ -106,7 +107,14 @@ void syncCloseTcpThreadPool(void *param) { SPoolObj * pPool = param; SThreadObj *pThread; + pPool->stop = 1; + +#ifdef WINDOWS + closesocket(pPool->acceptFd); +#else shutdown(pPool->acceptFd, SHUT_RD); +#endif + pthread_join(pPool->thread, NULL); for (int32_t i = 0; i < pPool->info.numOfThreads; ++i) { @@ -257,6 +265,11 @@ static void *syncAcceptPeerTcpConnection(void *argv) { struct sockaddr_in clientAddr; socklen_t addrlen = sizeof(clientAddr); SOCKET connFd = accept(pPool->acceptFd, (struct sockaddr *)&clientAddr, &addrlen); + if (pPool->stop) { + sDebug("%p TCP server accept is stopped", pPool); + break; + } + if (connFd < 0) { if (errno == EINVAL) { sDebug("%p TCP server accept is exiting...", pPool); From 5d9ebb25b0f84c4543bb2cbb82cb2ff3b227fcd5 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 17 Jan 2021 20:47:53 +0800 Subject: [PATCH 21/62] TD-1207 --- src/dnode/src/dnodeSystem.c | 14 ++++++++------ src/os/inc/osSignal.h | 4 ++++ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/dnode/src/dnodeSystem.c b/src/dnode/src/dnodeSystem.c index bdaecf2e0f..3a50eca9e6 100644 --- a/src/dnode/src/dnodeSystem.c +++ b/src/dnode/src/dnodeSystem.c @@ -120,6 +120,7 @@ int32_t main(int32_t argc, char *argv[]) { taosSetSignal(SIGHUP, sigintHandler); taosSetSignal(SIGINT, sigintHandler); taosSetSignal(SIGABRT, sigintHandler); + taosSetSignal(SIGBREAK, sigintHandler); // Open /var/log/syslog file to record information. openlog("TDengine:", LOG_PID | LOG_CONS | LOG_NDELAY, LOG_LOCAL1); @@ -152,12 +153,6 @@ static void siguser1Handler(int32_t signum) { taosCfgDynamicOptions("debugFlag 1 static void siguser2Handler(int32_t signum) { taosCfgDynamicOptions("resetlog"); } static void sigintHandler(int32_t signum) { - // clean the system. - dInfo("shut down signal is %d", signum); - - syslog(LOG_INFO, "Shut down signal is %d", signum); - syslog(LOG_INFO, "Shutting down TDengine service..."); - // protect the application from receive another signal taosIgnSignal(SIGUSR1); taosIgnSignal(SIGUSR2); @@ -165,6 +160,13 @@ static void sigintHandler(int32_t signum) { taosIgnSignal(SIGHUP); taosIgnSignal(SIGINT); taosIgnSignal(SIGABRT); + taosIgnSignal(SIGBREAK); + + // clean the system. + dInfo("shut down signal is %d", signum); + + syslog(LOG_INFO, "Shut down signal is %d", signum); + syslog(LOG_INFO, "Shutting down TDengine service..."); // inform main thread to exit tsem_post(&exitSem); diff --git a/src/os/inc/osSignal.h b/src/os/inc/osSignal.h index 8b047b9e25..57582a8a28 100644 --- a/src/os/inc/osSignal.h +++ b/src/os/inc/osSignal.h @@ -44,6 +44,10 @@ extern "C" { #define SIGUSR2 1234 #endif +#ifndef SIGBREAK + #define SIGBREAK 1234 +#endif + typedef void (*FSignalHandler)(int32_t signum); void taosSetSignal(int32_t signum, FSignalHandler sigfp); void taosIgnSignal(int32_t signum); From 0c96f7653ec4d5c697ff6d504afc59bc40002fb2 Mon Sep 17 00:00:00 2001 From: freemine Date: Sun, 17 Jan 2021 21:40:39 +0800 Subject: [PATCH 22/62] 1. bugfix: eok.c: array of pointers rather than POD buffer 2. initial port of taosd for MacOSX --- cmake/define.inc | 2 + deps/CMakeLists.txt | 6 +- packaging/tools/install_client.sh | 2 +- packaging/tools/make_install.sh | 14 ++- src/balance/CMakeLists.txt | 4 + src/client/CMakeLists.txt | 22 +++++ src/cq/CMakeLists.txt | 12 ++- src/dnode/CMakeLists.txt | 41 +++++++- src/dnode/src/dnodeSystem.c | 4 + src/dnode/src/dnodeTelemetry.c | 7 ++ src/kit/shell/CMakeLists.txt | 13 ++- src/kit/taosdemo/CMakeLists.txt | 11 ++- src/kit/taosdemox/CMakeLists.txt | 34 +++++-- src/kit/taosdump/CMakeLists.txt | 12 ++- src/mnode/CMakeLists.txt | 13 ++- src/mnode/src/mnodeCluster.c | 7 ++ src/os/src/darwin/eok.c | 99 ++++++++++++++------ src/os/src/detail/CMakeLists.txt | 1 + src/plugins/http/CMakeLists.txt | 19 +++- src/plugins/http/src/httpServer.c | 46 +++++++++ src/plugins/monitor/CMakeLists.txt | 14 ++- src/plugins/mqtt/CMakeLists.txt | 12 +++ src/query/CMakeLists.txt | 5 + src/rpc/src/rpcTcp.c | 10 +- src/rpc/test/CMakeLists.txt | 14 +++ src/sync/CMakeLists.txt | 15 ++- src/sync/src/syncRetrieve.c | 2 +- src/sync/test/CMakeLists.txt | 1 - src/util/CMakeLists.txt | 3 +- src/vnode/CMakeLists.txt | 6 ++ src/wal/CMakeLists.txt | 8 +- src/wal/test/CMakeLists.txt | 8 ++ tests/comparisonTest/tdengine/CMakeLists.txt | 5 + tests/examples/c/CMakeLists.txt | 3 + tests/examples/c/demo.c | 2 +- 35 files changed, 413 insertions(+), 64 deletions(-) diff --git a/cmake/define.inc b/cmake/define.inc index 5d4d94ff42..ba12df6f0c 100755 --- a/cmake/define.inc +++ b/cmake/define.inc @@ -128,6 +128,8 @@ IF (TD_DARWIN_64) SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -Wno-missing-braces -fPIC -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") SET(DEBUG_FLAGS "-O0 -g3 -DDEBUG") SET(RELEASE_FLAGS "-Og") + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/lz4/inc) ENDIF () IF (TD_WINDOWS) diff --git a/deps/CMakeLists.txt b/deps/CMakeLists.txt index 1d725add21..1e59396c70 100644 --- a/deps/CMakeLists.txt +++ b/deps/CMakeLists.txt @@ -12,4 +12,8 @@ ADD_SUBDIRECTORY(MsvcLibX) IF (TD_LINUX AND TD_MQTT) ADD_SUBDIRECTORY(MQTT-C) -ENDIF () \ No newline at end of file +ENDIF () + +IF (TD_DARWIN AND TD_MQTT) + ADD_SUBDIRECTORY(MQTT-C) +ENDIF () diff --git a/packaging/tools/install_client.sh b/packaging/tools/install_client.sh index dd116e9bfb..84ca3b5131 100755 --- a/packaging/tools/install_client.sh +++ b/packaging/tools/install_client.sh @@ -21,7 +21,7 @@ else cd ${script_dir} script_dir="$(pwd)" data_dir="/var/lib/taos" - log_dir="~/TDengineLog" + log_dir=~/TDengineLog fi log_link_dir="/usr/local/taos/log" diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh index 831012851a..0727fe2a1e 100755 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -24,7 +24,7 @@ data_dir="/var/lib/taos" if [ "$osType" != "Darwin" ]; then log_dir="/var/log/taos" else - log_dir="~/TDengineLog" + log_dir=~/TDengineLog fi data_link_dir="/usr/local/taos/data" @@ -178,7 +178,9 @@ function install_bin() { function install_lib() { # Remove links ${csudo} rm -f ${lib_link_dir}/libtaos.* || : - ${csudo} rm -f ${lib64_link_dir}/libtaos.* || : + if [ "$osType" != "Darwin" ]; then + ${csudo} rm -f ${lib64_link_dir}/libtaos.* || : + fi if [ "$osType" != "Darwin" ]; then ${csudo} cp ${binary_dir}/build/lib/libtaos.so.${verNumber} ${install_main_dir}/driver && ${csudo} chmod 777 ${install_main_dir}/driver/* @@ -190,12 +192,14 @@ function install_lib() { ${csudo} ln -sf ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so fi else - ${csudo} cp ${binary_dir}/build/lib/libtaos.* ${install_main_dir}/driver && ${csudo} chmod 777 ${install_main_dir}/driver/* - ${csudo} ln -sf ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.1.dylib + ${csudo} cp -Rf ${binary_dir}/build/lib/libtaos.* ${install_main_dir}/driver && ${csudo} chmod 777 ${install_main_dir}/driver/* + ${csudo} ln -sf ${install_main_dir}/driver/libtaos.1.dylib ${lib_link_dir}/libtaos.1.dylib ${csudo} ln -sf ${lib_link_dir}/libtaos.1.dylib ${lib_link_dir}/libtaos.dylib fi - ${csudo} ldconfig + if [ "$osType" != "Darwin" ]; then + ${csudo} ldconfig + fi } function install_header() { diff --git a/src/balance/CMakeLists.txt b/src/balance/CMakeLists.txt index fdc43ea32f..a3ae74d825 100644 --- a/src/balance/CMakeLists.txt +++ b/src/balance/CMakeLists.txt @@ -11,3 +11,7 @@ AUX_SOURCE_DIRECTORY(src SRC) IF (TD_LINUX) ADD_LIBRARY(balance ${SRC}) ENDIF () + +IF (TD_DARWIN) + ADD_LIBRARY(balance ${SRC}) +ENDIF () diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index 3f8f9d2934..660ad564a5 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -28,6 +28,28 @@ IF (TD_LINUX) ADD_SUBDIRECTORY(tests) +ELSEIF (TD_DARWIN) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/linux) + + # set the static lib name + ADD_LIBRARY(taos_static STATIC ${SRC}) + TARGET_LINK_LIBRARIES(taos_static common query trpc tutil pthread m) + SET_TARGET_PROPERTIES(taos_static PROPERTIES OUTPUT_NAME "taos_static") + SET_TARGET_PROPERTIES(taos_static PROPERTIES CLEAN_DIRECT_OUTPUT 1) + + # generate dynamic library (*.dylib) + ADD_LIBRARY(taos SHARED ${SRC}) + TARGET_LINK_LIBRARIES(taos common query trpc tutil pthread m) + SET_TARGET_PROPERTIES(taos PROPERTIES CLEAN_DIRECT_OUTPUT 1) + + #set version of .dylib + #VERSION dylib version + #SOVERSION dylib version + #MESSAGE(STATUS "build version ${TD_VER_NUMBER}") + SET_TARGET_PROPERTIES(taos PROPERTIES VERSION ${TD_VER_NUMBER} SOVERSION 1) + + ADD_SUBDIRECTORY(tests) + ELSEIF (TD_WINDOWS) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/windows) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/windows/win32) diff --git a/src/cq/CMakeLists.txt b/src/cq/CMakeLists.txt index 9da831c9c1..34146241a5 100644 --- a/src/cq/CMakeLists.txt +++ b/src/cq/CMakeLists.txt @@ -10,7 +10,17 @@ IF (TD_LINUX) ADD_LIBRARY(tcq ${SRC}) IF (TD_SOMODE_STATIC) TARGET_LINK_LIBRARIES(tcq tutil common taos_static) - ELSE () + ELSE () + TARGET_LINK_LIBRARIES(tcq tutil common taos) + ENDIF () + ADD_SUBDIRECTORY(test) +ENDIF () + +IF (TD_DARWIN) + ADD_LIBRARY(tcq ${SRC}) + IF (TD_SOMODE_STATIC) + TARGET_LINK_LIBRARIES(tcq tutil common taos_static) + ELSE () TARGET_LINK_LIBRARIES(tcq tutil common taos) ENDIF () ADD_SUBDIRECTORY(test) diff --git a/src/dnode/CMakeLists.txt b/src/dnode/CMakeLists.txt index 699ca00a25..905ad9d920 100644 --- a/src/dnode/CMakeLists.txt +++ b/src/dnode/CMakeLists.txt @@ -23,7 +23,7 @@ IF (TD_LINUX) IF (TD_ACCOUNT) TARGET_LINK_LIBRARIES(taosd account) ENDIF () - + IF (TD_GRANT) TARGET_LINK_LIBRARIES(taosd grant) ENDIF () @@ -47,3 +47,42 @@ IF (TD_LINUX) COMMENT "prepare taosd environment") ADD_CUSTOM_TARGET(${PREPARE_ENV_TARGET} ALL WORKING_DIRECTORY ${TD_EXECUTABLE_OUTPUT_PATH} DEPENDS ${PREPARE_ENV_CMD}) ENDIF () + +IF (TD_DARWIN) + ADD_EXECUTABLE(taosd ${SRC}) + TARGET_LINK_LIBRARIES(taosd mnode monitor http tsdb twal vnode cJson lz4 balance sync) + + IF (TD_SOMODE_STATIC) + TARGET_LINK_LIBRARIES(taosd taos_static) + ELSE () + TARGET_LINK_LIBRARIES(taosd taos) + ENDIF () + + IF (TD_ACCOUNT) + TARGET_LINK_LIBRARIES(taosd account) + ENDIF () + + IF (TD_GRANT) + TARGET_LINK_LIBRARIES(taosd grant) + ENDIF () + + IF (TD_MQTT) + TARGET_LINK_LIBRARIES(taosd mqtt) + ENDIF () + + # SET(PREPARE_ENV_CMD "prepare_env_cmd") + # SET(PREPARE_ENV_TARGET "prepare_env_target") + # ADD_CUSTOM_COMMAND(OUTPUT ${PREPARE_ENV_CMD} + # POST_BUILD + # COMMAND echo "make test directory" + # DEPENDS taosd + # COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/cfg/ + # COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/log/ + # COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/data/ + # COMMAND ${CMAKE_COMMAND} -E echo dataDir ${TD_TESTS_OUTPUT_DIR}/data > ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg + # COMMAND ${CMAKE_COMMAND} -E echo logDir ${TD_TESTS_OUTPUT_DIR}/log >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg + # COMMAND ${CMAKE_COMMAND} -E echo charset UTF-8 >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg + # COMMENT "prepare taosd environment") + # ADD_CUSTOM_TARGET(${PREPARE_ENV_TARGET} ALL WORKING_DIRECTORY ${TD_EXECUTABLE_OUTPUT_PATH} DEPENDS ${PREPARE_ENV_CMD}) +ENDIF () + diff --git a/src/dnode/src/dnodeSystem.c b/src/dnode/src/dnodeSystem.c index a4d7e791e6..893049df2e 100644 --- a/src/dnode/src/dnodeSystem.c +++ b/src/dnode/src/dnodeSystem.c @@ -160,7 +160,11 @@ static void signal_handler(int32_t signum, siginfo_t *sigInfo, void *context) { syslog(LOG_INFO, "Shut down signal is %d", signum); syslog(LOG_INFO, "Shutting down TDengine service..."); // clean the system. +#ifdef __APPLE__ + dInfo("shut down signal is %d, sender PID:%d", signum, sigInfo->si_pid); +#else // __APPLE__ dInfo("shut down signal is %d, sender PID:%d cmdline:%s", signum, sigInfo->si_pid, taosGetCmdlineByPID(sigInfo->si_pid)); +#endif // protect the application from receive another signal struct sigaction act = {{0}}; diff --git a/src/dnode/src/dnodeTelemetry.c b/src/dnode/src/dnodeTelemetry.c index ff9598ecc5..1f7084c3b5 100644 --- a/src/dnode/src/dnodeTelemetry.c +++ b/src/dnode/src/dnodeTelemetry.c @@ -236,6 +236,13 @@ static void sendTelemetryReport() { taosCloseSocket(fd); } +#ifdef __APPLE__ +static int sem_timedwait(tsem_t *sem, struct timespec *to) { + fprintf(stderr, "%s[%d]%s(): not implemented yet!\n", basename(__FILE__), __LINE__, __func__); + abort(); +} +#endif + static void* telemetryThread(void* param) { struct timespec end = {0}; clock_gettime(CLOCK_REALTIME, &end); diff --git a/src/kit/shell/CMakeLists.txt b/src/kit/shell/CMakeLists.txt index 45da99e572..c4f3cc5696 100644 --- a/src/kit/shell/CMakeLists.txt +++ b/src/kit/shell/CMakeLists.txt @@ -9,14 +9,14 @@ IF (TD_LINUX) AUX_SOURCE_DIRECTORY(./src SRC) LIST(REMOVE_ITEM SRC ./src/shellWindows.c) LIST(REMOVE_ITEM SRC ./src/shellDarwin.c) - ADD_EXECUTABLE(shell ${SRC}) - + ADD_EXECUTABLE(shell ${SRC}) + IF (TD_SOMODE_STATIC) TARGET_LINK_LIBRARIES(shell taos_static) ELSE () TARGET_LINK_LIBRARIES(shell taos) ENDIF () - + SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME taos) ELSEIF (TD_WINDOWS) LIST(APPEND SRC ./src/shellEngine.c) @@ -27,7 +27,7 @@ ELSEIF (TD_WINDOWS) IF (TD_POWER) SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME power) - ELSE () + ELSE () SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME taos) ENDIF () ELSEIF (TD_DARWIN) @@ -37,7 +37,10 @@ ELSEIF (TD_DARWIN) LIST(APPEND SRC ./src/shellCommand.c) LIST(APPEND SRC ./src/shellImport.c) ADD_EXECUTABLE(shell ${SRC}) - TARGET_LINK_LIBRARIES(shell taos_static) + # linking with dylib + TARGET_LINK_LIBRARIES(shell taos) + # linking taos statically + # TARGET_LINK_LIBRARIES(shell taos_static) SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME taos) ENDIF () diff --git a/src/kit/taosdemo/CMakeLists.txt b/src/kit/taosdemo/CMakeLists.txt index 91c743939c..f74dbc2de4 100644 --- a/src/kit/taosdemo/CMakeLists.txt +++ b/src/kit/taosdemo/CMakeLists.txt @@ -7,7 +7,7 @@ INCLUDE_DIRECTORIES(inc) IF (TD_LINUX) AUX_SOURCE_DIRECTORY(. SRC) ADD_EXECUTABLE(taosdemo ${SRC}) - + IF (TD_SOMODE_STATIC) TARGET_LINK_LIBRARIES(taosdemo taos_static) ELSE () @@ -17,4 +17,13 @@ ELSEIF (TD_WINDOWS) AUX_SOURCE_DIRECTORY(. SRC) ADD_EXECUTABLE(taosdemo ${SRC}) TARGET_LINK_LIBRARIES(taosdemo taos_static) +ELSEIF (TD_DARWIN) + AUX_SOURCE_DIRECTORY(. SRC) + ADD_EXECUTABLE(taosdemo ${SRC}) + + IF (TD_SOMODE_STATIC) + TARGET_LINK_LIBRARIES(taosdemo taos_static) + ELSE () + TARGET_LINK_LIBRARIES(taosdemo taos) + ENDIF () ENDIF () diff --git a/src/kit/taosdemox/CMakeLists.txt b/src/kit/taosdemox/CMakeLists.txt index 7db6c04b28..3993cb0feb 100644 --- a/src/kit/taosdemox/CMakeLists.txt +++ b/src/kit/taosdemox/CMakeLists.txt @@ -4,22 +4,44 @@ PROJECT(TDengine) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/libcurl/include) -IF (TD_LINUX) +IF (TD_LINUX) AUX_SOURCE_DIRECTORY(. SRC) ADD_EXECUTABLE(taosdemox ${SRC}) - - #find_program(HAVE_CURL NAMES curl) - IF ((NOT TD_ARM_64) AND (NOT TD_ARM_32)) + + #find_program(HAVE_CURL NAMES curl) + IF ((NOT TD_ARM_64) AND (NOT TD_ARM_32)) ADD_DEFINITIONS(-DTD_LOWA_CURL) LINK_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/libcurl/lib) ADD_LIBRARY(curl STATIC IMPORTED) SET_PROPERTY(TARGET curl PROPERTY IMPORTED_LOCATION ${TD_COMMUNITY_DIR}/deps/libcurl/lib/libcurl.a) - TARGET_LINK_LIBRARIES(taosdemox curl) + TARGET_LINK_LIBRARIES(taosdemox curl) ENDIF () - + IF (TD_SOMODE_STATIC) TARGET_LINK_LIBRARIES(taosdemox taos_static cJson) ELSE () TARGET_LINK_LIBRARIES(taosdemox taos cJson) ENDIF () ENDIF () + +IF (TD_DARWIN) + # missing a few dependencies, such as + # AUX_SOURCE_DIRECTORY(. SRC) + # ADD_EXECUTABLE(taosdemox ${SRC}) + # + # #find_program(HAVE_CURL NAMES curl) + # IF ((NOT TD_ARM_64) AND (NOT TD_ARM_32)) + # ADD_DEFINITIONS(-DTD_LOWA_CURL) + # LINK_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/libcurl/lib) + # ADD_LIBRARY(curl STATIC IMPORTED) + # SET_PROPERTY(TARGET curl PROPERTY IMPORTED_LOCATION ${TD_COMMUNITY_DIR}/deps/libcurl/lib/libcurl.a) + # TARGET_LINK_LIBRARIES(taosdemox curl) + # ENDIF () + # + # IF (TD_SOMODE_STATIC) + # TARGET_LINK_LIBRARIES(taosdemox taos_static cJson) + # ELSE () + # TARGET_LINK_LIBRARIES(taosdemox taos cJson) + # ENDIF () +ENDIF () + diff --git a/src/kit/taosdump/CMakeLists.txt b/src/kit/taosdump/CMakeLists.txt index dc9ac6d4a7..b50ad85c08 100644 --- a/src/kit/taosdump/CMakeLists.txt +++ b/src/kit/taosdump/CMakeLists.txt @@ -12,5 +12,15 @@ IF (TD_LINUX) TARGET_LINK_LIBRARIES(taosdump taos_static) ELSE () TARGET_LINK_LIBRARIES(taosdump taos) - ENDIF () + ENDIF () +ENDIF () + +IF (TD_DARWIN) + # missing for macosx + # ADD_EXECUTABLE(taosdump ${SRC}) + # IF (TD_SOMODE_STATIC) + # TARGET_LINK_LIBRARIES(taosdump taos_static) + # ELSE () + # TARGET_LINK_LIBRARIES(taosdump taos) + # ENDIF () ENDIF () diff --git a/src/mnode/CMakeLists.txt b/src/mnode/CMakeLists.txt index 4123098694..ae4a37320c 100644 --- a/src/mnode/CMakeLists.txt +++ b/src/mnode/CMakeLists.txt @@ -9,4 +9,15 @@ IF (TD_LINUX) AUX_SOURCE_DIRECTORY(src SRC) ADD_LIBRARY(mnode ${SRC}) -ENDIF () \ No newline at end of file +ENDIF () + +IF (TD_DARWIN) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/dnode/inc) + + INCLUDE_DIRECTORIES(inc) + AUX_SOURCE_DIRECTORY(src SRC) + + ADD_LIBRARY(mnode ${SRC}) +ENDIF () + diff --git a/src/mnode/src/mnodeCluster.c b/src/mnode/src/mnodeCluster.c index a35e304810..08bf20e1a8 100644 --- a/src/mnode/src/mnodeCluster.c +++ b/src/mnode/src/mnodeCluster.c @@ -138,6 +138,13 @@ void mnodeDecClusterRef(SClusterObj *pCluster) { sdbDecRef(tsClusterSdb, pCluster); } +#ifdef __APPLE__ +bool taosGetSystemUid(char *uid) { + fprintf(stderr, "%s[%d]%s(): not implemented yet!\n", basename(__FILE__), __LINE__, __func__); + abort(); + return false; +} +#endif // __APPLE__ static int32_t mnodeCreateCluster() { int32_t numOfClusters = sdbGetNumOfRows(tsClusterSdb); if (numOfClusters != 0) return TSDB_CODE_SUCCESS; diff --git a/src/os/src/darwin/eok.c b/src/os/src/darwin/eok.c index a6dceb3df3..1be5ec4ac1 100644 --- a/src/os/src/darwin/eok.c +++ b/src/os/src/darwin/eok.c @@ -103,9 +103,9 @@ struct eok_event_s { typedef struct eoks_s eoks_t; struct eoks_s { pthread_mutex_t lock; - ep_over_kq_t *eoks; - int neoks; - ep_over_kq_t *eoks_free; + ep_over_kq_t **eoks; // note: this memory leaks when process terminates + int neoks; // we can add an extra api to let user clean + ep_over_kq_t *eoks_free; // currently, we just keep it simple stupid }; static eoks_t eoks = { @@ -297,7 +297,7 @@ int epoll_create(int size) { struct epoll_event ev = {0}; ev.events = EPOLLIN; ev.data.ptr = &eok_dummy; - D("epoll_create epfd:[%d]", eok->idx); + D("epoll_create epfd:[%d] and sv0[%d]", eok->idx, eok->sv[0]); if (epoll_ctl(eok->idx, EPOLL_CTL_ADD, eok->sv[0], &ev)) { e = errno; epoll_close(eok->idx); @@ -517,8 +517,8 @@ int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) eok_event_t *ev = (eok_event_t*)kev->udata; A(kev->ident == ev->fd, "internal logic error"); if (kev->flags & EV_ERROR) { - D("error when processing change list for fd[%d], error[%s], kev_flags:[%04x:%s]", - ev->fd, strerror(kev->data), kev->flags, kev_flags_str(kev->flags, 0)); + D("epfd[%d] error when processing change list for fd[%d], error[%s], kev_flags:[%04x:%s]", + epfd, ev->fd, strerror(kev->data), kev->flags, kev_flags_str(kev->flags, 0)); } switch (kev->filter) { case EVFILT_READ: { @@ -528,11 +528,11 @@ int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) char c = '\0'; A(1==recv(kev->ident, &c, 1, 0), "internal logic error"); A(0==memcmp(&c, "1", 1), "internal logic error"); - D("wokenup"); + D("epfd[%d] wokenup", epfd); continue; } else { if (ev->changed==3) { - D("already requested to delete for fd[%d]", ev->fd); + D("epfd[%d] already requested to delete for fd[%d]", epfd, ev->fd); // TODO: write a unit test for this case // EV_DELETE? continue; @@ -550,7 +550,8 @@ int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) } // rounded to what user care pev.events = pev.events & ev->epev.events; - D("events found for fd[%d]: [%04x:%s], which was registered: [%04x:%s], kev_flags: [%04x:%s]", + D("epfd[%d] events found for fd[%d]: [%04x:%s], which was registered: [%04x:%s], kev_flags: [%04x:%s]", + epfd, ev->fd, pev.events, events_str(pev.events, 0), ev->epev.events, events_str(ev->epev.events, 1), kev->flags, kev_flags_str(kev->flags, 2)); @@ -573,7 +574,7 @@ int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) while (p) { eok_event_t *next = p->next; if (p->changed==3) { - D("removing registered event for fd[%d]: [%04x:%s]", p->fd, p->epev.events, events_str(p->epev.events, 0)); + D("epfd[%d] removing registered event for fd[%d]: [%04x:%s]", epfd, p->fd, p->epev.events, events_str(p->epev.events, 0)); eok_free_ev(eok, p); } p = next; @@ -591,13 +592,13 @@ int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) } } while (cnts==0); if (cnts>0) { - D("kevent64 waiting done with [%d] events", cnts); + D("kevent64 epfd[%d] waiting done with [%d] events", epfd, cnts); } A(0==pthread_mutex_unlock(&eok->lock), ""); if (e) { errno = e; - E("epoll_wait failed"); + E("epfd[%d] epoll_wait failed", epfd); return -1; } @@ -752,7 +753,7 @@ static int eok_chgs_refresh(ep_over_kq_t *eok, eok_event_t *oev, eok_event_t *ev eok->kchanges[eok->ichanges++] = *kwev; ++n; } - D("add #changes[%d] for fd[%d], and now #changes/registers [%d/%d]", n, ev->fd, eok->ichanges, eok->evs_count); + D("epfd[%d]: add #changes[%d] for fd[%d], and now #changes/registers [%d/%d]", eok->idx, n, ev->fd, eok->ichanges, eok->evs_count); return 0; } @@ -764,14 +765,21 @@ static ep_over_kq_t* eoks_alloc(void) { if (eoks.eoks_free) { eok = eoks.eoks_free; eoks.eoks_free = eok->next; + A(eoks.eoks, "internal logic error"); + A(eok->idx>=0 && eok->idxidx)==NULL, "internal logic error"); + *(eoks.eoks + eok->idx) = eok; eok->next = NULL; + eok->stopping = 0; break; } - ep_over_kq_t *p = (ep_over_kq_t*)realloc(eoks.eoks, sizeof(*p) * (eoks.neoks+1)); - if (!p) break; - eoks.eoks = p; - eok = eoks.eoks + eoks.neoks; - memset(eok, 0, sizeof(*eok)); + eok = (ep_over_kq_t*)calloc(1, sizeof(*eok)); + if (!eok) break; + eok->idx = -1; + ep_over_kq_t **ar = (ep_over_kq_t**)realloc(eoks.eoks, sizeof(**ar) * (eoks.neoks+1)); + if (!ar) break; + eoks.eoks = ar; + *(eoks.eoks + eoks.neoks) = eok; eok->idx = eoks.neoks; eok->kq = -1; eok->sv[0] = -1; @@ -784,6 +792,11 @@ static ep_over_kq_t* eoks_alloc(void) { errno = ENOMEM; return NULL; } + if (eok->idx==-1) { + free(eok); + errno = ENOMEM; + return NULL; + } if (eok->lock_valid) { return eok; } @@ -801,23 +814,50 @@ static ep_over_kq_t* eoks_alloc(void) { static void eoks_free(ep_over_kq_t *eok) { A(0==pthread_mutex_lock(&eoks.lock), ""); do { + A(eok->idx>=0 && eok->idxnext==NULL, "internal logic error"); // leave eok->kchanges as is A(eok->ichanges==0, "internal logic error"); A(eok->waiting==0, "internal logic error"); - if (eok->evs_count==1) { - A(eok->evs_head && eok->evs_tail && eok->evs_head==eok->evs_tail, "internal logic error"); - A(eok->evs_head->fd==eok->sv[0] && eok->sv[0]!=-1 && eok->sv[1]!=-1, "internal logic error"); - // fd is critical system resource - close(eok->sv[0]); - eok->sv[0] = -1; - close(eok->sv[1]); - eok->sv[1] = -1; - eok_free_ev(eok, eok->evs_head); + eok_event_t *ev = eok->evs_head; + while (ev) { + eok_event_t *next = ev->next; + if (ev->fd==eok->sv[0]) { + // fd is critical system resource + close(eok->sv[0]); + eok->sv[0] = -1; + close(eok->sv[1]); + eok->sv[1] = -1; + eok_free_ev(eok, ev); + } else { + // user forget calling epoll_ctl(EPOLL_CTL_DEL) before calling epoll_close/close? + // calling close(ev->fd) here smells really bad +#ifdef LET_IT_BE + // we just let it be and reclaim ev + eok_free_ev(eok, ev); +#else + // panic otherwise, if LET_IT_BE not defined + A(eok->evs_head==NULL && eok->evs_tail==NULL && eok->evs_count==0, + "epfd[%d] fd[%d]: internal logic error: have you epoll_ctl(EPOLL_CTL_DEL) everything before calling epoll_close?", + eok->idx, ev->fd); +#endif + } + ev = next; } - A(eok->evs_head==NULL && eok->evs_tail==NULL && eok->evs_count==0, "internal logic error"); + // if (eok->evs_count==1) { + // A(eok->evs_head && eok->evs_tail && eok->evs_head==eok->evs_tail, "internal logic error"); + // A(eok->evs_head->fd==eok->sv[0] && eok->sv[0]!=-1 && eok->sv[1]!=-1, "internal logic error"); + // // fd is critical system resource + // close(eok->sv[0]); + // eok->sv[0] = -1; + // close(eok->sv[1]); + // eok->sv[1] = -1; + // eok_free_ev(eok, eok->evs_head); + // } + A(eok->evs_head==NULL && eok->evs_tail==NULL && eok->evs_count==0, + "internal logic error: have you epoll_ctl(EPOLL_CTL_DEL) everything before calling epoll_close?"); A(eok->sv[0]==-1 && eok->sv[1]==-1, "internal logic error"); if (eok->kq!=-1) { close(eok->kq); @@ -825,6 +865,7 @@ static void eoks_free(ep_over_kq_t *eok) { } eok->next = eoks.eoks_free; eoks.eoks_free = eok; + *(eoks.eoks + eok->idx) = NULL; } while (0); A(0==pthread_mutex_unlock(&eoks.lock), ""); } @@ -837,7 +878,7 @@ static ep_over_kq_t* eoks_find(int epfd) { break; } A(eoks.eoks, "internal logic error"); - eok = eoks.eoks + epfd; + eok = *(eoks.eoks + epfd); A(eok->next==NULL, "internal logic error"); A(eok->lock_valid, "internal logic error"); } while (0); diff --git a/src/os/src/detail/CMakeLists.txt b/src/os/src/detail/CMakeLists.txt index e4052a4fb3..1c5e55a522 100644 --- a/src/os/src/detail/CMakeLists.txt +++ b/src/os/src/detail/CMakeLists.txt @@ -13,3 +13,4 @@ TARGET_LINK_LIBRARIES(osdetail os) IF (TD_ARM_32 OR TD_LINUX_32) TARGET_LINK_LIBRARIES(osdetail atomic) ENDIF () + diff --git a/src/plugins/http/CMakeLists.txt b/src/plugins/http/CMakeLists.txt index 56c3c25d7c..274f50d24a 100644 --- a/src/plugins/http/CMakeLists.txt +++ b/src/plugins/http/CMakeLists.txt @@ -8,14 +8,29 @@ INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) INCLUDE_DIRECTORIES(inc) AUX_SOURCE_DIRECTORY(src SRC) - + IF (TD_LINUX) ADD_LIBRARY(http ${SRC}) TARGET_LINK_LIBRARIES(http z) IF (TD_SOMODE_STATIC) TARGET_LINK_LIBRARIES(http taos_static) - ELSE () + ELSE () + TARGET_LINK_LIBRARIES(http taos) + ENDIF () + + IF (TD_ADMIN) + TARGET_LINK_LIBRARIES(http admin) + ENDIF () +ENDIF () + +IF (TD_DARWIN) + ADD_LIBRARY(http ${SRC}) + TARGET_LINK_LIBRARIES(http z) + + IF (TD_SOMODE_STATIC) + TARGET_LINK_LIBRARIES(http taos_static) + ELSE () TARGET_LINK_LIBRARIES(http taos) ENDIF () diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index 1cc73aef06..97a5b4bc23 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -31,11 +31,36 @@ static bool httpReadData(HttpContext *pContext); +#ifdef __APPLE__ +static int sv_dummy = 0; +#endif + static void httpStopThread(HttpThread* pThread) { pThread->stop = true; // signal the thread to stop, try graceful method first, // and use pthread_cancel when failed +#ifdef __APPLE__ + int sv[2]; + sv[0] = sv[1] = -1; + int r = socketpair(PF_LOCAL, SOCK_STREAM, 0, sv); + do { + if (r) break; + struct epoll_event ev = {0}; + ev.events = EPOLLIN; + ev.data.ptr = &sv_dummy; + pThread->stop = true; + r = epoll_ctl(pThread->pollFd, EPOLL_CTL_ADD, sv[0], &ev); + if (r) break; + if (1!=send(sv[1], "1", 1, 0)) { + r = -1; + break; + } + } while (0); + if (r) { + pthread_cancel(pThread->thread); + } +#else // __APPLE__ struct epoll_event event = { .events = EPOLLIN }; eventfd_t fd = eventfd(1, 0); if (fd == -1) { @@ -46,11 +71,23 @@ static void httpStopThread(HttpThread* pThread) { httpError("%s, failed to call epoll_ctl, will call pthread_cancel instead, which may result in data corruption: %s", pThread->label, strerror(errno)); pthread_cancel(pThread->thread); } +#endif // __APPLE__ pthread_join(pThread->thread, NULL); +#ifdef __APPLE__ + if (sv[0]!=-1) { + close(sv[0]); + sv[0] = -1; + } + if (sv[1]!=-1) { + close(sv[1]); + sv[1] = -1; + } +#else // __APPLE__ if (fd != -1) { close(fd); } +#endif // __APPLE__ #ifdef __APPLE__ epoll_close(pThread->pollFd); @@ -97,6 +134,15 @@ static void httpProcessHttpData(void *param) { if (fdNum <= 0) continue; for (int32_t i = 0; i < fdNum; ++i) { +#ifdef __APPLE__ + if (events[i].data.ptr == &sv_dummy) { + // no need to drain the recv buffer of sv[0] + // since there's only one time to send at most 1 byte to sv[0] + // btw, pThread->stop shall be already set, thus never reached here + httpDebug("if you see this line, there's internal logic error"); + continue; + } +#endif // __APPLE__ pContext = httpGetContext(events[i].data.ptr); if (pContext == NULL) { httpError("context:%p, is already released, close connect", events[i].data.ptr); diff --git a/src/plugins/monitor/CMakeLists.txt b/src/plugins/monitor/CMakeLists.txt index 90189c9d75..6b26c0447a 100644 --- a/src/plugins/monitor/CMakeLists.txt +++ b/src/plugins/monitor/CMakeLists.txt @@ -8,10 +8,20 @@ AUX_SOURCE_DIRECTORY(./src SRC) IF (TD_LINUX) ADD_LIBRARY(monitor ${SRC}) - + IF (TD_SOMODE_STATIC) TARGET_LINK_LIBRARIES(monitor taos_static) ELSE () TARGET_LINK_LIBRARIES(monitor taos) ENDIF () -ENDIF () +ENDIF () + +IF (TD_DARWIN) + ADD_LIBRARY(monitor ${SRC}) + + IF (TD_SOMODE_STATIC) + TARGET_LINK_LIBRARIES(monitor taos_static) + ELSE () + TARGET_LINK_LIBRARIES(monitor taos) + ENDIF () +ENDIF () diff --git a/src/plugins/mqtt/CMakeLists.txt b/src/plugins/mqtt/CMakeLists.txt index 3761f70134..b6de421517 100644 --- a/src/plugins/mqtt/CMakeLists.txt +++ b/src/plugins/mqtt/CMakeLists.txt @@ -18,3 +18,15 @@ IF (TD_LINUX) TARGET_LINK_LIBRARIES(mqtt taos) ENDIF () ENDIF () + +IF (TD_DARWIN) + ADD_LIBRARY(mqtt ${SRC}) + TARGET_LINK_LIBRARIES(mqtt cJson mqttc) + + IF (TD_SOMODE_STATIC) + TARGET_LINK_LIBRARIES(mqtt taos_static) + ELSE () + TARGET_LINK_LIBRARIES(mqtt taos) + ENDIF () +ENDIF () + diff --git a/src/query/CMakeLists.txt b/src/query/CMakeLists.txt index e403251858..967e86de3c 100644 --- a/src/query/CMakeLists.txt +++ b/src/query/CMakeLists.txt @@ -14,3 +14,8 @@ IF (TD_LINUX) TARGET_LINK_LIBRARIES(query m rt) ADD_SUBDIRECTORY(tests) ENDIF () + +IF (TD_DARWIN) + TARGET_LINK_LIBRARIES(query m) + ADD_SUBDIRECTORY(tests) +ENDIF () diff --git a/src/rpc/src/rpcTcp.c b/src/rpc/src/rpcTcp.c index 2bdc58390d..ba90f93073 100644 --- a/src/rpc/src/rpcTcp.c +++ b/src/rpc/src/rpcTcp.c @@ -308,7 +308,10 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int num, void * pthread_attr_destroy(&thattr); if (code != 0) { #ifdef __APPLE__ - epoll_close(pThreadObj->pollFd); + if (pThreadObj->pollFd!=-1) { + epoll_close(pThreadObj->pollFd); + pThreadObj->pollFd = -1; + } #else taosCloseSocket(pThreadObj->pollFd); #endif @@ -517,7 +520,10 @@ static void *taosProcessTcpData(void *param) { } #ifdef __APPLE__ - if (pThreadObj->pollFd >=0) epoll_close(pThreadObj->pollFd); + if (pThreadObj->pollFd >=0) { + epoll_close(pThreadObj->pollFd); + pThreadObj->pollFd = -1; + } #else if (pThreadObj->pollFd >=0) taosCloseSocket(pThreadObj->pollFd); #endif diff --git a/src/rpc/test/CMakeLists.txt b/src/rpc/test/CMakeLists.txt index 9a4bcc353d..e923105860 100644 --- a/src/rpc/test/CMakeLists.txt +++ b/src/rpc/test/CMakeLists.txt @@ -16,3 +16,17 @@ IF (TD_LINUX) ADD_EXECUTABLE(rserver ${SERVER_SRC}) TARGET_LINK_LIBRARIES(rserver trpc) ENDIF () + +IF (TD_DARWIN) + LIST(APPEND CLIENT_SRC ./rclient.c) + ADD_EXECUTABLE(rclient ${CLIENT_SRC}) + TARGET_LINK_LIBRARIES(rclient trpc) + + LIST(APPEND SCLIENT_SRC ./rsclient.c) + ADD_EXECUTABLE(rsclient ${SCLIENT_SRC}) + TARGET_LINK_LIBRARIES(rsclient trpc) + + LIST(APPEND SERVER_SRC ./rserver.c) + ADD_EXECUTABLE(rserver ${SERVER_SRC}) + TARGET_LINK_LIBRARIES(rserver trpc) +ENDIF () diff --git a/src/sync/CMakeLists.txt b/src/sync/CMakeLists.txt index 6a53380841..3721163f79 100644 --- a/src/sync/CMakeLists.txt +++ b/src/sync/CMakeLists.txt @@ -3,7 +3,7 @@ PROJECT(TDengine) INCLUDE_DIRECTORIES(inc) AUX_SOURCE_DIRECTORY(src SRC) - + IF (TD_LINUX) LIST(REMOVE_ITEM SRC src/syncArbitrator.c) ADD_LIBRARY(sync ${SRC}) @@ -16,3 +16,16 @@ IF (TD_LINUX) #ADD_SUBDIRECTORY(test) ENDIF () + +IF (TD_DARWIN) + LIST(REMOVE_ITEM SRC src/syncArbitrator.c) + ADD_LIBRARY(sync ${SRC}) + TARGET_LINK_LIBRARIES(sync tutil pthread common) + + LIST(APPEND BIN_SRC src/syncArbitrator.c) + LIST(APPEND BIN_SRC src/syncTcp.c) + ADD_EXECUTABLE(tarbitrator ${BIN_SRC}) + TARGET_LINK_LIBRARIES(tarbitrator sync common osdetail tutil) + + #ADD_SUBDIRECTORY(test) +ENDIF () diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c index cb2379583f..7678a7d284 100644 --- a/src/sync/src/syncRetrieve.c +++ b/src/sync/src/syncRetrieve.c @@ -14,7 +14,7 @@ */ #define _DEFAULT_SOURCE -#include +// #include #include "os.h" #include "taoserror.h" #include "tlog.h" diff --git a/src/sync/test/CMakeLists.txt b/src/sync/test/CMakeLists.txt index 256e87580d..ab2e6c307b 100644 --- a/src/sync/test/CMakeLists.txt +++ b/src/sync/test/CMakeLists.txt @@ -13,4 +13,3 @@ IF (TD_LINUX) TARGET_LINK_LIBRARIES(syncServer sync trpc common) ENDIF () - diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index 48b4d76561..3606aea76b 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -5,7 +5,7 @@ INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/rpc/inc) AUX_SOURCE_DIRECTORY(src SRC) ADD_LIBRARY(tutil ${SRC}) TARGET_LINK_LIBRARIES(tutil pthread osdetail lz4 z) - + IF (TD_LINUX) TARGET_LINK_LIBRARIES(tutil m rt) # ADD_SUBDIRECTORY(tests) @@ -28,5 +28,6 @@ IF (TD_LINUX) ELSEIF (TD_WINDOWS) TARGET_LINK_LIBRARIES(tutil iconv regex winmm IPHLPAPI ws2_32 wepoll) ELSEIF(TD_DARWIN) + TARGET_LINK_LIBRARIES(tutil m) TARGET_LINK_LIBRARIES(tutil iconv) ENDIF() diff --git a/src/vnode/CMakeLists.txt b/src/vnode/CMakeLists.txt index c953883361..3a26172411 100644 --- a/src/vnode/CMakeLists.txt +++ b/src/vnode/CMakeLists.txt @@ -13,3 +13,9 @@ IF (TD_LINUX) ADD_LIBRARY(vnode ${SRC}) TARGET_LINK_LIBRARIES(vnode tsdb tcq) ENDIF () + +IF (TD_DARWIN) + ADD_LIBRARY(vnode ${SRC}) + TARGET_LINK_LIBRARIES(vnode tsdb tcq) +ENDIF () + diff --git a/src/wal/CMakeLists.txt b/src/wal/CMakeLists.txt index 681bed5425..d3cd86df50 100644 --- a/src/wal/CMakeLists.txt +++ b/src/wal/CMakeLists.txt @@ -8,4 +8,10 @@ IF (TD_LINUX) ADD_LIBRARY(twal ${SRC}) TARGET_LINK_LIBRARIES(twal tutil common) ADD_SUBDIRECTORY(test) -ENDIF () +ENDIF () + +IF (TD_DARWIN) + ADD_LIBRARY(twal ${SRC}) + TARGET_LINK_LIBRARIES(twal tutil common) + ADD_SUBDIRECTORY(test) +ENDIF () diff --git a/src/wal/test/CMakeLists.txt b/src/wal/test/CMakeLists.txt index b8338b1738..aec0602ac0 100644 --- a/src/wal/test/CMakeLists.txt +++ b/src/wal/test/CMakeLists.txt @@ -10,4 +10,12 @@ IF (TD_LINUX) ENDIF () +IF (TD_DARWIN) + INCLUDE_DIRECTORIES(../inc) + + LIST(APPEND WALTEST_SRC ./waltest.c) + ADD_EXECUTABLE(waltest ${WALTEST_SRC}) + TARGET_LINK_LIBRARIES(waltest twal osdetail tutil) + +ENDIF () diff --git a/tests/comparisonTest/tdengine/CMakeLists.txt b/tests/comparisonTest/tdengine/CMakeLists.txt index aaa18592ed..a12e36ab6b 100644 --- a/tests/comparisonTest/tdengine/CMakeLists.txt +++ b/tests/comparisonTest/tdengine/CMakeLists.txt @@ -5,3 +5,8 @@ IF (TD_LINUX) add_executable(tdengineTest tdengineTest.c) target_link_libraries(tdengineTest taos_static tutil common pthread) ENDIF() + +IF (TD_DARWIN) + add_executable(tdengineTest tdengineTest.c) + target_link_libraries(tdengineTest taos_static tutil common pthread) +ENDIF() diff --git a/tests/examples/c/CMakeLists.txt b/tests/examples/c/CMakeLists.txt index 7cacefda57..5af7935c6b 100644 --- a/tests/examples/c/CMakeLists.txt +++ b/tests/examples/c/CMakeLists.txt @@ -10,6 +10,9 @@ IF (TD_LINUX) ENDIF () IF (TD_DARWIN) INCLUDE_DIRECTORIES(. ${TD_COMMUNITY_DIR}/src/inc ${TD_COMMUNITY_DIR}/src/client/inc ${TD_COMMUNITY_DIR}/inc) + AUX_SOURCE_DIRECTORY(. SRC) + ADD_EXECUTABLE(demo demo.c) + TARGET_LINK_LIBRARIES(demo taos_static trpc tutil pthread ) ADD_EXECUTABLE(epoll epoll.c) TARGET_LINK_LIBRARIES(epoll taos_static trpc tutil pthread ) ENDIF () diff --git a/tests/examples/c/demo.c b/tests/examples/c/demo.c index 54e81d33b9..f5ad37acef 100644 --- a/tests/examples/c/demo.c +++ b/tests/examples/c/demo.c @@ -86,7 +86,7 @@ void Test(TAOS *taos, char *qstr, int index) { int i = 0; for (i = 0; i < 10; ++i) { - sprintf(qstr, "insert into m1 values (%" PRId64 ", %d, %d, %d, %d, %f, %lf, '%s')", 1546300800000 + i * 1000, i, i, i, i*10000000, i*1.0, i*2.0, "hello"); + sprintf(qstr, "insert into m1 values (%" PRId64 ", %d, %d, %d, %d, %f, %lf, '%s')", 1546300800000LL + i * 1000, i, i, i, i*10000000, i*1.0, i*2.0, "hello"); printf("qstr: %s\n", qstr); // note: how do you wanna do if taos_query returns non-NULL From 7a78290715a86124ae7e2fc59191f8a544e7e144 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 17 Jan 2021 21:45:40 +0800 Subject: [PATCH 23/62] TD-1207 change socket to int32 --- src/dnode/src/dnodeCheck.c | 8 ++++---- src/dnode/src/dnodeTelemetry.c | 2 +- src/os/inc/osFile.h | 2 +- src/os/inc/osSocket.h | 5 ++--- src/os/src/darwin/darwinFile.c | 2 +- src/os/src/darwin/darwinSocket.c | 2 +- src/os/src/detail/osFile.c | 2 +- src/os/src/detail/osSocket.c | 4 ++-- src/os/src/windows/wFile.c | 2 +- src/os/src/windows/wSocket.c | 4 ++-- src/plugins/http/inc/httpInt.h | 4 ++-- src/plugins/http/src/httpServer.c | 2 +- src/rpc/src/rpcTcp.c | 20 ++++++++++---------- src/rpc/src/rpcUdp.c | 2 +- src/sync/inc/syncInt.h | 4 ++-- src/sync/inc/syncTcp.h | 4 ++-- src/sync/src/syncArbitrator.c | 10 +++++----- src/sync/src/syncMain.c | 8 ++++---- src/sync/src/syncTcp.c | 14 +++++++------- src/util/inc/tsocket.h | 20 ++++++++++---------- src/util/src/tnettest.c | 24 ++++++++++++------------ src/util/src/tsocket.c | 28 ++++++++++++++-------------- 22 files changed, 86 insertions(+), 87 deletions(-) diff --git a/src/dnode/src/dnodeCheck.c b/src/dnode/src/dnodeCheck.c index 8955fb5643..2c8dcfdcc9 100644 --- a/src/dnode/src/dnodeCheck.c +++ b/src/dnode/src/dnodeCheck.c @@ -30,10 +30,10 @@ static SCheckItem tsCheckItem[TSDB_CHECK_ITEM_MAX] = {{0}}; int64_t tsMinFreeMemSizeForStart = 0; static int32_t bindTcpPort(int32_t port) { - SOCKET serverSocket; + int32_t serverSocket; struct sockaddr_in server_addr; - if ((serverSocket = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0) { + if ((serverSocket = ( int32_t)socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0) { dError("socket() fail: %s", strerror(errno)); return -1; } @@ -60,10 +60,10 @@ static int32_t bindTcpPort(int32_t port) { } static int32_t bindUdpPort(int32_t port) { - SOCKET serverSocket; + int32_t serverSocket; struct sockaddr_in server_addr; - if ((serverSocket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0) { + if ((serverSocket = (int32_t)socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0) { dError("socket() fail: %s", strerror(errno)); return -1; } diff --git a/src/dnode/src/dnodeTelemetry.c b/src/dnode/src/dnodeTelemetry.c index 9291831884..2834ea3391 100644 --- a/src/dnode/src/dnodeTelemetry.c +++ b/src/dnode/src/dnodeTelemetry.c @@ -200,7 +200,7 @@ static void sendTelemetryReport() { dTrace("failed to get IP address of " TELEMETRY_SERVER ", reason:%s", strerror(errno)); return; } - SOCKET fd = taosOpenTcpClientSocket(ip, TELEMETRY_PORT, 0); + int32_t fd = taosOpenTcpClientSocket(ip, TELEMETRY_PORT, 0); if (fd < 0) { dTrace("failed to create socket for telemetry, reason:%s", strerror(errno)); return; diff --git a/src/os/inc/osFile.h b/src/os/inc/osFile.h index 19cc78472c..d72f9cefb9 100644 --- a/src/os/inc/osFile.h +++ b/src/os/inc/osFile.h @@ -39,7 +39,7 @@ int32_t taosRenameFile(char *fullPath, char *suffix, char delimiter, char **dstP } // TAOS_OS_FUNC_FILE_SENDIFLE -int64_t taosSendFile(SOCKET dfd, int32_t sfd, int64_t *offset, int64_t size); +int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t *offset, int64_t size); int64_t taosFSendFile(FILE *outfile, FILE *infile, int64_t *offset, int64_t size); #ifdef TAOS_RANDOM_FILE_FAIL diff --git a/src/os/inc/osSocket.h b/src/os/inc/osSocket.h index 4e2671ce2b..9683adf45a 100644 --- a/src/os/inc/osSocket.h +++ b/src/os/inc/osSocket.h @@ -33,7 +33,6 @@ extern "C" { x = FD_INITIALIZER; \ } \ } - typedef int32_t SOCKET; #endif #ifndef TAOS_OS_DEF_EPOLL @@ -58,13 +57,13 @@ extern "C" { #endif // TAOS_OS_FUNC_SOCKET -int32_t taosSetNonblocking(SOCKET sock, int32_t on); +int32_t taosSetNonblocking(int32_t sock, int32_t on); void taosIgnSIGPIPE(); void taosBlockSIGPIPE(); void taosSetMaskSIGPIPE(); // TAOS_OS_FUNC_SOCKET_SETSOCKETOPT -int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen); +int32_t taosSetSockOpt(int32_t socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen); // TAOS_OS_FUNC_SOCKET_INET uint32_t taosInetAddr(char *ipAddr); diff --git a/src/os/src/darwin/darwinFile.c b/src/os/src/darwin/darwinFile.c index 1e77cd68d8..00c1d8532f 100644 --- a/src/os/src/darwin/darwinFile.c +++ b/src/os/src/darwin/darwinFile.c @@ -51,7 +51,7 @@ int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t co return writeLen; } -int64_t taosSendFile(SOCKET dfd, int32_t sfd, int64_t* offset, int64_t count) { +int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t* offset, int64_t count) { lseek(sfd, (int32_t)(*offset), 0); int64_t writeLen = 0; uint8_t buffer[_SEND_FILE_STEP_] = { 0 }; diff --git a/src/os/src/darwin/darwinSocket.c b/src/os/src/darwin/darwinSocket.c index 69a4666d34..9137d4b125 100644 --- a/src/os/src/darwin/darwinSocket.c +++ b/src/os/src/darwin/darwinSocket.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "os.h" -int taosSetSockOpt(SOCKET socketfd, int level, int optname, void *optval, int optlen) { +int taosSetSockOpt(int32_t socketfd, int level, int optname, void *optval, int optlen) { if (level == SOL_SOCKET && optname == SO_SNDBUF) { return 0; } diff --git a/src/os/src/detail/osFile.c b/src/os/src/detail/osFile.c index bb68622731..9560ce59bb 100644 --- a/src/os/src/detail/osFile.c +++ b/src/os/src/detail/osFile.c @@ -121,7 +121,7 @@ int64_t taosLSeekImp(int32_t fd, int64_t offset, int32_t whence) { #ifndef TAOS_OS_FUNC_FILE_SENDIFLE -int64_t taosSendFile(SOCKET dfd, int32_t sfd, int64_t *offset, int64_t size) { +int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t *offset, int64_t size) { int64_t leftbytes = size; int64_t sentbytes; diff --git a/src/os/src/detail/osSocket.c b/src/os/src/detail/osSocket.c index d03e8086bf..f929b3e717 100644 --- a/src/os/src/detail/osSocket.c +++ b/src/os/src/detail/osSocket.c @@ -19,7 +19,7 @@ #ifndef TAOS_OS_FUNC_SOCKET -int32_t taosSetNonblocking(SOCKET sock, int32_t on) { +int32_t taosSetNonblocking(int32_t sock, int32_t on) { int32_t flags = 0; if ((flags = fcntl(sock, F_GETFL, 0)) < 0) { uError("fcntl(F_GETFL) error: %d (%s)\n", errno, strerror(errno)); @@ -67,7 +67,7 @@ void taosSetMaskSIGPIPE() { #ifndef TAOS_OS_FUNC_SOCKET_SETSOCKETOPT -int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen) { +int32_t taosSetSockOpt(int32_t socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen) { return setsockopt(socketfd, level, optname, optval, (socklen_t)optlen); } diff --git a/src/os/src/windows/wFile.c b/src/os/src/windows/wFile.c index 4ad195dc6f..e388e5fc67 100644 --- a/src/os/src/windows/wFile.c +++ b/src/os/src/windows/wFile.c @@ -76,7 +76,7 @@ int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t co return writeLen; } -int64_t taosSendFile(SOCKET dfd, int32_t sfd, int64_t *offset, int64_t count) { +int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t *offset, int64_t count) { if (offset != NULL) lseek(sfd, (int32_t)(*offset), 0); int64_t writeLen = 0; diff --git a/src/os/src/windows/wSocket.c b/src/os/src/windows/wSocket.c index 4e6ee15e77..679842a7e9 100644 --- a/src/os/src/windows/wSocket.c +++ b/src/os/src/windows/wSocket.c @@ -34,7 +34,7 @@ void taosWinSocketInit() { } } -int32_t taosSetNonblocking(SOCKET sock, int32_t on) { +int32_t taosSetNonblocking(int32_t sock, int32_t on) { u_long mode; if (on) { mode = 1; @@ -50,7 +50,7 @@ void taosIgnSIGPIPE() {} void taosBlockSIGPIPE() {} void taosSetMaskSIGPIPE() {} -int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen) { +int32_t taosSetSockOpt(int32_t socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen) { if (level == SOL_SOCKET && optname == TCP_KEEPCNT) { return 0; } diff --git a/src/plugins/http/inc/httpInt.h b/src/plugins/http/inc/httpInt.h index 00a7bcf6c5..c23cfe0637 100644 --- a/src/plugins/http/inc/httpInt.h +++ b/src/plugins/http/inc/httpInt.h @@ -166,7 +166,7 @@ typedef struct HttpThread { HttpContext * pHead; pthread_mutex_t threadMutex; bool stop; - SOCKET pollFd; + int32_t pollFd; int32_t numOfContexts; int32_t threadId; char label[HTTP_LABEL_SIZE]; @@ -179,7 +179,7 @@ typedef struct HttpServer { uint16_t serverPort; int8_t stop; int8_t reserve; - SOCKET fd; + int32_t fd; int32_t numOfThreads; int32_t methodScannerLen; int32_t requestNum; diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index 5a04a021cd..72e0c7129e 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -260,7 +260,7 @@ bool httpInitConnect() { return false; } - pThread->pollFd = (SOCKET)epoll_create(HTTP_MAX_EVENTS); // size does not matter + pThread->pollFd = (int32_t)epoll_create(HTTP_MAX_EVENTS); // size does not matter if (pThread->pollFd < 0) { httpError("http thread:%s, failed to create HTTP epoll", pThread->label); pthread_mutex_destroy(&(pThread->threadMutex)); diff --git a/src/rpc/src/rpcTcp.c b/src/rpc/src/rpcTcp.c index 4cc7530784..6b3682a441 100644 --- a/src/rpc/src/rpcTcp.c +++ b/src/rpc/src/rpcTcp.c @@ -31,7 +31,7 @@ typedef struct SFdObj { void *signature; - SOCKET fd; // TCP socket FD + int32_t fd; // TCP socket FD int closedByApp; // 1: already closed by App void *thandle; // handle from upper layer, like TAOS uint32_t ip; @@ -47,7 +47,7 @@ typedef struct SThreadObj { pthread_mutex_t mutex; uint32_t ip; bool stop; - SOCKET pollFd; + int32_t pollFd; int numOfFds; int threadId; char label[TSDB_LABEL_LEN]; @@ -56,7 +56,7 @@ typedef struct SThreadObj { } SThreadObj; typedef struct { - SOCKET fd; + int32_t fd; uint32_t ip; uint16_t port; int8_t stop; @@ -69,7 +69,7 @@ typedef struct { } SServerObj; static void *taosProcessTcpData(void *param); -static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, SOCKET fd); +static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, int32_t fd); static void taosFreeFdObj(SFdObj *pFdObj); static void taosReportBrokenLink(SFdObj *pFdObj); static void *taosAcceptTcpConnection(void *arg); @@ -134,7 +134,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread break; } - pThreadObj->pollFd = (SOCKET)epoll_create(10); // size does not matter + pThreadObj->pollFd = (int32_t)epoll_create(10); // size does not matter if (pThreadObj->pollFd < 0) { tError("%s failed to create TCP epoll", label); code = -1; @@ -227,7 +227,7 @@ void taosCleanUpTcpServer(void *handle) { } static void *taosAcceptTcpConnection(void *arg) { - SOCKET connFd = -1; + int32_t connFd = -1; struct sockaddr_in caddr; int threadId = 0; SThreadObj *pThreadObj; @@ -238,7 +238,7 @@ static void *taosAcceptTcpConnection(void *arg) { while (1) { socklen_t addrlen = sizeof(caddr); - connFd = accept(pServerObj->fd, (struct sockaddr *)&caddr, &addrlen); + connFd = (int32_t)accept(pServerObj->fd, (struct sockaddr *)&caddr, &addrlen); if (pServerObj->stop) { tDebug("%s TCP server stop accepting new connections", pServerObj->label); break; @@ -306,7 +306,7 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int num, void * return NULL; } - pThreadObj->pollFd = (SOCKET)epoll_create(10); // size does not matter + pThreadObj->pollFd = (int32_t)epoll_create(10); // size does not matter if (pThreadObj->pollFd < 0) { tError("%s failed to create TCP client epoll", label); free(pThreadObj); @@ -351,7 +351,7 @@ void taosCleanUpTcpClient(void *chandle) { void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port) { SThreadObj * pThreadObj = shandle; - SOCKET fd = taosOpenTcpClientSocket(ip, port, pThreadObj->ip); + int32_t fd = taosOpenTcpClientSocket(ip, port, pThreadObj->ip); if (fd < 0) return NULL; struct sockaddr_in sin; @@ -541,7 +541,7 @@ static void *taosProcessTcpData(void *param) { return NULL; } -static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, SOCKET fd) { +static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, int32_t fd) { struct epoll_event event; SFdObj *pFdObj = (SFdObj *)calloc(sizeof(SFdObj), 1); diff --git a/src/rpc/src/rpcUdp.c b/src/rpc/src/rpcUdp.c index 22301fcecc..30a2fb2899 100644 --- a/src/rpc/src/rpcUdp.c +++ b/src/rpc/src/rpcUdp.c @@ -31,7 +31,7 @@ typedef struct { int index; - SOCKET fd; + int32_t fd; uint16_t port; // peer port uint16_t localPort; // local port char label[TSDB_LABEL_LEN]; // copy from udpConnSet; diff --git a/src/sync/inc/syncInt.h b/src/sync/inc/syncInt.h index e43140d4e6..eef687d647 100644 --- a/src/sync/inc/syncInt.h +++ b/src/sync/inc/syncInt.h @@ -82,8 +82,8 @@ typedef struct SsyncPeer { uint64_t sversion; // track the peer version in retrieve process uint64_t lastFileVer; // track the file version while retrieve uint64_t lastWalVer; // track the wal version while retrieve - SOCKET syncFd; - SOCKET peerFd; // forward FD + int32_t syncFd; + int32_t peerFd; // forward FD int32_t numOfRetrieves; // number of retrieves tried int32_t fileChanged; // a flag to indicate file is changed during retrieving process int32_t refCount; diff --git a/src/sync/inc/syncTcp.h b/src/sync/inc/syncTcp.h index b322c3440c..d4674fee6b 100644 --- a/src/sync/inc/syncTcp.h +++ b/src/sync/inc/syncTcp.h @@ -27,12 +27,12 @@ typedef struct { int32_t bufferSize; void (*processBrokenLink)(int64_t handleId); int32_t (*processIncomingMsg)(int64_t handleId, void *buffer); - void (*processIncomingConn)(SOCKET fd, uint32_t ip); + void (*processIncomingConn)(int32_t fd, uint32_t ip); } SPoolInfo; void *syncOpenTcpThreadPool(SPoolInfo *pInfo); void syncCloseTcpThreadPool(void *); -void *syncAllocateTcpConn(void *, int64_t rid, SOCKET connFd); +void *syncAllocateTcpConn(void *, int64_t rid, int32_t connFd); void syncFreeTcpConn(void *); #ifdef __cplusplus diff --git a/src/sync/src/syncArbitrator.c b/src/sync/src/syncArbitrator.c index 4fc5e2196c..d65a7b3043 100644 --- a/src/sync/src/syncArbitrator.c +++ b/src/sync/src/syncArbitrator.c @@ -28,16 +28,16 @@ #include "syncTcp.h" static void arbSignalHandler(int32_t signum); -static void arbProcessIncommingConnection(SOCKET connFd, uint32_t sourceIp); +static void arbProcessIncommingConnection(int32_t connFd, uint32_t sourceIp); static void arbProcessBrokenLink(int64_t rid); static int32_t arbProcessPeerMsg(int64_t rid, void *buffer); static tsem_t tsArbSem; static void * tsArbTcpPool; typedef struct { - char id[TSDB_EP_LEN + 24]; - SOCKET nodeFd; - void * pConn; + char id[TSDB_EP_LEN + 24]; + int32_t nodeFd; + void * pConn; } SNodeConn; int32_t main(int32_t argc, char *argv[]) { @@ -106,7 +106,7 @@ int32_t main(int32_t argc, char *argv[]) { return 0; } -static void arbProcessIncommingConnection(SOCKET connFd, uint32_t sourceIp) { +static void arbProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { char ipstr[24]; tinet_ntoa(ipstr, sourceIp); sDebug("peer TCP connection from ip:%s", ipstr); diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index b43f5824be..2cfd393aad 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -45,7 +45,7 @@ static void syncCheckPeerConnection(void *param, void *tmrId); static int32_t syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack, int8_t type, uint16_t tranId); static void syncProcessBrokenLink(int64_t rid); static int32_t syncProcessPeerMsg(int64_t rid, void *buffer); -static void syncProcessIncommingConnection(SOCKET connFd, uint32_t sourceIp); +static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp); static void syncRemovePeer(SSyncPeer *pPeer); static void syncAddArbitrator(SSyncNode *pNode); static void syncFreeNode(void *); @@ -1114,8 +1114,8 @@ static void syncSetupPeerConnection(SSyncPeer *pPeer) { return; } - SOCKET connFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0); - if (connFd < 0) { + int32_t connFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0); + if ((int32_t)connFd < 0) { sDebug("%s, failed to open tcp socket since %s", pPeer->id, strerror(errno)); taosTmrReset(syncCheckPeerConnection, SYNC_CHECK_INTERVAL, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer); return; @@ -1179,7 +1179,7 @@ static void syncCreateRestoreDataThread(SSyncPeer *pPeer) { } } -static void syncProcessIncommingConnection(SOCKET connFd, uint32_t sourceIp) { +static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { char ipstr[24]; int32_t i; diff --git a/src/sync/src/syncTcp.c b/src/sync/src/syncTcp.c index 72ba8e26b2..1210caa84c 100644 --- a/src/sync/src/syncTcp.c +++ b/src/sync/src/syncTcp.c @@ -35,7 +35,7 @@ typedef struct SThreadObj { pthread_t thread; bool stop; - SOCKET pollFd; + int32_t pollFd; int32_t numOfFds; struct SPoolObj *pPool; } SThreadObj; @@ -45,14 +45,14 @@ typedef struct SPoolObj { SThreadObj **pThread; pthread_t thread; int32_t nextId; - SOCKET acceptFd; // FD for accept new connection + int32_t acceptFd; // FD for accept new connection int8_t stop; } SPoolObj; typedef struct { SThreadObj *pThread; int64_t handleId; - SOCKET fd; + int32_t fd; int32_t closedByApp; } SConnObj; @@ -128,7 +128,7 @@ void syncCloseTcpThreadPool(void *param) { tfree(pPool); } -void *syncAllocateTcpConn(void *param, int64_t rid, SOCKET connFd) { +void *syncAllocateTcpConn(void *param, int64_t rid, int32_t connFd) { struct epoll_event event; SPoolObj *pPool = param; @@ -264,13 +264,13 @@ static void *syncAcceptPeerTcpConnection(void *argv) { while (1) { struct sockaddr_in clientAddr; socklen_t addrlen = sizeof(clientAddr); - SOCKET connFd = accept(pPool->acceptFd, (struct sockaddr *)&clientAddr, &addrlen); + int32_t connFd = (int32_t)accept(pPool->acceptFd, (struct sockaddr *)&clientAddr, &addrlen); if (pPool->stop) { sDebug("%p TCP server accept is stopped", pPool); break; } - if (connFd < 0) { + if ((int32_t)connFd < 0) { if (errno == EINVAL) { sDebug("%p TCP server accept is exiting...", pPool); break; @@ -298,7 +298,7 @@ static SThreadObj *syncGetTcpThread(SPoolObj *pPool) { if (pThread == NULL) return NULL; pThread->pPool = pPool; - pThread->pollFd = (SOCKET)epoll_create(10); // size does not matter + pThread->pollFd = (int32_t)epoll_create(10); // size does not matter if (pThread->pollFd < 0) { tfree(pThread); return NULL; diff --git a/src/util/inc/tsocket.h b/src/util/inc/tsocket.h index 6b449d6bc9..fc0d71c4d8 100644 --- a/src/util/inc/tsocket.h +++ b/src/util/inc/tsocket.h @@ -28,17 +28,17 @@ extern "C" { #define EPOLLWAKEUP (1u << 29) #endif -int32_t taosReadn(SOCKET sock, char *buffer, int32_t len); -int32_t taosWriteMsg(SOCKET fd, void *ptr, int32_t nbytes); -int32_t taosReadMsg(SOCKET fd, void *ptr, int32_t nbytes); -int32_t taosNonblockwrite(SOCKET fd, char *ptr, int32_t nbytes); -int32_t taosCopyFds(SOCKET sfd, SOCKET dfd, int64_t len); -int32_t taosSetNonblocking(SOCKET sock, int32_t on); +int32_t taosReadn(int32_t sock, char *buffer, int32_t len); +int32_t taosWriteMsg(int32_t fd, void *ptr, int32_t nbytes); +int32_t taosReadMsg(int32_t fd, void *ptr, int32_t nbytes); +int32_t taosNonblockwrite(int32_t fd, char *ptr, int32_t nbytes); +int32_t taosCopyFds(int32_t sfd, int32_t dfd, int64_t len); +int32_t taosSetNonblocking(int32_t sock, int32_t on); -SOCKET taosOpenUdpSocket(uint32_t localIp, uint16_t localPort); -SOCKET taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp); -SOCKET taosOpenTcpServerSocket(uint32_t ip, uint16_t port); -int32_t taosKeepTcpAlive(SOCKET sockFd); +int32_t taosOpenUdpSocket(uint32_t localIp, uint16_t localPort); +int32_t taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp); +int32_t taosOpenTcpServerSocket(uint32_t ip, uint16_t port); +int32_t taosKeepTcpAlive(int32_t sockFd); int32_t taosGetFqdn(char *); uint32_t taosGetIpv4FromFqdn(const char *); diff --git a/src/util/src/tnettest.c b/src/util/src/tnettest.c index e1b834b949..70c38d36dc 100644 --- a/src/util/src/tnettest.c +++ b/src/util/src/tnettest.c @@ -39,7 +39,7 @@ typedef struct { static void *taosNetBindUdpPort(void *sarg) { STestInfo *pinfo = (STestInfo *)sarg; int32_t port = pinfo->port; - SOCKET serverSocket; + int32_t serverSocket; char buffer[BUFFER_SIZE]; int32_t iDataNum; socklen_t sin_size; @@ -48,7 +48,7 @@ static void *taosNetBindUdpPort(void *sarg) { struct sockaddr_in server_addr; struct sockaddr_in clientAddr; - if ((serverSocket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0) { + if ((serverSocket = (int32_t)socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0) { uError("failed to create UDP socket since %s", strerror(errno)); return NULL; } @@ -104,14 +104,14 @@ static void *taosNetBindTcpPort(void *sarg) { struct sockaddr_in server_addr; struct sockaddr_in clientAddr; - STestInfo *pinfo = sarg; + STestInfo *pinfo = sarg; int32_t port = pinfo->port; - SOCKET serverSocket; + int32_t serverSocket; int32_t addr_len = sizeof(clientAddr); - SOCKET client; + int32_t client; char buffer[BUFFER_SIZE]; - if ((serverSocket = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0) { + if ((serverSocket = (int32_t)socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0) { uError("failed to create TCP socket since %s", strerror(errno)); return NULL; } @@ -148,7 +148,7 @@ static void *taosNetBindTcpPort(void *sarg) { uInfo("TCP server at port:%d is listening", port); while (1) { - client = accept(serverSocket, (struct sockaddr *)&clientAddr, (socklen_t *)&addr_len); + client = (int32_t)accept(serverSocket, (struct sockaddr *)&clientAddr, (socklen_t *)&addr_len); if (client < 0) { uDebug("TCP: failed to accept at port:%d since %s", port, strerror(errno)); continue; @@ -178,10 +178,10 @@ static void *taosNetBindTcpPort(void *sarg) { } static int32_t taosNetCheckTcpPort(STestInfo *info) { - SOCKET clientSocket; - char buffer[BUFFER_SIZE] = {0}; + int32_t clientSocket; + char buffer[BUFFER_SIZE] = {0}; - if ((clientSocket = socket(AF_INET, SOCK_STREAM, 0)) < 0) { + if ((clientSocket = (int32_t)socket(AF_INET, SOCK_STREAM, 0)) < 0) { uError("failed to create TCP client socket since %s", strerror(errno)); return -1; } @@ -226,14 +226,14 @@ static int32_t taosNetCheckTcpPort(STestInfo *info) { } static int32_t taosNetCheckUdpPort(STestInfo *info) { - SOCKET clientSocket; + int32_t clientSocket; char buffer[BUFFER_SIZE] = {0}; int32_t iDataNum = 0; int32_t bufSize = 1024000; struct sockaddr_in serverAddr; - if ((clientSocket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0) { + if ((clientSocket = (int32_t)socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0) { uError("failed to create udp client socket since %s", strerror(errno)); return -1; } diff --git a/src/util/src/tsocket.c b/src/util/src/tsocket.c index 49b69ea0a1..d41c1518b8 100644 --- a/src/util/src/tsocket.c +++ b/src/util/src/tsocket.c @@ -102,7 +102,7 @@ uint32_t ip2uint(const char *const ip_addr) { return *((uint32_t *)ip); } -int32_t taosWriteMsg(SOCKET fd, void *buf, int32_t nbytes) { +int32_t taosWriteMsg(int32_t fd, void *buf, int32_t nbytes) { int32_t nleft, nwritten; char * ptr = (char *)buf; @@ -128,7 +128,7 @@ int32_t taosWriteMsg(SOCKET fd, void *buf, int32_t nbytes) { return (nbytes - nleft); } -int32_t taosReadMsg(SOCKET fd, void *buf, int32_t nbytes) { +int32_t taosReadMsg(int32_t fd, void *buf, int32_t nbytes) { int32_t nleft, nread; char * ptr = (char *)buf; @@ -159,7 +159,7 @@ int32_t taosReadMsg(SOCKET fd, void *buf, int32_t nbytes) { return (nbytes - nleft); } -int32_t taosNonblockwrite(SOCKET fd, char *ptr, int32_t nbytes) { +int32_t taosNonblockwrite(int32_t fd, char *ptr, int32_t nbytes) { taosSetNonblocking(fd, 1); int32_t nleft, nwritten, nready; @@ -201,7 +201,7 @@ int32_t taosNonblockwrite(SOCKET fd, char *ptr, int32_t nbytes) { return (nbytes - nleft); } -int32_t taosReadn(SOCKET fd, char *ptr, int32_t nbytes) { +int32_t taosReadn(int32_t fd, char *ptr, int32_t nbytes) { int32_t nread, nready, nleft = nbytes; fd_set fset; @@ -239,9 +239,9 @@ int32_t taosReadn(SOCKET fd, char *ptr, int32_t nbytes) { return (nbytes - nleft); } -SOCKET taosOpenUdpSocket(uint32_t ip, uint16_t port) { +int32_t taosOpenUdpSocket(uint32_t ip, uint16_t port) { struct sockaddr_in localAddr; - SOCKET sockFd; + int32_t sockFd; int32_t bufSize = 1024000; uDebug("open udp socket:0x%x:%hu", ip, port); @@ -279,14 +279,14 @@ SOCKET taosOpenUdpSocket(uint32_t ip, uint16_t port) { return sockFd; } -SOCKET taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clientIp) { - SOCKET sockFd = 0; +int32_t taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clientIp) { + int32_t sockFd = 0; int32_t ret; struct sockaddr_in serverAddr, clientAddr; int32_t bufSize = 1024 * 1024; - sockFd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP); - + sockFd = (int32_t)socket(PF_INET, SOCK_STREAM, IPPROTO_TCP); + if (sockFd <= 2) { uError("failed to open the socket: %d (%s)", errno, strerror(errno)); taosCloseSocketNoCheck(sockFd); @@ -346,7 +346,7 @@ SOCKET taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clie return sockFd; } -int32_t taosKeepTcpAlive(SOCKET sockFd) { +int32_t taosKeepTcpAlive(int32_t sockFd) { int32_t alive = 1; if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_KEEPALIVE, (void *)&alive, sizeof(alive)) < 0) { uError("fd:%d setsockopt SO_KEEPALIVE failed: %d (%s)", sockFd, errno, strerror(errno)); @@ -394,9 +394,9 @@ int32_t taosKeepTcpAlive(SOCKET sockFd) { return 0; } -SOCKET taosOpenTcpServerSocket(uint32_t ip, uint16_t port) { +int32_t taosOpenTcpServerSocket(uint32_t ip, uint16_t port) { struct sockaddr_in serverAdd; - SOCKET sockFd; + int32_t sockFd; int32_t reuse; uDebug("open tcp server socket:0x%x:%hu", ip, port); @@ -449,7 +449,7 @@ void tinet_ntoa(char *ipstr, uint32_t ip) { #define COPY_SIZE 32768 // sendfile shall be used -int32_t taosCopyFds(SOCKET sfd, SOCKET dfd, int64_t len) { +int32_t taosCopyFds(int32_t sfd, int32_t dfd, int64_t len) { int64_t leftLen; int32_t readLen, writeLen; char temp[COPY_SIZE]; From 0ef43ca365355fdc40c7c333a26bebc254593b63 Mon Sep 17 00:00:00 2001 From: freemine Date: Sun, 17 Jan 2021 21:53:02 +0800 Subject: [PATCH 24/62] literal big integer --- tests/examples/c/demo.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/examples/c/demo.c b/tests/examples/c/demo.c index f5ad37acef..9ad3e6f902 100644 --- a/tests/examples/c/demo.c +++ b/tests/examples/c/demo.c @@ -86,7 +86,7 @@ void Test(TAOS *taos, char *qstr, int index) { int i = 0; for (i = 0; i < 10; ++i) { - sprintf(qstr, "insert into m1 values (%" PRId64 ", %d, %d, %d, %d, %f, %lf, '%s')", 1546300800000LL + i * 1000, i, i, i, i*10000000, i*1.0, i*2.0, "hello"); + sprintf(qstr, "insert into m1 values (%" PRId64 ", %d, %d, %d, %d, %f, %lf, '%s')", (uint64_t)(1546300800000 + i * 1000), i, i, i, i*10000000, i*1.0, i*2.0, "hello"); printf("qstr: %s\n", qstr); // note: how do you wanna do if taos_query returns non-NULL From f6e18e01d1a567f2d57e267eed2bea101bc77e8e Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 18 Jan 2021 09:41:35 +0800 Subject: [PATCH 25/62] TD-1207 --- deps/zlib-1.2.11/src/gzlib.c | 4 ++++ src/os/inc/osDef.h | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/deps/zlib-1.2.11/src/gzlib.c b/deps/zlib-1.2.11/src/gzlib.c index f3bd685be0..4c135aabff 100644 --- a/deps/zlib-1.2.11/src/gzlib.c +++ b/deps/zlib-1.2.11/src/gzlib.c @@ -5,6 +5,10 @@ #include #include "gzguts.h" +#ifndef O_BINARY + #define O_BINARY 0 +#endif + #if defined(_WIN32) && !defined(__BORLANDC__) && !defined(__MINGW32__) # define LSEEK _lseeki64 #else diff --git a/src/os/inc/osDef.h b/src/os/inc/osDef.h index d718bef6da..bdbe95a75d 100644 --- a/src/os/inc/osDef.h +++ b/src/os/inc/osDef.h @@ -20,6 +20,10 @@ extern "C" { #endif +#ifndef O_BINARY + #define O_BINARY 0 +#endif + #ifndef STDERR_FILENO #define STDERR_FILENO (2) #endif From 1eace47c8570775e08b2131ce498aa7bdfdcdebe Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 18 Jan 2021 19:52:42 +0800 Subject: [PATCH 26/62] TD-1207 --- src/os/inc/osDef.h | 6 +- tests/script/jenkins/wbasic.txt | 393 ++++++++++++++++++++++++++++++++ tests/script/sh/cfg.bat | 42 ++++ tests/script/sh/deploy.bat | 231 ++++++++----------- tests/script/sh/exec.bat | 134 +++-------- tests/script/sh/stop_dnodes.bat | 25 +- tests/script/wtest.bat | 43 ++-- tests/tsim/src/simExe.c | 26 +++ 8 files changed, 614 insertions(+), 286 deletions(-) create mode 100644 tests/script/jenkins/wbasic.txt create mode 100644 tests/script/sh/cfg.bat diff --git a/src/os/inc/osDef.h b/src/os/inc/osDef.h index bdbe95a75d..e5b5e0f895 100644 --- a/src/os/inc/osDef.h +++ b/src/os/inc/osDef.h @@ -20,8 +20,10 @@ extern "C" { #endif -#ifndef O_BINARY - #define O_BINARY 0 +#ifndef WINDOWS + #ifndef O_BINARY + #define O_BINARY 0 + #endif #endif #ifndef STDERR_FILENO diff --git a/tests/script/jenkins/wbasic.txt b/tests/script/jenkins/wbasic.txt new file mode 100644 index 0000000000..984b15fc5a --- /dev/null +++ b/tests/script/jenkins/wbasic.txt @@ -0,0 +1,393 @@ +wtest.bat -f issue/TD-2677.sim +wtest.bat -f issue/TD-2680.sim +wtest.bat -f issue/TD-2713.sim + +wtest.bat -f general/alter/cached_schema_after_alter.sim +wtest.bat -f general/alter/count.sim +wtest.bat -f general/alter/dnode.sim +wtest.bat -f general/alter/import.sim +wtest.bat -f general/alter/insert1.sim +wtest.bat -f general/alter/insert2.sim +wtest.bat -f general/alter/metrics.sim +wtest.bat -f general/alter/table.sim + +wtest.bat -f general/cache/new_metrics.sim +wtest.bat -f general/cache/restart_metrics.sim +wtest.bat -f general/cache/restart_table.sim + +wtest.bat -f general/connection/connection.sim + +wtest.bat -f general/column/commit.sim +wtest.bat -f general/column/metrics.sim +wtest.bat -f general/column/table.sim + +wtest.bat -f general/compress/commitlog.sim +wtest.bat -f general/compress/compress.sim +wtest.bat -f general/compress/compress2.sim +wtest.bat -f general/compress/uncompress.sim + +wtest.bat -f general/compute/avg.sim +wtest.bat -f general/compute/bottom.sim +wtest.bat -f general/compute/count.sim +wtest.bat -f general/compute/diff.sim +wtest.bat -f general/compute/diff2.sim +wtest.bat -f general/compute/first.sim +wtest.bat -f general/compute/interval.sim +wtest.bat -f general/compute/last.sim +wtest.bat -f general/compute/leastsquare.sim +wtest.bat -f general/compute/max.sim +wtest.bat -f general/compute/min.sim +wtest.bat -f general/compute/null.sim +wtest.bat -f general/compute/percentile.sim +wtest.bat -f general/compute/stddev.sim +wtest.bat -f general/compute/sum.sim +wtest.bat -f general/compute/top.sim + +wtest.bat -f general/db/alter_option.sim +wtest.bat -f general/db/alter_tables_d2.sim +wtest.bat -f general/db/alter_tables_v1.sim +wtest.bat -f general/db/alter_tables_v4.sim +wtest.bat -f general/db/alter_vgroups.sim +wtest.bat -f general/db/basic.sim +wtest.bat -f general/db/basic1.sim +wtest.bat -f general/db/basic2.sim +wtest.bat -f general/db/basic3.sim +wtest.bat -f general/db/basic4.sim +wtest.bat -f general/db/basic5.sim +wtest.bat -f general/db/delete_reuse1.sim +wtest.bat -f general/db/delete_reuse2.sim +wtest.bat -f general/db/delete_reusevnode.sim +wtest.bat -f general/db/delete_reusevnode2.sim +wtest.bat -f general/db/delete_writing1.sim +wtest.bat -f general/db/delete_writing2.sim +wtest.bat -f general/db/delete.sim +wtest.bat -f general/db/len.sim +wtest.bat -f general/db/repeat.sim +wtest.bat -f general/db/tables.sim +wtest.bat -f general/db/vnodes.sim +wtest.bat -f general/db/nosuchfile.sim + +wtest.bat -f general/field/2.sim +wtest.bat -f general/field/3.sim +wtest.bat -f general/field/4.sim +wtest.bat -f general/field/5.sim +wtest.bat -f general/field/6.sim +wtest.bat -f general/field/bigint.sim +wtest.bat -f general/field/binary.sim +wtest.bat -f general/field/bool.sim +wtest.bat -f general/field/single.sim +wtest.bat -f general/field/smallint.sim +wtest.bat -f general/field/tinyint.sim + +wtest.bat -f general/http/autocreate.sim +wtest.bat -f general/http/chunked.sim +wtest.bat -f general/http/gzip.sim +wtest.bat -f general/http/restful.sim +wtest.bat -f general/http/restful_insert.sim +wtest.bat -f general/http/restful_limit.sim +wtest.bat -f general/http/restful_full.sim +wtest.bat -f general/http/prepare.sim +wtest.bat -f general/http/telegraf.sim +wtest.bat -f general/http/grafana_bug.sim +wtest.bat -f general/http/grafana.sim + +wtest.bat -f general/import/basic.sim +wtest.bat -f general/import/commit.sim +wtest.bat -f general/import/large.sim +wtest.bat -f general/import/replica1.sim + +wtest.bat -f general/insert/basic.sim +wtest.bat -f general/insert/insert_drop.sim +wtest.bat -f general/insert/query_block1_memory.sim +wtest.bat -f general/insert/query_block2_memory.sim +wtest.bat -f general/insert/query_block1_file.sim +wtest.bat -f general/insert/query_block2_file.sim +wtest.bat -f general/insert/query_file_memory.sim +wtest.bat -f general/insert/query_multi_file.sim +wtest.bat -f general/insert/tcp.sim + +wtest.bat -f general/parser/alter.sim +wtest.bat -f general/parser/alter1.sim +wtest.bat -f general/parser/alter_stable.sim +wtest.bat -f general/parser/auto_create_tb.sim +wtest.bat -f general/parser/auto_create_tb_drop_tb.sim +wtest.bat -f general/parser/col_arithmetic_operation.sim +wtest.bat -f general/parser/columnValue.sim +wtest.bat -f general/parser/commit.sim +wtest.bat -f general/parser/create_db.sim +wtest.bat -f general/parser/create_mt.sim +wtest.bat -f general/parser/create_tb.sim +wtest.bat -f general/parser/dbtbnameValidate.sim +wtest.bat -f general/parser/import_commit1.sim +wtest.bat -f general/parser/import_commit2.sim +wtest.bat -f general/parser/import_commit3.sim +wtest.bat -f general/parser/insert_tb.sim +wtest.bat -f general/parser/first_last.sim +wtest.bat -f general/parser/lastrow.sim +wtest.bat -f general/parser/nchar.sim +wtest.bat -f general/parser/null_char.sim +wtest.bat -f general/parser/single_row_in_tb.sim +wtest.bat -f general/parser/select_from_cache_disk.sim +wtest.bat -f general/parser/mixed_blocks.sim +wtest.bat -f general/parser/selectResNum.sim +wtest.bat -f general/parser/limit.sim +wtest.bat -f general/parser/limit1.sim +wtest.bat -f general/parser/limit1_tblocks100.sim +wtest.bat -f general/parser/select_across_vnodes.sim +wtest.bat -f general/parser/slimit1.sim +wtest.bat -f general/parser/tbnameIn.sim +wtest.bat -f general/parser/projection_limit_offset.sim +wtest.bat -f general/parser/limit2.sim +wtest.bat -f general/parser/fill.sim +wtest.bat -f general/parser/fill_stb.sim +wtest.bat -f general/parser/where.sim +wtest.bat -f general/parser/slimit.sim +wtest.bat -f general/parser/select_with_tags.sim +wtest.bat -f general/parser/interp.sim +wtest.bat -f general/parser/tags_dynamically_specifiy.sim +wtest.bat -f general/parser/groupby.sim +wtest.bat -f general/parser/set_tag_vals.sim +wtest.bat -f general/parser/tags_filter.sim +wtest.bat -f general/parser/slimit_alter_tags.sim +wtest.bat -f general/parser/join.sim +wtest.bat -f general/parser/join_multivnode.sim +wtest.bat -f general/parser/binary_escapeCharacter.sim +wtest.bat -f general/parser/repeatAlter.sim +wtest.bat -f general/parser/union.sim +wtest.bat -f general/parser/topbot.sim +wtest.bat -f general/parser/function.sim + +wtest.bat -f general/stable/disk.sim +wtest.bat -f general/stable/dnode3.sim +wtest.bat -f general/stable/metrics.sim +wtest.bat -f general/stable/refcount.sim +wtest.bat -f general/stable/show.sim +wtest.bat -f general/stable/values.sim +wtest.bat -f general/stable/vnode3.sim + +wtest.bat -f general/table/autocreate.sim +wtest.bat -f general/table/basic1.sim +wtest.bat -f general/table/basic2.sim +wtest.bat -f general/table/basic3.sim +wtest.bat -f general/table/bigint.sim +wtest.bat -f general/table/binary.sim +wtest.bat -f general/table/bool.sim +wtest.bat -f general/table/column_name.sim +wtest.bat -f general/table/column_num.sim +wtest.bat -f general/table/column_value.sim +wtest.bat -f general/table/column2.sim +wtest.bat -f general/table/date.sim +wtest.bat -f general/table/db.table.sim +wtest.bat -f general/table/delete_reuse1.sim +wtest.bat -f general/table/delete_reuse2.sim +wtest.bat -f general/table/delete_writing.sim +wtest.bat -f general/table/describe.sim +wtest.bat -f general/table/double.sim +wtest.bat -f general/table/fill.sim +wtest.bat -f general/table/float.sim +wtest.bat -f general/table/int.sim +wtest.bat -f general/table/limit.sim +wtest.bat -f general/table/smallint.sim +wtest.bat -f general/table/table_len.sim +wtest.bat -f general/table/table.sim +wtest.bat -f general/table/tinyint.sim +wtest.bat -f general/table/vgroup.sim + +wtest.bat -f general/tag/3.sim +wtest.bat -f general/tag/4.sim +wtest.bat -f general/tag/5.sim +wtest.bat -f general/tag/6.sim +wtest.bat -f general/tag/add.sim +wtest.bat -f general/tag/bigint.sim +wtest.bat -f general/tag/binary_binary.sim +wtest.bat -f general/tag/binary.sim +wtest.bat -f general/tag/bool_binary.sim +wtest.bat -f general/tag/bool_int.sim +wtest.bat -f general/tag/bool.sim +wtest.bat -f general/tag/change.sim +wtest.bat -f general/tag/column.sim +wtest.bat -f general/tag/commit.sim +wtest.bat -f general/tag/create.sim +wtest.bat -f general/tag/delete.sim +wtest.bat -f general/tag/double.sim +wtest.bat -f general/tag/filter.sim +wtest.bat -f general/tag/float.sim +wtest.bat -f general/tag/int_binary.sim +wtest.bat -f general/tag/int_float.sim +wtest.bat -f general/tag/int.sim +wtest.bat -f general/tag/set.sim +wtest.bat -f general/tag/smallint.sim +wtest.bat -f general/tag/tinyint.sim + +wtest.bat -f general/user/authority.sim +wtest.bat -f general/user/monitor.sim +wtest.bat -f general/user/pass_alter.sim +wtest.bat -f general/user/pass_len.sim +wtest.bat -f general/user/user_create.sim +wtest.bat -f general/user/user_len.sim + +wtest.bat -f general/vector/metrics_field.sim +wtest.bat -f general/vector/metrics_mix.sim +wtest.bat -f general/vector/metrics_query.sim +wtest.bat -f general/vector/metrics_tag.sim +wtest.bat -f general/vector/metrics_time.sim +wtest.bat -f general/vector/multi.sim +wtest.bat -f general/vector/single.sim +wtest.bat -f general/vector/table_field.sim +wtest.bat -f general/vector/table_mix.sim +wtest.bat -f general/vector/table_query.sim +wtest.bat -f general/vector/table_time.sim + +wtest.bat -f general/wal/sync.sim +wtest.bat -f general/wal/kill.sim +wtest.bat -f general/wal/maxtables.sim + +wtest.bat -f unique/account/account_create.sim +wtest.bat -f unique/account/account_delete.sim +wtest.bat -f unique/account/account_len.sim +wtest.bat -f unique/account/authority.sim +wtest.bat -f unique/account/basic.sim +wtest.bat -f unique/account/paras.sim +wtest.bat -f unique/account/pass_alter.sim +wtest.bat -f unique/account/pass_len.sim +wtest.bat -f unique/account/usage.sim +wtest.bat -f unique/account/user_create.sim +wtest.bat -f unique/account/user_len.sim + +wtest.bat -f unique/big/balance.sim +wtest.bat -f unique/big/maxvnodes.sim +wtest.bat -f unique/big/tcp.sim + +wtest.bat -f unique/cluster/alter.sim +wtest.bat -f unique/cluster/balance1.sim +wtest.bat -f unique/cluster/balance2.sim +wtest.bat -f unique/cluster/balance3.sim +wtest.bat -f unique/cluster/cache.sim +wtest.bat -f unique/cluster/vgroup100.sim + +wtest.bat -f unique/column/replica3.sim + +wtest.bat -f unique/db/commit.sim +wtest.bat -f unique/db/delete.sim +wtest.bat -f unique/db/delete_part.sim +wtest.bat -f unique/db/replica_add12.sim +wtest.bat -f unique/db/replica_add13.sim +wtest.bat -f unique/db/replica_add23.sim +wtest.bat -f unique/db/replica_reduce21.sim +wtest.bat -f unique/db/replica_reduce32.sim +wtest.bat -f unique/db/replica_reduce31.sim +wtest.bat -f unique/db/replica_part.sim + +wtest.bat -f unique/dnode/alternativeRole.sim +wtest.bat -f unique/dnode/monitor.sim +wtest.bat -f unique/dnode/monitor_bug.sim +wtest.bat -f unique/dnode/simple.sim +wtest.bat -f unique/dnode/balance1.sim +wtest.bat -f unique/dnode/balance2.sim +wtest.bat -f unique/dnode/balance3.sim +wtest.bat -f unique/dnode/balancex.sim +wtest.bat -f unique/dnode/data1.sim +wtest.bat -f unique/dnode/m2.sim +wtest.bat -f unique/dnode/m3.sim +wtest.bat -f unique/dnode/lossdata.sim +wtest.bat -f unique/dnode/offline1.sim +wtest.bat -f unique/dnode/offline2.sim +wtest.bat -f unique/dnode/offline3.sim +wtest.bat -f unique/dnode/reason.sim +wtest.bat -f unique/dnode/remove1.sim +wtest.bat -f unique/dnode/remove2.sim +wtest.bat -f unique/dnode/vnode_clean.sim + +wtest.bat -f unique/http/admin.sim +wtest.bat -f unique/http/opentsdb.sim + +wtest.bat -f unique/import/replica2.sim +wtest.bat -f unique/import/replica3.sim + +wtest.bat -f unique/stable/balance_replica1.sim +wtest.bat -f unique/stable/dnode2_stop.sim +wtest.bat -f unique/stable/dnode2.sim +wtest.bat -f unique/stable/dnode3.sim +wtest.bat -f unique/stable/replica2_dnode4.sim +wtest.bat -f unique/stable/replica2_vnode3.sim +wtest.bat -f unique/stable/replica3_dnode6.sim +wtest.bat -f unique/stable/replica3_vnode3.sim + +wtest.bat -f unique/mnode/mgmt20.sim +wtest.bat -f unique/mnode/mgmt21.sim +wtest.bat -f unique/mnode/mgmt22.sim +wtest.bat -f unique/mnode/mgmt23.sim +wtest.bat -f unique/mnode/mgmt24.sim +wtest.bat -f unique/mnode/mgmt25.sim +wtest.bat -f unique/mnode/mgmt26.sim +wtest.bat -f unique/mnode/mgmt30.sim +wtest.bat -f unique/mnode/mgmt33.sim +wtest.bat -f unique/mnode/mgmt34.sim +wtest.bat -f unique/mnode/mgmtr2.sim + +wtest.bat -f unique/vnode/many.sim +wtest.bat -f unique/vnode/replica2_basic2.sim +wtest.bat -f unique/vnode/replica2_repeat.sim +wtest.bat -f unique/vnode/replica3_basic.sim +wtest.bat -f unique/vnode/replica3_repeat.sim +wtest.bat -f unique/vnode/replica3_vgroup.sim + +wtest.bat -f general/stream/metrics_del.sim +wtest.bat -f general/stream/metrics_replica1_vnoden.sim +wtest.bat -f general/stream/restart_stream.sim +wtest.bat -f general/stream/stream_3.sim +wtest.bat -f general/stream/stream_restart.sim +wtest.bat -f general/stream/table_del.sim +wtest.bat -f general/stream/table_replica1_vnoden.sim + +wtest.bat -f unique/arbitrator/check_cluster_cfg_para.sim +#wtest.bat -f unique/arbitrator/dn2_mn1_cache_file_sync.sim +wtest.bat -f unique/arbitrator/dn3_mn1_full_createTableFail.sim +wtest.bat -f unique/arbitrator/dn3_mn1_multiCreateDropTable.sim +#wtest.bat -f unique/arbitrator/dn3_mn1_nw_disable_timeout_autoDropDnode.sim +#wtest.bat -f unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim +wtest.bat -f unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim +wtest.bat -f unique/arbitrator/dn3_mn1_replica_change.sim +#wtest.bat -f unique/arbitrator/dn3_mn1_stopDnode_timeout.sim +# lower the priority while file corruption +#wtest.bat -f unique/arbitrator/dn3_mn1_vnode_change.sim +#wtest.bat -f unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim +#wtest.bat -f unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim +#wtest.bat -f unique/arbitrator/dn3_mn1_vnode_createErrData_online.sim +wtest.bat -f unique/arbitrator/dn3_mn1_vnode_noCorruptFile_offline.sim +wtest.bat -f unique/arbitrator/dn3_mn1_vnode_delDir.sim +wtest.bat -f unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim +wtest.bat -f unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim +wtest.bat -f unique/arbitrator/dn3_mn1_vnode_nomaster.sim +wtest.bat -f unique/arbitrator/dn3_mn2_killDnode.sim +wtest.bat -f unique/arbitrator/insert_duplicationTs.sim +wtest.bat -f unique/arbitrator/offline_replica2_alterTable_online.sim +wtest.bat -f unique/arbitrator/offline_replica2_alterTag_online.sim +wtest.bat -f unique/arbitrator/offline_replica2_createTable_online.sim +wtest.bat -f unique/arbitrator/offline_replica2_dropDb_online.sim +wtest.bat -f unique/arbitrator/offline_replica2_dropTable_online.sim +wtest.bat -f unique/arbitrator/offline_replica3_alterTable_online.sim +wtest.bat -f unique/arbitrator/offline_replica3_alterTag_online.sim +wtest.bat -f unique/arbitrator/offline_replica3_createTable_online.sim +wtest.bat -f unique/arbitrator/offline_replica3_dropDb_online.sim +wtest.bat -f unique/arbitrator/offline_replica3_dropTable_online.sim +wtest.bat -f unique/arbitrator/replica_changeWithArbitrator.sim +wtest.bat -f unique/arbitrator/sync_replica2_alterTable_add.sim +wtest.bat -f unique/arbitrator/sync_replica2_alterTable_drop.sim + +wtest.bat -f unique/arbitrator/sync_replica2_dropDb.sim +wtest.bat -f unique/arbitrator/sync_replica2_dropTable.sim +wtest.bat -f unique/arbitrator/sync_replica3_alterTable_add.sim +wtest.bat -f unique/arbitrator/sync_replica3_alterTable_drop.sim +wtest.bat -f unique/arbitrator/sync_replica3_dropDb.sim +wtest.bat -f unique/arbitrator/sync_replica3_dropTable.sim + +wtest.bat -f unique/migrate/mn2_vn2_repl2_rmMnodeDir.sim +wtest.bat -f unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim +wtest.bat -f unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir_stopAll_starAll.sim +wtest.bat -f unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim + +wtest.bat -f general/connection/test_old_data.sim +wtest.bat -f unique/dnode/datatrans_3node.sim +wtest.bat -f unique/dnode/datatrans_3node_2.sim + diff --git a/tests/script/sh/cfg.bat b/tests/script/sh/cfg.bat new file mode 100644 index 0000000000..49ab34383a --- /dev/null +++ b/tests/script/sh/cfg.bat @@ -0,0 +1,42 @@ +@echo off + +if %1 == -n set NODE_NAME=%2 +if %1 == -c set CONFIG_NAME=%2 +if %1 == -v set CONFIG_VALUE=%2 +if %3 == -n set NODE_NAME=%4 +if %3 == -c set CONFIG_NAME=%4 +if %3 == -v set CONFIG_VALUE=%4 +if %5 == -n set NODE_NAME=%6 +if %5 == -c set CONFIG_NAME=%6 +if %5 == -v set CONFIG_VALUE=%6 + +rem echo NODE_NAME: %NODE_NAME% +rem echo NODE: %NODE% + +set SCRIPT_DIR=%~dp0..\ +rem echo SCRIPT_DIR: %SCRIPT_DIR% + +set BUILD_DIR=%SCRIPT_DIR%..\..\..\debug\build\bin\ +set TSIM=%BUILD_DIR%tsim +rem echo BUILD_DIR: %BUILD_DIR% +rem echo TSIM: %TSIM% + +set SIM_DIR=%SCRIPT_DIR%..\..\..\sim\ +rem echo SIM_DIR: %SIM_DIR% + +set NODE_DIR=%SIM_DIR%%NODE_NAME%\ +rem echo NODE_DIR: %NODE_DIR% + +set CFG_DIR=%NODE_DIR%cfg\ +rem echo CFG_DIR: %CFG_DIR% + +set LOG_DIR=%NODE_DIR%log\ +rem echo LOG_DIR: %LOG_DIR% + +set DATA_DIR=%NODE_DIR%data\ +rem echo DATA_DIR: %DATA_DIR% + +set TAOS_CFG=%CFG_DIR%taos.cfg +rem echo TAOS_CFG: %TAOS_CFG% + +echo %CONFIG_NAME% %CONFIG_VALUE% >> %TAOS_CFG% diff --git a/tests/script/sh/deploy.bat b/tests/script/sh/deploy.bat index 9b61a33d45..0b419b1e9b 100644 --- a/tests/script/sh/deploy.bat +++ b/tests/script/sh/deploy.bat @@ -1,155 +1,106 @@ -#!/bin/bash +@echo off -echo "Executing deploy.sh" +rem echo Executing deploy.sh -if [ $# != 4 ]; then - echo "argument list need input : " - echo " -n nodeName" - echo " -i nodePort" - exit 1 -fi +if %1 == -n set NODE_NAME=%2 +if %1 == -i set NODE=%2 +if %3 == -n set NODE_NAME=%4 +if %3 == -i set NODE=%4 -NODE_NAME= -NODE= -while getopts "n:i:" arg -do - case $arg in - n) - NODE_NAME=$OPTARG - ;; - i) - NODE=$OPTARG - ;; - ?) - echo "unkonw argument" - ;; - esac -done +rem echo NODE_NAME: %NODE_NAME% +rem echo NODE: %NODE% -SCRIPT_DIR=`dirname $0` -cd $SCRIPT_DIR/../ -SCRIPT_DIR=`pwd` -echo "SCRIPT_DIR: $SCRIPT_DIR" +set SCRIPT_DIR=%~dp0..\ +rem echo SCRIPT_DIR: %SCRIPT_DIR% -IN_TDINTERNAL="community" -if [[ "$SCRIPT_DIR" == *"$IN_TDINTERNAL"* ]]; then - cd ../../.. -else - cd ../../ -fi +set BUILD_DIR=%SCRIPT_DIR%..\..\..\debug\build\bin\ +set TSIM=%BUILD_DIR%tsim +rem echo BUILD_DIR: %BUILD_DIR% +rem echo TSIM: %TSIM% -TAOS_DIR=`pwd` -TAOSD_DIR=`find . -name "taosd"|grep bin|head -n1` +set SIM_DIR=%SCRIPT_DIR%..\..\..\sim\ +rem echo SIM_DIR: %SIM_DIR% -if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then - BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2,3` -else - BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2` -fi +set NODE_DIR=%SIM_DIR%%NODE_NAME%\ +rem echo NODE_DIR: %NODE_DIR% -BUILD_DIR=$TAOS_DIR/$BIN_DIR/build +set CFG_DIR=%NODE_DIR%cfg\ +rem echo CFG_DIR: %CFG_DIR% -SIM_DIR=$TAOS_DIR/sim +set LOG_DIR=%NODE_DIR%log\ +rem echo LOG_DIR: %LOG_DIR% -NODE_DIR=$SIM_DIR/$NODE_NAME -EXE_DIR=$BUILD_DIR/bin -CFG_DIR=$NODE_DIR/cfg -LOG_DIR=$NODE_DIR/log -DATA_DIR=$NODE_DIR/data +set DATA_DIR=%NODE_DIR%data\ +rem echo DATA_DIR: %DATA_DIR% -rm -rf $NODE_DIR +set TAOS_CFG=%CFG_DIR%taos.cfg +rem echo TAOS_CFG: %TAOS_CFG% -mkdir -p $SIM_DIR -mkdir -p $NODE_DIR -mkdir -p $LOG_DIR -mkdir -p $DATA_DIR +if not exist %SIM_DIR% mkdir %SIM_DIR% +if not exist %NODE_DIR% mkdir %NODE_DIR% +if exist %CFG_DIR% rmdir /s/q %CFG_DIR% +if exist %LOG_DIR% rmdir /s/q %LOG_DIR% +if exist %DATA_DIR% rmdir /s/q %DATA_DIR% +if not exist %CFG_DIR% mkdir %CFG_DIR% +if not exist %LOG_DIR% mkdir %LOG_DIR% +if not exist %DATA_DIR% mkdir %DATA_DIR% -#cp -rf $TAOS_DIR/cfg $NODE_DIR/ -mkdir -p $CFG_DIR +if %NODE% == 1 set NODE=7100 +if %NODE% == 2 set NODE=7200 +if %NODE% == 3 set NODE=7300 +if %NODE% == 4 set NODE=7400 +if %NODE% == 5 set NODE=7500 +if %NODE% == 6 set NODE=7600 +if %NODE% == 7 set NODE=7700 +if %NODE% == 8 set NODE=7800 -#allow normal user to read/write log -chmod -R 777 $NODE_DIR - -TAOS_CFG=$NODE_DIR/cfg/taos.cfg -touch -f $TAOS_CFG - -TAOS_FLAG=$SIM_DIR/tsim/flag -if [ -f "$TAOS_FLAG" ] ; then - TAOS_CFG=/etc/taos/taos.cfg - DATA_DIR=/var/lib/taos - LOG_DIR=/var/log/taos - sudo rm -f /etc/taos/*.cfg - sudo cp -rf $TAOS_DIR/cfg/*.cfg /etc/taos - sudo rm -rf $DATA_DIR - sudo rm -rf $LOG_DIR -fi - -HOSTNAME=`hostname -f` - -if [ $NODE -eq 1 ]; then - NODE=7100 -elif [ $NODE -eq 2 ]; then - NODE=7200 -elif [ $NODE -eq 3 ]; then - NODE=7300 -elif [ $NODE -eq 4 ]; then - NODE=7400 -elif [ $NODE -eq 5 ]; then - NODE=7500 -elif [ $NODE -eq 6 ]; then - NODE=7600 -elif [ $NODE -eq 7 ]; then - NODE=7700 -elif [ $NODE -eq 8 ]; then - NODE=7800 -fi - -echo " " >> $TAOS_CFG -echo "firstEp ${HOSTNAME}:7100" >> $TAOS_CFG -echo "secondEp ${HOSTNAME}:7200" >> $TAOS_CFG -echo "serverPort ${NODE}" >> $TAOS_CFG -echo "dataDir $DATA_DIR" >> $TAOS_CFG -echo "logDir $LOG_DIR" >> $TAOS_CFG -echo "debugFlag 0" >> $TAOS_CFG -echo "mDebugFlag 143" >> $TAOS_CFG -echo "sdbDebugFlag 143" >> $TAOS_CFG -echo "dDebugFlag 143" >> $TAOS_CFG -echo "vDebugFlag 143" >> $TAOS_CFG -echo "tsdbDebugFlag 143" >> $TAOS_CFG -echo "cDebugFlag 143" >> $TAOS_CFG -echo "jnidebugFlag 143" >> $TAOS_CFG -echo "odbcdebugFlag 143" >> $TAOS_CFG -echo "httpDebugFlag 143" >> $TAOS_CFG -echo "monDebugFlag 143" >> $TAOS_CFG -echo "mqttDebugFlag 143" >> $TAOS_CFG -echo "qdebugFlag 143" >> $TAOS_CFG -echo "rpcDebugFlag 143" >> $TAOS_CFG -echo "tmrDebugFlag 131" >> $TAOS_CFG -echo "udebugFlag 143" >> $TAOS_CFG -echo "sdebugFlag 143" >> $TAOS_CFG -echo "wdebugFlag 143" >> $TAOS_CFG -echo "cqdebugFlag 143" >> $TAOS_CFG -echo "monitor 0" >> $TAOS_CFG -echo "monitorInterval 1" >> $TAOS_CFG -echo "http 0" >> $TAOS_CFG -echo "slaveQuery 0" >> $TAOS_CFG -echo "numOfThreadsPerCore 2.0" >> $TAOS_CFG -echo "defaultPass taosdata" >> $TAOS_CFG -echo "numOfLogLines 20000000" >> $TAOS_CFG -echo "mnodeEqualVnodeNum 0" >> $TAOS_CFG -echo "balanceInterval 1" >> $TAOS_CFG -echo "clog 2" >> $TAOS_CFG -#echo "cache 1" >> $TAOS_CFG -echo "days 10" >> $TAOS_CFG -echo "statusInterval 1" >> $TAOS_CFG -echo "maxVgroupsPerDb 4" >> $TAOS_CFG -echo "minTablesPerVnode 4" >> $TAOS_CFG -echo "maxTablesPerVnode 1000" >> $TAOS_CFG -echo "tableIncStepPerVnode 10000" >> $TAOS_CFG -echo "asyncLog 0" >> $TAOS_CFG -echo "numOfMnodes 1" >> $TAOS_CFG -echo "locale en_US.UTF-8" >> $TAOS_CFG -echo "fsync 0" >> $TAOS_CFG -echo "telemetryReporting 0" >> $TAOS_CFG -echo " " >> $TAOS_CFG +set "fqdn=" +for /f "skip=1" %%A in ( + 'wmic computersystem get caption' +) do if not defined fqdn set "fqdn=%%A" +echo firstEp %fqdn% > %TAOS_CFG% +echo fqdn %fqdn% >> %TAOS_CFG% +echo serverPort %NODE% >> %TAOS_CFG% +echo dataDir %DATA_DIR% >> %TAOS_CFG% +echo logDir %LOG_DIR% >> %TAOS_CFG% +echo debugFlag 0 >> %TAOS_CFG% +echo mDebugFlag 143 >> %TAOS_CFG% +echo sdbDebugFlag 143 >> %TAOS_CFG% +echo dDebugFlag 143 >> %TAOS_CFG% +echo vDebugFlag 143 >> %TAOS_CFG% +echo tsdbDebugFlag 143 >> %TAOS_CFG% +echo cDebugFlag 143 >> %TAOS_CFG% +echo jnidebugFlag 143 >> %TAOS_CFG% +echo odbcdebugFlag 143 >> %TAOS_CFG% +echo httpDebugFlag 143 >> %TAOS_CFG% +echo monDebugFlag 143 >> %TAOS_CFG% +echo mqttDebugFlag 143 >> %TAOS_CFG% +echo qdebugFlag 143 >> %TAOS_CFG% +echo rpcDebugFlag 143 >> %TAOS_CFG% +echo tmrDebugFlag 131 >> %TAOS_CFG% +echo udebugFlag 143 >> %TAOS_CFG% +echo sdebugFlag 143 >> %TAOS_CFG% +echo wdebugFlag 143 >> %TAOS_CFG% +echo cqdebugFlag 143 >> %TAOS_CFG% +echo monitor 0 >> %TAOS_CFG% +echo monitorInterval 1 >> %TAOS_CFG% +echo http 0 >> %TAOS_CFG% +echo slaveQuery 0 >> %TAOS_CFG% +echo numOfThreadsPerCore 2.0 >> %TAOS_CFG% +echo defaultPass taosdata >> %TAOS_CFG% +echo numOfLogLines 20000000 >> %TAOS_CFG% +echo mnodeEqualVnodeNum 0 >> %TAOS_CFG% +echo balanceInterval 1 >> %TAOS_CFG% +echo clog 2 >> %TAOS_CFG% +echo days 10 >> %TAOS_CFG% +echo statusInterval 1 >> %TAOS_CFG% +echo maxVgroupsPerDb 4 >> %TAOS_CFG% +echo minTablesPerVnode 4 >> %TAOS_CFG% +echo maxTablesPerVnode 1000 >> %TAOS_CFG% +echo tableIncStepPerVnode 10000 >> %TAOS_CFG% +echo asyncLog 0 >> %TAOS_CFG% +echo numOfMnodes 1 >> %TAOS_CFG% +echo locale en_US.UTF-8 >> %TAOS_CFG% +echo fsync 0 >> %TAOS_CFG% +echo telemetryReporting 0 >> %TAOS_CFG% diff --git a/tests/script/sh/exec.bat b/tests/script/sh/exec.bat index 1c84a6b10e..47b7910210 100644 --- a/tests/script/sh/exec.bat +++ b/tests/script/sh/exec.bat @@ -1,115 +1,41 @@ -#!/bin/bash +@echo off -# if [ $# != 4 || $# != 5 ]; then - # echo "argument list need input : " - # echo " -n nodeName" - # echo " -s start/stop" - # echo " -c clear" - # exit 1 -# fi +rem echo Executing exec.sh -NODE_NAME= -EXEC_OPTON= -CLEAR_OPTION="false" -while getopts "n:s:u:x:ct" arg -do - case $arg in - n) - NODE_NAME=$OPTARG - ;; - s) - EXEC_OPTON=$OPTARG - ;; - c) - CLEAR_OPTION="clear" - ;; - t) - SHELL_OPTION="true" - ;; - u) - USERS=$OPTARG - ;; - x) - SIGNAL=$OPTARG - ;; - ?) - echo "unkown argument" - ;; - esac -done +if %1 == -n set NODE_NAME=%2 +if %1 == -s set EXEC_OPTON=%2 +if %3 == -n set NODE_NAME=%4 +if %3 == -s set EXEC_OPTON=%4 -SCRIPT_DIR=`dirname $0` -cd $SCRIPT_DIR/../ -SCRIPT_DIR=`pwd` +rem echo NODE_NAME: %NODE_NAME% +rem echo NODE: %EXEC_OPTON% -IN_TDINTERNAL="community" -if [[ "$SCRIPT_DIR" == *"$IN_TDINTERNAL"* ]]; then - cd ../../.. -else - cd ../../ -fi +set SCRIPT_DIR=%~dp0..\ +rem echo SCRIPT_DIR: %SCRIPT_DIR% -TAOS_DIR=`pwd` -TAOSD_DIR=`find . -name "taosd"|grep bin|head -n1` +set BUILD_DIR=%SCRIPT_DIR%..\..\..\debug\build\bin\ +set TAOSD=%BUILD_DIR%taosd +rem echo BUILD_DIR: %BUILD_DIR% +rem echo TAOSD: %TAOSD% -if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then - BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2,3` -else - BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2` -fi +set SIM_DIR=%SCRIPT_DIR%..\..\..\sim\ +rem echo SIM_DIR: %SIM_DIR% -BUILD_DIR=$TAOS_DIR/$BIN_DIR/build +set NODE_DIR=%SIM_DIR%%NODE_NAME%\ +rem echo NODE_DIR: %NODE_DIR% -SIM_DIR=$TAOS_DIR/sim -NODE_DIR=$SIM_DIR/$NODE_NAME -EXE_DIR=$BUILD_DIR/bin -CFG_DIR=$NODE_DIR/cfg -LOG_DIR=$NODE_DIR/log -DATA_DIR=$NODE_DIR/data -MGMT_DIR=$NODE_DIR/data/mgmt -TSDB_DIR=$NODE_DIR/data/tsdb +set CFG_DIR=%NODE_DIR%cfg\ +rem echo CFG_DIR: %CFG_DIR% -TAOS_CFG=$NODE_DIR/cfg/taos.cfg +set TAOS_CFG=%CFG_DIR%taos.cfg +rem echo TAOS_CFG: %TAOS_CFG% -echo ------------ $EXEC_OPTON $NODE_NAME - -TAOS_FLAG=$SIM_DIR/tsim/flag -if [ -f "$TAOS_FLAG" ]; then - EXE_DIR=/usr/local/bin/taos -fi - -if [ "$CLEAR_OPTION" = "clear" ]; then - echo rm -rf $MGMT_DIR $TSDB_DIR - rm -rf $TSDB_DIR - rm -rf $MGMT_DIR -fi - -if [ "$EXEC_OPTON" = "start" ]; then - echo "ExcuteCmd:" $EXE_DIR/taosd -c $CFG_DIR - - if [ "$SHELL_OPTION" = "true" ]; then - TT=`date +%s` - mkdir ${LOG_DIR}/${TT} - nohup valgrind --log-file=${LOG_DIR}/${TT}/valgrind.log --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 & - else - nohup $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 & - fi - -else - #relative path - RCFG_DIR=sim/$NODE_NAME/cfg - PID=`ps -ef|grep taosd | grep $RCFG_DIR | grep -v grep | awk '{print $2}'` - while [ -n "$PID" ] - do - if [ "$SIGNAL" = "SIGKILL" ]; then - echo try to kill by signal SIGKILL - kill -9 $PID - else - echo try to kill by signal SIGINT - kill -SIGINT $PID - fi - sleep 1 - PID=`ps -ef|grep taosd | grep $RCFG_DIR | grep -v grep | awk '{print $2}'` - done -fi +if %EXEC_OPTON% == start ( + echo start %TAOSD% -c %CFG_DIR% + start %TAOSD% -c %CFG_DIR% +) +if %EXEC_OPTON% == stop ( + rem echo wmic process where "name='taosd.exe' and CommandLine like '%%%NODE_NAME%%%'" list INSTANCE + wmic process where "name='taosd.exe' and CommandLine like '%%%NODE_NAME%%%'" call terminate > NUL 2>&1 +) diff --git a/tests/script/sh/stop_dnodes.bat b/tests/script/sh/stop_dnodes.bat index 1a6e7153c3..e44b7b4e29 100644 --- a/tests/script/sh/stop_dnodes.bat +++ b/tests/script/sh/stop_dnodes.bat @@ -1,24 +1,5 @@ -#!/bin/sh +@echo off -PID=`ps -ef|grep /usr/bin/taosd | grep -v grep | awk '{print $2}'` -if [ -n "$PID" ]; then - echo systemctl stop taosd - systemctl stop taosd -fi - -PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` -while [ -n "$PID" ]; do - echo kill -9 $PID - pkill -9 taosd - fuser -k -n tcp 6030 - PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` -done - -PID=`ps -ef|grep -w tarbitrator | grep -v grep | awk '{print $2}'` -while [ -n "$PID" ]; do - echo kill -9 $PID - pkill -9 tarbitrator - fuser -k -n tcp 6040 - PID=`ps -ef|grep -w tarbitrator | grep -v grep | awk '{print $2}'` -done +rem echo taskkill /F /IM taosd.exe +taskkill /F /IM taosd.exe > NUL 2>&1 \ No newline at end of file diff --git a/tests/script/wtest.bat b/tests/script/wtest.bat index 8ca1f22518..a89c1df67a 100644 --- a/tests/script/wtest.bat +++ b/tests/script/wtest.bat @@ -1,29 +1,30 @@ @echo off echo TDengine in windows -echo Start TDengine Testing Case ... +rem echo Start TDengine Testing Case ... set "SCRIPT_DIR=%~dp0" -echo SCRIPT_DIR: %SCRIPT_DIR% +rem echo SCRIPT_DIR: %SCRIPT_DIR% -set "BUILD_DIR=%~dp0..\..\debug\build\bin" -set "TSIM=%~dp0..\..\debug\build\bin\tsim" -echo BUILD_DIR: %BUILD_DIR% +set "BUILD_DIR=%SCRIPT_DIR%..\..\..\debug\build\bin\" +set "TSIM=%BUILD_DIR%tsim" +rem echo BUILD_DIR: %BUILD_DIR% +rem echo TSIM: %TSIM% -set "SIM_DIR=%~dp0..\..\sim" -echo SIM_DIR: %SIM_DIR% +set "SIM_DIR=%SCRIPT_DIR%..\..\..\sim\" +rem echo SIM_DIR: %SIM_DIR% -set "TSIM_DIR=%~dp0..\..\sim\tsim" -echo TSIM_DIR: %TSIM_DIR% +set "TSIM_DIR=%SIM_DIR%tsim\" +rem echo TSIM_DIR: %TSIM_DIR% -set "CFG_DIR=%~dp0..\..\sim\tsim\cfg" -echo CFG_DIR: %CFG_DIR% +set "CFG_DIR=%TSIM_DIR%cfg\" +rem echo CFG_DIR: %CFG_DIR% -set "LOG_DIR=%~dp0..\..\sim\tsim\log" -echo LOG_DIR: %LOG_DIR% +set "LOG_DIR=%TSIM_DIR%log\" +rem echo LOG_DIR: %LOG_DIR% -set "TAOS_CFG=%~dp0..\..\sim\tsim\cfg\taos.cfg" -echo TAOS_CFG: %TAOS_CFG% +set "TAOS_CFG=%CFG_DIR%taos.cfg" +rem echo TAOS_CFG: %TAOS_CFG% if not exist %SIM_DIR% mkdir %SIM_DIR% if not exist %TSIM_DIR% mkdir %TSIM_DIR% @@ -32,7 +33,13 @@ if exist %LOG_DIR% rmdir /s/q %LOG_DIR% if not exist %CFG_DIR% mkdir %CFG_DIR% if not exist %LOG_DIR% mkdir %LOG_DIR% -echo firstEp localhost > %TAOS_CFG% +set "fqdn=" +for /f "skip=1" %%A in ( + 'wmic computersystem get caption' +) do if not defined fqdn set "fqdn=%%A" + +echo firstEp %fqdn% > %TAOS_CFG% +echo fqdn %fqdn% >> %TAOS_CFG% echo serverPort 7100 >> %TAOS_CFG% echo logDir %LOG_DIR% >> %TAOS_CFG% echo scriptDir %SCRIPT_DIR% >> %TAOS_CFG% @@ -49,7 +56,7 @@ echo enableCoreFile 1 >> %TAOS_CFG% set "FILE_NAME=testSuite.sim" if "%1" == "-f" set "FILE_NAME=%2" -echo FILE_NAME: %FILE_NAME% +rem echo FILE_NAME: %FILE_NAME% echo ExcuteCmd: %tsim% -c %CFG_DIR% -f %FILE_NAME% -%tsim% -c %CFG_DIR% -f %FILE_NAME% \ No newline at end of file +%TSIM% -c %CFG_DIR% -f %FILE_NAME% \ No newline at end of file diff --git a/tests/tsim/src/simExe.c b/tests/tsim/src/simExe.c index 10e87eefe6..83ca46599c 100644 --- a/tests/tsim/src/simExe.c +++ b/tests/tsim/src/simExe.c @@ -301,11 +301,37 @@ bool simExecuteRunBackCmd(SScript *script, char *option) { return true; } +void simReplaceShToBat(char *dst) { + char* sh = strstr(dst, ".sh"); + if (sh != NULL) { + int32_t dstLen = (int32_t)strlen(dst); + char *end = dst + dstLen; + *(end + 1) = 0; + + for (char *p = end; p >= sh; p--) { + *(p + 1) = *p; + } + + sh[0] = '.'; + sh[1] = 'b'; + sh[2] = 'a'; + sh[3] = 't'; + sh[4] = ' '; + } + + simDebug("system cmd is %s", dst); +} + bool simExecuteSystemCmd(SScript *script, char *option) { char buf[4096] = {0}; +#ifndef WINDOWS sprintf(buf, "cd %s; ", tsScriptDir); simVisuallizeOption(script, option, buf + strlen(buf)); +#else + sprintf(buf, "%s%s", tsScriptDir, option); + simReplaceShToBat(buf); +#endif simLogSql(buf, true); int32_t code = system(buf); From d0534df8c01ef7958337a57644d9d3fe59702dd4 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 19 Jan 2021 15:26:44 +0800 Subject: [PATCH 27/62] TD-1207 --- src/dnode/src/dnodeSystem.c | 3 +++ src/os/inc/osSignal.h | 2 +- src/os/src/windows/wSignal.c | 13 ++++++++++--- tests/script/sh/deploy.bat | 2 +- tests/script/sh/exec.bat | 9 ++++++++- tests/script/wtest.bat | 2 +- 6 files changed, 24 insertions(+), 7 deletions(-) diff --git a/src/dnode/src/dnodeSystem.c b/src/dnode/src/dnodeSystem.c index 3a50eca9e6..dcbf249a51 100644 --- a/src/dnode/src/dnodeSystem.c +++ b/src/dnode/src/dnodeSystem.c @@ -145,6 +145,8 @@ int32_t main(int32_t argc, char *argv[]) { syslog(LOG_INFO, "Shut down TDengine service successfully"); dInfo("TDengine is shut down!"); closelog(); + + tsem_post(&exitSem); return EXIT_SUCCESS; } @@ -170,4 +172,5 @@ static void sigintHandler(int32_t signum) { // inform main thread to exit tsem_post(&exitSem); + tsem_wait(&exitSem); } \ No newline at end of file diff --git a/src/os/inc/osSignal.h b/src/os/inc/osSignal.h index 57582a8a28..8016c49ba8 100644 --- a/src/os/inc/osSignal.h +++ b/src/os/inc/osSignal.h @@ -29,7 +29,7 @@ extern "C" { #endif #ifndef SIGHUP - #define SIGHUP 1234 + #define SIGHUP 1230 #endif #ifndef SIGCHLD diff --git a/src/os/src/windows/wSignal.c b/src/os/src/windows/wSignal.c index 8a3c2cfbd4..3988f0c6e1 100644 --- a/src/os/src/windows/wSignal.c +++ b/src/os/src/windows/wSignal.c @@ -16,18 +16,25 @@ #define _DEFAULT_SOURCE #include "os.h" #include +#include void taosSetSignal(int32_t signum, FSignalHandler sigfp) { if (signum == SIGUSR1) return; - signal(signum, sigfp); + + // SIGHUP doesn't exist in windows, we handle it in the way of ctrlhandler + if (signum == SIGHUP) { + SetConsoleCtrlHandler((PHANDLER_ROUTINE)sigfp, TRUE); + } else { + signal(signum, sigfp); + } } void taosIgnSignal(int32_t signum) { - if (signum == SIGUSR1) return; + if (signum == SIGUSR1 || signum == SIGHUP) return; signal(signum, SIG_IGN); } void taosDflSignal(int32_t signum) { - if (signum == SIGUSR1) return; + if (signum == SIGUSR1 || signum == SIGHUP) return; signal(signum, SIG_DFL); } diff --git a/tests/script/sh/deploy.bat b/tests/script/sh/deploy.bat index 0b419b1e9b..7a81761e5b 100644 --- a/tests/script/sh/deploy.bat +++ b/tests/script/sh/deploy.bat @@ -54,7 +54,7 @@ if %NODE% == 6 set NODE=7600 if %NODE% == 7 set NODE=7700 if %NODE% == 8 set NODE=7800 -set "fqdn=" +rem set "fqdn=" for /f "skip=1" %%A in ( 'wmic computersystem get caption' ) do if not defined fqdn set "fqdn=%%A" diff --git a/tests/script/sh/exec.bat b/tests/script/sh/exec.bat index 47b7910210..9ad6667644 100644 --- a/tests/script/sh/exec.bat +++ b/tests/script/sh/exec.bat @@ -37,5 +37,12 @@ if %EXEC_OPTON% == start ( if %EXEC_OPTON% == stop ( rem echo wmic process where "name='taosd.exe' and CommandLine like '%%%NODE_NAME%%%'" list INSTANCE - wmic process where "name='taosd.exe' and CommandLine like '%%%NODE_NAME%%%'" call terminate > NUL 2>&1 + rem wmic process where "name='taosd.exe' and CommandLine like '%%%NODE_NAME%%%'" call terminate > NUL 2>&1 + + for /f "tokens=1 skip=1" %%A in ( + 'wmic process where "name='taosd.exe'" get processId ' + ) do ( + rem echo taskkill /IM %%A + taskkill /IM %%A > NUL 2>&1 + ) ) diff --git a/tests/script/wtest.bat b/tests/script/wtest.bat index a89c1df67a..0b5cdda527 100644 --- a/tests/script/wtest.bat +++ b/tests/script/wtest.bat @@ -33,7 +33,7 @@ if exist %LOG_DIR% rmdir /s/q %LOG_DIR% if not exist %CFG_DIR% mkdir %CFG_DIR% if not exist %LOG_DIR% mkdir %LOG_DIR% -set "fqdn=" +rem set "fqdn=" for /f "skip=1" %%A in ( 'wmic computersystem get caption' ) do if not defined fqdn set "fqdn=%%A" From fffe865871c100e2a1a257fba55ac77b9d8d9da4 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 20 Jan 2021 09:40:47 +0800 Subject: [PATCH 28/62] TD-1207 --- src/dnode/src/dnodeSystem.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/dnode/src/dnodeSystem.c b/src/dnode/src/dnodeSystem.c index dcbf249a51..b7ca6f053c 100644 --- a/src/dnode/src/dnodeSystem.c +++ b/src/dnode/src/dnodeSystem.c @@ -146,7 +146,9 @@ int32_t main(int32_t argc, char *argv[]) { dInfo("TDengine is shut down!"); closelog(); +#ifdef WINDOWS tsem_post(&exitSem); +#endif return EXIT_SUCCESS; } @@ -172,5 +174,7 @@ static void sigintHandler(int32_t signum) { // inform main thread to exit tsem_post(&exitSem); +#ifdef WINDOWS tsem_wait(&exitSem); +#endif } \ No newline at end of file From d73cdd656dd6705d84efb275aae17e9010ff52ea Mon Sep 17 00:00:00 2001 From: freemine Date: Wed, 20 Jan 2021 10:05:53 +0800 Subject: [PATCH 29/62] 1. bugFix: typo in eok.c which results in memory leakage 2. for the sake of SIGALRM used in the base code, add taos_block_sigalrm and so forth 3. TODO: getaddrinfo would block a few unnecessary seconds when host ~ *.local better give warning/suggestion message to user, or just bypass getaddrinfo and fail the function --- src/os/inc/eok.h | 4 ++++ src/os/inc/osDarwin.h | 2 ++ src/os/src/darwin/darwinTimer.c | 13 +++++++++++++ src/os/src/darwin/eok.c | 16 +++++++--------- src/os/src/detail/osTimer.c | 6 +++++- 5 files changed, 31 insertions(+), 10 deletions(-) diff --git a/src/os/inc/eok.h b/src/os/inc/eok.h index 4d38e8e632..0874ca975b 100644 --- a/src/os/inc/eok.h +++ b/src/os/inc/eok.h @@ -22,6 +22,8 @@ extern "C" { #endif +#ifdef __APPLE__ + enum EPOLL_EVENTS { EPOLLIN = 0x001, @@ -81,6 +83,8 @@ int epoll_ctl(int epfd, int op, int fd, struct epoll_event *event); int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout); int epoll_close(int epfd); +#endif // __APPLE__ + #ifdef __cplusplus } #endif diff --git a/src/os/inc/osDarwin.h b/src/os/inc/osDarwin.h index 52bb661f89..935708eb1b 100644 --- a/src/os/inc/osDarwin.h +++ b/src/os/inc/osDarwin.h @@ -104,6 +104,8 @@ int64_t tsosStr2int64(char *str); #include "eok.h" +void taos_block_sigalrm(void); + diff --git a/src/os/src/darwin/darwinTimer.c b/src/os/src/darwin/darwinTimer.c index 5fe65fb99e..121e2e9427 100644 --- a/src/os/src/darwin/darwinTimer.c +++ b/src/os/src/darwin/darwinTimer.c @@ -34,3 +34,16 @@ void taosUninitTimer() { setitimer(ITIMER_REAL, &tv, NULL); } +void taos_block_sigalrm(void) { + // since SIGALRM has been used + // consideration: any better solution? + static __thread int already_set = 0; + if (!already_set) { + sigset_t set; + sigemptyset(&set); + sigaddset(&set, SIGALRM); + pthread_sigmask(SIG_BLOCK, &set, NULL); + already_set = 1; + } +} + diff --git a/src/os/src/darwin/eok.c b/src/os/src/darwin/eok.c index 1be5ec4ac1..9c6a263a40 100644 --- a/src/os/src/darwin/eok.c +++ b/src/os/src/darwin/eok.c @@ -15,15 +15,11 @@ #include "eok.h" -#include -#include -#include -#include -#include -#include +#include "os.h" + #include -#include -#include + +#define LET_IT_BE #ifdef ENABLE_LOG #define D(fmt, ...) fprintf(stderr, "%s[%d]%s(): " fmt "\n", basename(__FILE__), __LINE__, __func__, ##__VA_ARGS__) @@ -414,6 +410,8 @@ int epoll_ctl(int epfd, int op, int fd, struct epoll_event *event) { static struct timespec do_timespec_diff(struct timespec *from, struct timespec *to); int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) { + taos_block_sigalrm(); + int e = 0; if (!events) { errno = EINVAL; @@ -699,7 +697,7 @@ static void eok_free_ev(ep_over_kq_t *eok, eok_event_t *ev) { else eok->evs_tail = ev->prev; ev->prev = NULL; ev->next = eok->evs_free; - eok->evs_free = ev->next; + eok->evs_free = ev; eok->evs_count -= 1; } diff --git a/src/os/src/detail/osTimer.c b/src/os/src/detail/osTimer.c index 1d3ba30def..c1135ce292 100644 --- a/src/os/src/detail/osTimer.c +++ b/src/os/src/detail/osTimer.c @@ -116,6 +116,9 @@ void taosUninitTimer() { pthread_sigmask(SIG_BLOCK, &set, NULL); */ void taosMsleep(int mseconds) { +#ifdef __APPLE__ + taos_block_sigalrm(); +#endif // __APPLE__ #if 1 usleep(mseconds * 1000); #else @@ -136,6 +139,7 @@ void taosMsleep(int mseconds) { /* pthread_sigmask(SIG_UNBLOCK, &set, NULL); */ #endif + } -#endif \ No newline at end of file +#endif From 05ce993084f68699cc8cd3bd95218756f7db0973 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 21 Jan 2021 15:05:09 +0800 Subject: [PATCH 30/62] TD-1207 --- tests/script/sh/deploy.bat | 2 +- tests/script/sh/exec.bat | 4 ++-- tests/script/sh/stop_dnodes.bat | 1 + tests/tsim/src/simSystem.c | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/script/sh/deploy.bat b/tests/script/sh/deploy.bat index 7a81761e5b..04c7b8a660 100644 --- a/tests/script/sh/deploy.bat +++ b/tests/script/sh/deploy.bat @@ -59,7 +59,7 @@ for /f "skip=1" %%A in ( 'wmic computersystem get caption' ) do if not defined fqdn set "fqdn=%%A" -echo firstEp %fqdn% > %TAOS_CFG% +echo firstEp %fqdn%:7100 > %TAOS_CFG% echo fqdn %fqdn% >> %TAOS_CFG% echo serverPort %NODE% >> %TAOS_CFG% echo dataDir %DATA_DIR% >> %TAOS_CFG% diff --git a/tests/script/sh/exec.bat b/tests/script/sh/exec.bat index 9ad6667644..6651c7aa8f 100644 --- a/tests/script/sh/exec.bat +++ b/tests/script/sh/exec.bat @@ -40,9 +40,9 @@ if %EXEC_OPTON% == stop ( rem wmic process where "name='taosd.exe' and CommandLine like '%%%NODE_NAME%%%'" call terminate > NUL 2>&1 for /f "tokens=1 skip=1" %%A in ( - 'wmic process where "name='taosd.exe'" get processId ' + 'wmic process where "name='taosd.exe' and CommandLine like '%%%NODE_NAME%%%'" get processId ' ) do ( rem echo taskkill /IM %%A taskkill /IM %%A > NUL 2>&1 - ) + ) ) diff --git a/tests/script/sh/stop_dnodes.bat b/tests/script/sh/stop_dnodes.bat index e44b7b4e29..ab7af2ca92 100644 --- a/tests/script/sh/stop_dnodes.bat +++ b/tests/script/sh/stop_dnodes.bat @@ -2,4 +2,5 @@ rem echo taskkill /F /IM taosd.exe +wmic process where "name='taosd.exe'" call terminate > NUL 2>&1 taskkill /F /IM taosd.exe > NUL 2>&1 \ No newline at end of file diff --git a/tests/tsim/src/simSystem.c b/tests/tsim/src/simSystem.c index bf47c56718..65086eace5 100644 --- a/tests/tsim/src/simSystem.c +++ b/tests/tsim/src/simSystem.c @@ -78,8 +78,8 @@ char *simParseHostName(char *varName) { } bool simSystemInit() { - taosGetFqdn(simHostName); taos_init(); + taosGetFqdn(simHostName); simInitsimCmdList(); memset(simScriptList, 0, sizeof(SScript *) * MAX_MAIN_SCRIPT_NUM); return true; From cd79df1de2050f311f926607c3b2c5619ee266ab Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 21 Jan 2021 19:32:01 +0800 Subject: [PATCH 31/62] TD-1207 --- src/rpc/src/rpcUdp.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/rpc/src/rpcUdp.c b/src/rpc/src/rpcUdp.c index 30a2fb2899..7281647136 100644 --- a/src/rpc/src/rpcUdp.c +++ b/src/rpc/src/rpcUdp.c @@ -140,6 +140,7 @@ void taosStopUdpConnection(void *handle) { pConn = pSet->udpConn + i; if (pConn->fd >=0) shutdown(pConn->fd, SHUT_RDWR); if (pConn->fd >=0) taosCloseSocket(pConn->fd); + pConn->fd = -1; } for (int i = 0; i < pSet->threads; ++i) { @@ -197,9 +198,16 @@ static void *taosRecvUdpData(void *param) { while (1) { dataLen = recvfrom(pConn->fd, pConn->buffer, RPC_MAX_UDP_SIZE, 0, (struct sockaddr *)&sourceAdd, &addLen); - if(dataLen <= 0) { - tDebug("%s UDP socket was closed, exiting(%s)", pConn->label, strerror(errno)); - break; + if (dataLen <= 0) { + tDebug("%s UDP socket was closed, exiting(%s), dataLen:%d fd:%d", pConn->label, strerror(errno), (int32_t)dataLen, + pConn->fd); + + //for windows usage, remote shutdown also returns - 1 in windows client + if (pConn->fd == -1) { + break; + } else { + continue; + } } port = ntohs(sourceAdd.sin_port); From fa7e9bd6d262afbf6547dcd49e4db4170c2e2ed7 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 22 Jan 2021 11:59:14 +0800 Subject: [PATCH 32/62] TD-1207 --- src/plugins/http/src/httpAuth.c | 8 +-- src/plugins/http/src/httpContext.c | 11 ++- src/plugins/http/src/httpGcHandle.c | 2 +- src/plugins/http/src/httpGcJson.c | 27 ++++---- src/plugins/http/src/httpGzip.c | 92 +++++++++++++------------ src/plugins/http/src/httpHandle.c | 2 +- src/plugins/http/src/httpJson.c | 47 ++++++------- src/plugins/http/src/httpParser.c | 96 +++++++++++++++------------ src/plugins/http/src/httpQueue.c | 14 ++-- src/plugins/http/src/httpResp.c | 5 +- src/plugins/http/src/httpRestHandle.c | 1 - src/plugins/http/src/httpRestJson.c | 23 ++++--- src/plugins/http/src/httpServer.c | 34 +++++----- src/plugins/http/src/httpSession.c | 5 +- src/plugins/http/src/httpSql.c | 10 +-- src/plugins/http/src/httpSystem.c | 6 +- src/plugins/http/src/httpTgHandle.c | 45 +++++++------ src/plugins/http/src/httpUtil.c | 55 +++++++-------- tests/script/general/http/restful.sim | 2 - 19 files changed, 247 insertions(+), 238 deletions(-) diff --git a/src/plugins/http/src/httpAuth.c b/src/plugins/http/src/httpAuth.c index 973f69f24e..2ef5406823 100644 --- a/src/plugins/http/src/httpAuth.c +++ b/src/plugins/http/src/httpAuth.c @@ -26,7 +26,7 @@ int32_t httpParseBasicAuthToken(HttpContext *pContext, char *token, int32_t len) { token[len] = '\0'; int32_t outlen = 0; - char *base64 = (char *)base64_decode(token, len, &outlen); + char * base64 = (char *)base64_decode(token, len, &outlen); if (base64 == NULL || outlen == 0) { httpError("context:%p, fd:%d, basic token:%s parsed error", pContext, pContext->fd, token); free(base64); @@ -49,7 +49,7 @@ int32_t httpParseBasicAuthToken(HttpContext *pContext, char *token, int32_t len) strncpy(pContext->user, base64, (size_t)user_len); pContext->user[user_len] = 0; - char *password = user + 1; + char * password = user + 1; int32_t pass_len = (int32_t)((base64 + outlen) - password); if (pass_len < 1 || pass_len >= HTTP_PASSWORD_LEN) { httpError("context:%p, fd:%d, basic token:%s parse password error", pContext, pContext->fd, token); @@ -66,7 +66,7 @@ int32_t httpParseBasicAuthToken(HttpContext *pContext, char *token, int32_t len) int32_t httpParseTaosdAuthToken(HttpContext *pContext, char *token, int32_t len) { token[len] = '\0'; - int32_t outlen = 0; + int32_t outlen = 0; unsigned char *base64 = base64_decode(token, len, &outlen); if (base64 == NULL || outlen == 0) { httpError("context:%p, fd:%d, taosd token:%s parsed error", pContext, pContext->fd, token); @@ -97,7 +97,7 @@ int32_t httpParseTaosdAuthToken(HttpContext *pContext, char *token, int32_t len) } int32_t httpGenTaosdAuthToken(HttpContext *pContext, char *token, int32_t maxLen) { - char buffer[sizeof(pContext->user) + sizeof(pContext->pass)] = {0}; + char buffer[sizeof(pContext->user) + sizeof(pContext->pass)] = {0}; size_t size = sizeof(pContext->user); tstrncpy(buffer, pContext->user, size); size = sizeof(pContext->pass); diff --git a/src/plugins/http/src/httpContext.c b/src/plugins/http/src/httpContext.c index 266228e7ac..db77c479c0 100644 --- a/src/plugins/http/src/httpContext.c +++ b/src/plugins/http/src/httpContext.c @@ -48,7 +48,7 @@ static void httpDestroyContext(void *data) { httpRemoveContextFromEpoll(pContext); httpReleaseSession(pContext); atomic_sub_fetch_32(&pThread->numOfContexts, 1); - + httpDebug("context:%p, is destroyed, refCount:%d data:%p thread:%s numOfContexts:%d", pContext, pContext->refCount, data, pContext->pThread->label, pContext->pThread->numOfContexts); pContext->pThread = 0; @@ -100,9 +100,7 @@ const char *httpContextStateStr(HttpContextState state) { } } -void httpNotifyContextClose(HttpContext *pContext) { - shutdown(pContext->fd, SHUT_WR); -} +void httpNotifyContextClose(HttpContext *pContext) { shutdown(pContext->fd, SHUT_WR); } bool httpAlterContextState(HttpContext *pContext, HttpContextState srcState, HttpContextState destState) { return (atomic_val_compare_exchange_32(&pContext->state, srcState, destState) == srcState); @@ -123,8 +121,8 @@ HttpContext *httpCreateContext(int32_t fd) { pContext->ppContext = ppContext; httpDebug("context:%p, fd:%d, is created, data:%p", pContext, fd, ppContext); - // set the ref to 0 - taosCacheRelease(tsHttpServer.contextCache, (void**)&ppContext, false); + // set the ref to 0 + taosCacheRelease(tsHttpServer.contextCache, (void **)&ppContext, false); return pContext; } @@ -174,7 +172,6 @@ bool httpInitContext(HttpContext *pContext) { pContext->encodeMethod = NULL; memset(&pContext->singleCmd, 0, sizeof(HttpSqlCmd)); - httpTrace("context:%p, fd:%d, parsed:%d", pContext, pContext->fd, pContext->parsed); return true; } diff --git a/src/plugins/http/src/httpGcHandle.c b/src/plugins/http/src/httpGcHandle.c index 4c10249672..925c74e7cd 100644 --- a/src/plugins/http/src/httpGcHandle.c +++ b/src/plugins/http/src/httpGcHandle.c @@ -192,7 +192,7 @@ bool gcProcessQueryRequest(HttpContext* pContext) { break; } - cJSON* alias = cJSON_GetObjectItem(query, "alias"); + cJSON* alias = cJSON_GetObjectItem(query, "alias"); int32_t aliasBuffer = -1; if (!(alias == NULL || alias->valuestring == NULL || strlen(alias->valuestring) == 0)) { aliasBuffer = httpAddToSqlCmdBuffer(pContext, alias->valuestring); diff --git a/src/plugins/http/src/httpGcJson.c b/src/plugins/http/src/httpGcJson.c index 3a053afd87..8c223a1500 100644 --- a/src/plugins/http/src/httpGcJson.c +++ b/src/plugins/http/src/httpGcJson.c @@ -86,7 +86,7 @@ bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, JsonBuf *jsonBuf = httpMallocJsonBuf(pContext); if (jsonBuf == NULL) return false; - int32_t num_fields = taos_num_fields(result); + int32_t num_fields = taos_num_fields(result); TAOS_FIELD *fields = taos_fetch_fields(result); if (num_fields == 0) { return false; @@ -101,7 +101,7 @@ bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, // such as select count(*) count(*) from sys.cpu group by ipaddr interval(1d) int32_t dataFields = -1; int32_t groupFields = -1; - bool hasTimestamp = fields[0].type == TSDB_DATA_TYPE_TIMESTAMP; + bool hasTimestamp = fields[0].type == TSDB_DATA_TYPE_TIMESTAMP; if (hasTimestamp) { dataFields = 1; if (num_fields > 2) groupFields = num_fields - 1; @@ -125,15 +125,15 @@ bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, cmd->numOfRows--; continue; } - int32_t* length = taos_fetch_lengths(result); + int32_t *length = taos_fetch_lengths(result); // for group by if (groupFields != -1) { - char target[HTTP_GC_TARGET_SIZE] = {0}; + char target[HTTP_GC_TARGET_SIZE] = {0}; int32_t len; - len = snprintf(target,HTTP_GC_TARGET_SIZE,"%s{",aliasBuffer); - for (int32_t i = dataFields + 1; istate) { + switch (gzip->state) { case EHTTP_GZIP_READY: { inflateEnd(gzip->gzip); } break; - default: break; + default: + break; } if (gzip->gzip) { free(gzip->gzip); @@ -60,43 +59,43 @@ static void ehttp_gzip_cleanup(ehttp_gzip_t *gzip) { gzip->state = EHTTP_GZIP_CLOSED; } -ehttp_gzip_t* ehttp_gzip_create_decompressor(ehttp_gzip_conf_t conf, ehttp_gzip_callbacks_t callbacks, void *arg) { - ehttp_gzip_t *gzip = (ehttp_gzip_t*)calloc(1, sizeof(*gzip)); +ehttp_gzip_t *ehttp_gzip_create_decompressor(ehttp_gzip_conf_t conf, ehttp_gzip_callbacks_t callbacks, void *arg) { + ehttp_gzip_t *gzip = (ehttp_gzip_t *)calloc(1, sizeof(*gzip)); if (!gzip) return NULL; do { - gzip->conf = conf; - gzip->callbacks = callbacks; - gzip->arg = arg; + gzip->conf = conf; + gzip->callbacks = callbacks; + gzip->arg = arg; if (gzip->callbacks.on_data == NULL) gzip->callbacks.on_data = dummy_on_data; - gzip->gzip = (z_stream*)calloc(1, sizeof(*gzip->gzip)); + gzip->gzip = (z_stream *)calloc(1, sizeof(*gzip->gzip)); if (gzip->conf.get_header) { - gzip->header = (gz_header*)calloc(1, sizeof(*gzip->header)); + gzip->header = (gz_header *)calloc(1, sizeof(*gzip->header)); } - if (gzip->conf.chunk_size<=0) gzip->conf.chunk_size = EHTTP_GZIP_CHUNK_SIZE_DEFAULT; - gzip->chunk = (char*)malloc(gzip->conf.chunk_size); + if (gzip->conf.chunk_size <= 0) gzip->conf.chunk_size = EHTTP_GZIP_CHUNK_SIZE_DEFAULT; + gzip->chunk = (char *)malloc(gzip->conf.chunk_size); if (!gzip->gzip || (gzip->conf.get_header && !gzip->header) || !gzip->chunk) break; - gzip->gzip->zalloc = Z_NULL; - gzip->gzip->zfree = Z_NULL; - gzip->gzip->opaque = Z_NULL; + gzip->gzip->zalloc = Z_NULL; + gzip->gzip->zfree = Z_NULL; + gzip->gzip->opaque = Z_NULL; - // 863 windowBits can also be greater than 15 for optional gzip decoding. Add - // 864 32 to windowBits to enable zlib and gzip decoding with automatic header - // 865 detection, or add 16 to decode only the gzip format (the zlib format will - // 866 return a Z_DATA_ERROR). If a gzip stream is being decoded, strm->adler is a - // 867 CRC-32 instead of an Adler-32. Unlike the gunzip utility and gzread() (see - // 868 below), inflate() will not automatically decode concatenated gzip streams. - // 869 inflate() will return Z_STREAM_END at the end of the gzip stream. The state - // 870 would need to be reset to continue decoding a subsequent gzip stream. - int32_t ret = inflateInit2(gzip->gzip, 32); // 32/16? 32/16 + MAX_WBITS + // 863 windowBits can also be greater than 15 for optional gzip decoding. Add + // 864 32 to windowBits to enable zlib and gzip decoding with automatic header + // 865 detection, or add 16 to decode only the gzip format (the zlib format will + // 866 return a Z_DATA_ERROR). If a gzip stream is being decoded, strm->adler is a + // 867 CRC-32 instead of an Adler-32. Unlike the gunzip utility and gzread() (see + // 868 below), inflate() will not automatically decode concatenated gzip streams. + // 869 inflate() will return Z_STREAM_END at the end of the gzip stream. The state + // 870 would need to be reset to continue decoding a subsequent gzip stream. + int32_t ret = inflateInit2(gzip->gzip, 32); // 32/16? 32/16 + MAX_WBITS if (ret != Z_OK) break; if (gzip->header) { ret = inflateGetHeader(gzip->gzip, gzip->header); } if (ret != Z_OK) break; - gzip->gzip->next_out = (z_const Bytef*)gzip->chunk; - gzip->gzip->avail_out = gzip->conf.chunk_size; + gzip->gzip->next_out = (z_const Bytef *)gzip->chunk; + gzip->gzip->avail_out = gzip->conf.chunk_size; gzip->state = EHTTP_GZIP_READY; return gzip; } while (0); @@ -105,7 +104,7 @@ ehttp_gzip_t* ehttp_gzip_create_decompressor(ehttp_gzip_conf_t conf, ehttp_gzip_ return NULL; } -ehttp_gzip_t* ehttp_gzip_create_compressor(ehttp_gzip_conf_t conf, ehttp_gzip_callbacks_t callbacks, void *arg); +ehttp_gzip_t *ehttp_gzip_create_compressor(ehttp_gzip_conf_t conf, ehttp_gzip_callbacks_t callbacks, void *arg); void ehttp_gzip_destroy(ehttp_gzip_t *gzip) { ehttp_gzip_cleanup(gzip); @@ -129,16 +128,16 @@ int32_t ehttp_gzip_write(ehttp_gzip_t *gzip, const char *buf, int32_t len) { } if (ret != Z_OK && ret != Z_STREAM_END) return -1; - if (gzip->gzip->avail_out>0) { - if (ret!=Z_STREAM_END) continue; + if (gzip->gzip->avail_out > 0) { + if (ret != Z_STREAM_END) continue; } - int32_t len = (int32_t)(gzip->gzip->next_out - (z_const Bytef*)gzip->chunk); + int32_t len = (int32_t)(gzip->gzip->next_out - (z_const Bytef *)gzip->chunk); gzip->gzip->next_out[0] = '\0'; gzip->callbacks.on_data(gzip, gzip->arg, gzip->chunk, len); - gzip->gzip->next_out = (z_const Bytef*)gzip->chunk; - gzip->gzip->avail_out = gzip->conf.chunk_size; + gzip->gzip->next_out = (z_const Bytef *)gzip->chunk; + gzip->gzip->avail_out = gzip->conf.chunk_size; } return 0; @@ -147,21 +146,20 @@ int32_t ehttp_gzip_write(ehttp_gzip_t *gzip, const char *buf, int32_t len) { int32_t ehttp_gzip_finish(ehttp_gzip_t *gzip) { if (gzip->state != EHTTP_GZIP_READY) return -1; - gzip->gzip->next_in = NULL; - gzip->gzip->avail_in = 0; + gzip->gzip->next_in = NULL; + gzip->gzip->avail_in = 0; int32_t ret; ret = inflate(gzip->gzip, Z_FINISH); if (ret != Z_STREAM_END) return -1; - int32_t len = (int32_t)(gzip->gzip->next_out - (z_const Bytef*)gzip->chunk); + int32_t len = (int32_t)(gzip->gzip->next_out - (z_const Bytef *)gzip->chunk); gzip->gzip->next_out[0] = '\0'; gzip->callbacks.on_data(gzip, gzip->arg, gzip->chunk, len); - gzip->gzip->next_out = NULL; - gzip->gzip->avail_out = 0; + gzip->gzip->next_out = NULL; + gzip->gzip->avail_out = 0; return 0; } - diff --git a/src/plugins/http/src/httpHandle.c b/src/plugins/http/src/httpHandle.c index 7c56507514..ad79e24061 100644 --- a/src/plugins/http/src/httpHandle.c +++ b/src/plugins/http/src/httpHandle.c @@ -48,7 +48,7 @@ bool httpProcessData(HttpContext* pContext) { /* * httpCloseContextByApp has been called when parsing the error */ - //httpCloseContextByApp(pContext); + // httpCloseContextByApp(pContext); } else { httpProcessRequest(pContext); } diff --git a/src/plugins/http/src/httpJson.c b/src/plugins/http/src/httpJson.c index 132553ffe1..b120496898 100644 --- a/src/plugins/http/src/httpJson.c +++ b/src/plugins/http/src/httpJson.c @@ -44,20 +44,21 @@ int32_t httpWriteBufByFd(struct HttpContext* pContext, const char* buf, int32_t int32_t writeLen = 0; do { - if (pContext->fd > 2){ + if (pContext->fd > 2) { len = (int32_t)taosSend(pContext->fd, buf + writeLen, (size_t)(sz - writeLen), MSG_NOSIGNAL); - } - else { + } else { return sz; } if (len < 0) { - httpDebug("context:%p, fd:%d, socket write errno:%d:%s, times:%d", pContext, pContext->fd, errno, strerror(errno), countWait); + httpDebug("context:%p, fd:%d, socket write errno:%d:%s, times:%d", pContext, pContext->fd, errno, strerror(errno), + countWait); if (++countWait > HTTP_WRITE_RETRY_TIMES) break; taosMsleep(HTTP_WRITE_WAIT_TIME_MS); continue; } else if (len == 0) { - httpDebug("context:%p, fd:%d, socket write errno:%d:%s, connect already closed", pContext, pContext->fd, errno, strerror(errno)); + httpDebug("context:%p, fd:%d, socket write errno:%d:%s, connect already closed", pContext, pContext->fd, errno, + strerror(errno)); break; } else { countWait = 0; @@ -80,7 +81,7 @@ int32_t httpWriteBuf(struct HttpContext* pContext, const char* buf, int32_t sz) return writeSz; } -int32_t httpWriteBufNoTrace(struct HttpContext *pContext, const char *buf, int32_t sz) { +int32_t httpWriteBufNoTrace(struct HttpContext* pContext, const char* buf, int32_t sz) { int32_t writeSz = httpWriteBufByFd(pContext, buf, sz); if (writeSz != sz) { httpError("context:%p, fd:%d, dataSize:%d, writeSize:%d, failed to send response", pContext, pContext->fd, sz, @@ -92,8 +93,8 @@ int32_t httpWriteBufNoTrace(struct HttpContext *pContext, const char *buf, int32 int32_t httpWriteJsonBufBody(JsonBuf* buf, bool isTheLast) { int32_t remain = 0; - char sLen[24]; - int32_t srcLen = (int32_t) (buf->lst - buf->buf); + char sLen[24]; + int32_t srcLen = (int32_t)(buf->lst - buf->buf); if (buf->pContext->fd <= 0) { httpTrace("context:%p, fd:%d, write json body error", buf->pContext, buf->pContext->fd); @@ -113,21 +114,21 @@ int32_t httpWriteJsonBufBody(JsonBuf* buf, bool isTheLast) { httpTrace("context:%p, fd:%d, no data need dump", buf->pContext, buf->pContext->fd); return 0; // there is no data to dump. } else { - int32_t len = sprintf(sLen, "%d\r\n", srcLen); - httpTrace("context:%p, fd:%d, write body, chunkSize:%d, response:\n%s", buf->pContext, buf->pContext->fd, - srcLen, buf->buf); + int32_t len = sprintf(sLen, "%x\r\n", srcLen); + httpTrace("context:%p, fd:%d, write body, chunkSize:%d, response:\n%s", buf->pContext, buf->pContext->fd, srcLen, + buf->buf); httpWriteBufNoTrace(buf->pContext, sLen, len); remain = httpWriteBufNoTrace(buf->pContext, buf->buf, srcLen); } } else { - char compressBuf[JSON_BUFFER_SIZE] = {0}; + char compressBuf[JSON_BUFFER_SIZE] = {0}; int32_t compressBufLen = JSON_BUFFER_SIZE; int32_t ret = httpGzipCompress(buf->pContext, buf->buf, srcLen, compressBuf, &compressBufLen, isTheLast); if (ret == 0) { if (compressBufLen > 0) { int32_t len = sprintf(sLen, "%x\r\n", compressBufLen); - httpTrace("context:%p, fd:%d, write body, chunkSize:%d, compressSize:%d, last:%d, response:\n%s", - buf->pContext, buf->pContext->fd, srcLen, compressBufLen, isTheLast, buf->buf); + httpTrace("context:%p, fd:%d, write body, chunkSize:%d, compressSize:%d, last:%d, response:\n%s", buf->pContext, + buf->pContext->fd, srcLen, compressBufLen, isTheLast, buf->buf); httpWriteBufNoTrace(buf->pContext, sLen, len); remain = httpWriteBufNoTrace(buf->pContext, (const char*)compressBuf, compressBufLen); } else { @@ -154,8 +155,8 @@ void httpWriteJsonBufHead(JsonBuf* buf) { buf->pContext->fd = -1; } - char msg[1024] = {0}; - int32_t len = -1; + char msg[1024] = {0}; + int32_t len = -1; if (buf->pContext->parser->acceptEncodingGzip == 0 || !tsHttpEnableCompress) { len = sprintf(msg, httpRespTemplate[HTTP_RESPONSE_CHUNKED_UN_COMPRESS], httpVersionStr[buf->pContext->parser->httpVersion], @@ -256,16 +257,16 @@ void httpJsonInt64(JsonBuf* buf, int64_t num) { } void httpJsonTimestamp(JsonBuf* buf, int64_t t, bool us) { - char ts[35] = {0}; - struct tm *ptm; - int32_t precision = 1000; + char ts[35] = {0}; + struct tm* ptm; + int32_t precision = 1000; if (us) { precision = 1000000; } time_t tt = t / precision; ptm = localtime(&tt); - int32_t length = (int32_t) strftime(ts, 35, "%Y-%m-%d %H:%M:%S", ptm); + int32_t length = (int32_t)strftime(ts, 35, "%Y-%m-%d %H:%M:%S", ptm); if (us) { length += snprintf(ts + length, 8, ".%06" PRId64, t % precision); } else { @@ -276,9 +277,9 @@ void httpJsonTimestamp(JsonBuf* buf, int64_t t, bool us) { } void httpJsonUtcTimestamp(JsonBuf* buf, int64_t t, bool us) { - char ts[40] = {0}; - struct tm *ptm; - int32_t precision = 1000; + char ts[40] = {0}; + struct tm* ptm; + int32_t precision = 1000; if (us) { precision = 1000000; } diff --git a/src/plugins/http/src/httpParser.c b/src/plugins/http/src/httpParser.c index 331ca079fa..5ebe9df96f 100644 --- a/src/plugins/http/src/httpParser.c +++ b/src/plugins/http/src/httpParser.c @@ -130,7 +130,7 @@ static int32_t httpAppendString(HttpString *str, const char *s, int32_t len) { static void httpClearString(HttpString *str) { if (str->str) { str->str[0] = '\0'; - str->pos = 0; + str->pos = 0; } } @@ -237,7 +237,6 @@ static int32_t httpOnParseHeaderField(HttpParser *parser, const char *key, const } httpTrace("context:%p, fd:%d, keepAlive:%d", pContext, pContext->fd, pContext->parser->keepAlive); } - #if 0 else if (0 == strcasecmp(key, "Content-Encoding")) { if (0 == strcmp(val, "gzip")) { @@ -246,7 +245,7 @@ static int32_t httpOnParseHeaderField(HttpParser *parser, const char *key, const } return 0; } - #endif +#endif else if (0 == strcasecmp(key, "Transfer-Encoding") || 0 == strcasecmp(key, "Content-Encoding")) { if (strstr(val, "gzip")) { @@ -349,7 +348,7 @@ static int32_t httpOnBody(HttpParser *parser, const char *chunk, int32_t len) { newSize = MIN(newSize, HTTP_BUFFER_SIZE); buf->str = realloc(buf->str, newSize); buf->size = newSize; - + if (buf->str == NULL) { httpError("context:%p, fd:%d, failed parse body, realloc %d failed", pContext, pContext->fd, buf->size); httpOnError(parser, 0, TSDB_CODE_HTTP_NO_ENOUGH_MEMORY); @@ -410,9 +409,7 @@ static int32_t httpPopStack(HttpParser *parser) { return 0; } -static void httpClearStack(HttpStack *stack) { - stack->pos = 0; -} +static void httpClearStack(HttpStack *stack) { stack->pos = 0; } static int32_t httpCleanupStack(HttpStack *stack) { free(stack->stacks); @@ -451,7 +448,7 @@ void httpInitParser(HttpParser *parser) { free(parser->key); parser->key = NULL; free(parser->val); parser->val = NULL; free(parser->authContent); parser->authContent = NULL; - + httpClearStack(&parser->stacks); httpClearString(&parser->str); httpClearString(&parser->body); @@ -497,7 +494,7 @@ void httpDestroyParser(HttpParser *parser) { free(parser->key); parser->key = NULL; free(parser->val); parser->val = NULL; free(parser->authContent); parser->authContent = NULL; - + httpCleanupStack(&parser->stacks); httpCleanupString(&parser->str); httpCleanupString(&parser->body); @@ -513,25 +510,36 @@ void httpDestroyParser(HttpParser *parser) { free(parser); } -#define is_token(c) (strchr("!#$%&'*+-.^_`|~", c) || isdigit(c) || isalpha(c)) +#define is_token(c) (strchr("!#$%&'*+-.^_`|~", c) || isdigit(c) || isalpha(c)) char *httpDecodeUrl(const char *enc) { - int32_t ok = 1; + int32_t ok = 1; HttpString str = {0}; while (*enc) { char *p = strchr(enc, '%'); if (!p) break; int32_t hex, cnt; - int32_t n = sscanf(p+1, "%2x%n", &hex, &cnt); - if (n!=1 && cnt !=2) { ok = 0; break; } - if (httpAppendString(&str, enc, (int32_t)(p-enc))) { ok = 0; break; } + int32_t n = sscanf(p + 1, "%2x%n", &hex, &cnt); + if (n != 1 && cnt != 2) { + ok = 0; + break; + } + if (httpAppendString(&str, enc, (int32_t)(p - enc))) { + ok = 0; + break; + } char c = (char)hex; - if (httpAppendString(&str, &c, 1)) { ok = 0; break; } - enc = p+3; + if (httpAppendString(&str, &c, 1)) { + ok = 0; + break; + } + enc = p + 3; } char *dec = NULL; if (ok && *enc) { - if (httpAppendString(&str, enc, (int32_t)strlen(enc))) { ok = 0; } + if (httpAppendString(&str, enc, (int32_t)strlen(enc))) { + ok = 0; + } } if (ok) { dec = str.str; @@ -542,13 +550,13 @@ char *httpDecodeUrl(const char *enc) { } static void httpOnData(ehttp_gzip_t *gzip, void *arg, const char *buf, int32_t len) { - HttpParser *parser = (HttpParser*)arg; + HttpParser *parser = (HttpParser *)arg; httpOnBody(parser, buf, len); } static int32_t httpParserOnBegin(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { if (c == 'G' || c == 'P' || c == 'H' || c == 'D' || c == 'C' || c == 'O' || c == 'T') { if (httpAppendString(&parser->str, &c, 1)) { @@ -570,7 +578,7 @@ static int32_t httpParserOnBegin(HttpParser *parser, HTTP_PARSER_STATE state, co static int32_t httpParserOnRquestOrResponse(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { if (parser->str.pos == 1) { if (c == 'T' && parser->str.str[0] == 'H') { @@ -608,7 +616,7 @@ static int32_t httpParserOnRquestOrResponse(HttpParser *parser, HTTP_PARSER_STAT static int32_t httpParserOnMethod(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { if (isalnum(c) || strchr("!#$%&'*+-.^_`|~", c)) { if (httpAppendString(&parser->str, &c, 1)) { @@ -637,7 +645,7 @@ static int32_t httpParserOnMethod(HttpParser *parser, HTTP_PARSER_STATE state, c static int32_t httpParserOnTarget(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { if (!isspace(c) && c != '\r' && c != '\n') { if (httpAppendString(&parser->str, &c, 1)) { @@ -664,7 +672,7 @@ static int32_t httpParserOnTarget(HttpParser *parser, HTTP_PARSER_STATE state, c static int32_t httpParserOnVersion(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { const char *prefix = "HTTP/1."; int32_t len = (int32_t)strlen(prefix); @@ -727,7 +735,7 @@ static int32_t httpParserOnVersion(HttpParser *parser, HTTP_PARSER_STATE state, static int32_t httpParserOnSp(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { if (c == ' ') { httpPopStack(parser); @@ -742,7 +750,7 @@ static int32_t httpParserOnSp(HttpParser *parser, HTTP_PARSER_STATE state, const static int32_t httpParserOnStatusCode(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { if (isdigit(c)) { if (httpAppendString(&parser->str, &c, 1)) { @@ -767,7 +775,7 @@ static int32_t httpParserOnStatusCode(HttpParser *parser, HTTP_PARSER_STATE stat static int32_t httpParserOnReasonPhrase(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { if (c == '\r') { parser->reasonPhrase = strdup(parser->str.str); @@ -808,10 +816,10 @@ static int32_t httpParserPostProcess(HttpParser *parser) { static int32_t httpParserOnCrlf(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { - const char *s = "\r\n"; - int32_t len = (int32_t)strlen(s); + const char *s = "\r\n"; + int32_t len = (int32_t)strlen(s); if (s[parser->str.pos] != c) { httpError("context:%p, fd:%d, parser state:%d, unexpected char:[%c]%02x", pContext, pContext->fd, state, c, c); ok = -1; @@ -838,7 +846,7 @@ static int32_t httpParserOnCrlf(HttpParser *parser, HTTP_PARSER_STATE state, con static int32_t httpParserOnHeader(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { if (c == '\r') { httpPopStack(parser); @@ -876,7 +884,7 @@ static int32_t httpParserOnHeader(HttpParser *parser, HTTP_PARSER_STATE state, c static int32_t httpParserOnHeaderKey(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { if (isalnum(c) || strchr("!#$%&'*+-.^_`|~", c)) { if (httpAppendString(&parser->str, &c, 1)) { @@ -888,7 +896,7 @@ static int32_t httpParserOnHeaderKey(HttpParser *parser, HTTP_PARSER_STATE state break; } if (c == ':') { - parser->key = strdup(parser->str.str); + parser->key = strdup(parser->str.str); if (!parser->key) { httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); ok = -1; @@ -908,7 +916,7 @@ static int32_t httpParserOnHeaderKey(HttpParser *parser, HTTP_PARSER_STATE state static int32_t httpParserOnHeaderVal(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { if (c != '\r' && c != '\n' && (!isspace(c) || parser->str.pos > 0)) { if (httpAppendString(&parser->str, &c, 1)) { @@ -935,10 +943,10 @@ static int32_t httpParserOnHeaderVal(HttpParser *parser, HTTP_PARSER_STATE state static int32_t httpParserOnChunkSize(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; - int32_t bytes; - int32_t len; - int32_t n; + int32_t ok = 0; + int32_t bytes; + int32_t len; + int32_t n; do { if (isxdigit(c)) { if (httpAppendString(&parser->str, &c, 1)) { @@ -985,7 +993,7 @@ static int32_t httpParserOnChunkSize(HttpParser *parser, HTTP_PARSER_STATE state static int32_t httpParserOnChunk(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { if (httpAppendString(&parser->str, &c, 1)) { httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); @@ -1019,7 +1027,7 @@ static int32_t httpParserOnChunk(HttpParser *parser, HTTP_PARSER_STATE state, co static int32_t httpParserOnEnd(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { ok = -1; httpError("context:%p, fd:%d, parser state:%d, unexpected char:[%c]%02x", pContext, pContext->fd, state, c, c); @@ -1029,8 +1037,8 @@ static int32_t httpParserOnEnd(HttpParser *parser, HTTP_PARSER_STATE state, cons } static int32_t httpParseChar(HttpParser *parser, const char c, int32_t *again) { - HttpContext *pContext = parser->pContext; - int32_t ok = 0; + HttpContext * pContext = parser->pContext; + int32_t ok = 0; HTTP_PARSER_STATE state = httpTopStack(parser); do { if (state == HTTP_PARSER_BEGIN) { @@ -1119,9 +1127,9 @@ static int32_t httpParseChar(HttpParser *parser, const char c, int32_t *again) { int32_t httpParseBuf(HttpParser *parser, const char *buf, int32_t len) { HttpContext *pContext = parser->pContext; - const char *p = buf; - int32_t ret = 0; - int32_t i = 0; + const char * p = buf; + int32_t ret = 0; + int32_t i = 0; while (i < len) { int32_t again = 0; diff --git a/src/plugins/http/src/httpQueue.c b/src/plugins/http/src/httpQueue.c index 7ae54717f3..aebba97fb8 100644 --- a/src/plugins/http/src/httpQueue.c +++ b/src/plugins/http/src/httpQueue.c @@ -38,16 +38,16 @@ typedef struct { } SHttpWorkerPool; typedef struct { - void * param; - void * result; - int32_t code; - int32_t rows; + void * param; + void * result; + int32_t code; + int32_t rows; FHttpResultFp fp; } SHttpResult; static SHttpWorkerPool tsHttpPool; -static taos_qset tsHttpQset; -static taos_queue tsHttpQueue; +static taos_qset tsHttpQset; +static taos_queue tsHttpQueue; void httpDispatchToResultQueue(void *param, TAOS_RES *result, int32_t code, int32_t rows, FHttpResultFp fp) { if (tsHttpQueue != NULL) { @@ -105,7 +105,7 @@ static bool httpAllocateResultQueue() { httpDebug("http result worker:%d is launched, total:%d", pWorker->workerId, tsHttpPool.num); } - httpInfo("http result queue is opened"); + httpInfo("http result queue is opened"); return true; } diff --git a/src/plugins/http/src/httpResp.c b/src/plugins/http/src/httpResp.c index 72604e79b7..37eef2bfad 100644 --- a/src/plugins/http/src/httpResp.c +++ b/src/plugins/http/src/httpResp.c @@ -160,8 +160,9 @@ void httpSendTaosdInvalidSqlErrorResp(HttpContext *pContext, char *errMsg) { if (temp[i] == '\"') { temp[i] = '\''; } else if (temp[i] == '\n') { - temp[i] = ' '; - } else {} + temp[i] = ' '; + } else { + } } httpSendErrorRespImp(pContext, httpCode, "Bad Request", TSDB_CODE_TSC_INVALID_SQL & 0XFFFF, temp); diff --git a/src/plugins/http/src/httpRestHandle.c b/src/plugins/http/src/httpRestHandle.c index 112137b3df..8999fb879e 100644 --- a/src/plugins/http/src/httpRestHandle.c +++ b/src/plugins/http/src/httpRestHandle.c @@ -95,7 +95,6 @@ bool restProcessSqlRequest(HttpContext* pContext, int32_t timestampFmt) { return false; } - /* * for async test * diff --git a/src/plugins/http/src/httpRestJson.c b/src/plugins/http/src/httpRestJson.c index a5b156bffc..baa61117be 100644 --- a/src/plugins/http/src/httpRestJson.c +++ b/src/plugins/http/src/httpRestJson.c @@ -83,7 +83,8 @@ void restStartSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result) httpJsonToken(jsonBuf, JsonArrStt); } -bool restBuildSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows, int32_t timestampFormat) { +bool restBuildSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows, + int32_t timestampFormat) { JsonBuf *jsonBuf = httpMallocJsonBuf(pContext); if (jsonBuf == NULL) return false; @@ -95,7 +96,7 @@ bool restBuildSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, if (row == NULL) { continue; } - int32_t* length = taos_fetch_lengths(result); + int32_t *length = taos_fetch_lengths(result); // data row array begin httpJsonItemToken(jsonBuf); @@ -131,15 +132,17 @@ bool restBuildSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, break; case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: - httpJsonStringForTransMean(jsonBuf, (char*)row[i], length[i]); + httpJsonStringForTransMean(jsonBuf, (char *)row[i], length[i]); break; case TSDB_DATA_TYPE_TIMESTAMP: if (timestampFormat == REST_TIMESTAMP_FMT_LOCAL_STRING) { - httpJsonTimestamp(jsonBuf, *((int64_t *)row[i]), taos_result_precision(result) == TSDB_TIME_PRECISION_MICRO); + httpJsonTimestamp(jsonBuf, *((int64_t *)row[i]), + taos_result_precision(result) == TSDB_TIME_PRECISION_MICRO); } else if (timestampFormat == REST_TIMESTAMP_FMT_TIMESTAMP) { httpJsonInt64(jsonBuf, *((int64_t *)row[i])); } else { - httpJsonUtcTimestamp(jsonBuf, *((int64_t *)row[i]), taos_result_precision(result) == TSDB_TIME_PRECISION_MICRO); + httpJsonUtcTimestamp(jsonBuf, *((int64_t *)row[i]), + taos_result_precision(result) == TSDB_TIME_PRECISION_MICRO); } break; default: @@ -148,8 +151,8 @@ bool restBuildSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, } // data row array end - httpJsonToken(jsonBuf, JsonArrEnd); - cmd->numOfRows ++; + httpJsonToken(jsonBuf, JsonArrEnd); + cmd->numOfRows++; if (pContext->fd <= 0) { httpError("context:%p, fd:%d, user:%s, conn closed, abort retrieve", pContext, pContext->fd, pContext->user); @@ -168,15 +171,15 @@ bool restBuildSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, } bool restBuildSqlTimestampJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows) { - return restBuildSqlJson(pContext,cmd, result, numOfRows, REST_TIMESTAMP_FMT_TIMESTAMP); + return restBuildSqlJson(pContext, cmd, result, numOfRows, REST_TIMESTAMP_FMT_TIMESTAMP); } bool restBuildSqlLocalTimeStringJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows) { - return restBuildSqlJson(pContext,cmd, result, numOfRows, REST_TIMESTAMP_FMT_LOCAL_STRING); + return restBuildSqlJson(pContext, cmd, result, numOfRows, REST_TIMESTAMP_FMT_LOCAL_STRING); } bool restBuildSqlUtcTimeStringJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows) { - return restBuildSqlJson(pContext,cmd, result, numOfRows, REST_TIMESTAMP_FMT_UTC_STRING); + return restBuildSqlJson(pContext, cmd, result, numOfRows, REST_TIMESTAMP_FMT_UTC_STRING); } void restStopSqlJson(HttpContext *pContext, HttpSqlCmd *cmd) { diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index 505b658743..c4a03d44ea 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -27,19 +27,21 @@ static bool httpReadData(HttpContext *pContext); -static void httpStopThread(HttpThread* pThread) { +static void httpStopThread(HttpThread *pThread) { pThread->stop = true; // signal the thread to stop, try graceful method first, // and use pthread_cancel when failed - struct epoll_event event = { .events = EPOLLIN }; - eventfd_t fd = eventfd(1, 0); + struct epoll_event event = {.events = EPOLLIN}; + eventfd_t fd = eventfd(1, 0); if (fd == -1) { - httpError("%s, failed to create eventfd, will call pthread_cancel instead, which may result in data corruption: %s", pThread->label, strerror(errno)); + httpError("%s, failed to create eventfd, will call pthread_cancel instead, which may result in data corruption: %s", + pThread->label, strerror(errno)); pThread->stop = true; pthread_cancel(pThread->thread); } else if (epoll_ctl(pThread->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) { - httpError("%s, failed to call epoll_ctl, will call pthread_cancel instead, which may result in data corruption: %s", pThread->label, strerror(errno)); + httpError("%s, failed to call epoll_ctl, will call pthread_cancel instead, which may result in data corruption: %s", + pThread->label, strerror(errno)); pthread_cancel(pThread->thread); } @@ -61,7 +63,7 @@ void httpCleanUpConnect() { } for (int32_t i = 0; i < pServer->numOfThreads; ++i) { - HttpThread* pThread = pServer->pThreads + i; + HttpThread *pThread = pServer->pThreads + i; if (pThread != NULL) { httpStopThread(pThread); } @@ -71,8 +73,8 @@ void httpCleanUpConnect() { } static void httpProcessHttpData(void *param) { - HttpServer *pServer = &tsHttpServer; - HttpThread *pThread = (HttpThread *)param; + HttpServer * pServer = &tsHttpServer; + HttpThread * pThread = (HttpThread *)param; HttpContext *pContext; int32_t fdNum; @@ -92,8 +94,8 @@ static void httpProcessHttpData(void *param) { pContext = httpGetContext(events[i].data.ptr); if (pContext == NULL) { httpError("context:%p, is already released, close connect", events[i].data.ptr); - //epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, events[i].data.fd, NULL); - //taosClose(events[i].data.fd); + // epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, events[i].data.fd, NULL); + // taosClose(events[i].data.fd); continue; } @@ -200,7 +202,7 @@ static void *httpAcceptHttpConnection(void *arg) { taosCloseSocket(connFd); continue; } -#endif +#endif taosKeepTcpAlive(connFd); taosSetNonblocking(connFd, 1); @@ -210,15 +212,15 @@ static void *httpAcceptHttpConnection(void *arg) { pContext = httpCreateContext(connFd); if (pContext == NULL) { - httpError("fd:%d, ip:%s:%u, no enough resource to allocate http context", connFd, taosInetNtoa(clientAddr.sin_addr), - htons(clientAddr.sin_port)); + httpError("fd:%d, ip:%s:%u, no enough resource to allocate http context", connFd, + taosInetNtoa(clientAddr.sin_addr), htons(clientAddr.sin_port)); taosCloseSocket(connFd); continue; } pContext->pThread = pThread; sprintf(pContext->ipstr, "%s:%u", taosInetNtoa(clientAddr.sin_addr), htons(clientAddr.sin_port)); - + struct epoll_event event; event.events = EPOLLIN | EPOLLPRI | EPOLLWAKEUP | EPOLLERR | EPOLLHUP | EPOLLRDHUP; event.data.ptr = pContext; @@ -276,7 +278,7 @@ bool httpInitConnect() { if (pthread_create(&(pThread->thread), &thattr, (void *)httpProcessHttpData, (void *)(pThread)) != 0) { httpError("http thread:%s, failed to create HTTP process data thread, reason:%s", pThread->label, strerror(errno)); - pthread_mutex_destroy(&(pThread->threadMutex)); + pthread_mutex_destroy(&(pThread->threadMutex)); return false; } pthread_attr_destroy(&thattr); @@ -307,7 +309,7 @@ static bool httpReadData(HttpContext *pContext) { } if (pParser->parsed) { - httpDebug("context:%p, fd:%d, not in ready state, parsed:%d", pContext, pContext->fd, pParser->parsed); + httpDebug("context:%p, fd:%d, not in ready state, parsed:%d", pContext, pContext->fd, pParser->parsed); return false; } diff --git a/src/plugins/http/src/httpSession.c b/src/plugins/http/src/httpSession.c index 35ce0160b2..2e1ee7df2f 100644 --- a/src/plugins/http/src/httpSession.c +++ b/src/plugins/http/src/httpSession.c @@ -35,7 +35,8 @@ void httpCreateSession(HttpContext *pContext, void *taos) { session.refCount = 1; int32_t len = snprintf(session.id, HTTP_SESSION_ID_LEN, "%s.%s", pContext->user, pContext->pass); - pContext->session = taosCachePut(server->sessionCache, session.id, len, &session, sizeof(HttpSession), tsHttpSessionExpire * 1000); + pContext->session = + taosCachePut(server->sessionCache, session.id, len, &session, sizeof(HttpSession), tsHttpSessionExpire * 1000); // void *temp = pContext->session; // taosCacheRelease(server->sessionCache, (void **)&temp, false); @@ -56,7 +57,7 @@ static void httpFetchSessionImp(HttpContext *pContext) { HttpServer *server = &tsHttpServer; pthread_mutex_lock(&server->serverMutex); - char sessionId[HTTP_SESSION_ID_LEN]; + char sessionId[HTTP_SESSION_ID_LEN]; int32_t len = snprintf(sessionId, HTTP_SESSION_ID_LEN, "%s.%s", pContext->user, pContext->pass); pContext->session = taosCacheAcquireByKey(server->sessionCache, sessionId, len); diff --git a/src/plugins/http/src/httpSql.c b/src/plugins/http/src/httpSql.c index cc8e9e86e3..4e9b54b7bd 100644 --- a/src/plugins/http/src/httpSql.c +++ b/src/plugins/http/src/httpSql.c @@ -64,7 +64,7 @@ void httpProcessMultiSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int32 } taos_free_result(result); - + if (singleCmd->cmdReturnType == HTTP_CMD_RETURN_TYPE_WITH_RETURN && encode->stopJsonFp) { (encode->stopJsonFp)(pContext, singleCmd); } @@ -82,7 +82,7 @@ void httpProcessMultiSqlCallBackImp(void *param, TAOS_RES *result, int32_t code, HttpContext *pContext = (HttpContext *)param; if (pContext == NULL) return; - HttpSqlCmds *multiCmds = pContext->multiCmds; + HttpSqlCmds * multiCmds = pContext->multiCmds; HttpEncodeMethod *encode = pContext->encodeMethod; HttpSqlCmd *singleCmd = multiCmds->cmds + multiCmds->pos; @@ -269,8 +269,8 @@ void httpProcessSingleSqlCallBackImp(void *param, TAOS_RES *result, int32_t code pContext->user, tstrerror(code), pObj, taos_errstr(pObj)); httpSendTaosdInvalidSqlErrorResp(pContext, taos_errstr(pObj)); } else { - httpError("context:%p, fd:%d, user:%s, query error, code:%s, sqlObj:%p", pContext, pContext->fd, - pContext->user, tstrerror(code), pObj); + httpError("context:%p, fd:%d, user:%s, query error, code:%s, sqlObj:%p", pContext, pContext->fd, pContext->user, + tstrerror(code), pObj); httpSendErrorResp(pContext, code); } taos_free_result(result); @@ -381,7 +381,7 @@ void httpExecCmd(HttpContext *pContext) { void httpProcessRequestCb(void *param, TAOS_RES *result, int32_t code) { HttpContext *pContext = param; taos_free_result(result); - + if (pContext == NULL) return; if (code < 0) { diff --git a/src/plugins/http/src/httpSystem.c b/src/plugins/http/src/httpSystem.c index df39c080a5..4721451529 100644 --- a/src/plugins/http/src/httpSystem.c +++ b/src/plugins/http/src/httpSystem.c @@ -110,10 +110,8 @@ void httpCleanUpSystem() { pthread_mutex_destroy(&tsHttpServer.serverMutex); tfree(tsHttpServer.pThreads); tsHttpServer.pThreads = NULL; - + tsHttpServer.status = HTTP_SERVER_CLOSED; } -int32_t httpGetReqCount() { - return atomic_exchange_32(&tsHttpServer.requestNum, 0); -} +int32_t httpGetReqCount() { return atomic_exchange_32(&tsHttpServer.requestNum, 0); } diff --git a/src/plugins/http/src/httpTgHandle.c b/src/plugins/http/src/httpTgHandle.c index 7a26cae97c..c1d006ff5a 100644 --- a/src/plugins/http/src/httpTgHandle.c +++ b/src/plugins/http/src/httpTgHandle.c @@ -137,8 +137,8 @@ void tgInitSchemas(int32_t size) { } void tgParseSchemaMetric(cJSON *metric) { - STgSchema schema = {0}; - bool parsedOk = true; + STgSchema schema = {0}; + bool parsedOk = true; // name cJSON *name = cJSON_GetObjectItem(metric, "name"); @@ -186,7 +186,7 @@ void tgParseSchemaMetric(cJSON *metric) { schema.tbName = calloc(tbnameLen + 1, 1); strcpy(schema.tbName, tbname->valuestring); - // fields + // fields cJSON *fields = cJSON_GetObjectItem(metric, "fields"); if (fields == NULL) { goto ParseEnd; @@ -227,14 +227,14 @@ ParseEnd: } } -int32_t tgParseSchema(const char *content, char*fileName) { +int32_t tgParseSchema(const char *content, char *fileName) { cJSON *root = cJSON_Parse(content); if (root == NULL) { httpError("failed to parse telegraf schema file:%s, invalid json format, content:%s", fileName, content); return -1; } int32_t size = 0; - cJSON *metrics = cJSON_GetObjectItem(root, "metrics"); + cJSON * metrics = cJSON_GetObjectItem(root, "metrics"); if (metrics != NULL) { size = cJSON_GetArraySize(metrics); if (size <= 0) { @@ -277,7 +277,7 @@ int32_t tgReadSchema(char *fileName) { rewind(fp); char * content = (char *)calloc(contentSize + 1, 1); int32_t result = (int32_t)fread(content, 1, contentSize, fp); - + if (result != contentSize) { httpError("failed to read telegraf schema file:%s", fileName); fclose(fp); @@ -296,7 +296,7 @@ int32_t tgReadSchema(char *fileName) { } void tgInitHandle(HttpServer *pServer) { - char fileName[TSDB_FILENAME_LEN*2] = {0}; + char fileName[TSDB_FILENAME_LEN * 2] = {0}; sprintf(fileName, "%s/taos.telegraf.cfg", configDir); if (tgReadSchema(fileName) <= 0) { tgFreeSchemas(); @@ -308,9 +308,7 @@ void tgInitHandle(HttpServer *pServer) { httpAddMethod(pServer, &tgDecodeMethod); } -void tgCleanupHandle() { - tgFreeSchemas(); -} +void tgCleanupHandle() { tgFreeSchemas(); } bool tgGetUserFromUrl(HttpContext *pContext) { HttpParser *pParser = pContext->parser; @@ -357,7 +355,7 @@ char *tgGetStableName(char *stname, cJSON *fields, int32_t fieldsSize) { bool schemaMatched = true; for (int32_t f = 0; f < schema->fieldNum; ++f) { char *fieldName = schema->fields[f]; - bool fieldMatched = false; + bool fieldMatched = false; for (int32_t i = 0; i < fieldsSize; i++) { cJSON *field = cJSON_GetArrayItem(fields, i); @@ -469,9 +467,9 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) { } /* - * tag size may be larget than TSDB_COL_NAME_LEN - * we keep the first TSDB_COL_NAME_LEN bytes - */ + * tag size may be larget than TSDB_COL_NAME_LEN + * we keep the first TSDB_COL_NAME_LEN bytes + */ if (0) { if (strlen(tag->string) >= TSDB_COL_NAME_LEN) { httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TAG_NAME_SIZE); @@ -540,9 +538,9 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) { return false; } /* - * tag size may be larget than TSDB_COL_NAME_LEN - * we keep the first TSDB_COL_NAME_LEN bytes - */ + * tag size may be larget than TSDB_COL_NAME_LEN + * we keep the first TSDB_COL_NAME_LEN bytes + */ if (0) { if (strlen(field->string) >= TSDB_COL_NAME_LEN) { httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_FIELD_NAME_SIZE); @@ -578,8 +576,8 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) { table_cmd->cmdType = HTTP_CMD_TYPE_INSERT; // order by tag name - cJSON *orderedTags[TG_MAX_SORT_TAG_SIZE] = {0}; - int32_t orderTagsLen = 0; + cJSON * orderedTags[TG_MAX_SORT_TAG_SIZE] = {0}; + int32_t orderTagsLen = 0; for (int32_t i = 0; i < tagsSize; ++i) { cJSON *tag = cJSON_GetArrayItem(tags, i); orderedTags[orderTagsLen++] = tag; @@ -603,7 +601,8 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) { if (tsTelegrafUseFieldNum == 0) { table_cmd->stable = stable_cmd->stable = httpAddToSqlCmdBuffer(pContext, "%s", stname); } else { - table_cmd->stable = stable_cmd->stable = httpAddToSqlCmdBuffer(pContext, "%s_%d_%d", stname, fieldsSize, orderTagsLen); + table_cmd->stable = stable_cmd->stable = + httpAddToSqlCmdBuffer(pContext, "%s_%d_%d", stname, fieldsSize, orderTagsLen); } table_cmd->stable = stable_cmd->stable = httpShrinkTableName(pContext, table_cmd->stable, httpGetCmdsString(pContext, table_cmd->stable)); @@ -627,9 +626,11 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) { // table name if (tsTelegrafUseFieldNum == 0) { - table_cmd->table = stable_cmd->table = httpAddToSqlCmdBufferNoTerminal(pContext, "%s_%s", stname, host->valuestring); + table_cmd->table = stable_cmd->table = + httpAddToSqlCmdBufferNoTerminal(pContext, "%s_%s", stname, host->valuestring); } else { - table_cmd->table = stable_cmd->table = httpAddToSqlCmdBufferNoTerminal(pContext, "%s_%d_%d_%s", stname, fieldsSize, orderTagsLen, host->valuestring); + table_cmd->table = stable_cmd->table = + httpAddToSqlCmdBufferNoTerminal(pContext, "%s_%d_%d_%s", stname, fieldsSize, orderTagsLen, host->valuestring); } for (int32_t i = 0; i < orderTagsLen; ++i) { cJSON *tag = orderedTags[i]; diff --git a/src/plugins/http/src/httpUtil.c b/src/plugins/http/src/httpUtil.c index 51333d2118..17d61e9e3d 100644 --- a/src/plugins/http/src/httpUtil.c +++ b/src/plugins/http/src/httpUtil.c @@ -160,8 +160,7 @@ bool httpMallocMultiCmds(HttpContext *pContext, int32_t cmdSize, int32_t bufferS free(multiCmds->cmds); multiCmds->cmds = (HttpSqlCmd *)malloc((size_t)cmdSize * sizeof(HttpSqlCmd)); if (multiCmds->cmds == NULL) { - httpError("context:%p, fd:%d, user:%s, malloc cmds:%d error", pContext, pContext->fd, - pContext->user, cmdSize); + httpError("context:%p, fd:%d, user:%s, malloc cmds:%d error", pContext, pContext->fd, pContext->user, cmdSize); return false; } multiCmds->maxSize = (int16_t)cmdSize; @@ -350,38 +349,40 @@ char *httpGetCmdsString(HttpContext *pContext, int32_t pos) { } int32_t httpGzipDeCompress(char *srcData, int32_t nSrcData, char *destData, int32_t *nDestData) { - int32_t err = 0; + int32_t err = 0; z_stream gzipStream = {0}; static char dummyHead[2] = { - 0x8 + 0x7 * 0x10, - (((0x8 + 0x7 * 0x10) * 0x100 + 30) / 31 * 31) & 0xFF, + 0x8 + 0x7 * 0x10, + (((0x8 + 0x7 * 0x10) * 0x100 + 30) / 31 * 31) & 0xFF, }; - gzipStream.zalloc = (alloc_func) 0; - gzipStream.zfree = (free_func) 0; - gzipStream.opaque = (voidpf) 0; - gzipStream.next_in = (Bytef *) srcData; + gzipStream.zalloc = (alloc_func)0; + gzipStream.zfree = (free_func)0; + gzipStream.opaque = (voidpf)0; + gzipStream.next_in = (Bytef *)srcData; gzipStream.avail_in = 0; - gzipStream.next_out = (Bytef *) destData; + gzipStream.next_out = (Bytef *)destData; if (inflateInit2(&gzipStream, 47) != Z_OK) { return -1; } while (gzipStream.total_out < *nDestData && gzipStream.total_in < nSrcData) { - gzipStream.avail_in = gzipStream.avail_out = nSrcData; //1 + gzipStream.avail_in = gzipStream.avail_out = nSrcData; // 1 if ((err = inflate(&gzipStream, Z_NO_FLUSH)) == Z_STREAM_END) { break; } if (err != Z_OK) { if (err == Z_DATA_ERROR) { - gzipStream.next_in = (Bytef *) dummyHead; + gzipStream.next_in = (Bytef *)dummyHead; gzipStream.avail_in = sizeof(dummyHead); if ((err = inflate(&gzipStream, Z_NO_FLUSH)) != Z_OK) { return -2; } - } else return -3; + } else { + return -3; + } } } @@ -394,23 +395,25 @@ int32_t httpGzipDeCompress(char *srcData, int32_t nSrcData, char *destData, int3 } int32_t httpGzipCompressInit(HttpContext *pContext) { - pContext->gzipStream.zalloc = (alloc_func) 0; - pContext->gzipStream.zfree = (free_func) 0; - pContext->gzipStream.opaque = (voidpf) 0; - if (deflateInit2(&pContext->gzipStream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, MAX_WBITS + 16, 8, Z_DEFAULT_STRATEGY) != Z_OK) { + pContext->gzipStream.zalloc = (alloc_func)0; + pContext->gzipStream.zfree = (free_func)0; + pContext->gzipStream.opaque = (voidpf)0; + if (deflateInit2(&pContext->gzipStream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, MAX_WBITS + 16, 8, Z_DEFAULT_STRATEGY) != + Z_OK) { return -1; } return 0; } -int32_t httpGzipCompress(HttpContext *pContext, char *srcData, int32_t nSrcData, char *destData, int32_t *nDestData, bool isTheLast) { +int32_t httpGzipCompress(HttpContext *pContext, char *srcData, int32_t nSrcData, char *destData, int32_t *nDestData, + bool isTheLast) { int32_t err = 0; - int32_t lastTotalLen = (int32_t) (pContext->gzipStream.total_out); - pContext->gzipStream.next_in = (Bytef *) srcData; - pContext->gzipStream.avail_in = (uLong) nSrcData; - pContext->gzipStream.next_out = (Bytef *) destData; - pContext->gzipStream.avail_out = (uLong) (*nDestData); + int32_t lastTotalLen = (int32_t)(pContext->gzipStream.total_out); + pContext->gzipStream.next_in = (Bytef *)srcData; + pContext->gzipStream.avail_in = (uLong)nSrcData; + pContext->gzipStream.next_out = (Bytef *)destData; + pContext->gzipStream.avail_out = (uLong)(*nDestData); while (pContext->gzipStream.avail_in != 0) { if (deflate(&pContext->gzipStream, Z_FULL_FLUSH) != Z_OK) { @@ -442,12 +445,12 @@ int32_t httpGzipCompress(HttpContext *pContext, char *srcData, int32_t nSrcData, } } - *nDestData = (int32_t) (pContext->gzipStream.total_out) - lastTotalLen; + *nDestData = (int32_t)(pContext->gzipStream.total_out) - lastTotalLen; return 0; } -bool httpUrlMatch(HttpContext* pContext, int32_t pos, char* cmp) { - HttpParser* pParser = pContext->parser; +bool httpUrlMatch(HttpContext *pContext, int32_t pos, char *cmp) { + HttpParser *pParser = pContext->parser; if (pos < 0 || pos >= HTTP_MAX_URL) { return false; diff --git a/tests/script/general/http/restful.sim b/tests/script/general/http/restful.sim index a06e899d93..fdde975238 100644 --- a/tests/script/general/http/restful.sim +++ b/tests/script/general/http/restful.sim @@ -1,12 +1,10 @@ system sh/stop_dnodes.sh -sleep 2000 system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/cfg.sh -n dnode1 -c http -v 1 system sh/cfg.sh -n dnode1 -c httpEnableRecordSql -v 1 system sh/exec.sh -n dnode1 -s start -sleep 2000 sql connect print ============================ dnode1 start From 0694e69dda7a590c3ad804ba8d66a67c10ca4b51 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Fri, 22 Jan 2021 15:16:33 +0800 Subject: [PATCH 33/62] test for mail --- tests/pytest/insert/unsigenedTinyint.py | 106 ++++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 tests/pytest/insert/unsigenedTinyint.py diff --git a/tests/pytest/insert/unsigenedTinyint.py b/tests/pytest/insert/unsigenedTinyint.py new file mode 100644 index 0000000000..1c0634b69a --- /dev/null +++ b/tests/pytest/insert/unsigenedTinyint.py @@ -0,0 +1,106 @@ +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + + tdLog.info('=============== step1') + tdLog.info('create table tb (ts timestamp, speed tinyint unsigned)') + tdSql.execute('create table tb (ts timestamp, speed tinyint unsigned)') + tdLog.info("insert into tb values (now, NULL)") + tdSql.execute("insert into tb values (now, NULL)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + tdLog.info('tdSql.checkData(0, 1, null)') + tdSql.checkData(0, 1, None) + tdLog.info('=============== step2') + tdLog.info("insert into tb values (now+1m, -1) -x step2") + tdSql.error("insert into tb values (now+1m, -1) ") + tdLog.info("insert into tb values (now+1m, NULL)") + tdSql.execute("insert into tb values (now+1m, NULL)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(2)') + tdSql.checkRows(2) + tdLog.info('tdSql.checkData(0, 1, null)') + tdSql.checkData(0, 1, None) + tdLog.info('=============== step3') + tdLog.info("insert into tb values (now+2m, 254)") + tdSql.execute("insert into tb values (now+2m, 254)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(3)') + tdSql.checkRows(3) + tdLog.info('tdSql.checkData(0, 1, 254)') + tdSql.checkData(0, 1, 254) + tdLog.info('=============== step4') + tdLog.info("insert into tb values (now+3m, 255) -x step4") + tdSql.error("insert into tb values (now+3m, 255)") + tdLog.info("insert into tb values (now+3m, NULL)") + tdSql.execute("insert into tb values (now+3m, NULL)") + tdLog.info('select * from tb') + tdSql.query('select * from tb') + tdLog.info('tdSql.checkRow(4)') + tdSql.checkRows(4) + tdLog.info('tdSql.checkData(0, 1, null)') + tdSql.checkData(0, 1, None) + tdLog.info('=============== step5') + tdLog.info("insert into tb values (now+4m, a2)") + tdSql.error("insert into tb values (now+4m, a2)") + tdLog.info("insert into tb values (now+4m, 0)") + tdSql.execute("insert into tb values (now+4m, 0)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(5)') + tdSql.checkRows(5) + tdLog.info('tdSql.checkData(0, 1, 0)') + tdSql.checkData(0, 1, 0) + tdLog.info('=============== step6') + tdLog.info("insert into tb values (now+5m, 2a)") + tdSql.error("insert into tb values (now+5m, 2a)") + tdLog.info("insert into tb values (now+5m, 2)") + tdSql.execute("insert into tb values (now+5m, 2)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(6)') + tdSql.checkRows(6) + tdLog.info('tdSql.checkData(0, 1, 2)') + tdSql.checkData(0, 1, 2) + tdLog.info('=============== step7') + tdLog.info("insert into tb values (now+6m, 2a'1)") + tdSql.error("insert into tb values (now+6m, 2a'1)") + tdLog.info("insert into tb values (now+6m, 2)") + tdSql.execute("insert into tb values (now+6m, 2)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(7)') + tdSql.checkRows(7) + tdLog.info('tdSql.checkData(0, 1, 2)') + tdSql.checkData(0, 1, 2) + tdLog.info('drop database db') + tdSql.execute('drop database db') + tdLog.info('show databases') + tdSql.query('show databases') + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) +# convert end + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From b304cd96c0c8d03a43940353e2c6ee9aa86ad33b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 22 Jan 2021 16:11:33 +0800 Subject: [PATCH 34/62] TD-1207 --- src/os/inc/osDef.h | 2 +- src/wal/src/walMgmt.c | 4 ++-- tests/tsim/src/simMain.c | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/os/inc/osDef.h b/src/os/inc/osDef.h index e5b5e0f895..a21b721565 100644 --- a/src/os/inc/osDef.h +++ b/src/os/inc/osDef.h @@ -100,7 +100,7 @@ extern "C" { #elif defined(__GNUC__) && !defined(threadlocal) #define threadlocal __thread #else - #define threadlocal + #define threadlocal __declspec( thread ) #endif #ifdef __cplusplus diff --git a/src/wal/src/walMgmt.c b/src/wal/src/walMgmt.c index 62b066500c..39ce2657aa 100644 --- a/src/wal/src/walMgmt.c +++ b/src/wal/src/walMgmt.c @@ -44,7 +44,7 @@ int32_t walInit() { return code; } - wInfo("wal module is initialized, refId:%d", tsWal.refId); + wInfo("wal module is initialized, rsetId:%d", tsWal.refId); return code; } @@ -203,7 +203,7 @@ static int32_t walCreateThread() { } pthread_attr_destroy(&thAttr); - wDebug("wal thread is launched"); + wDebug("wal thread is launched, thread:0x%08" PRIx64, taosGetPthreadId(tsWal.thread)); return TSDB_CODE_SUCCESS; } diff --git a/tests/tsim/src/simMain.c b/tests/tsim/src/simMain.c index 6c9c492a5a..251b4f80bb 100644 --- a/tests/tsim/src/simMain.c +++ b/tests/tsim/src/simMain.c @@ -64,7 +64,7 @@ int32_t main(int32_t argc, char *argv[]) { simExecuteScript(script); int32_t ret = simExecSuccess ? 0 : -1; - simError("execute result %d", ret); + simInfo("execute result %d", ret); return ret; } \ No newline at end of file From 33228ee5897341e1e6413858d182ad6442be9fbd Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 22 Jan 2021 19:37:17 +0800 Subject: [PATCH 35/62] TD-1207 --- src/dnode/src/dnodeSystem.c | 16 ++++++++++------ src/kit/shell/src/shellMain.c | 2 +- src/os/inc/osSignal.h | 2 +- src/os/src/detail/osSignal.c | 6 ++++++ src/os/src/windows/wSignal.c | 4 +++- src/sync/src/syncArbitrator.c | 4 ++-- tests/tsim/src/simMain.c | 2 +- 7 files changed, 24 insertions(+), 12 deletions(-) diff --git a/src/dnode/src/dnodeSystem.c b/src/dnode/src/dnodeSystem.c index b7ca6f053c..ad5950ab5b 100644 --- a/src/dnode/src/dnodeSystem.c +++ b/src/dnode/src/dnodeSystem.c @@ -20,9 +20,9 @@ #include "dnodeMain.h" static tsem_t exitSem; -static void siguser1Handler(int32_t signum); -static void siguser2Handler(int32_t signum); -static void sigintHandler(int32_t signum); +static void siguser1Handler(int32_t signum, void *sigInfo, void *context); +static void siguser2Handler(int32_t signum, void *sigInfo, void *context); +static void sigintHandler(int32_t signum, void *sigInfo, void *context); int32_t main(int32_t argc, char *argv[]) { int dump_config = 0; @@ -152,11 +152,11 @@ int32_t main(int32_t argc, char *argv[]) { return EXIT_SUCCESS; } -static void siguser1Handler(int32_t signum) { taosCfgDynamicOptions("debugFlag 143"); } +static void siguser1Handler(int32_t signum, void *sigInfo, void *context) { taosCfgDynamicOptions("debugFlag 143"); } -static void siguser2Handler(int32_t signum) { taosCfgDynamicOptions("resetlog"); } +static void siguser2Handler(int32_t signum, void *sigInfo, void *context) { taosCfgDynamicOptions("resetlog"); } -static void sigintHandler(int32_t signum) { +static void sigintHandler(int32_t signum, void *sigInfo, void *context) { // protect the application from receive another signal taosIgnSignal(SIGUSR1); taosIgnSignal(SIGUSR2); @@ -169,6 +169,10 @@ static void sigintHandler(int32_t signum) { // clean the system. dInfo("shut down signal is %d", signum); +#ifndef WINDOWS + dInfo("sender PID:%d cmdline:%s",((siginfo_t *)sigInfo)->si_pid, taosGetCmdlineByPID(sigInfo->si_pid)); +#endif + syslog(LOG_INFO, "Shut down signal is %d", signum); syslog(LOG_INFO, "Shutting down TDengine service..."); diff --git a/src/kit/shell/src/shellMain.c b/src/kit/shell/src/shellMain.c index c7a0dab2dd..49de42796c 100644 --- a/src/kit/shell/src/shellMain.c +++ b/src/kit/shell/src/shellMain.c @@ -21,7 +21,7 @@ pthread_t pid; static tsem_t cancelSem; -void shellQueryInterruptHandler(int32_t signum) { +void shellQueryInterruptHandler(int32_t signum, void *sigInfo, void *context) { tsem_post(&cancelSem); } diff --git a/src/os/inc/osSignal.h b/src/os/inc/osSignal.h index 8016c49ba8..c13cd83178 100644 --- a/src/os/inc/osSignal.h +++ b/src/os/inc/osSignal.h @@ -48,7 +48,7 @@ extern "C" { #define SIGBREAK 1234 #endif -typedef void (*FSignalHandler)(int32_t signum); +typedef void (*FSignalHandler)(int32_t signum, void *sigInfo, void *context); void taosSetSignal(int32_t signum, FSignalHandler sigfp); void taosIgnSignal(int32_t signum); void taosDflSignal(int32_t signum); diff --git a/src/os/src/detail/osSignal.c b/src/os/src/detail/osSignal.c index c97a3343de..e1a0e84e7f 100644 --- a/src/os/src/detail/osSignal.c +++ b/src/os/src/detail/osSignal.c @@ -20,10 +20,16 @@ #include "tulog.h" #ifndef TAOS_OS_FUNC_SIGNAL +typedef void (*FLinuxSignalHandler)(int32_t signum, siginfo_t *sigInfo, void *context); void taosSetSignal(int32_t signum, FSignalHandler sigfp) { struct sigaction act = {{0}}; +#if 1 + act.sa_flags = SA_SIGINFO; + act.sa_sigaction = (FLinuxSignalHandler)sigfp; +#else act.sa_handler = sigfp; +#endif sigaction(signum, &act, NULL); } diff --git a/src/os/src/windows/wSignal.c b/src/os/src/windows/wSignal.c index 3988f0c6e1..9de6b5f343 100644 --- a/src/os/src/windows/wSignal.c +++ b/src/os/src/windows/wSignal.c @@ -18,6 +18,8 @@ #include #include +typedef void (*FWinSignalHandler)(int32_t signum); + void taosSetSignal(int32_t signum, FSignalHandler sigfp) { if (signum == SIGUSR1) return; @@ -25,7 +27,7 @@ void taosSetSignal(int32_t signum, FSignalHandler sigfp) { if (signum == SIGHUP) { SetConsoleCtrlHandler((PHANDLER_ROUTINE)sigfp, TRUE); } else { - signal(signum, sigfp); + signal(signum, (FWinSignalHandler)sigfp); } } diff --git a/src/sync/src/syncArbitrator.c b/src/sync/src/syncArbitrator.c index d65a7b3043..24760ce1a0 100644 --- a/src/sync/src/syncArbitrator.c +++ b/src/sync/src/syncArbitrator.c @@ -27,7 +27,7 @@ #include "syncInt.h" #include "syncTcp.h" -static void arbSignalHandler(int32_t signum); +static void arbSignalHandler(int32_t signum, void *sigInfo, void *context); static void arbProcessIncommingConnection(int32_t connFd, uint32_t sourceIp); static void arbProcessBrokenLink(int64_t rid); static int32_t arbProcessPeerMsg(int64_t rid, void *buffer); @@ -170,7 +170,7 @@ static int32_t arbProcessPeerMsg(int64_t rid, void *buffer) { return 0; } -static void arbSignalHandler(int32_t signum) { +static void arbSignalHandler(int32_t signum, void *sigInfo, void *context) { taosIgnSignal(SIGTERM); taosIgnSignal(SIGINT); taosIgnSignal(SIGABRT); diff --git a/tests/tsim/src/simMain.c b/tests/tsim/src/simMain.c index 251b4f80bb..990ea71413 100644 --- a/tests/tsim/src/simMain.c +++ b/tests/tsim/src/simMain.c @@ -22,7 +22,7 @@ bool simAsyncQuery = false; bool simExecSuccess = false; -void simHandleSignal(int32_t signo) { +void simHandleSignal(int32_t signo, void *sigInfo, void *context) { simSystemCleanUp(); exit(1); } From df96fd1b3dcdb055fdfbdb81284c6d1bc62d1f97 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 22 Jan 2021 19:47:25 +0800 Subject: [PATCH 36/62] compile error --- src/dnode/src/dnodeSystem.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dnode/src/dnodeSystem.c b/src/dnode/src/dnodeSystem.c index ad5950ab5b..33ebc8b64f 100644 --- a/src/dnode/src/dnodeSystem.c +++ b/src/dnode/src/dnodeSystem.c @@ -170,7 +170,7 @@ static void sigintHandler(int32_t signum, void *sigInfo, void *context) { dInfo("shut down signal is %d", signum); #ifndef WINDOWS - dInfo("sender PID:%d cmdline:%s",((siginfo_t *)sigInfo)->si_pid, taosGetCmdlineByPID(sigInfo->si_pid)); + dInfo("sender PID:%d cmdline:%s", ((siginfo_t *)sigInfo)->si_pid, taosGetCmdlineByPID(((siginfo_t *)sigInfo)->si_pid)); #endif syslog(LOG_INFO, "Shut down signal is %d", signum); From 0be7fe43cab50799c2c6937eae72a7c1253706f9 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Sat, 23 Jan 2021 04:25:52 +0800 Subject: [PATCH 37/62] fix bug --- src/common/src/tvariant.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/common/src/tvariant.c b/src/common/src/tvariant.c index 95aa576889..fa95e17f32 100644 --- a/src/common/src/tvariant.c +++ b/src/common/src/tvariant.c @@ -775,7 +775,7 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu return -1; } } else { - wcsncpy((wchar_t *)p, pVariant->wpz, pVariant->nLen); + memcpy(p, pVariant->wpz, pVariant->nLen); newlen = pVariant->nLen; } @@ -867,4 +867,4 @@ int32_t tVariantTypeSetType(tVariant *pVariant, char type) { } return 0; -} \ No newline at end of file +} From d6c7c58be1dde5b4d3a74a36cad497d2277300e2 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Sat, 23 Jan 2021 06:39:38 +0800 Subject: [PATCH 38/62] fix bug --- src/client/src/tscSQLParser.c | 80 +++++++++++++++++++++++++++++++++++ src/common/src/tvariant.c | 4 +- src/inc/taosdef.h | 5 +++ 3 files changed, 87 insertions(+), 2 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index f9eedff7eb..6b24dcd858 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -4300,6 +4300,78 @@ static void doAddJoinTagsColumnsIntoTagList(SSqlCmd* pCmd, SQueryInfo* pQueryInf } } +static int32_t validateTagCondExpr(SSqlCmd* pCmd, tExprNode *p) { + const char *msg1 = "tag type mismatch"; + const char *msg2 = "invalid tag operator"; + const char* msg3 = "not supported filter condition"; + + do { + if (p->nodeType != TSQL_NODE_EXPR) { + break; + } + + if (!p->_node.pLeft || !p->_node.pRight) { + break; + } + + if (IS_ARITHMETIC_OPTR(p->_node.optr)) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + } + + if (!IS_RELATION_OPTR(p->_node.optr)) { + break; + } + + tVariant * vVariant = NULL; + int32_t schemaType = -1; + + if (p->_node.pLeft->nodeType == TSQL_NODE_VALUE && p->_node.pRight->nodeType == TSQL_NODE_COL) { + if (!p->_node.pRight->pSchema) { + break; + } + + vVariant = p->_node.pLeft->pVal->nType; + schemaType = p->_node.pRight->pSchema->type; + } else if (p->_node.pLeft->nodeType == TSQL_NODE_COL && p->_node.pRight->nodeType == TSQL_NODE_VALUE) { + if (!p->_node.pLeft->pSchema) { + break; + } + + vVariant = p->_node.pRight->pVal->nType; + schemaType = p->_node.pLeft->pSchema->type; + } else { + break; + } + + if (schemaType >= TSDB_DATA_TYPE_TINYINT && schemaType <= TSDB_DATA_TYPE_BIGINT) { + schemaType = TSDB_DATA_TYPE_BIGINT; + } else if (schemaType == TSDB_DATA_TYPE_FLOAT || schemaType == TSDB_DATA_TYPE_DOUBLE) { + schemaType = TSDB_DATA_TYPE_DOUBLE; + } + + int32_t retVal = TSDB_CODE_SUCCESS; + if (schemaType == TSDB_DATA_TYPE_BINARY) { + char *tmp = (int64_t)calloc(1, (vVariant->nLen + 1) + TSDB_NCHAR_SIZE); + retVal = tVariantDump(vVariant, tmp, schemaType, false); + free(tmp); + } else if (schemaType == TSDB_DATA_TYPE_NCHAR) { + // pRight->val.nLen + 1 is larger than the actual nchar string length + char *tmp = (int64_t)calloc(1, (vVariant->nLen + 1) * TSDB_NCHAR_SIZE); + retVal = tVariantDump(vVariant, tmp, schemaType, false); + free(tmp); + } else { + double tmp; + retVal = tVariantDump(vVariant, (char*)&tmp, schemaType, false); + } + + if (retVal != TSDB_CODE_SUCCESS) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + } + }while (0); + + return TSDB_CODE_SUCCESS; +} + static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondExpr* pCondExpr, tSQLExpr** pExpr) { int32_t ret = TSDB_CODE_SUCCESS; @@ -4342,6 +4414,10 @@ static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondE tsSetSTableQueryCond(&pQueryInfo->tagCond, uid, &bw); doCompactQueryExpr(pExpr); + if (ret == TSDB_CODE_SUCCESS) { + ret = validateTagCondExpr(pCmd, p); + } + tSqlExprDestroy(p1); tExprTreeDestroy(p, NULL); @@ -4349,6 +4425,10 @@ static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondE if (pQueryInfo->tagCond.pCond != NULL && taosArrayGetSize(pQueryInfo->tagCond.pCond) > 0 && !UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "filter on tag not supported for normal table"); } + + if (ret) { + break; + } } pCondExpr->pTagCond = NULL; diff --git a/src/common/src/tvariant.c b/src/common/src/tvariant.c index 95aa576889..2f2750df4f 100644 --- a/src/common/src/tvariant.c +++ b/src/common/src/tvariant.c @@ -430,7 +430,7 @@ static FORCE_INLINE int32_t convertToInteger(tVariant *pVariant, int64_t *result } errno = 0; - if (IS_SIGNED_NUMERIC_TYPE(pVariant->nType)) { + if (IS_SIGNED_NUMERIC_TYPE(pVariant->nType) || (pVariant->nType == TSDB_DATA_TYPE_BOOL)) { *result = pVariant->i64; } else if (IS_UNSIGNED_NUMERIC_TYPE(pVariant->nType)) { *result = pVariant->u64; @@ -867,4 +867,4 @@ int32_t tVariantTypeSetType(tVariant *pVariant, char type) { } return 0; -} \ No newline at end of file +} diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 6fea049074..9f3c31f225 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -163,6 +163,11 @@ do { \ #define TSDB_BINARY_OP_MULTIPLY 32 #define TSDB_BINARY_OP_DIVIDE 33 #define TSDB_BINARY_OP_REMAINDER 34 + + +#define IS_RELATION_OPTR(op) (((op) >= TSDB_RELATION_LESS) && ((op) <= TSDB_RELATION_IN)) +#define IS_ARITHMETIC_OPTR(op) (((op) >= TSDB_BINARY_OP_ADD) && ((op) <= TSDB_BINARY_OP_REMAINDER)) + #define TS_PATH_DELIMITER_LEN 1 #define TSDB_UNI_LEN 24 From 94eb0e1e71b713d66d00274242d7667408095154 Mon Sep 17 00:00:00 2001 From: freemine Date: Sat, 23 Jan 2021 08:44:32 +0800 Subject: [PATCH 39/62] 1. __APPLE__ 2. tsem_xxx for mac --- src/client/src/tscUtil.c | 4 ++-- src/dnode/src/dnodeSystem.c | 2 +- src/dnode/src/dnodeTelemetry.c | 2 +- src/mnode/src/mnodeCluster.c | 1 + src/os/inc/osDarwin.h | 10 +++++----- src/os/inc/osDef.h | 5 +++-- src/plugins/http/src/httpServer.c | 2 +- src/rpc/src/rpcTcp.c | 13 ++++++++----- src/sync/src/syncTcp.c | 8 ++++---- src/tsdb/inc/tsdbMain.h | 4 ++-- src/tsdb/src/tsdbCommit.c | 4 ++-- src/tsdb/src/tsdbMain.c | 12 ++++++------ src/tsdb/src/tsdbMemTable.c | 8 ++++---- src/util/src/tsocket.c | 2 +- tests/examples/c/epoll.c | 4 ++-- 15 files changed, 43 insertions(+), 38 deletions(-) diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 19a1e99c33..bb2a81db66 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -2520,9 +2520,9 @@ bool tscSetSqlOwner(SSqlObj* pSql) { // set the sql object owner #ifdef __APPLE__ pthread_t threadId = (pthread_t)taosGetSelfPthreadId(); -#else +#else // __APPLE__ uint64_t threadId = taosGetSelfPthreadId(); -#endif +#endif // __APPLE__ if (atomic_val_compare_exchange_64(&pSql->owner, 0, threadId) != 0) { pRes->code = TSDB_CODE_QRY_IN_EXEC; return false; diff --git a/src/dnode/src/dnodeSystem.c b/src/dnode/src/dnodeSystem.c index 893049df2e..7a218f901e 100644 --- a/src/dnode/src/dnodeSystem.c +++ b/src/dnode/src/dnodeSystem.c @@ -164,7 +164,7 @@ static void signal_handler(int32_t signum, siginfo_t *sigInfo, void *context) { dInfo("shut down signal is %d, sender PID:%d", signum, sigInfo->si_pid); #else // __APPLE__ dInfo("shut down signal is %d, sender PID:%d cmdline:%s", signum, sigInfo->si_pid, taosGetCmdlineByPID(sigInfo->si_pid)); -#endif +#endif // __APPLE__ // protect the application from receive another signal struct sigaction act = {{0}}; diff --git a/src/dnode/src/dnodeTelemetry.c b/src/dnode/src/dnodeTelemetry.c index 1f7084c3b5..b2e8eac267 100644 --- a/src/dnode/src/dnodeTelemetry.c +++ b/src/dnode/src/dnodeTelemetry.c @@ -241,7 +241,7 @@ static int sem_timedwait(tsem_t *sem, struct timespec *to) { fprintf(stderr, "%s[%d]%s(): not implemented yet!\n", basename(__FILE__), __LINE__, __func__); abort(); } -#endif +#endif // __APPLE__ static void* telemetryThread(void* param) { struct timespec end = {0}; diff --git a/src/mnode/src/mnodeCluster.c b/src/mnode/src/mnodeCluster.c index 1b153247f0..5dee0e1c26 100644 --- a/src/mnode/src/mnodeCluster.c +++ b/src/mnode/src/mnodeCluster.c @@ -145,6 +145,7 @@ bool taosGetSystemUid(char *uid) { return false; } #endif // __APPLE__ + static int32_t mnodeCreateCluster() { int32_t numOfClusters = sdbGetNumOfRows(tsClusterSdb); if (numOfClusters != 0) return TSDB_CODE_SUCCESS; diff --git a/src/os/inc/osDarwin.h b/src/os/inc/osDarwin.h index 935708eb1b..0a77ed6895 100644 --- a/src/os/inc/osDarwin.h +++ b/src/os/inc/osDarwin.h @@ -75,11 +75,11 @@ extern "C" { #define TAOS_OS_FUNC_FILE_SENDIFLE #define TAOS_OS_FUNC_SEMPHONE - #define tsem_t dispatch_semaphore_t - int tsem_init(dispatch_semaphore_t *sem, int pshared, unsigned int value); - int tsem_wait(dispatch_semaphore_t *sem); - int tsem_post(dispatch_semaphore_t *sem); - int tsem_destroy(dispatch_semaphore_t *sem); + typedef struct tsem_s *tsem_t; + int tsem_init(tsem_t *sem, int pshared, unsigned int value); + int tsem_wait(tsem_t *sem); + int tsem_post(tsem_t *sem); + int tsem_destroy(tsem_t *sem); #define TAOS_OS_FUNC_SOCKET_SETSOCKETOPT #define TAOS_OS_FUNC_STRING_STR2INT64 diff --git a/src/os/inc/osDef.h b/src/os/inc/osDef.h index d718bef6da..f646940d3a 100644 --- a/src/os/inc/osDef.h +++ b/src/os/inc/osDef.h @@ -90,11 +90,12 @@ extern "C" { #ifdef _ISOC11_SOURCE #define threadlocal _Thread_local #elif defined(__APPLE__) - #define threadlocal + #define threadlocal __thread #elif defined(__GNUC__) && !defined(threadlocal) #define threadlocal __thread #else - #define threadlocal + // #define threadlocal + #error please follow with the target platform's thread-local implementation #endif #ifdef __cplusplus diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index 97a5b4bc23..6a8eb556d8 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -33,7 +33,7 @@ static bool httpReadData(HttpContext *pContext); #ifdef __APPLE__ static int sv_dummy = 0; -#endif +#endif // __APPLE__ static void httpStopThread(HttpThread* pThread) { pThread->stop = true; diff --git a/src/rpc/src/rpcTcp.c b/src/rpc/src/rpcTcp.c index ba90f93073..d64bdb42aa 100644 --- a/src/rpc/src/rpcTcp.c +++ b/src/rpc/src/rpcTcp.c @@ -312,9 +312,9 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int num, void * epoll_close(pThreadObj->pollFd); pThreadObj->pollFd = -1; } -#else +#else // __APPLE__ taosCloseSocket(pThreadObj->pollFd); -#endif +#endif // __APPLE__ free(pThreadObj); terrno = TAOS_SYSTEM_ERROR(errno); tError("%s failed to create TCP read data thread(%s)", label, strerror(errno)); @@ -477,7 +477,10 @@ static void *taosProcessTcpData(void *param) { SFdObj *pFdObj; struct epoll_event events[maxEvents]; SRecvInfo recvInfo; - + +#ifdef __APPLE__ + taos_block_sigalrm(); +#endif // __APPLE__ while (1) { int fdNum = epoll_wait(pThreadObj->pollFd, events, maxEvents, TAOS_EPOLL_WAIT_TIME); if (pThreadObj->stop) { @@ -524,9 +527,9 @@ static void *taosProcessTcpData(void *param) { epoll_close(pThreadObj->pollFd); pThreadObj->pollFd = -1; } -#else +#else // __APPLE__ if (pThreadObj->pollFd >=0) taosCloseSocket(pThreadObj->pollFd); -#endif +#endif // __APPLE__ while (pThreadObj->pHead) { SFdObj *pFdObj = pThreadObj->pHead; diff --git a/src/sync/src/syncTcp.c b/src/sync/src/syncTcp.c index 1e5761316a..e15f71e905 100644 --- a/src/sync/src/syncTcp.c +++ b/src/sync/src/syncTcp.c @@ -235,9 +235,9 @@ static void *syncProcessTcpData(void *param) { #ifdef __APPLE__ epoll_close(pThread->pollFd); -#else +#else // __APPLE__ close(pThread->pollFd); -#endif +#endif // __APPLE__ tfree(pThread); tfree(buffer); return NULL; @@ -296,9 +296,9 @@ static SThreadObj *syncGetTcpThread(SPoolObj *pPool) { if (ret != 0) { #ifdef __APPLE__ epoll_close(pThread->pollFd); -#else +#else // __APPLE__ close(pThread->pollFd); -#endif +#endif // __APPLE__ tfree(pThread); return NULL; } diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 984839162e..f3e69d9210 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -235,9 +235,9 @@ typedef struct { STsdbFileH* tsdbFileH; #ifdef __APPLE__ sem_t *readyToCommit; -#else +#else // __APPLE__ sem_t readyToCommit; -#endif +#endif // __APPLE__ pthread_mutex_t mutex; bool repoLocked; int32_t code; // Commit code diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c index 9551d1b3a5..1c8a661c0a 100644 --- a/src/tsdb/src/tsdbCommit.c +++ b/src/tsdb/src/tsdbCommit.c @@ -168,9 +168,9 @@ static void tsdbEndCommit(STsdbRepo *pRepo, int eno) { tsdbUnRefMemTable(pRepo, pIMem); #ifdef __APPLE__ sem_post(pRepo->readyToCommit); -#else +#else // __APPLE__ sem_post(&(pRepo->readyToCommit)); -#endif +#endif // __APPLE__ } static int tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey) { diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 5493188c09..c770f0ff42 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -148,9 +148,9 @@ int tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit) { tsdbAsyncCommit(pRepo); #ifdef __APPLE__ sem_wait(pRepo->readyToCommit); -#else +#else // __APPLE__ sem_wait(&(pRepo->readyToCommit)); -#endif +#endif // __APPLE__ terrno = pRepo->code; } tsdbUnRefMemTable(pRepo, pRepo->mem); @@ -654,14 +654,14 @@ static STsdbRepo *tsdbNewRepo(char *rootDir, STsdbAppH *pAppH, STsdbCfg *pCfg) { terrno = TAOS_SYSTEM_ERROR(code); goto _err; } -#else +#else // __APPLE__ code = sem_init(&(pRepo->readyToCommit), 0, 1); if (code != 0) { code = errno; terrno = TAOS_SYSTEM_ERROR(code); goto _err; } -#endif +#endif // __APPLE__ pRepo->repoLocked = false; @@ -709,9 +709,9 @@ static void tsdbFreeRepo(STsdbRepo *pRepo) { tfree(pRepo->rootDir); #ifdef __APPLE__ sem_close(pRepo->readyToCommit); -#else +#else // __APPLE__ sem_destroy(&(pRepo->readyToCommit)); -#endif +#endif // __APPLE__ pthread_mutex_destroy(&pRepo->mutex); free(pRepo); } diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 202405df00..206a3332d5 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -209,9 +209,9 @@ int tsdbAsyncCommit(STsdbRepo *pRepo) { #ifdef __APPLE__ sem_wait(pRepo->readyToCommit); -#else +#else // __APPLE__ sem_wait(&(pRepo->readyToCommit)); -#endif +#endif // __APPLE__ ASSERT(pRepo->imem == NULL); @@ -236,10 +236,10 @@ int tsdbSyncCommit(TSDB_REPO_T *repo) { #ifdef __APPLE__ sem_wait(pRepo->readyToCommit); sem_post(pRepo->readyToCommit); -#else +#else // __APPLE__ sem_wait(&(pRepo->readyToCommit)); sem_post(&(pRepo->readyToCommit)); -#endif +#endif // __APPLE__ if (pRepo->code != TSDB_CODE_SUCCESS) { terrno = pRepo->code; diff --git a/src/util/src/tsocket.c b/src/util/src/tsocket.c index cfd948265b..4bfd7a855b 100644 --- a/src/util/src/tsocket.c +++ b/src/util/src/tsocket.c @@ -376,7 +376,7 @@ int32_t taosKeepTcpAlive(SOCKET sockFd) { taosCloseSocket(sockFd); return -1; } -#endif +#endif // __APPLE__ int32_t nodelay = 1; if (taosSetSockOpt(sockFd, IPPROTO_TCP, TCP_NODELAY, (void *)&nodelay, sizeof(nodelay)) < 0) { diff --git a/tests/examples/c/epoll.c b/tests/examples/c/epoll.c index eb66156c14..4f65ea478e 100644 --- a/tests/examples/c/epoll.c +++ b/tests/examples/c/epoll.c @@ -15,9 +15,9 @@ #ifdef __APPLE__ #include "eok.h" -#else +#else // __APPLE__ #include -#endif +#endif // __APPLE__ #include #include #include From 4e774660b6de2f0035f8735c3a16c2f3b705e5a2 Mon Sep 17 00:00:00 2001 From: freemine Date: Sat, 23 Jan 2021 08:45:44 +0800 Subject: [PATCH 40/62] tsem_xxx for mac --- src/os/src/darwin/darwinSemphone.c | 88 ++++++++++++++++++++++++------ 1 file changed, 72 insertions(+), 16 deletions(-) diff --git a/src/os/src/darwin/darwinSemphone.c b/src/os/src/darwin/darwinSemphone.c index 97ff543789..d8e60cd2d5 100644 --- a/src/os/src/darwin/darwinSemphone.c +++ b/src/os/src/darwin/darwinSemphone.c @@ -16,25 +16,81 @@ #define _DEFAULT_SOURCE #include "os.h" -int tsem_init(dispatch_semaphore_t *sem, int pshared, unsigned int value) { - *sem = dispatch_semaphore_create(value); - if (*sem == NULL) { - return -1; - } else { +struct tsem_s { + dispatch_semaphore_t sem; + volatile unsigned int valid:1; +}; + +int tsem_wait(tsem_t *sem); +int tsem_post(tsem_t *sem); +int tsem_destroy(tsem_t *sem); +int tsem_init(tsem_t *sem, int pshared, unsigned int value) { + fprintf(stderr, "==%s[%d]%s():[%p]==creating\n", basename(__FILE__), __LINE__, __func__, sem); + if (*sem) { + fprintf(stderr, "==%s[%d]%s():[%p]==already initialized\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } + struct tsem_s *p = (struct tsem_s*)calloc(1, sizeof(*p)); + if (!p) { + fprintf(stderr, "==%s[%d]%s():[%p]==out of memory\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } + + p->sem = dispatch_semaphore_create(value); + if (p->sem == NULL) { + fprintf(stderr, "==%s[%d]%s():[%p]==not created\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } + p->valid = 1; + + *sem = p; + + return 0; +} + +int tsem_wait(tsem_t *sem) { + if (!*sem) { + fprintf(stderr, "==%s[%d]%s():[%p]==not initialized\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } + struct tsem_s *p = *sem; + if (!p->valid) { + fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } + return dispatch_semaphore_wait(p->sem, DISPATCH_TIME_FOREVER); +} + +int tsem_post(tsem_t *sem) { + if (!*sem) { + fprintf(stderr, "==%s[%d]%s():[%p]==not initialized\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } + struct tsem_s *p = *sem; + if (!p->valid) { + fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } + return dispatch_semaphore_signal(p->sem); +} + +int tsem_destroy(tsem_t *sem) { + fprintf(stderr, "==%s[%d]%s():[%p]==destroying\n", basename(__FILE__), __LINE__, __func__, sem); + if (!*sem) { + fprintf(stderr, "==%s[%d]%s():[%p]==not initialized\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); return 0; } -} + struct tsem_s *p = *sem; + if (!p->valid) { + fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } -int tsem_wait(dispatch_semaphore_t *sem) { - dispatch_semaphore_wait(*sem, DISPATCH_TIME_FOREVER); + p->valid = 0; + free(p); + + *sem = NULL; return 0; } -int tsem_post(dispatch_semaphore_t *sem) { - dispatch_semaphore_signal(*sem); - return 0; -} - -int tsem_destroy(dispatch_semaphore_t *sem) { - return 0; -} From 79e4925dc73231190c2395214ada2d540ea30bf0 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 23 Jan 2021 12:00:02 +0800 Subject: [PATCH 41/62] [TD-2696] : fix RFC3339 timestamp bug. --- src/os/src/detail/osTime.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/os/src/detail/osTime.c b/src/os/src/detail/osTime.c index d6877790a8..d9d070218e 100644 --- a/src/os/src/detail/osTime.c +++ b/src/os/src/detail/osTime.c @@ -43,7 +43,7 @@ */ int64_t user_mktime64(const unsigned int year0, const unsigned int mon0, const unsigned int day, const unsigned int hour, - const unsigned int min, const unsigned int sec) + const unsigned int min, const unsigned int sec, int64_t timezone) { unsigned int mon = mon0, year = year0; @@ -61,12 +61,6 @@ int64_t user_mktime64(const unsigned int year0, const unsigned int mon0, res = res*24; res = ((res + hour) * 60 + min) * 60 + sec; -#ifdef _MSC_VER -#if _MSC_VER >= 1900 - int64_t timezone = _timezone; -#endif -#endif - return (res + timezone); } @@ -219,7 +213,7 @@ int32_t parseTimeWithTz(char* timestr, int64_t* time, int32_t timePrec) { /* mktime will be affected by TZ, set by using taos_options */ #ifdef WINDOWS - int64_t seconds = user_mktime64(tm.tm_year+1900, tm.tm_mon+1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); + int64_t seconds = user_mktime64(tm.tm_year+1900, tm.tm_mon+1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec, 0); //int64_t seconds = gmtime(&tm); #else int64_t seconds = timegm(&tm); @@ -276,7 +270,13 @@ int32_t parseLocaltime(char* timestr, int64_t* time, int32_t timePrec) { return -1; } - int64_t seconds = user_mktime64(tm.tm_year+1900, tm.tm_mon+1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); +#ifdef _MSC_VER +#if _MSC_VER >= 1900 + int64_t timezone = _timezone; +#endif +#endif + + int64_t seconds = user_mktime64(tm.tm_year+1900, tm.tm_mon+1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec, timezone); int64_t fraction = 0; @@ -574,4 +574,4 @@ const char* fmtts(int64_t ts) { } return buf; -} \ No newline at end of file +} From b15d8fefaa9f3d35a3e0eade89e5bbb178f097bf Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 23 Jan 2021 12:55:47 +0800 Subject: [PATCH 42/62] [TD-2829] : refine Windows compiling section of README.md --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 566b3e5c8b..fafcdc8321 100644 --- a/README.md +++ b/README.md @@ -133,7 +133,9 @@ cmake .. -G "NMake Makefiles" nmake ``` -If you use the Visual Studio 2019, please open a command window by executing "cmd.exe". +If you use the Visual Studio 2019 or 2017: + +please open a command window by executing "cmd.exe". Please specify "x64" for 64 bits Windows or specify "x86" is for 32 bits Windows when you execute vcvarsall.bat. ``` mkdir debug && cd debug @@ -142,7 +144,7 @@ cmake .. -G "NMake Makefiles" nmake ``` -Or, you can open a command window by clicking Visual Studio 2019 menu "Tools -> Command Line -> Developer Command Prompt" or "Tools -> Command Line -> Developer PowerShell" then execute commands as follows: +Or, you can simply open a command window by clicking Windows Start -> "Visual Studio < 2019 | 2017 >" folder -> "x64 Native Tools Command Prompt for VS < 2019 | 2017 >" or "x86 Native Tools Command Prompt for VS < 2019 | 2017 >" depends what architecture your Windows is, then execute commands as follows: ``` mkdir debug && cd debug cmake .. -G "NMake Makefiles" From 67e2a153b05cc54508165fce1b4e192a81086939 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 23 Jan 2021 23:02:21 +0800 Subject: [PATCH 43/62] [TD-225]fix compiler error. --- src/tsdb/src/tsdbRead.c | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 39030b15d7..4045e302c7 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -2148,16 +2148,7 @@ bool tsdbNextDataBlockWithoutMerge(TsdbQueryHandleT* pHandle) { size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); assert(numOfTables > 0); - if (pQueryHandle->type == TSDB_QUERY_TYPE_EXTERNAL) { - SMemRef* pMemRef = pQueryHandle->pMemRef; - tsdbMayTakeMemSnapshot(pQueryHandle); - bool ret = getNeighborRows(pQueryHandle); - tsdbMayUnTakeMemSnapshot(pQueryHandle); - - // restore the pMemRef - pQueryHandle->pMemRef = pMemRef; - return ret; - } else if (pQueryHandle->type == TSDB_QUERY_TYPE_LAST && pQueryHandle->cachelastrow) { + if (pQueryHandle->type == TSDB_QUERY_TYPE_LAST && pQueryHandle->cachelastrow) { // the last row is cached in buffer, return it directly. // here note that the pQueryHandle->window must be the TS_INITIALIZER int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pQueryHandle)); From 9c58bfbd79720ba8eb5fb27c55e95547535cd1be Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 23 Jan 2021 23:16:01 +0800 Subject: [PATCH 44/62] [TD-225]update the sim. --- tests/script/general/parser/stableOp.sim | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/script/general/parser/stableOp.sim b/tests/script/general/parser/stableOp.sim index 133de95b9c..f4b1bd4b8a 100644 --- a/tests/script/general/parser/stableOp.sim +++ b/tests/script/general/parser/stableOp.sim @@ -1,6 +1,5 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start @@ -78,7 +77,6 @@ if $rows != 0 then return -1 endi - sql DROP STABLE $stb sql show stables From ce50e131bc35b23f9db111736c0a55c59135f958 Mon Sep 17 00:00:00 2001 From: freemine Date: Sun, 24 Jan 2021 09:29:13 +0800 Subject: [PATCH 45/62] 1. kqueue rather than SIGALRM for the timer 2. semaphore rather than dispatch_semaphore for tsem_xxx 3. fail-fast comments 4. niche --- src/client/src/tscSql.c | 4 + src/client/src/tscUtil.c | 4 + src/os/src/darwin/darwinSemphone.c | 201 +++++++++++++++++++++++++++-- src/os/src/darwin/darwinTimer.c | 74 +++++++++++ src/os/src/darwin/eok.c | 37 +++--- 5 files changed, 297 insertions(+), 23 deletions(-) diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index fa7c1f83ba..ef9b079cfc 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -295,6 +295,10 @@ void taos_close(TAOS *taos) { tscDebug("%p HB is freed", pHb); taosReleaseRef(tscObjRef, pHb->self); +#ifdef __APPLE__ + // to satisfy later tsem_destroy in taos_free_result + tsem_init(&pHb->rspSem, 0, 0); +#endif // __APPLE__ taos_free_result(pHb); } } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 8dcb7425a8..e1521b4aa1 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1933,6 +1933,10 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, __async_cb_func_t fp, void* param, in } if (tscAddSubqueryInfo(pCmd) != TSDB_CODE_SUCCESS) { +#ifdef __APPLE__ + // to satisfy later tsem_destroy in taos_free_result + tsem_init(&pNew->rspSem, 0, 0); +#endif // __APPLE__ tscFreeSqlObj(pNew); return NULL; } diff --git a/src/os/src/darwin/darwinSemphone.c b/src/os/src/darwin/darwinSemphone.c index d8e60cd2d5..a9e470bd70 100644 --- a/src/os/src/darwin/darwinSemphone.c +++ b/src/os/src/darwin/darwinSemphone.c @@ -13,19 +13,76 @@ * along with this program. If not, see . */ +// fail-fast or let-it-crash philosophy +// https://en.wikipedia.org/wiki/Fail-fast +// https://stackoverflow.com/questions/4393197/erlangs-let-it-crash-philosophy-applicable-elsewhere +// experimentally, we follow log-and-crash here + #define _DEFAULT_SOURCE #include "os.h" +// #define SEM_USE_PTHREAD +// #define SEM_USE_POSIX +#define SEM_USE_SEM + +#ifdef SEM_USE_SEM +#include +#include +#include +#include + +static pthread_t sem_thread; +static pthread_once_t sem_once; +static task_t sem_port; +static volatile int sem_inited = 0; +static semaphore_t sem_exit; + +static void* sem_thread_routine(void *arg) { + (void)arg; + sem_port = mach_task_self(); + kern_return_t ret = semaphore_create(sem_port, &sem_exit, SYNC_POLICY_FIFO, 0); + if (ret != KERN_SUCCESS) { + fprintf(stderr, "==%s[%d]%s()==failed to create sem_exit\n", basename(__FILE__), __LINE__, __func__); + sem_inited = -1; + return NULL; + } + sem_inited = 1; + semaphore_wait(sem_exit); + return NULL; +} + +static void once_init(void) { + int r = 0; + r = pthread_create(&sem_thread, NULL, sem_thread_routine, NULL); + if (r) { + fprintf(stderr, "==%s[%d]%s()==failed to create thread\n", basename(__FILE__), __LINE__, __func__); + return; + } + while (sem_inited==0) { + ; + } +} +#endif + struct tsem_s { +#ifdef SEM_USE_PTHREAD + pthread_mutex_t lock; + pthread_cond_t cond; + volatile int64_t val; +#elif defined(SEM_USE_POSIX) + size_t id; + sem_t *sem; +#elif defined(SEM_USE_SEM) + semaphore_t sem; +#else // SEM_USE_PTHREAD dispatch_semaphore_t sem; +#endif // SEM_USE_PTHREAD + volatile unsigned int valid:1; }; -int tsem_wait(tsem_t *sem); -int tsem_post(tsem_t *sem); -int tsem_destroy(tsem_t *sem); int tsem_init(tsem_t *sem, int pshared, unsigned int value) { - fprintf(stderr, "==%s[%d]%s():[%p]==creating\n", basename(__FILE__), __LINE__, __func__, sem); + // fprintf(stderr, "==%s[%d]%s():[%p]==creating\n", basename(__FILE__), __LINE__, __func__, sem); if (*sem) { fprintf(stderr, "==%s[%d]%s():[%p]==already initialized\n", basename(__FILE__), __LINE__, __func__, sem); abort(); @@ -36,11 +93,61 @@ int tsem_init(tsem_t *sem, int pshared, unsigned int value) { abort(); } +#ifdef SEM_USE_PTHREAD + int r = pthread_mutex_init(&p->lock, NULL); + do { + if (r) break; + r = pthread_cond_init(&p->cond, NULL); + if (r) { + pthread_mutex_destroy(&p->lock); + break; + } + p->val = value; + } while (0); + if (r) { + fprintf(stderr, "==%s[%d]%s():[%p]==not created\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } +#elif defined(SEM_USE_POSIX) + static size_t tick = 0; + do { + size_t id = atomic_add_fetch_64(&tick, 1); + if (id==SEM_VALUE_MAX) { + atomic_store_64(&tick, 0); + id = 0; + } + char name[NAME_MAX-4]; + snprintf(name, sizeof(name), "/t%ld", id); + p->sem = sem_open(name, O_CREAT|O_EXCL, pshared, value); + p->id = id; + if (p->sem!=SEM_FAILED) break; + int e = errno; + if (e==EEXIST) continue; + if (e==EINTR) continue; + fprintf(stderr, "==%s[%d]%s():[%p]==not created[%d]%s\n", basename(__FILE__), __LINE__, __func__, sem, e, strerror(e)); + abort(); + } while (p->sem==SEM_FAILED); +#elif defined(SEM_USE_SEM) + pthread_once(&sem_once, once_init); + if (sem_inited!=1) { + fprintf(stderr, "==%s[%d]%s():[%p]==internal resource init failed\n", basename(__FILE__), __LINE__, __func__, sem); + errno = ENOMEM; + return -1; + } + kern_return_t ret = semaphore_create(sem_port, &p->sem, SYNC_POLICY_FIFO, 0); + if (ret != KERN_SUCCESS) { + fprintf(stderr, "==%s[%d]%s():[%p]==semophore_create failed\n", basename(__FILE__), __LINE__, __func__, sem); + // we fail-fast here, because we have less-doc about semaphore_create for the moment + abort(); + } +#else // SEM_USE_PTHREAD p->sem = dispatch_semaphore_create(value); if (p->sem == NULL) { fprintf(stderr, "==%s[%d]%s():[%p]==not created\n", basename(__FILE__), __LINE__, __func__, sem); abort(); } +#endif // SEM_USE_PTHREAD + p->valid = 1; *sem = p; @@ -58,7 +165,30 @@ int tsem_wait(tsem_t *sem) { fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", basename(__FILE__), __LINE__, __func__, sem); abort(); } +#ifdef SEM_USE_PTHREAD + if (pthread_mutex_lock(&p->lock)) { + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } + p->val -= 1; + if (p->val < 0) { + if (pthread_cond_wait(&p->cond, &p->lock)) { + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } + } + if (pthread_mutex_unlock(&p->lock)) { + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } + return 0; +#elif defined(SEM_USE_POSIX) + return sem_wait(p->sem); +#elif defined(SEM_USE_SEM) + return semaphore_wait(p->sem); +#else // SEM_USE_PTHREAD return dispatch_semaphore_wait(p->sem, DISPATCH_TIME_FOREVER); +#endif // SEM_USE_PTHREAD } int tsem_post(tsem_t *sem) { @@ -71,21 +201,76 @@ int tsem_post(tsem_t *sem) { fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", basename(__FILE__), __LINE__, __func__, sem); abort(); } +#ifdef SEM_USE_PTHREAD + if (pthread_mutex_lock(&p->lock)) { + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } + p->val += 1; + if (p->val <= 0) { + if (pthread_cond_signal(&p->cond)) { + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } + } + if (pthread_mutex_unlock(&p->lock)) { + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } + return 0; +#elif defined(SEM_USE_POSIX) + return sem_post(p->sem); +#elif defined(SEM_USE_SEM) + return semaphore_signal(p->sem); +#else // SEM_USE_PTHREAD return dispatch_semaphore_signal(p->sem); +#endif // SEM_USE_PTHREAD } int tsem_destroy(tsem_t *sem) { - fprintf(stderr, "==%s[%d]%s():[%p]==destroying\n", basename(__FILE__), __LINE__, __func__, sem); + // fprintf(stderr, "==%s[%d]%s():[%p]==destroying\n", basename(__FILE__), __LINE__, __func__, sem); if (!*sem) { - fprintf(stderr, "==%s[%d]%s():[%p]==not initialized\n", basename(__FILE__), __LINE__, __func__, sem); - abort(); + // fprintf(stderr, "==%s[%d]%s():[%p]==not initialized\n", basename(__FILE__), __LINE__, __func__, sem); + // abort(); return 0; } struct tsem_s *p = *sem; if (!p->valid) { - fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", basename(__FILE__), __LINE__, __func__, sem); + // fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", basename(__FILE__), __LINE__, __func__, sem); + // abort(); + return 0; + } +#ifdef SEM_USE_PTHREAD + if (pthread_mutex_lock(&p->lock)) { + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); abort(); } + p->valid = 0; + if (pthread_cond_destroy(&p->cond)) { + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } + if (pthread_mutex_unlock(&p->lock)) { + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } + if (pthread_mutex_destroy(&p->lock)) { + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } +#elif defined(SEM_USE_POSIX) + char name[NAME_MAX-4]; + snprintf(name, sizeof(name), "/t%ld", p->id); + int r = sem_unlink(name); + if (r) { + int e = errno; + fprintf(stderr, "==%s[%d]%s():[%p]==unlink failed[%d]%s\n", basename(__FILE__), __LINE__, __func__, sem, e, strerror(e)); + abort(); + } +#elif defined(SEM_USE_SEM) + semaphore_destroy(sem_port, p->sem); +#else // SEM_USE_PTHREAD +#endif // SEM_USE_PTHREAD p->valid = 0; free(p); diff --git a/src/os/src/darwin/darwinTimer.c b/src/os/src/darwin/darwinTimer.c index 121e2e9427..ee1becc91a 100644 --- a/src/os/src/darwin/darwinTimer.c +++ b/src/os/src/darwin/darwinTimer.c @@ -13,9 +13,82 @@ * along with this program. If not, see . */ +// fail-fast or let-it-crash philosophy +// https://en.wikipedia.org/wiki/Fail-fast +// https://stackoverflow.com/questions/4393197/erlangs-let-it-crash-philosophy-applicable-elsewhere +// experimentally, we follow log-and-crash here + #define _DEFAULT_SOURCE #include "os.h" +#if 1 +#include + +static void (*timer_callback)(int); +static int timer_ms = 0; +static pthread_t timer_thread; +static int timer_kq = -1; +static volatile int timer_stop = 0; + +static void* timer_routine(void *arg) { + (void)arg; + + int r = 0; + struct timespec to = {0}; + to.tv_sec = timer_ms / 1000; + to.tv_nsec = (timer_ms % 1000) * 1000000; + while (!timer_stop) { + struct kevent64_s kev[10] = {0}; + r = kevent64(timer_kq, NULL, 0, kev, sizeof(kev)/sizeof(kev[0]), 0, &to); + if (r!=0) { + fprintf(stderr, "==%s[%d]%s()==kevent64 failed\n", basename(__FILE__), __LINE__, __func__); + abort(); + } + timer_callback(SIGALRM); // just mock + } + + return NULL; +} + +int taosInitTimer(void (*callback)(int), int ms) { + int r = 0; + timer_ms = ms; + timer_callback = callback; + + timer_kq = kqueue(); + if (timer_kq==-1) { + fprintf(stderr, "==%s[%d]%s()==failed to create timer kq\n", basename(__FILE__), __LINE__, __func__); + // since no caller of this func checks the return value for the moment + abort(); + } + + r = pthread_create(&timer_thread, NULL, timer_routine, NULL); + if (r) { + fprintf(stderr, "==%s[%d]%s()==failed to create timer thread\n", basename(__FILE__), __LINE__, __func__); + // since no caller of this func checks the return value for the moment + abort(); + } + return 0; +} + +void taosUninitTimer() { + int r = 0; + timer_stop = 1; + r = pthread_join(timer_thread, NULL); + if (r) { + fprintf(stderr, "==%s[%d]%s()==failed to join timer thread\n", basename(__FILE__), __LINE__, __func__); + // since no caller of this func checks the return value for the moment + abort(); + } + close(timer_kq); + timer_kq = -1; +} + +void taos_block_sigalrm(void) { + // we don't know if there's any specific API for SIGALRM to deliver to specific thread + // this implementation relies on kqueue rather than SIGALRM +} +#else int taosInitTimer(void (*callback)(int), int ms) { signal(SIGALRM, callback); @@ -46,4 +119,5 @@ void taos_block_sigalrm(void) { already_set = 1; } } +#endif diff --git a/src/os/src/darwin/eok.c b/src/os/src/darwin/eok.c index 9c6a263a40..f731aaf330 100644 --- a/src/os/src/darwin/eok.c +++ b/src/os/src/darwin/eok.c @@ -13,13 +13,18 @@ * along with this program. If not, see . */ +// fail-fast or let-it-crash philosophy +// https://en.wikipedia.org/wiki/Fail-fast +// https://stackoverflow.com/questions/4393197/erlangs-let-it-crash-philosophy-applicable-elsewhere +// experimentally, we follow log-and-crash here + #include "eok.h" #include "os.h" #include -#define LET_IT_BE +// #define BALANCE_CHECK_WHEN_CLOSE #ifdef ENABLE_LOG #define D(fmt, ...) fprintf(stderr, "%s[%d]%s(): " fmt "\n", basename(__FILE__), __LINE__, __func__, ##__VA_ARGS__) @@ -99,16 +104,16 @@ struct eok_event_s { typedef struct eoks_s eoks_t; struct eoks_s { pthread_mutex_t lock; - ep_over_kq_t **eoks; // note: this memory leaks when process terminates - int neoks; // we can add an extra api to let user clean - ep_over_kq_t *eoks_free; // currently, we just keep it simple stupid + ep_over_kq_t **eoks; // note: this memory leaks when process terminates + int neoks; // we can add an extra api to let user clean + ep_over_kq_t *eoks_free_list; // currently, we just keep it simple stupid }; static eoks_t eoks = { - .lock = PTHREAD_MUTEX_INITIALIZER, - .eoks = NULL, - .neoks = 0, - .eoks_free = NULL, + .lock = PTHREAD_MUTEX_INITIALIZER, + .eoks = NULL, + .neoks = 0, + .eoks_free_list = NULL, }; #ifdef ENABLE_LOG @@ -760,9 +765,9 @@ static ep_over_kq_t* eoks_alloc(void) { A(0==pthread_mutex_lock(&eoks.lock), ""); do { - if (eoks.eoks_free) { - eok = eoks.eoks_free; - eoks.eoks_free = eok->next; + if (eoks.eoks_free_list) { + eok = eoks.eoks_free_list; + eoks.eoks_free_list = eok->next; A(eoks.eoks, "internal logic error"); A(eok->idx>=0 && eok->idxidx)==NULL, "internal logic error"); @@ -820,10 +825,12 @@ static void eoks_free(ep_over_kq_t *eok) { A(eok->waiting==0, "internal logic error"); eok_event_t *ev = eok->evs_head; + int sv_closed = 0; while (ev) { eok_event_t *next = ev->next; if (ev->fd==eok->sv[0]) { // fd is critical system resource + A(sv_closed==0, "internal logic error"); close(eok->sv[0]); eok->sv[0] = -1; close(eok->sv[1]); @@ -832,11 +839,11 @@ static void eoks_free(ep_over_kq_t *eok) { } else { // user forget calling epoll_ctl(EPOLL_CTL_DEL) before calling epoll_close/close? // calling close(ev->fd) here smells really bad -#ifdef LET_IT_BE +#ifndef BALANCE_CHECK_WHEN_CLOSE // we just let it be and reclaim ev eok_free_ev(eok, ev); #else - // panic otherwise, if LET_IT_BE not defined + // panic otherwise, if BALANCE_CHECK_WHEN_CLOSE is defined A(eok->evs_head==NULL && eok->evs_tail==NULL && eok->evs_count==0, "epfd[%d] fd[%d]: internal logic error: have you epoll_ctl(EPOLL_CTL_DEL) everything before calling epoll_close?", eok->idx, ev->fd); @@ -861,8 +868,8 @@ static void eoks_free(ep_over_kq_t *eok) { close(eok->kq); eok->kq = -1; } - eok->next = eoks.eoks_free; - eoks.eoks_free = eok; + eok->next = eoks.eoks_free_list; + eoks.eoks_free_list = eok; *(eoks.eoks + eok->idx) = NULL; } while (0); A(0==pthread_mutex_unlock(&eoks.lock), ""); From 15b7973e48caf52b7a22e621fe9fa654bfc6a8a6 Mon Sep 17 00:00:00 2001 From: freemine Date: Sun, 24 Jan 2021 09:44:47 +0800 Subject: [PATCH 46/62] typo --- src/os/inc/osDef.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/os/inc/osDef.h b/src/os/inc/osDef.h index f646940d3a..e15c45ad14 100644 --- a/src/os/inc/osDef.h +++ b/src/os/inc/osDef.h @@ -95,7 +95,7 @@ extern "C" { #define threadlocal __thread #else // #define threadlocal - #error please follow with the target platform's thread-local implementation + #error please follow with the thread-local implementation on the target platform #endif #ifdef __cplusplus From a368b7b40afccb1bf6b4568f2db3c06354aa496b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 24 Jan 2021 10:24:44 +0800 Subject: [PATCH 47/62] TD-1207 change int32_t to socket --- src/common/inc/tdataformat.h | 22 +++++++++---------- src/dnode/src/dnodeCheck.c | 16 +++++++------- src/dnode/src/dnodeTelemetry.c | 2 +- src/os/inc/osFile.h | 2 +- src/os/inc/osSocket.h | 7 ++++-- src/os/inc/osWindows.h | 6 +++++ src/os/src/darwin/darwinFile.c | 2 +- src/os/src/darwin/darwinSocket.c | 2 +- src/os/src/detail/osFile.c | 2 +- src/os/src/detail/osSocket.c | 4 ++-- src/os/src/linux/linuxEnv.c | 4 ---- src/os/src/windows/wFile.c | 2 +- src/os/src/windows/wSocket.c | 4 ++-- src/os/src/windows/wSysinfo.c | 2 +- src/plugins/http/inc/httpContext.h | 2 +- src/plugins/http/inc/httpInt.h | 7 +++--- src/plugins/http/src/httpContext.c | 10 ++++++--- src/plugins/http/src/httpServer.c | 12 +++++----- src/rpc/src/rpcTcp.c | 35 ++++++++++++------------------ src/rpc/src/rpcUdp.c | 2 +- src/sync/inc/syncInt.h | 4 ++-- src/sync/inc/syncTcp.h | 4 ++-- src/sync/src/syncArbitrator.c | 6 ++--- src/sync/src/syncMain.c | 8 +++---- src/sync/src/syncTcp.c | 24 +++++++------------- src/util/inc/tsocket.h | 24 +++++++++----------- src/util/src/tnettest.c | 21 +++++++++--------- src/util/src/tsocket.c | 30 ++++++++++++------------- 28 files changed, 128 insertions(+), 138 deletions(-) diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index b6ad99f02a..4999fbb9bf 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -27,23 +27,23 @@ extern "C" { #endif -#define STR_TO_VARSTR(x, str) \ - do { \ - VarDataLenT __len = (int32_t)strlen(str); \ - *(VarDataLenT *)(x) = __len; \ - memcpy(varDataVal(x), (str), __len); \ +#define STR_TO_VARSTR(x, str) \ + do { \ + VarDataLenT __len = (int32_t)strlen(str); \ + *(VarDataLenT *)(x) = __len; \ + memcpy(varDataVal(x), (str), __len); \ } while (0); -#define STR_WITH_MAXSIZE_TO_VARSTR(x, str, _maxs) \ - do { \ +#define STR_WITH_MAXSIZE_TO_VARSTR(x, str, _maxs) \ + do { \ char *_e = stpncpy(varDataVal(x), (str), (_maxs)-VARSTR_HEADER_SIZE); \ - varDataSetLen(x, (_e - (x)-VARSTR_HEADER_SIZE)); \ + varDataSetLen(x, (_e - (x)-VARSTR_HEADER_SIZE)); \ } while (0) #define STR_WITH_SIZE_TO_VARSTR(x, str, _size) \ do { \ - *(VarDataLenT *)(x) = (int32_t)(_size); \ - memcpy(varDataVal(x), (str), (_size)); \ + *(VarDataLenT *)(x) = (int32_t)(_size); \ + memcpy(varDataVal(x), (str), (_size)); \ } while (0); // ----------------- TSDB COLUMN DEFINITION @@ -156,7 +156,7 @@ static FORCE_INLINE int tkeyComparFn(const void *tkey1, const void *tkey2) { * +----------+----------+---------------------------------+---------------------------------+ * | len | sversion | First part | Second part | * +----------+----------+---------------------------------+---------------------------------+ - * + * * NOTE: timestamp in this row structure is TKEY instead of TSKEY */ typedef void *SDataRow; diff --git a/src/dnode/src/dnodeCheck.c b/src/dnode/src/dnodeCheck.c index 2c8dcfdcc9..94d2360950 100644 --- a/src/dnode/src/dnodeCheck.c +++ b/src/dnode/src/dnodeCheck.c @@ -29,11 +29,11 @@ typedef struct { static SCheckItem tsCheckItem[TSDB_CHECK_ITEM_MAX] = {{0}}; int64_t tsMinFreeMemSizeForStart = 0; -static int32_t bindTcpPort(int32_t port) { - int32_t serverSocket; +static int32_t bindTcpPort(int16_t port) { + SOCKET serverSocket; struct sockaddr_in server_addr; - if ((serverSocket = ( int32_t)socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0) { + if ((serverSocket = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0) { dError("socket() fail: %s", strerror(errno)); return -1; } @@ -59,11 +59,11 @@ static int32_t bindTcpPort(int32_t port) { return 0; } -static int32_t bindUdpPort(int32_t port) { - int32_t serverSocket; +static int32_t bindUdpPort(int16_t port) { + SOCKET serverSocket; struct sockaddr_in server_addr; - if ((serverSocket = (int32_t)socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0) { + if ((serverSocket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0) { dError("socket() fail: %s", strerror(errno)); return -1; } @@ -85,9 +85,9 @@ static int32_t bindUdpPort(int32_t port) { static int32_t dnodeCheckNetwork() { int32_t ret; - int32_t startPort = tsServerPort; + int16_t startPort = tsServerPort; - for (int32_t port = startPort; port < startPort + 12; port++) { + for (int16_t port = startPort; port < startPort + 12; port++) { ret = bindTcpPort(port); if (0 != ret) { dError("failed to tcp bind port %d, quit", port); diff --git a/src/dnode/src/dnodeTelemetry.c b/src/dnode/src/dnodeTelemetry.c index d1b7127dda..7a9cf45431 100644 --- a/src/dnode/src/dnodeTelemetry.c +++ b/src/dnode/src/dnodeTelemetry.c @@ -200,7 +200,7 @@ static void sendTelemetryReport() { dTrace("failed to get IP address of " TELEMETRY_SERVER ", reason:%s", strerror(errno)); return; } - int32_t fd = taosOpenTcpClientSocket(ip, TELEMETRY_PORT, 0); + SOCKET fd = taosOpenTcpClientSocket(ip, TELEMETRY_PORT, 0); if (fd < 0) { dTrace("failed to create socket for telemetry, reason:%s", strerror(errno)); return; diff --git a/src/os/inc/osFile.h b/src/os/inc/osFile.h index d72f9cefb9..19cc78472c 100644 --- a/src/os/inc/osFile.h +++ b/src/os/inc/osFile.h @@ -39,7 +39,7 @@ int32_t taosRenameFile(char *fullPath, char *suffix, char delimiter, char **dstP } // TAOS_OS_FUNC_FILE_SENDIFLE -int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t *offset, int64_t size); +int64_t taosSendFile(SOCKET dfd, int32_t sfd, int64_t *offset, int64_t size); int64_t taosFSendFile(FILE *outfile, FILE *infile, int64_t *offset, int64_t size); #ifdef TAOS_RANDOM_FILE_FAIL diff --git a/src/os/inc/osSocket.h b/src/os/inc/osSocket.h index 9683adf45a..111fca712c 100644 --- a/src/os/inc/osSocket.h +++ b/src/os/inc/osSocket.h @@ -37,6 +37,9 @@ extern "C" { #ifndef TAOS_OS_DEF_EPOLL #define TAOS_EPOLL_WAIT_TIME 500 + typedef int32_t SOCKET; + typedef SOCKET EpollFd; + #define EpollClose(pollFd) taosCloseSocket(pollFd) #endif #ifdef TAOS_RANDOM_NETWORK_FAIL @@ -57,13 +60,13 @@ extern "C" { #endif // TAOS_OS_FUNC_SOCKET -int32_t taosSetNonblocking(int32_t sock, int32_t on); +int32_t taosSetNonblocking(SOCKET sock, int32_t on); void taosIgnSIGPIPE(); void taosBlockSIGPIPE(); void taosSetMaskSIGPIPE(); // TAOS_OS_FUNC_SOCKET_SETSOCKETOPT -int32_t taosSetSockOpt(int32_t socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen); +int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen); // TAOS_OS_FUNC_SOCKET_INET uint32_t taosInetAddr(char *ipAddr); diff --git a/src/os/inc/osWindows.h b/src/os/inc/osWindows.h index a984a5ba88..1f3b1b02e3 100644 --- a/src/os/inc/osWindows.h +++ b/src/os/inc/osWindows.h @@ -93,6 +93,12 @@ typedef SOCKET eventfd_t; #define TAOS_OS_DEF_EPOLL #define TAOS_EPOLL_WAIT_TIME 100 + typedef SOCKET EpollFd; + #define EpollClose(pollFd) epoll_close(pollFd) + +#ifndef EPOLLWAKEUP + #define EPOLLWAKEUP (1u << 29) +#endif #define TAOS_OS_DEF_ZU #define PRIzu "ld" diff --git a/src/os/src/darwin/darwinFile.c b/src/os/src/darwin/darwinFile.c index 00c1d8532f..1e77cd68d8 100644 --- a/src/os/src/darwin/darwinFile.c +++ b/src/os/src/darwin/darwinFile.c @@ -51,7 +51,7 @@ int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t co return writeLen; } -int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t* offset, int64_t count) { +int64_t taosSendFile(SOCKET dfd, int32_t sfd, int64_t* offset, int64_t count) { lseek(sfd, (int32_t)(*offset), 0); int64_t writeLen = 0; uint8_t buffer[_SEND_FILE_STEP_] = { 0 }; diff --git a/src/os/src/darwin/darwinSocket.c b/src/os/src/darwin/darwinSocket.c index 9137d4b125..69a4666d34 100644 --- a/src/os/src/darwin/darwinSocket.c +++ b/src/os/src/darwin/darwinSocket.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "os.h" -int taosSetSockOpt(int32_t socketfd, int level, int optname, void *optval, int optlen) { +int taosSetSockOpt(SOCKET socketfd, int level, int optname, void *optval, int optlen) { if (level == SOL_SOCKET && optname == SO_SNDBUF) { return 0; } diff --git a/src/os/src/detail/osFile.c b/src/os/src/detail/osFile.c index 9560ce59bb..bb68622731 100644 --- a/src/os/src/detail/osFile.c +++ b/src/os/src/detail/osFile.c @@ -121,7 +121,7 @@ int64_t taosLSeekImp(int32_t fd, int64_t offset, int32_t whence) { #ifndef TAOS_OS_FUNC_FILE_SENDIFLE -int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t *offset, int64_t size) { +int64_t taosSendFile(SOCKET dfd, int32_t sfd, int64_t *offset, int64_t size) { int64_t leftbytes = size; int64_t sentbytes; diff --git a/src/os/src/detail/osSocket.c b/src/os/src/detail/osSocket.c index f929b3e717..d03e8086bf 100644 --- a/src/os/src/detail/osSocket.c +++ b/src/os/src/detail/osSocket.c @@ -19,7 +19,7 @@ #ifndef TAOS_OS_FUNC_SOCKET -int32_t taosSetNonblocking(int32_t sock, int32_t on) { +int32_t taosSetNonblocking(SOCKET sock, int32_t on) { int32_t flags = 0; if ((flags = fcntl(sock, F_GETFL, 0)) < 0) { uError("fcntl(F_GETFL) error: %d (%s)\n", errno, strerror(errno)); @@ -67,7 +67,7 @@ void taosSetMaskSIGPIPE() { #ifndef TAOS_OS_FUNC_SOCKET_SETSOCKETOPT -int32_t taosSetSockOpt(int32_t socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen) { +int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen) { return setsockopt(socketfd, level, optname, optval, (socklen_t)optlen); } diff --git a/src/os/src/linux/linuxEnv.c b/src/os/src/linux/linuxEnv.c index abcdabcfd5..e3eadbc94b 100644 --- a/src/os/src/linux/linuxEnv.c +++ b/src/os/src/linux/linuxEnv.c @@ -43,7 +43,6 @@ void osInit() { char cmdline[1024]; char* taosGetCmdlineByPID(int pid) { -#if 0 sprintf(cmdline, "/proc/%d/cmdline", pid); FILE* f = fopen(cmdline, "r"); if (f) { @@ -55,7 +54,4 @@ char* taosGetCmdlineByPID(int pid) { fclose(f); } return cmdline; -#else - return ""; -#endif } diff --git a/src/os/src/windows/wFile.c b/src/os/src/windows/wFile.c index e388e5fc67..4ad195dc6f 100644 --- a/src/os/src/windows/wFile.c +++ b/src/os/src/windows/wFile.c @@ -76,7 +76,7 @@ int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t co return writeLen; } -int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t *offset, int64_t count) { +int64_t taosSendFile(SOCKET dfd, int32_t sfd, int64_t *offset, int64_t count) { if (offset != NULL) lseek(sfd, (int32_t)(*offset), 0); int64_t writeLen = 0; diff --git a/src/os/src/windows/wSocket.c b/src/os/src/windows/wSocket.c index 679842a7e9..4e6ee15e77 100644 --- a/src/os/src/windows/wSocket.c +++ b/src/os/src/windows/wSocket.c @@ -34,7 +34,7 @@ void taosWinSocketInit() { } } -int32_t taosSetNonblocking(int32_t sock, int32_t on) { +int32_t taosSetNonblocking(SOCKET sock, int32_t on) { u_long mode; if (on) { mode = 1; @@ -50,7 +50,7 @@ void taosIgnSIGPIPE() {} void taosBlockSIGPIPE() {} void taosSetMaskSIGPIPE() {} -int32_t taosSetSockOpt(int32_t socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen) { +int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen) { if (level == SOL_SOCKET && optname == TCP_KEEPCNT) { return 0; } diff --git a/src/os/src/windows/wSysinfo.c b/src/os/src/windows/wSysinfo.c index 9a79f5a6bf..082aaaf5d8 100644 --- a/src/os/src/windows/wSysinfo.c +++ b/src/os/src/windows/wSysinfo.c @@ -66,7 +66,7 @@ bool taosGetSysMemory(float *memoryUsedMB) { bool taosGetProcMemory(float *memoryUsedMB) { unsigned bytes_used = 0; -#if defined(_WIN32) && defined(_MSC_VER) +#if defined(_WIN64) && defined(_MSC_VER) PROCESS_MEMORY_COUNTERS pmc; HANDLE cur_proc = GetCurrentProcess(); diff --git a/src/plugins/http/inc/httpContext.h b/src/plugins/http/inc/httpContext.h index 260858c5cc..b016da2dd3 100644 --- a/src/plugins/http/inc/httpContext.h +++ b/src/plugins/http/inc/httpContext.h @@ -22,7 +22,7 @@ bool httpInitContexts(); void httpCleanupContexts(); const char *httpContextStateStr(HttpContextState state); -HttpContext *httpCreateContext(int32_t fd); +HttpContext *httpCreateContext(SOCKET fd); bool httpInitContext(HttpContext *pContext); HttpContext *httpGetContext(void * pContext); void httpReleaseContext(HttpContext *pContext, bool clearRes); diff --git a/src/plugins/http/inc/httpInt.h b/src/plugins/http/inc/httpInt.h index 73e9d30924..634468f3cc 100644 --- a/src/plugins/http/inc/httpInt.h +++ b/src/plugins/http/inc/httpInt.h @@ -16,6 +16,7 @@ #ifndef TDENGINE_HTTP_INT_H #define TDENGINE_HTTP_INT_H +#include "os.h" #include #include "pthread.h" #include "semaphore.h" @@ -140,7 +141,7 @@ typedef enum { typedef struct HttpContext { int32_t refCount; - int32_t fd; + SOCKET fd; uint32_t accessTimes; uint32_t lastAccessTime; int32_t state; @@ -167,7 +168,7 @@ typedef struct HttpThread { HttpContext * pHead; pthread_mutex_t threadMutex; bool stop; - int32_t pollFd; + EpollFd pollFd; int32_t numOfContexts; int32_t threadId; char label[HTTP_LABEL_SIZE]; @@ -180,7 +181,7 @@ typedef struct HttpServer { uint16_t serverPort; int8_t stop; int8_t reserve; - int32_t fd; + SOCKET fd; int32_t numOfThreads; int32_t methodScannerLen; int32_t requestNum; diff --git a/src/plugins/http/src/httpContext.c b/src/plugins/http/src/httpContext.c index db77c479c0..f71a84a5af 100644 --- a/src/plugins/http/src/httpContext.c +++ b/src/plugins/http/src/httpContext.c @@ -35,14 +35,18 @@ static void httpRemoveContextFromEpoll(HttpContext *pContext) { HttpThread *pThread = pContext->pThread; if (pContext->fd >= 0) { epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, pContext->fd, NULL); - int32_t fd = atomic_val_compare_exchange_32(&pContext->fd, pContext->fd, -1); +#ifdef WINDOWS + SOCKET fd = atomic_val_compare_exchange_32(&pContext->fd, pContext->fd, -1); +#else + SOCKET fd = atomic_val_compare_exchange_64(&pContext->fd, pContext->fd, -1); +#endif taosCloseSocket(fd); } } static void httpDestroyContext(void *data) { HttpContext *pContext = *(HttpContext **)data; - if (pContext->fd > 0) taosClose(pContext->fd); + if (pContext->fd > 0) taosCloseSocket(pContext->fd); HttpThread *pThread = pContext->pThread; httpRemoveContextFromEpoll(pContext); @@ -106,7 +110,7 @@ bool httpAlterContextState(HttpContext *pContext, HttpContextState srcState, Htt return (atomic_val_compare_exchange_32(&pContext->state, srcState, destState) == srcState); } -HttpContext *httpCreateContext(int32_t fd) { +HttpContext *httpCreateContext(SOCKET fd) { HttpContext *pContext = calloc(1, sizeof(HttpContext)); if (pContext == NULL) return NULL; diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index c4a03d44ea..932f3e3808 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -50,7 +50,7 @@ static void httpStopThread(HttpThread *pThread) { taosCloseSocket(fd); } - taosCloseSocket(pThread->pollFd); + EpollClose(pThread->pollFd); pthread_mutex_destroy(&(pThread->threadMutex)); } @@ -152,7 +152,7 @@ static void httpProcessHttpData(void *param) { } static void *httpAcceptHttpConnection(void *arg) { - int32_t connFd = -1; + SOCKET connFd = -1; struct sockaddr_in clientAddr; int32_t threadId = 0; HttpServer * pServer = &tsHttpServer; @@ -175,7 +175,7 @@ static void *httpAcceptHttpConnection(void *arg) { while (1) { socklen_t addrlen = sizeof(clientAddr); - connFd = (int32_t)accept(pServer->fd, (struct sockaddr *)&clientAddr, &addrlen); + connFd = accept(pServer->fd, (struct sockaddr *)&clientAddr, &addrlen); if (pServer->stop) { httpDebug("http server:%s socket stop, exiting...", pServer->label); break; @@ -227,7 +227,7 @@ static void *httpAcceptHttpConnection(void *arg) { if (epoll_ctl(pThread->pollFd, EPOLL_CTL_ADD, connFd, &event) < 0) { httpError("context:%p, fd:%d, ip:%s, thread:%s, failed to add http fd for epoll, error:%s", pContext, connFd, pContext->ipstr, pThread->label, strerror(errno)); - taosClose(pContext->fd); + taosCloseSocket(pContext->fd); httpReleaseContext(pContext, true); continue; } @@ -265,8 +265,8 @@ bool httpInitConnect() { return false; } - pThread->pollFd = (int32_t)epoll_create(HTTP_MAX_EVENTS); // size does not matter - if (pThread->pollFd < 0) { + pThread->pollFd = (EpollFd)epoll_create(HTTP_MAX_EVENTS); // size does not matter + if (pThread->pollFd <= 0) { httpError("http thread:%s, failed to create HTTP epoll", pThread->label); pthread_mutex_destroy(&(pThread->threadMutex)); return false; diff --git a/src/rpc/src/rpcTcp.c b/src/rpc/src/rpcTcp.c index 1d2dbe281e..acd81e94a4 100644 --- a/src/rpc/src/rpcTcp.c +++ b/src/rpc/src/rpcTcp.c @@ -21,21 +21,14 @@ #include "rpcLog.h" #include "rpcHead.h" #include "rpcTcp.h" -#ifdef WINDOWS -#include "wepoll.h" -#endif - -#ifndef EPOLLWAKEUP - #define EPOLLWAKEUP (1u << 29) -#endif typedef struct SFdObj { void *signature; - int32_t fd; // TCP socket FD - int closedByApp; // 1: already closed by App + SOCKET fd; // TCP socket FD void *thandle; // handle from upper layer, like TAOS uint32_t ip; uint16_t port; + int16_t closedByApp; // 1: already closed by App struct SThreadObj *pThreadObj; struct SFdObj *prev; struct SFdObj *next; @@ -47,7 +40,7 @@ typedef struct SThreadObj { pthread_mutex_t mutex; uint32_t ip; bool stop; - int32_t pollFd; + EpollFd pollFd; int numOfFds; int threadId; char label[TSDB_LABEL_LEN]; @@ -56,7 +49,7 @@ typedef struct SThreadObj { } SThreadObj; typedef struct { - int32_t fd; + SOCKET fd; uint32_t ip; uint16_t port; int8_t stop; @@ -69,7 +62,7 @@ typedef struct { } SServerObj; static void *taosProcessTcpData(void *param); -static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, int32_t fd); +static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, SOCKET fd); static void taosFreeFdObj(SFdObj *pFdObj); static void taosReportBrokenLink(SFdObj *pFdObj); static void *taosAcceptTcpConnection(void *arg); @@ -134,7 +127,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread break; } - pThreadObj->pollFd = (int32_t)epoll_create(10); // size does not matter + pThreadObj->pollFd = (EpollFd)epoll_create(10); // size does not matter if (pThreadObj->pollFd < 0) { tError("%s failed to create TCP epoll", label); code = -1; @@ -227,7 +220,7 @@ void taosCleanUpTcpServer(void *handle) { } static void *taosAcceptTcpConnection(void *arg) { - int32_t connFd = -1; + SOCKET connFd = -1; struct sockaddr_in caddr; int threadId = 0; SThreadObj *pThreadObj; @@ -238,7 +231,7 @@ static void *taosAcceptTcpConnection(void *arg) { while (1) { socklen_t addrlen = sizeof(caddr); - connFd = (int32_t)accept(pServerObj->fd, (struct sockaddr *)&caddr, &addrlen); + connFd = accept(pServerObj->fd, (struct sockaddr *)&caddr, &addrlen); if (pServerObj->stop) { tDebug("%s TCP server stop accepting new connections", pServerObj->label); break; @@ -306,7 +299,7 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int num, void * return NULL; } - pThreadObj->pollFd = (int32_t)epoll_create(10); // size does not matter + pThreadObj->pollFd = (EpollFd)epoll_create(10); // size does not matter if (pThreadObj->pollFd < 0) { tError("%s failed to create TCP client epoll", label); free(pThreadObj); @@ -321,7 +314,7 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int num, void * int code = pthread_create(&(pThreadObj->thread), &thattr, taosProcessTcpData, (void *)(pThreadObj)); pthread_attr_destroy(&thattr); if (code != 0) { - taosCloseSocket(pThreadObj->pollFd); + EpollClose(pThreadObj->pollFd); free(pThreadObj); terrno = TAOS_SYSTEM_ERROR(errno); tError("%s failed to create TCP read data thread(%s)", label, strerror(errno)); @@ -351,8 +344,8 @@ void taosCleanUpTcpClient(void *chandle) { void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port) { SThreadObj * pThreadObj = shandle; - int32_t fd = taosOpenTcpClientSocket(ip, port, pThreadObj->ip); - if (fd < 0) return NULL; + SOCKET fd = taosOpenTcpClientSocket(ip, port, pThreadObj->ip); + if (fd <= 0) return NULL; struct sockaddr_in sin; uint16_t localPort = 0; @@ -526,7 +519,7 @@ static void *taosProcessTcpData(void *param) { if (pThreadObj->stop) break; } - if (pThreadObj->pollFd >=0) taosCloseSocket(pThreadObj->pollFd); + if (pThreadObj->pollFd >=0) EpollClose(pThreadObj->pollFd); while (pThreadObj->pHead) { SFdObj *pFdObj = pThreadObj->pHead; @@ -541,7 +534,7 @@ static void *taosProcessTcpData(void *param) { return NULL; } -static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, int32_t fd) { +static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, SOCKET fd) { struct epoll_event event; SFdObj *pFdObj = (SFdObj *)calloc(sizeof(SFdObj), 1); diff --git a/src/rpc/src/rpcUdp.c b/src/rpc/src/rpcUdp.c index 862d9b1f5c..2599bca075 100644 --- a/src/rpc/src/rpcUdp.c +++ b/src/rpc/src/rpcUdp.c @@ -31,7 +31,7 @@ typedef struct { int index; - int32_t fd; + SOCKET fd; uint16_t port; // peer port uint16_t localPort; // local port char label[TSDB_LABEL_LEN]; // copy from udpConnSet; diff --git a/src/sync/inc/syncInt.h b/src/sync/inc/syncInt.h index eef687d647..e43140d4e6 100644 --- a/src/sync/inc/syncInt.h +++ b/src/sync/inc/syncInt.h @@ -82,8 +82,8 @@ typedef struct SsyncPeer { uint64_t sversion; // track the peer version in retrieve process uint64_t lastFileVer; // track the file version while retrieve uint64_t lastWalVer; // track the wal version while retrieve - int32_t syncFd; - int32_t peerFd; // forward FD + SOCKET syncFd; + SOCKET peerFd; // forward FD int32_t numOfRetrieves; // number of retrieves tried int32_t fileChanged; // a flag to indicate file is changed during retrieving process int32_t refCount; diff --git a/src/sync/inc/syncTcp.h b/src/sync/inc/syncTcp.h index d4674fee6b..b322c3440c 100644 --- a/src/sync/inc/syncTcp.h +++ b/src/sync/inc/syncTcp.h @@ -27,12 +27,12 @@ typedef struct { int32_t bufferSize; void (*processBrokenLink)(int64_t handleId); int32_t (*processIncomingMsg)(int64_t handleId, void *buffer); - void (*processIncomingConn)(int32_t fd, uint32_t ip); + void (*processIncomingConn)(SOCKET fd, uint32_t ip); } SPoolInfo; void *syncOpenTcpThreadPool(SPoolInfo *pInfo); void syncCloseTcpThreadPool(void *); -void *syncAllocateTcpConn(void *, int64_t rid, int32_t connFd); +void *syncAllocateTcpConn(void *, int64_t rid, SOCKET connFd); void syncFreeTcpConn(void *); #ifdef __cplusplus diff --git a/src/sync/src/syncArbitrator.c b/src/sync/src/syncArbitrator.c index 24760ce1a0..9fb6b0ddb7 100644 --- a/src/sync/src/syncArbitrator.c +++ b/src/sync/src/syncArbitrator.c @@ -28,7 +28,7 @@ #include "syncTcp.h" static void arbSignalHandler(int32_t signum, void *sigInfo, void *context); -static void arbProcessIncommingConnection(int32_t connFd, uint32_t sourceIp); +static void arbProcessIncommingConnection(SOCKET connFd, uint32_t sourceIp); static void arbProcessBrokenLink(int64_t rid); static int32_t arbProcessPeerMsg(int64_t rid, void *buffer); static tsem_t tsArbSem; @@ -36,7 +36,7 @@ static void * tsArbTcpPool; typedef struct { char id[TSDB_EP_LEN + 24]; - int32_t nodeFd; + SOCKET nodeFd; void * pConn; } SNodeConn; @@ -106,7 +106,7 @@ int32_t main(int32_t argc, char *argv[]) { return 0; } -static void arbProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { +static void arbProcessIncommingConnection(SOCKET connFd, uint32_t sourceIp) { char ipstr[24]; tinet_ntoa(ipstr, sourceIp); sDebug("peer TCP connection from ip:%s", ipstr); diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index 606809e82b..d698432176 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -45,7 +45,7 @@ static void syncCheckPeerConnection(void *param, void *tmrId); static int32_t syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack, int8_t type, uint16_t tranId); static void syncProcessBrokenLink(int64_t rid); static int32_t syncProcessPeerMsg(int64_t rid, void *buffer); -static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp); +static void syncProcessIncommingConnection(SOCKET connFd, uint32_t sourceIp); static void syncRemovePeer(SSyncPeer *pPeer); static void syncAddArbitrator(SSyncNode *pNode); static void syncFreeNode(void *); @@ -1114,8 +1114,8 @@ static void syncSetupPeerConnection(SSyncPeer *pPeer) { return; } - int32_t connFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0); - if ((int32_t)connFd < 0) { + SOCKET connFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0); + if (connFd <= 0) { sDebug("%s, failed to open tcp socket since %s", pPeer->id, strerror(errno)); taosTmrReset(syncCheckPeerConnection, SYNC_CHECK_INTERVAL, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer); return; @@ -1179,7 +1179,7 @@ static void syncCreateRestoreDataThread(SSyncPeer *pPeer) { } } -static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { +static void syncProcessIncommingConnection(SOCKET connFd, uint32_t sourceIp) { char ipstr[24]; int32_t i; diff --git a/src/sync/src/syncTcp.c b/src/sync/src/syncTcp.c index 1210caa84c..50c52623f7 100644 --- a/src/sync/src/syncTcp.c +++ b/src/sync/src/syncTcp.c @@ -24,18 +24,10 @@ #include "syncInt.h" #include "syncTcp.h" -#ifdef WINDOWS -#include "wepoll.h" -#endif - -#ifndef EPOLLWAKEUP - #define EPOLLWAKEUP (1u << 29) -#endif - typedef struct SThreadObj { pthread_t thread; bool stop; - int32_t pollFd; + SOCKET pollFd; int32_t numOfFds; struct SPoolObj *pPool; } SThreadObj; @@ -45,14 +37,14 @@ typedef struct SPoolObj { SThreadObj **pThread; pthread_t thread; int32_t nextId; - int32_t acceptFd; // FD for accept new connection + SOCKET acceptFd; // FD for accept new connection int8_t stop; } SPoolObj; typedef struct { SThreadObj *pThread; int64_t handleId; - int32_t fd; + SOCKET fd; int32_t closedByApp; } SConnObj; @@ -128,7 +120,7 @@ void syncCloseTcpThreadPool(void *param) { tfree(pPool); } -void *syncAllocateTcpConn(void *param, int64_t rid, int32_t connFd) { +void *syncAllocateTcpConn(void *param, int64_t rid, SOCKET connFd) { struct epoll_event event; SPoolObj *pPool = param; @@ -249,7 +241,7 @@ static void *syncProcessTcpData(void *param) { sDebug("%p TCP epoll thread exits", pThread); - taosCloseSocket(pThread->pollFd); + EpollClose(pThread->pollFd); tfree(pThread); tfree(buffer); return NULL; @@ -264,13 +256,13 @@ static void *syncAcceptPeerTcpConnection(void *argv) { while (1) { struct sockaddr_in clientAddr; socklen_t addrlen = sizeof(clientAddr); - int32_t connFd = (int32_t)accept(pPool->acceptFd, (struct sockaddr *)&clientAddr, &addrlen); + SOCKET connFd = accept(pPool->acceptFd, (struct sockaddr *)&clientAddr, &addrlen); if (pPool->stop) { sDebug("%p TCP server accept is stopped", pPool); break; } - if ((int32_t)connFd < 0) { + if (connFd < 0) { if (errno == EINVAL) { sDebug("%p TCP server accept is exiting...", pPool); break; @@ -298,7 +290,7 @@ static SThreadObj *syncGetTcpThread(SPoolObj *pPool) { if (pThread == NULL) return NULL; pThread->pPool = pPool; - pThread->pollFd = (int32_t)epoll_create(10); // size does not matter + pThread->pollFd = (EpollFd)epoll_create(10); // size does not matter if (pThread->pollFd < 0) { tfree(pThread); return NULL; diff --git a/src/util/inc/tsocket.h b/src/util/inc/tsocket.h index fc0d71c4d8..35b591b61e 100644 --- a/src/util/inc/tsocket.h +++ b/src/util/inc/tsocket.h @@ -24,21 +24,17 @@ extern "C" { #include "wepoll.h" #endif -#ifndef EPOLLWAKEUP - #define EPOLLWAKEUP (1u << 29) -#endif +int32_t taosReadn(SOCKET sock, char *buffer, int32_t len); +int32_t taosWriteMsg(SOCKET fd, void *ptr, int32_t nbytes); +int32_t taosReadMsg(SOCKET fd, void *ptr, int32_t nbytes); +int32_t taosNonblockwrite(SOCKET fd, char *ptr, int32_t nbytes); +int32_t taosCopyFds(SOCKET sfd, int32_t dfd, int64_t len); +int32_t taosSetNonblocking(SOCKET sock, int32_t on); -int32_t taosReadn(int32_t sock, char *buffer, int32_t len); -int32_t taosWriteMsg(int32_t fd, void *ptr, int32_t nbytes); -int32_t taosReadMsg(int32_t fd, void *ptr, int32_t nbytes); -int32_t taosNonblockwrite(int32_t fd, char *ptr, int32_t nbytes); -int32_t taosCopyFds(int32_t sfd, int32_t dfd, int64_t len); -int32_t taosSetNonblocking(int32_t sock, int32_t on); - -int32_t taosOpenUdpSocket(uint32_t localIp, uint16_t localPort); -int32_t taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp); -int32_t taosOpenTcpServerSocket(uint32_t ip, uint16_t port); -int32_t taosKeepTcpAlive(int32_t sockFd); +SOCKET taosOpenUdpSocket(uint32_t localIp, uint16_t localPort); +SOCKET taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp); +SOCKET taosOpenTcpServerSocket(uint32_t ip, uint16_t port); +int32_t taosKeepTcpAlive(SOCKET sockFd); int32_t taosGetFqdn(char *); uint32_t taosGetIpv4FromFqdn(const char *); diff --git a/src/util/src/tnettest.c b/src/util/src/tnettest.c index 70c38d36dc..fe6dfb493d 100644 --- a/src/util/src/tnettest.c +++ b/src/util/src/tnettest.c @@ -39,7 +39,7 @@ typedef struct { static void *taosNetBindUdpPort(void *sarg) { STestInfo *pinfo = (STestInfo *)sarg; int32_t port = pinfo->port; - int32_t serverSocket; + SOCKET serverSocket; char buffer[BUFFER_SIZE]; int32_t iDataNum; socklen_t sin_size; @@ -48,7 +48,7 @@ static void *taosNetBindUdpPort(void *sarg) { struct sockaddr_in server_addr; struct sockaddr_in clientAddr; - if ((serverSocket = (int32_t)socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0) { + if ((serverSocket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0) { uError("failed to create UDP socket since %s", strerror(errno)); return NULL; } @@ -106,12 +106,12 @@ static void *taosNetBindTcpPort(void *sarg) { STestInfo *pinfo = sarg; int32_t port = pinfo->port; - int32_t serverSocket; + SOCKET serverSocket; int32_t addr_len = sizeof(clientAddr); - int32_t client; + SOCKET client; char buffer[BUFFER_SIZE]; - if ((serverSocket = (int32_t)socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0) { + if ((serverSocket = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0) { uError("failed to create TCP socket since %s", strerror(errno)); return NULL; } @@ -133,7 +133,6 @@ static void *taosNetBindTcpPort(void *sarg) { return NULL; } - if (taosKeepTcpAlive(serverSocket) < 0) { uError("failed to set tcp server keep-alive option since %s", strerror(errno)); taosCloseSocket(serverSocket); @@ -148,7 +147,7 @@ static void *taosNetBindTcpPort(void *sarg) { uInfo("TCP server at port:%d is listening", port); while (1) { - client = (int32_t)accept(serverSocket, (struct sockaddr *)&clientAddr, (socklen_t *)&addr_len); + client = accept(serverSocket, (struct sockaddr *)&clientAddr, (socklen_t *)&addr_len); if (client < 0) { uDebug("TCP: failed to accept at port:%d since %s", port, strerror(errno)); continue; @@ -178,10 +177,10 @@ static void *taosNetBindTcpPort(void *sarg) { } static int32_t taosNetCheckTcpPort(STestInfo *info) { - int32_t clientSocket; + SOCKET clientSocket; char buffer[BUFFER_SIZE] = {0}; - if ((clientSocket = (int32_t)socket(AF_INET, SOCK_STREAM, 0)) < 0) { + if ((clientSocket = socket(AF_INET, SOCK_STREAM, 0)) < 0) { uError("failed to create TCP client socket since %s", strerror(errno)); return -1; } @@ -226,14 +225,14 @@ static int32_t taosNetCheckTcpPort(STestInfo *info) { } static int32_t taosNetCheckUdpPort(STestInfo *info) { - int32_t clientSocket; + SOCKET clientSocket; char buffer[BUFFER_SIZE] = {0}; int32_t iDataNum = 0; int32_t bufSize = 1024000; struct sockaddr_in serverAddr; - if ((clientSocket = (int32_t)socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0) { + if ((clientSocket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0) { uError("failed to create udp client socket since %s", strerror(errno)); return -1; } diff --git a/src/util/src/tsocket.c b/src/util/src/tsocket.c index d41c1518b8..3636698162 100644 --- a/src/util/src/tsocket.c +++ b/src/util/src/tsocket.c @@ -102,7 +102,7 @@ uint32_t ip2uint(const char *const ip_addr) { return *((uint32_t *)ip); } -int32_t taosWriteMsg(int32_t fd, void *buf, int32_t nbytes) { +int32_t taosWriteMsg(SOCKET fd, void *buf, int32_t nbytes) { int32_t nleft, nwritten; char * ptr = (char *)buf; @@ -128,7 +128,7 @@ int32_t taosWriteMsg(int32_t fd, void *buf, int32_t nbytes) { return (nbytes - nleft); } -int32_t taosReadMsg(int32_t fd, void *buf, int32_t nbytes) { +int32_t taosReadMsg(SOCKET fd, void *buf, int32_t nbytes) { int32_t nleft, nread; char * ptr = (char *)buf; @@ -159,7 +159,7 @@ int32_t taosReadMsg(int32_t fd, void *buf, int32_t nbytes) { return (nbytes - nleft); } -int32_t taosNonblockwrite(int32_t fd, char *ptr, int32_t nbytes) { +int32_t taosNonblockwrite(SOCKET fd, char *ptr, int32_t nbytes) { taosSetNonblocking(fd, 1); int32_t nleft, nwritten, nready; @@ -201,7 +201,7 @@ int32_t taosNonblockwrite(int32_t fd, char *ptr, int32_t nbytes) { return (nbytes - nleft); } -int32_t taosReadn(int32_t fd, char *ptr, int32_t nbytes) { +int32_t taosReadn(SOCKET fd, char *ptr, int32_t nbytes) { int32_t nread, nready, nleft = nbytes; fd_set fset; @@ -239,9 +239,9 @@ int32_t taosReadn(int32_t fd, char *ptr, int32_t nbytes) { return (nbytes - nleft); } -int32_t taosOpenUdpSocket(uint32_t ip, uint16_t port) { +SOCKET taosOpenUdpSocket(uint32_t ip, uint16_t port) { struct sockaddr_in localAddr; - int32_t sockFd; + SOCKET sockFd; int32_t bufSize = 1024000; uDebug("open udp socket:0x%x:%hu", ip, port); @@ -251,7 +251,7 @@ int32_t taosOpenUdpSocket(uint32_t ip, uint16_t port) { localAddr.sin_addr.s_addr = ip; localAddr.sin_port = (uint16_t)htons(port); - if ((sockFd = (int32_t)socket(AF_INET, SOCK_DGRAM, 0)) <= 2) { + if ((sockFd = socket(AF_INET, SOCK_DGRAM, 0)) <= 2) { uError("failed to open udp socket: %d (%s)", errno, strerror(errno)); taosCloseSocketNoCheck(sockFd); return -1; @@ -279,13 +279,13 @@ int32_t taosOpenUdpSocket(uint32_t ip, uint16_t port) { return sockFd; } -int32_t taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clientIp) { - int32_t sockFd = 0; +SOCKET taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clientIp) { + SOCKET sockFd = 0; int32_t ret; struct sockaddr_in serverAddr, clientAddr; int32_t bufSize = 1024 * 1024; - sockFd = (int32_t)socket(PF_INET, SOCK_STREAM, IPPROTO_TCP); + sockFd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP); if (sockFd <= 2) { uError("failed to open the socket: %d (%s)", errno, strerror(errno)); @@ -346,7 +346,7 @@ int32_t taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t cli return sockFd; } -int32_t taosKeepTcpAlive(int32_t sockFd) { +int32_t taosKeepTcpAlive(SOCKET sockFd) { int32_t alive = 1; if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_KEEPALIVE, (void *)&alive, sizeof(alive)) < 0) { uError("fd:%d setsockopt SO_KEEPALIVE failed: %d (%s)", sockFd, errno, strerror(errno)); @@ -394,9 +394,9 @@ int32_t taosKeepTcpAlive(int32_t sockFd) { return 0; } -int32_t taosOpenTcpServerSocket(uint32_t ip, uint16_t port) { +SOCKET taosOpenTcpServerSocket(uint32_t ip, uint16_t port) { struct sockaddr_in serverAdd; - int32_t sockFd; + SOCKET sockFd; int32_t reuse; uDebug("open tcp server socket:0x%x:%hu", ip, port); @@ -406,7 +406,7 @@ int32_t taosOpenTcpServerSocket(uint32_t ip, uint16_t port) { serverAdd.sin_addr.s_addr = ip; serverAdd.sin_port = (uint16_t)htons(port); - if ((sockFd = (int32_t)socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) <= 2) { + if ((sockFd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) <= 2) { uError("failed to open TCP socket: %d (%s)", errno, strerror(errno)); taosCloseSocketNoCheck(sockFd); return -1; @@ -449,7 +449,7 @@ void tinet_ntoa(char *ipstr, uint32_t ip) { #define COPY_SIZE 32768 // sendfile shall be used -int32_t taosCopyFds(int32_t sfd, int32_t dfd, int64_t len) { +int32_t taosCopyFds(SOCKET sfd, int32_t dfd, int64_t len) { int64_t leftLen; int32_t readLen, writeLen; char temp[COPY_SIZE]; From d6a79a43167e6721fd5fd330739e42cc076ee517 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 24 Jan 2021 11:53:33 +0800 Subject: [PATCH 48/62] compile error in v2019 --- src/plugins/http/src/httpParser.c | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/src/plugins/http/src/httpParser.c b/src/plugins/http/src/httpParser.c index 5ebe9df96f..0de437c761 100644 --- a/src/plugins/http/src/httpParser.c +++ b/src/plugins/http/src/httpParser.c @@ -271,18 +271,15 @@ static int32_t httpOnParseHeaderField(HttpParser *parser, const char *key, const } else if (0 == strcasecmp(key, "Authorization")) { - char * t = NULL; - char * s = NULL; + char t[6] = {0}; + char s[129] = {0}; int32_t bytes = 0; - int32_t n = sscanf(val, "%ms %ms%n", &t, &s, &bytes); - if (n == 2 && t && s && bytes == strlen(val)) { + int32_t n = sscanf(val, "%5s %128s%n", t, s, &bytes); + if (n == 2 && t[0] && s[0] && bytes == strlen(val)) { if (strcmp(t, "Basic") == 0) { free(parser->authContent); - parser->authContent = s; + parser->authContent = strdup(s); parser->authType = HTTP_BASIC_AUTH; - s = NULL; - free(t); - free(s); httpTrace("context:%p, fd:%d, basic auth:%s", pContext, pContext->fd, parser->authContent); int32_t ok = httpParseBasicAuthToken(pContext, parser->authContent, (int32_t)strlen(parser->authContent)); if (ok != 0) { @@ -292,11 +289,8 @@ static int32_t httpOnParseHeaderField(HttpParser *parser, const char *key, const return 0; } else if (strcmp(t, "Taosd") == 0) { free(parser->authContent); - parser->authContent = s; + parser->authContent = strdup(s); parser->authType = HTTP_TAOSD_AUTH; - s = NULL; - free(t); - free(s); httpTrace("context:%p, fd:%d, taosd auth:%s", pContext, pContext->fd, parser->authContent); int32_t ok = httpParseTaosdAuthToken(pContext, parser->authContent, (int32_t)strlen(parser->authContent)); if (ok != 0) { @@ -308,16 +302,12 @@ static int32_t httpOnParseHeaderField(HttpParser *parser, const char *key, const parser->authType = HTTP_INVALID_AUTH; httpError("context:%p, fd:%d, invalid auth, t:%s s:%s", pContext, pContext->fd, t, s); httpOnError(parser, 0, TSDB_CODE_HTTP_INVALID_AUTH_TYPE); - free(t); - free(s); return -1; } } else { parser->authType = HTTP_INVALID_AUTH; httpError("context:%p, fd:%d, parse auth failed, t:%s s:%s", pContext, pContext->fd, t, s); httpOnError(parser, 0, TSDB_CODE_HTTP_INVALID_AUTH_FORMAT); - free(t); - free(s); return -1; } } From d95effe1fbee479d6ece65979694be0a8721672b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 24 Jan 2021 17:31:56 +0800 Subject: [PATCH 49/62] TD-1207 --- tests/script/general/http/grafana.sim | 2 +- tests/script/general/http/restful_full.sim | 8 +++++++- tests/script/unique/http/admin.sim | 4 ++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/script/general/http/grafana.sim b/tests/script/general/http/grafana.sim index 128994640d..ca2f62a368 100644 --- a/tests/script/general/http/grafana.sim +++ b/tests/script/general/http/grafana.sim @@ -84,7 +84,7 @@ endi system_content curl -H 'Authorization: Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJ3d3cudGFvc2RhdGEuY29tIiwicGFzcyI6InRhb3NkYXRhIiwic3ViIjoicm9vdCJ9.xPv3b5odlR7YF8G_QWASjIRbMtA5v4ItToJ35fFgi' -d 'show databases' 127.0.0.1:7111/grafana/root/1/login print 6-> $system_content -if $system_content != @{"status":"error","code":4386,"desc":"invalid type of Authorization"}@ then +if $system_content != @{"status":"error","code":4387,"desc":"invalid format of Authorization"}@ then return -1 endi diff --git a/tests/script/general/http/restful_full.sim b/tests/script/general/http/restful_full.sim index 69f8206347..12f8cc5c38 100644 --- a/tests/script/general/http/restful_full.sim +++ b/tests/script/general/http/restful_full.sim @@ -57,12 +57,18 @@ if $system_content != @{"status":"error","code":3,"desc":"Authentication failure endi #8 -system_content curl -H 'Authorization: Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJ3d3cudGFvc2RhdGEuY29tIiwicGFzcyI6InRhb3NkYXRhIiwic3ViIjoicm9vdCJ9.xPv3b5odlR7YF8G_QWASjIRbMtA5v4ItToJ35fFgi' -d 'show databases' 127.0.0.1:7111/rest/login/root/1 +system_content curl -H 'Authorization: Beare eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9' -d 'show databases' 127.0.0.1:7111/rest/login/root/1 print 8-> $system_content if $system_content != @{"status":"error","code":4386,"desc":"invalid type of Authorization"}@ then return -1 endi +system_content curl -H 'Authorization: Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJ3d3cudGFvc2RhdGEuY29tIiwicGFzcyI6InRhb3NkYXRhIiwic3ViIjoicm9vdCJ9.xPv3b5odlR7YF8G_QWASjIRbMtA5v4ItToJ35fFgi' -d 'show databases' 127.0.0.1:7111/rest/login/root/1 +print 8-> $system_content +if $system_content != @{"status":"error","code":4387,"desc":"invalid format of Authorization"}@ then + return -1 +endi + system_content curl -H 'Authorization: eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJ3d3cudGFvc2RhdGEuY29tIiwicGFzcyI6InRhb3NkYXRhIiwic3ViIjoicm9vdCJ9.xPv3b5odlR7YF8G_QWASjIRbMtA5v4ItToJ35fFgi' -d 'show databases' 127.0.0.1:7111/rest/login/root/1 print 9-> $system_content if $system_content != @{"status":"error","code":4387,"desc":"invalid format of Authorization"}@ then diff --git a/tests/script/unique/http/admin.sim b/tests/script/unique/http/admin.sim index acc28a4e13..1d67a7f86b 100644 --- a/tests/script/unique/http/admin.sim +++ b/tests/script/unique/http/admin.sim @@ -69,13 +69,13 @@ endi system_content curl -H 'Authorization: Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.' -d 'show databases' 127.0.0.1:7111/admin/login/root/1 print 7-> $system_content -if $system_content != @{"status":"error","code":4386,"desc":"invalid type of Authorization"}@ then +if $system_content != @{"status":"error","code":4387,"desc":"invalid format of Authorization"}@ then return -1 endi system_content curl -H 'Authorization: Taosd eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJ3d3cudGFvc2RhdGEuY29tIiwicGFzcyI6InRhb3NkYXRhIiwic3ViIjoicm9vdCJ9.xPv3b5odlR7YF8G_QWASjIRbMtA5v4ItToJ35fFgi' 127.0.0.1:7111/admin/login/root/1 print 8-> $system_content -if $system_content != @{"status":"error","code":4389,"desc":"invalid taosd Authorization"}@ then +if $system_content != @{"status":"error","code":4387,"desc":"invalid format of Authorization"}@ then return -1 endi From 8f488a0f99c3a9a7955f5427a5514c856a18c938 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 24 Jan 2021 17:57:55 +0800 Subject: [PATCH 50/62] compile error in 2019 --- src/plugins/http/src/httpUtil.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/http/src/httpUtil.c b/src/plugins/http/src/httpUtil.c index 17d61e9e3d..7f1e2a94d1 100644 --- a/src/plugins/http/src/httpUtil.c +++ b/src/plugins/http/src/httpUtil.c @@ -37,7 +37,7 @@ void httpTimeToString(time_t t, char *buf, int32_t buflen) { time_t tt = t / 1000; ptm = localtime(&tt); strftime(ts, 31, "%Y-%m-%d %H:%M:%S", ptm); - sprintf(buf, "%s.%03ld", ts, t % 1000); + sprintf(buf, "%s.%03" PRId64, ts, t % 1000); } int32_t httpAddToSqlCmdBuffer(HttpContext *pContext, const char *const format, ...) { From 39bd90915a620750ac3d4c81276d5bb5e0280031 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 25 Jan 2021 11:30:41 +0800 Subject: [PATCH 51/62] TD-1207 compile error in windows --- cmake/define.inc | 3 +++ src/common/inc/tdataformat.h | 18 +++++++++--------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/cmake/define.inc b/cmake/define.inc index 5d4d94ff42..9ee7b34e6a 100755 --- a/cmake/define.inc +++ b/cmake/define.inc @@ -139,6 +139,9 @@ IF (TD_WINDOWS) SET(CMAKE_GENERATOR "NMake Makefiles" CACHE INTERNAL "" FORCE) IF (NOT TD_GODLL) SET(COMMON_FLAGS "/nologo /WX /wd4018 /wd2220 /Oi /Oy- /Gm- /EHsc /MT /GS /Gy /fp:precise /Zc:wchar_t /Zc:forScope /Gd /errorReport:prompt /analyze-") + IF (MSVC AND (MSVC_VERSION GREATER_EQUAL 1900)) + SET(COMMON_FLAGS "${COMMON_FLAGS} /Wv:18") + ENDIF () SET(DEBUG_FLAGS "/Zi /W3 /GL") SET(RELEASE_FLAGS "/W0 /O3 /GL") ENDIF () diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index 4999fbb9bf..e842030b4c 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -27,11 +27,11 @@ extern "C" { #endif -#define STR_TO_VARSTR(x, str) \ - do { \ - VarDataLenT __len = (int32_t)strlen(str); \ - *(VarDataLenT *)(x) = __len; \ - memcpy(varDataVal(x), (str), __len); \ +#define STR_TO_VARSTR(x, str) \ + do { \ + VarDataLenT __len = (VarDataLenT)strlen(str); \ + *(VarDataLenT *)(x) = __len; \ + memcpy(varDataVal(x), (str), __len); \ } while (0); #define STR_WITH_MAXSIZE_TO_VARSTR(x, str, _maxs) \ @@ -40,10 +40,10 @@ extern "C" { varDataSetLen(x, (_e - (x)-VARSTR_HEADER_SIZE)); \ } while (0) -#define STR_WITH_SIZE_TO_VARSTR(x, str, _size) \ - do { \ - *(VarDataLenT *)(x) = (int32_t)(_size); \ - memcpy(varDataVal(x), (str), (_size)); \ +#define STR_WITH_SIZE_TO_VARSTR(x, str, _size) \ + do { \ + *(VarDataLenT *)(x) = (VarDataLenT)(_size); \ + memcpy(varDataVal(x), (str), (_size)); \ } while (0); // ----------------- TSDB COLUMN DEFINITION From 9efd58c62a006ab53f115167c2afad47db5e641b Mon Sep 17 00:00:00 2001 From: slguan Date: Mon, 25 Jan 2021 11:53:55 +0800 Subject: [PATCH 52/62] TD-2837 --- packaging/tools/install_client.sh | 2 +- packaging/tools/make_install.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packaging/tools/install_client.sh b/packaging/tools/install_client.sh index 84ca3b5131..d52428dc83 100755 --- a/packaging/tools/install_client.sh +++ b/packaging/tools/install_client.sh @@ -21,7 +21,7 @@ else cd ${script_dir} script_dir="$(pwd)" data_dir="/var/lib/taos" - log_dir=~/TDengineLog + log_dir=~/TDengine/log fi log_link_dir="/usr/local/taos/log" diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh index 0727fe2a1e..474b6f4619 100755 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -24,7 +24,7 @@ data_dir="/var/lib/taos" if [ "$osType" != "Darwin" ]; then log_dir="/var/log/taos" else - log_dir=~/TDengineLog + log_dir=~/TDengine/log fi data_link_dir="/usr/local/taos/data" From 7625e803d165778ac95627ec1afec949f5264c22 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 25 Jan 2021 13:23:55 +0800 Subject: [PATCH 53/62] TD-2837 --- CMakeLists.txt | 2 +- src/balance/CMakeLists.txt | 8 +-- src/cq/CMakeLists.txt | 24 ++----- src/dnode/CMakeLists.txt | 101 +++++++++-------------------- src/mnode/CMakeLists.txt | 23 ++----- src/os/inc/osDarwin.h | 9 +-- src/plugins/http/src/httpServer.c | 19 +++--- src/plugins/monitor/CMakeLists.txt | 22 ++----- src/rpc/src/rpcTcp.c | 14 +--- src/sync/src/syncTcp.c | 8 --- src/vnode/CMakeLists.txt | 12 +--- src/wal/CMakeLists.txt | 14 +--- 12 files changed, 68 insertions(+), 188 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7ac06c165d..315036d115 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -30,7 +30,7 @@ MESSAGE(STATUS "Community directory: " ${TD_COMMUNITY_DIR}) INCLUDE(cmake/input.inc) INCLUDE(cmake/platform.inc) -IF (TD_WINDOWS) +IF (TD_WINDOWS OR TD_DARWIN) SET(TD_SOMODE_STATIC TRUE) ENDIF () diff --git a/src/balance/CMakeLists.txt b/src/balance/CMakeLists.txt index 17eb34666e..bcb3769087 100644 --- a/src/balance/CMakeLists.txt +++ b/src/balance/CMakeLists.txt @@ -8,10 +8,4 @@ INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/inc) INCLUDE_DIRECTORIES(inc) AUX_SOURCE_DIRECTORY(src SRC) -IF (TD_LINUX OR TD_WINDOWS) - ADD_LIBRARY(balance ${SRC}) -ENDIF () - -IF (TD_DARWIN) - ADD_LIBRARY(balance ${SRC}) -ENDIF () +ADD_LIBRARY(balance ${SRC}) diff --git a/src/cq/CMakeLists.txt b/src/cq/CMakeLists.txt index d5d9116074..73d5eebd6d 100644 --- a/src/cq/CMakeLists.txt +++ b/src/cq/CMakeLists.txt @@ -6,22 +6,10 @@ INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR}/src SRC) -IF (TD_LINUX OR TD_WINDOWS) - ADD_LIBRARY(tcq ${SRC}) - IF (TD_SOMODE_STATIC) - TARGET_LINK_LIBRARIES(tcq tutil common taos_static) - ELSE () - TARGET_LINK_LIBRARIES(tcq tutil common taos) - ENDIF () - ADD_SUBDIRECTORY(test) -ENDIF () - -IF (TD_DARWIN) - ADD_LIBRARY(tcq ${SRC}) - IF (TD_SOMODE_STATIC) - TARGET_LINK_LIBRARIES(tcq tutil common taos_static) - ELSE () - TARGET_LINK_LIBRARIES(tcq tutil common taos) - ENDIF () - ADD_SUBDIRECTORY(test) +ADD_LIBRARY(tcq ${SRC}) +IF (TD_SOMODE_STATIC) + TARGET_LINK_LIBRARIES(tcq tutil common taos_static) +ELSE () + TARGET_LINK_LIBRARIES(tcq tutil common taos) ENDIF () +ADD_SUBDIRECTORY(test) diff --git a/src/dnode/CMakeLists.txt b/src/dnode/CMakeLists.txt index 8e902e1114..14ec98b8f8 100644 --- a/src/dnode/CMakeLists.txt +++ b/src/dnode/CMakeLists.txt @@ -10,79 +10,38 @@ INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/inc) INCLUDE_DIRECTORIES(inc) AUX_SOURCE_DIRECTORY(src SRC) -IF (TD_LINUX OR TD_WINDOWS) - ADD_EXECUTABLE(taosd ${SRC}) - TARGET_LINK_LIBRARIES(taosd mnode monitor http tsdb twal vnode cJson lz4 balance sync) +ADD_EXECUTABLE(taosd ${SRC}) +TARGET_LINK_LIBRARIES(taosd mnode monitor http tsdb twal vnode cJson lz4 balance sync) - IF (TD_SOMODE_STATIC) - TARGET_LINK_LIBRARIES(taosd taos_static) - ELSE () - TARGET_LINK_LIBRARIES(taosd taos) - ENDIF () - - IF (TD_ACCOUNT) - TARGET_LINK_LIBRARIES(taosd account) - ENDIF () - - IF (TD_GRANT) - TARGET_LINK_LIBRARIES(taosd grant) - ENDIF () - - IF (TD_LINUX AND TD_MQTT) - TARGET_LINK_LIBRARIES(taosd mqtt) - ENDIF () - - SET(PREPARE_ENV_CMD "prepare_env_cmd") - SET(PREPARE_ENV_TARGET "prepare_env_target") - ADD_CUSTOM_COMMAND(OUTPUT ${PREPARE_ENV_CMD} - POST_BUILD - COMMAND echo "make test directory" - DEPENDS taosd - COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/cfg/ - COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/log/ - COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/data/ - COMMAND ${CMAKE_COMMAND} -E echo dataDir ${TD_TESTS_OUTPUT_DIR}/data > ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg - COMMAND ${CMAKE_COMMAND} -E echo logDir ${TD_TESTS_OUTPUT_DIR}/log >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg - COMMAND ${CMAKE_COMMAND} -E echo charset UTF-8 >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg - COMMENT "prepare taosd environment") - ADD_CUSTOM_TARGET(${PREPARE_ENV_TARGET} ALL WORKING_DIRECTORY ${TD_EXECUTABLE_OUTPUT_PATH} DEPENDS ${PREPARE_ENV_CMD}) +IF (TD_SOMODE_STATIC) + TARGET_LINK_LIBRARIES(taosd taos_static) +ELSE () + TARGET_LINK_LIBRARIES(taosd taos) ENDIF () -IF (TD_DARWIN) - ADD_EXECUTABLE(taosd ${SRC}) - TARGET_LINK_LIBRARIES(taosd mnode monitor http tsdb twal vnode cJson lz4 balance sync) - - IF (TD_SOMODE_STATIC) - TARGET_LINK_LIBRARIES(taosd taos_static) - ELSE () - TARGET_LINK_LIBRARIES(taosd taos) - ENDIF () - - IF (TD_ACCOUNT) - TARGET_LINK_LIBRARIES(taosd account) - ENDIF () - - IF (TD_GRANT) - TARGET_LINK_LIBRARIES(taosd grant) - ENDIF () - - IF (TD_MQTT) - TARGET_LINK_LIBRARIES(taosd mqtt) - ENDIF () - - # SET(PREPARE_ENV_CMD "prepare_env_cmd") - # SET(PREPARE_ENV_TARGET "prepare_env_target") - # ADD_CUSTOM_COMMAND(OUTPUT ${PREPARE_ENV_CMD} - # POST_BUILD - # COMMAND echo "make test directory" - # DEPENDS taosd - # COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/cfg/ - # COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/log/ - # COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/data/ - # COMMAND ${CMAKE_COMMAND} -E echo dataDir ${TD_TESTS_OUTPUT_DIR}/data > ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg - # COMMAND ${CMAKE_COMMAND} -E echo logDir ${TD_TESTS_OUTPUT_DIR}/log >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg - # COMMAND ${CMAKE_COMMAND} -E echo charset UTF-8 >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg - # COMMENT "prepare taosd environment") - # ADD_CUSTOM_TARGET(${PREPARE_ENV_TARGET} ALL WORKING_DIRECTORY ${TD_EXECUTABLE_OUTPUT_PATH} DEPENDS ${PREPARE_ENV_CMD}) +IF (TD_ACCOUNT) + TARGET_LINK_LIBRARIES(taosd account) ENDIF () +IF (TD_GRANT) + TARGET_LINK_LIBRARIES(taosd grant) +ENDIF () + +IF ((TD_LINUX OR TD_WINDOWS) AND TD_MQTT) + TARGET_LINK_LIBRARIES(taosd mqtt) +ENDIF () + +SET(PREPARE_ENV_CMD "prepare_env_cmd") +SET(PREPARE_ENV_TARGET "prepare_env_target") +ADD_CUSTOM_COMMAND(OUTPUT ${PREPARE_ENV_CMD} + POST_BUILD + COMMAND echo "make test directory" + DEPENDS taosd + COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/cfg/ + COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/log/ + COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/data/ + COMMAND ${CMAKE_COMMAND} -E echo dataDir ${TD_TESTS_OUTPUT_DIR}/data > ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg + COMMAND ${CMAKE_COMMAND} -E echo logDir ${TD_TESTS_OUTPUT_DIR}/log >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg + COMMAND ${CMAKE_COMMAND} -E echo charset UTF-8 >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg + COMMENT "prepare taosd environment") +ADD_CUSTOM_TARGET(${PREPARE_ENV_TARGET} ALL WORKING_DIRECTORY ${TD_EXECUTABLE_OUTPUT_PATH} DEPENDS ${PREPARE_ENV_CMD}) diff --git a/src/mnode/CMakeLists.txt b/src/mnode/CMakeLists.txt index 7b25b1bec8..fffc82c6ef 100644 --- a/src/mnode/CMakeLists.txt +++ b/src/mnode/CMakeLists.txt @@ -1,23 +1,10 @@ CMAKE_MINIMUM_REQUIRED(VERSION 3.5) PROJECT(TDengine) -IF (TD_LINUX OR TD_WINDOWS) - INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) - INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/dnode/inc) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/dnode/inc) - INCLUDE_DIRECTORIES(inc) - AUX_SOURCE_DIRECTORY(src SRC) - - ADD_LIBRARY(mnode ${SRC}) -ENDIF () - -IF (TD_DARWIN) - INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) - INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/dnode/inc) - - INCLUDE_DIRECTORIES(inc) - AUX_SOURCE_DIRECTORY(src SRC) - - ADD_LIBRARY(mnode ${SRC}) -ENDIF () +INCLUDE_DIRECTORIES(inc) +AUX_SOURCE_DIRECTORY(src SRC) +ADD_LIBRARY(mnode ${SRC}) diff --git a/src/os/inc/osDarwin.h b/src/os/inc/osDarwin.h index 0a77ed6895..13f93456f2 100644 --- a/src/os/inc/osDarwin.h +++ b/src/os/inc/osDarwin.h @@ -106,10 +106,11 @@ int64_t tsosStr2int64(char *str); void taos_block_sigalrm(void); - - - - +#define TAOS_OS_DEF_EPOLL + #define TAOS_EPOLL_WAIT_TIME 500 + typedef int32_t SOCKET; + typedef SOCKET EpollFd; + #define EpollClose(pollFd) epoll_close(pollFd) #ifdef __cplusplus } diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index a0e5bede5a..a5f40fdc4c 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -48,7 +48,7 @@ static void httpStopThread(HttpThread *pThread) { pThread->stop = true; r = epoll_ctl(pThread->pollFd, EPOLL_CTL_ADD, sv[0], &ev); if (r) break; - if (1!=send(sv[1], "1", 1, 0)) { + if (1 != send(sv[1], "1", 1, 0)) { r = -1; break; } @@ -56,7 +56,7 @@ static void httpStopThread(HttpThread *pThread) { if (r) { pthread_cancel(pThread->thread); } -#else +#else struct epoll_event event = {.events = EPOLLIN}; eventfd_t fd = eventfd(1, 0); if (fd == -1) { @@ -69,29 +69,26 @@ static void httpStopThread(HttpThread *pThread) { pThread->label, strerror(errno)); pthread_cancel(pThread->thread); } -#endif // __APPLE__ +#endif // __APPLE__ pthread_join(pThread->thread, NULL); + #ifdef __APPLE__ - if (sv[0]!=-1) { + if (sv[0] != -1) { close(sv[0]); sv[0] = -1; } - if (sv[1]!=-1) { + if (sv[1] != -1) { close(sv[1]); sv[1] = -1; } -#else // __APPLE__ +#else // __APPLE__ if (fd != -1) { taosCloseSocket(fd); } -#endif // __APPLE__ +#endif // __APPLE__ -#ifdef __APPLE__ - epoll_close(pThread->pollFd); -#else EpollClose(pThread->pollFd); -#endif pthread_mutex_destroy(&(pThread->threadMutex)); } diff --git a/src/plugins/monitor/CMakeLists.txt b/src/plugins/monitor/CMakeLists.txt index 0e0bc19736..abab07e0cd 100644 --- a/src/plugins/monitor/CMakeLists.txt +++ b/src/plugins/monitor/CMakeLists.txt @@ -6,22 +6,10 @@ INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) AUX_SOURCE_DIRECTORY(./src SRC) -IF (TD_LINUX OR TD_WINDOWS) - ADD_LIBRARY(monitor ${SRC}) +ADD_LIBRARY(monitor ${SRC}) - IF (TD_SOMODE_STATIC) - TARGET_LINK_LIBRARIES(monitor taos_static) - ELSE () - TARGET_LINK_LIBRARIES(monitor taos) - ENDIF () -ENDIF () - -IF (TD_DARWIN) - ADD_LIBRARY(monitor ${SRC}) - - IF (TD_SOMODE_STATIC) - TARGET_LINK_LIBRARIES(monitor taos_static) - ELSE () - TARGET_LINK_LIBRARIES(monitor taos) - ENDIF () +IF (TD_SOMODE_STATIC) + TARGET_LINK_LIBRARIES(monitor taos_static) +ELSE () + TARGET_LINK_LIBRARIES(monitor taos) ENDIF () diff --git a/src/rpc/src/rpcTcp.c b/src/rpc/src/rpcTcp.c index cb8611bdfb..4fe41b1709 100644 --- a/src/rpc/src/rpcTcp.c +++ b/src/rpc/src/rpcTcp.c @@ -314,14 +314,8 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int num, void * int code = pthread_create(&(pThreadObj->thread), &thattr, taosProcessTcpData, (void *)(pThreadObj)); pthread_attr_destroy(&thattr); if (code != 0) { -#ifdef __APPLE__ - if (pThreadObj->pollFd!=-1) { - epoll_close(pThreadObj->pollFd); - pThreadObj->pollFd = -1; - } -#else // __APPLE__ EpollClose(pThreadObj->pollFd); -#endif // __APPLE__ + pThreadObj->pollFd = -1; free(pThreadObj); terrno = TAOS_SYSTEM_ERROR(errno); tError("%s failed to create TCP read data thread(%s)", label, strerror(errno)); @@ -529,14 +523,10 @@ static void *taosProcessTcpData(void *param) { if (pThreadObj->stop) break; } -#ifdef __APPLE__ if (pThreadObj->pollFd >=0) { - epoll_close(pThreadObj->pollFd); + EpollClose(pThreadObj->pollFd); pThreadObj->pollFd = -1; } -#else // __APPLE__ - if (pThreadObj->pollFd >=0) EpollClose(pThreadObj->pollFd); -#endif while (pThreadObj->pHead) { SFdObj *pFdObj = pThreadObj->pHead; diff --git a/src/sync/src/syncTcp.c b/src/sync/src/syncTcp.c index fb3a016791..22bdc7e74e 100644 --- a/src/sync/src/syncTcp.c +++ b/src/sync/src/syncTcp.c @@ -241,11 +241,7 @@ static void *syncProcessTcpData(void *param) { sDebug("%p TCP epoll thread exits", pThread); -#ifdef __APPLE__ - epoll_close(pThread->pollFd); -#else // __APPLE__ EpollClose(pThread->pollFd); -#endif // __APPLE__ tfree(pThread); tfree(buffer); return NULL; @@ -307,11 +303,7 @@ static SThreadObj *syncGetTcpThread(SPoolObj *pPool) { pthread_attr_destroy(&thattr); if (ret != 0) { -#ifdef __APPLE__ - epoll_close(pThread->pollFd); -#else // __APPLE__ EpollClose(pThread->pollFd); -#endif tfree(pThread); return NULL; } diff --git a/src/vnode/CMakeLists.txt b/src/vnode/CMakeLists.txt index 07106598a9..5d77d48ebf 100644 --- a/src/vnode/CMakeLists.txt +++ b/src/vnode/CMakeLists.txt @@ -10,13 +10,5 @@ INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/inc) INCLUDE_DIRECTORIES(inc) AUX_SOURCE_DIRECTORY(src SRC) -IF (TD_LINUX OR TD_WINDOWS) - ADD_LIBRARY(vnode ${SRC}) - TARGET_LINK_LIBRARIES(vnode tsdb tcq) -ENDIF () - -IF (TD_DARWIN) - ADD_LIBRARY(vnode ${SRC}) - TARGET_LINK_LIBRARIES(vnode tsdb tcq) -ENDIF () - +ADD_LIBRARY(vnode ${SRC}) +TARGET_LINK_LIBRARIES(vnode tsdb tcq) diff --git a/src/wal/CMakeLists.txt b/src/wal/CMakeLists.txt index e77d3cd329..6f35cb9ba7 100644 --- a/src/wal/CMakeLists.txt +++ b/src/wal/CMakeLists.txt @@ -4,14 +4,6 @@ PROJECT(TDengine) INCLUDE_DIRECTORIES(inc) AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR}/src SRC) -IF (TD_LINUX OR TD_WINDOWS) - ADD_LIBRARY(twal ${SRC}) - TARGET_LINK_LIBRARIES(twal tutil common) - ADD_SUBDIRECTORY(test) -ENDIF () - -IF (TD_DARWIN) - ADD_LIBRARY(twal ${SRC}) - TARGET_LINK_LIBRARIES(twal tutil common) - ADD_SUBDIRECTORY(test) -ENDIF () +ADD_LIBRARY(twal ${SRC}) +TARGET_LINK_LIBRARIES(twal tutil common) +ADD_SUBDIRECTORY(test) From d5a4e0c36876d8287eb3af805c8b21b3571a8d40 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Mon, 25 Jan 2021 13:59:58 +0800 Subject: [PATCH 54/62] [TD-2840]Add email notification for jenkins --- Jenkinsfile | 78 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index 73bb832d8e..74cef8f954 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -176,5 +176,83 @@ pipeline { } } } + post { + success { + emailext ( + subject: "PR-result: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'", + body: ''' + + + + + + + + + + + + +

+ 构建信息 +
+
    +
    +
  • 构建名称>>分支：${PROJECT_NAME}
    • +
  • 构建结果： Successful
    • +
  • 构建编号：${BUILD_NUMBER}
    • +
  • 触发用户：${CAUSE}
    • +
  • 提交信息：${CHANGE_TITLE}
    • +
  • 构建地址：${BUILD_URL}
    • +
  • 构建日志：${BUILD_URL}console
    • +
  • 变更集：${JELLY_SCRIPT}
    • +
    +
+
+ + ''', + to: "${CHANGE_AUTHOR_EMAIL}", + from: "support@taosdata.com" + ) + } + failure { + emailext ( + subject: "PR-result: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'", + body: ''' + + + + + + + + + + + + +

+ 构建信息 +
+
    +
    +
  • 构建名称>>分支：${PROJECT_NAME}
    • +
  • 构建结果： Successful
    • +
  • 构建编号：${BUILD_NUMBER}
    • +
  • 触发用户：${CAUSE}
    • +
  • 提交信息：${CHANGE_TITLE}
    • +
  • 构建地址：${BUILD_URL}
    • +
  • 构建日志：${BUILD_URL}console
    • +
  • 变更集：${JELLY_SCRIPT}
    • +
    +
+
+ + ''', + to: "${CHANGE_AUTHOR_EMAIL}", + from: "support@taosdata.com" + ) + } + } } From 7ef5e333b612fac4bb8f5df5c538c6eb259ae148 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Mon, 25 Jan 2021 14:16:44 +0800 Subject: [PATCH 55/62] fix some error --- Jenkinsfile | 4 +-- tests/pytest/tt.py | 27 +++++++++++++++++++ .../general/connection/test_old_data.sim | 6 ++--- 3 files changed, 32 insertions(+), 5 deletions(-) create mode 100644 tests/pytest/tt.py diff --git a/Jenkinsfile b/Jenkinsfile index 74cef8f954..0f07aeb659 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -211,7 +211,7 @@ pipeline { ''', - to: "${CHANGE_AUTHOR_EMAIL}", + to: "${env.CHANGE_AUTHOR_EMAIL}", from: "support@taosdata.com" ) } @@ -249,7 +249,7 @@ pipeline { ''', - to: "${CHANGE_AUTHOR_EMAIL}", + to: "${env.CHANGE_AUTHOR_EMAIL}", from: "support@taosdata.com" ) } diff --git a/tests/pytest/tt.py b/tests/pytest/tt.py new file mode 100644 index 0000000000..70bc70f581 --- /dev/null +++ b/tests/pytest/tt.py @@ -0,0 +1,27 @@ +import taos +import datetime +import random +# host= '127.0.0.1' +# user = 'root' +# password = 'taosdata' +# conn = taos.connect( +# host, +# user, +# password, +# ) +# cl = conn.cursor() +# cl.execute("select first(_c0),last(_c0) from test.st0 " ) +# dd1=datetime.datetime.now()+ datetime.timedelta(days=5) +# sql = 'select count(*) from test.st0 where ts <= ' + "'" + str(dd1) + "'" +# print(sql) +# cl.execute(sql) +# for data in cl: +# print(data[0]) +# # print(cl[0],cl[1]) +# # d1 = data[0] +# # d2 = data[1] + +# # print(d1+(d2-d1)/2) +# print(random.randrange(-100,100)) +# random.randrange(-100,100) +print(random.getrandbits(1)) \ No newline at end of file diff --git a/tests/script/general/connection/test_old_data.sim b/tests/script/general/connection/test_old_data.sim index 83df850f0b..c7cb1eeb77 100644 --- a/tests/script/general/connection/test_old_data.sim +++ b/tests/script/general/connection/test_old_data.sim @@ -27,6 +27,6 @@ if $rows != 7 then endi -system sh/exec.sh -n dnode1 -s stop -x SIGINT -system sh/exec.sh -n dnode2 -s stop -x SIGINT -system sh/exec.sh -n dnode3 -s stop -x SIGINT \ No newline at end of file +# system sh/exec.sh -n dnode1 -s stop -x SIGINT +# system sh/exec.sh -n dnode2 -s stop -x SIGINT +# system sh/exec.sh -n dnode3 -s stop -x SIGINT \ No newline at end of file From 85fbc21f7dfdb07cf49bc8a52e8b49f95a463a96 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Mon, 25 Jan 2021 14:23:55 +0800 Subject: [PATCH 56/62] revert --- Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0f07aeb659..516b179dce 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -176,7 +176,8 @@ pipeline { } } } - post { + post { + success { emailext ( subject: "PR-result: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'", From c906dd5c1e2c3b9340a50fac0610b4d282d565a3 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Mon, 25 Jan 2021 14:26:13 +0800 Subject: [PATCH 57/62] revert --- tests/pytest/tt.py | 27 ------------------- .../general/connection/test_old_data.sim | 6 ++--- 2 files changed, 3 insertions(+), 30 deletions(-) delete mode 100644 tests/pytest/tt.py diff --git a/tests/pytest/tt.py b/tests/pytest/tt.py deleted file mode 100644 index 70bc70f581..0000000000 --- a/tests/pytest/tt.py +++ /dev/null @@ -1,27 +0,0 @@ -import taos -import datetime -import random -# host= '127.0.0.1' -# user = 'root' -# password = 'taosdata' -# conn = taos.connect( -# host, -# user, -# password, -# ) -# cl = conn.cursor() -# cl.execute("select first(_c0),last(_c0) from test.st0 " ) -# dd1=datetime.datetime.now()+ datetime.timedelta(days=5) -# sql = 'select count(*) from test.st0 where ts <= ' + "'" + str(dd1) + "'" -# print(sql) -# cl.execute(sql) -# for data in cl: -# print(data[0]) -# # print(cl[0],cl[1]) -# # d1 = data[0] -# # d2 = data[1] - -# # print(d1+(d2-d1)/2) -# print(random.randrange(-100,100)) -# random.randrange(-100,100) -print(random.getrandbits(1)) \ No newline at end of file diff --git a/tests/script/general/connection/test_old_data.sim b/tests/script/general/connection/test_old_data.sim index c7cb1eeb77..83df850f0b 100644 --- a/tests/script/general/connection/test_old_data.sim +++ b/tests/script/general/connection/test_old_data.sim @@ -27,6 +27,6 @@ if $rows != 7 then endi -# system sh/exec.sh -n dnode1 -s stop -x SIGINT -# system sh/exec.sh -n dnode2 -s stop -x SIGINT -# system sh/exec.sh -n dnode3 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT \ No newline at end of file From c745ba0b81308ba9185848cd00a54e29e5827188 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Mon, 25 Jan 2021 14:51:57 +0800 Subject: [PATCH 58/62] fix bug --- src/client/src/tscSQLParser.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 6b24dcd858..3ffb48b5ca 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -4301,9 +4301,8 @@ static void doAddJoinTagsColumnsIntoTagList(SSqlCmd* pCmd, SQueryInfo* pQueryInf } static int32_t validateTagCondExpr(SSqlCmd* pCmd, tExprNode *p) { - const char *msg1 = "tag type mismatch"; - const char *msg2 = "invalid tag operator"; - const char* msg3 = "not supported filter condition"; + const char *msg1 = "invalid tag operator"; + const char* msg2 = "not supported filter condition"; do { if (p->nodeType != TSQL_NODE_EXPR) { @@ -4315,7 +4314,7 @@ static int32_t validateTagCondExpr(SSqlCmd* pCmd, tExprNode *p) { } if (IS_ARITHMETIC_OPTR(p->_node.optr)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } if (!IS_RELATION_OPTR(p->_node.optr)) { @@ -4330,14 +4329,14 @@ static int32_t validateTagCondExpr(SSqlCmd* pCmd, tExprNode *p) { break; } - vVariant = p->_node.pLeft->pVal->nType; + vVariant = p->_node.pLeft->pVal; schemaType = p->_node.pRight->pSchema->type; } else if (p->_node.pLeft->nodeType == TSQL_NODE_COL && p->_node.pRight->nodeType == TSQL_NODE_VALUE) { if (!p->_node.pLeft->pSchema) { break; } - vVariant = p->_node.pRight->pVal->nType; + vVariant = p->_node.pRight->pVal; schemaType = p->_node.pLeft->pSchema->type; } else { break; @@ -4351,12 +4350,12 @@ static int32_t validateTagCondExpr(SSqlCmd* pCmd, tExprNode *p) { int32_t retVal = TSDB_CODE_SUCCESS; if (schemaType == TSDB_DATA_TYPE_BINARY) { - char *tmp = (int64_t)calloc(1, (vVariant->nLen + 1) + TSDB_NCHAR_SIZE); + char *tmp = calloc(1, (vVariant->nLen + 1) + TSDB_NCHAR_SIZE); retVal = tVariantDump(vVariant, tmp, schemaType, false); free(tmp); } else if (schemaType == TSDB_DATA_TYPE_NCHAR) { // pRight->val.nLen + 1 is larger than the actual nchar string length - char *tmp = (int64_t)calloc(1, (vVariant->nLen + 1) * TSDB_NCHAR_SIZE); + char *tmp = calloc(1, (vVariant->nLen + 1) * TSDB_NCHAR_SIZE); retVal = tVariantDump(vVariant, tmp, schemaType, false); free(tmp); } else { @@ -4365,7 +4364,7 @@ static int32_t validateTagCondExpr(SSqlCmd* pCmd, tExprNode *p) { } if (retVal != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } }while (0); From 12f9959702a263b6142e1e496adb1742223bd790 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Mon, 25 Jan 2021 14:53:05 +0800 Subject: [PATCH 59/62] fix bug --- src/client/src/tscSQLParser.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 3ffb48b5ca..0ccc54ff0d 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -4350,7 +4350,7 @@ static int32_t validateTagCondExpr(SSqlCmd* pCmd, tExprNode *p) { int32_t retVal = TSDB_CODE_SUCCESS; if (schemaType == TSDB_DATA_TYPE_BINARY) { - char *tmp = calloc(1, (vVariant->nLen + 1) + TSDB_NCHAR_SIZE); + char *tmp = calloc(1, vVariant->nLen + TSDB_NCHAR_SIZE); retVal = tVariantDump(vVariant, tmp, schemaType, false); free(tmp); } else if (schemaType == TSDB_DATA_TYPE_NCHAR) { From 5d976a26a5d8457bc78f667a66385559e017b900 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Mon, 25 Jan 2021 17:00:22 +0800 Subject: [PATCH 60/62] fix bug --- src/client/src/tscSQLParser.c | 9 ++++++--- src/client/src/tscServer.c | 4 +++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 0ccc54ff0d..a23c9b75bb 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -921,16 +921,19 @@ int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SStrToken* pTableNam return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } } else { // get current DB name first, and then set it into path - char* t = getCurrentDBName(pSql); + char* t = cloneCurrentDBName(pSql); if (strlen(t) == 0) { return TSDB_CODE_TSC_DB_NOT_SELECTED; } code = tNameFromString(&pTableMetaInfo->name, t, T_NAME_ACCT | T_NAME_DB); if (code != 0) { + free(t); return TSDB_CODE_TSC_DB_NOT_SELECTED; } + free(t); + if (pTableName->n >= TSDB_TABLE_NAME_LEN) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } @@ -1244,8 +1247,8 @@ static bool has(SArray* pFieldList, int32_t startIdx, const char* name) { static char* getAccountId(SSqlObj* pSql) { return pSql->pTscObj->acctId; } -static char* getCurrentDBName(SSqlObj* pSql) { - return pSql->pTscObj->db; +static char* cloneCurrentDBName(SSqlObj* pSql) { + return strdup(pSql->pTscObj->db); } /* length limitation, strstr cannot be applied */ diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 8403876566..1ea19d5198 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -2165,7 +2165,9 @@ int tscProcessUseDbRsp(SSqlObj *pSql) { } int tscProcessDropDbRsp(SSqlObj *pSql) { - pSql->pTscObj->db[0] = 0; + //TODO LOCK DB WHEN MODIFY IT + //pSql->pTscObj->db[0] = 0; + taosHashEmpty(tscTableMetaInfo); return 0; } From acaefc098b6db21ad8a471f925c1e7489f62c9d2 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 25 Jan 2021 17:00:54 +0800 Subject: [PATCH 61/62] [TD-2832]: use httpDecodeUrl to parse reserved characters after percent-encoding --- src/plugins/http/src/httpParser.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/plugins/http/src/httpParser.c b/src/plugins/http/src/httpParser.c index b844834537..7fdd0a64f6 100644 --- a/src/plugins/http/src/httpParser.c +++ b/src/plugins/http/src/httpParser.c @@ -537,7 +537,7 @@ char *httpDecodeUrl(const char *enc) { dec = str.str; str.str = NULL; } - httpCleanupString(&str); + //httpCleanupString(&str); return dec; } @@ -648,7 +648,7 @@ static int32_t httpParserOnTarget(HttpParser *parser, HTTP_PARSER_STATE state, c } break; } - parser->target = strdup(parser->str.str); + parser->target = httpDecodeUrl(parser->str.str); if (!parser->target) { httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); ok = -1; @@ -717,6 +717,10 @@ static int32_t httpParserOnVersion(HttpParser *parser, HTTP_PARSER_STATE state, if (parser->method) { ok = httpOnRequestLine(parser, parser->method, parser->target, parser->version); + if (parser->target) { + free(parser->target); + parser->target = NULL; + } } httpClearString(&parser->str); From 0369b0bfa3c07ce5163fe3aab425bf50c91d8df9 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Mon, 25 Jan 2021 17:48:04 +0800 Subject: [PATCH 62/62] add lock for db access --- src/client/src/tscLocal.c | 3 +++ src/client/src/tscSQLParser.c | 8 ++++++-- src/client/src/tscServer.c | 21 ++++++++++++++++++--- 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index 4b1ab47730..bb015bce3d 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -749,7 +749,10 @@ static int32_t tscProcessCurrentUser(SSqlObj *pSql) { static int32_t tscProcessCurrentDB(SSqlObj *pSql) { char db[TSDB_DB_NAME_LEN] = {0}; + + pthread_mutex_lock(&pSql->pTscObj->mutex); extractDBName(pSql->pTscObj->db, db); + pthread_mutex_unlock(&pSql->pTscObj->mutex); SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, pSql->cmd.clauseIndex); diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index a23c9b75bb..fb48dc4519 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -60,7 +60,7 @@ static int32_t setShowInfo(SSqlObj* pSql, SSqlInfo* pInfo); static char* getAccountId(SSqlObj* pSql); static bool has(SArray* pFieldList, int32_t startIdx, const char* name); -static char* getCurrentDBName(SSqlObj* pSql); +static char* cloneCurrentDBName(SSqlObj* pSql); static bool hasSpecifyDB(SStrToken* pTableName); static bool validateTableColumnInfo(SArray* pFieldList, SSqlCmd* pCmd); static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pCmd); @@ -1248,7 +1248,11 @@ static bool has(SArray* pFieldList, int32_t startIdx, const char* name) { static char* getAccountId(SSqlObj* pSql) { return pSql->pTscObj->acctId; } static char* cloneCurrentDBName(SSqlObj* pSql) { - return strdup(pSql->pTscObj->db); + pthread_mutex_lock(&pSql->pTscObj->mutex); + char *p = strdup(pSql->pTscObj->db); + pthread_mutex_unlock(&pSql->pTscObj->mutex); + + return p; } /* length limitation, strstr cannot be applied */ diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 1ea19d5198..6035a270c5 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1250,8 +1250,10 @@ int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) { STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); - if (tNameIsEmpty(&pTableMetaInfo->name)) { - tstrncpy(pShowMsg->db, pObj->db, sizeof(pShowMsg->db)); + if (tNameIsEmpty(&pTableMetaInfo->name)) { + pthread_mutex_lock(&pObj->mutex); + tstrncpy(pShowMsg->db, pObj->db, sizeof(pShowMsg->db)); + pthread_mutex_unlock(&pObj->mutex); } else { tNameGetFullDbName(&pTableMetaInfo->name, pShowMsg->db); } @@ -1611,9 +1613,14 @@ int tscBuildConnectMsg(SSqlObj *pSql, SSqlInfo *pInfo) { // TODO refactor full_name char *db; // ugly code to move the space + + pthread_mutex_lock(&pObj->mutex); db = strstr(pObj->db, TS_PATH_DELIMITER); + db = (db == NULL) ? pObj->db : db + 1; tstrncpy(pConnect->db, db, sizeof(pConnect->db)); + pthread_mutex_unlock(&pObj->mutex); + tstrncpy(pConnect->clientVersion, version, sizeof(pConnect->clientVersion)); tstrncpy(pConnect->msgVersion, "", sizeof(pConnect->msgVersion)); @@ -2131,10 +2138,13 @@ int tscProcessConnectRsp(SSqlObj *pSql) { SConnectRsp *pConnect = (SConnectRsp *)pRes->pRsp; tstrncpy(pObj->acctId, pConnect->acctId, sizeof(pObj->acctId)); // copy acctId from response + + pthread_mutex_lock(&pObj->mutex); int32_t len = sprintf(temp, "%s%s%s", pObj->acctId, TS_PATH_DELIMITER, pObj->db); assert(len <= sizeof(pObj->db)); tstrncpy(pObj->db, temp, sizeof(pObj->db)); + pthread_mutex_unlock(&pObj->mutex); if (pConnect->epSet.numOfEps > 0) { tscEpSetHtons(&pConnect->epSet); @@ -2161,7 +2171,12 @@ int tscProcessConnectRsp(SSqlObj *pSql) { int tscProcessUseDbRsp(SSqlObj *pSql) { STscObj * pObj = pSql->pTscObj; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); - return tNameExtractFullName(&pTableMetaInfo->name, pObj->db); + + pthread_mutex_lock(&pObj->mutex); + int ret = tNameExtractFullName(&pTableMetaInfo->name, pObj->db); + pthread_mutex_unlock(&pObj->mutex); + + return ret; } int tscProcessDropDbRsp(SSqlObj *pSql) {