From 38c1f7d48d0b692e29a32c657f948b9901f8ead3 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 16 Oct 2023 20:20:04 +0800 Subject: [PATCH 01/11] fix: add lock for trans --- source/dnode/mnode/impl/inc/mndDef.h | 2 +- source/dnode/mnode/impl/src/mndTrans.c | 12 +++--------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index f4236964ca..e5f46c03cd 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -181,7 +181,7 @@ typedef struct { SArray* pRpcArray; SRWLatch lockRpcArray; int64_t mTraceId; - TdThreadMutex mutex; + int8_t lock; } STrans; typedef struct { diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 1d8dd5e345..f8d6cba171 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -470,7 +470,6 @@ void mndTransDropData(STrans *pTrans) { pTrans->param = NULL; pTrans->paramLen = 0; } - (void)taosThreadMutexDestroy(&pTrans->mutex); } static int32_t mndTransDelete(SSdb *pSdb, STrans *pTrans, bool callFunc) { @@ -543,10 +542,6 @@ STrans *mndAcquireTrans(SMnode *pMnode, int32_t transId) { STrans *pTrans = sdbAcquire(pMnode->pSdb, SDB_TRANS, &transId); if (pTrans == NULL) { terrno = TSDB_CODE_MND_TRANS_NOT_EXIST; - } else { - #ifdef WINDOWS - taosThreadMutexInit(&pTrans->mutex, NULL); - #endif } return pTrans; } @@ -582,7 +577,6 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnConflct conflict, pTrans->pRpcArray = taosArrayInit(1, sizeof(SRpcHandleInfo)); pTrans->mTraceId = pReq ? TRACE_GET_ROOTID(&pReq->info.traceId) : tGenIdPI64(); taosInitRWLatch(&pTrans->lockRpcArray); - taosThreadMutexInit(&pTrans->mutex, NULL); if (pTrans->redoActions == NULL || pTrans->undoActions == NULL || pTrans->commitActions == NULL || pTrans->pRpcArray == NULL) { @@ -1264,10 +1258,10 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans) int32_t numOfActions = taosArrayGetSize(pTrans->redoActions); if (numOfActions == 0) return code; - taosThreadMutexLock(&pTrans->mutex); + if (atomic_val_compare_exchange_8(&pTrans->lock, 0, 1) != 0) return code; if (pTrans->redoActionPos >= numOfActions) { - taosThreadMutexUnlock(&pTrans->mutex); + atomic_store_8(&pTrans->lock, 0); return code; } @@ -1339,7 +1333,7 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans) } } - taosThreadMutexUnlock(&pTrans->mutex); + atomic_store_8(&pTrans->lock, 0); return code; } From a30eedafecc8a320afb0fac4dbb25a115c532ecd Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 17 Oct 2023 11:06:06 +0800 Subject: [PATCH 02/11] fix: trans mutex init --- source/dnode/mnode/impl/inc/mndDef.h | 2 +- source/dnode/mnode/impl/src/mndTrans.c | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index e5f46c03cd..f4236964ca 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -181,7 +181,7 @@ typedef struct { SArray* pRpcArray; SRWLatch lockRpcArray; int64_t mTraceId; - int8_t lock; + TdThreadMutex mutex; } STrans; typedef struct { diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index f8d6cba171..76d6e21c64 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -428,6 +428,8 @@ static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans) { mInfo("trans:%d, perform insert action, row:%p stage:%s, callfunc:1, startFunc:%d", pTrans->id, pTrans, mndTransStr(pTrans->stage), pTrans->startFunc); + taosThreadMutexInit(&pTrans->mutex, NULL); + if (pTrans->startFunc > 0) { TransCbFp fp = mndTransGetCbFp(pTrans->startFunc); if (fp) { @@ -1258,10 +1260,10 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans) int32_t numOfActions = taosArrayGetSize(pTrans->redoActions); if (numOfActions == 0) return code; - if (atomic_val_compare_exchange_8(&pTrans->lock, 0, 1) != 0) return code; + taosThreadMutexLock(&pTrans->mutex); if (pTrans->redoActionPos >= numOfActions) { - atomic_store_8(&pTrans->lock, 0); + taosThreadMutexUnlock(&pTrans->mutex); return code; } @@ -1333,7 +1335,7 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans) } } - atomic_store_8(&pTrans->lock, 0); + taosThreadMutexUnlock(&pTrans->mutex); return code; } From 29c6daaff5efd137cc9d16f5aaa5a5869a766e45 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 17 Oct 2023 11:12:52 +0800 Subject: [PATCH 03/11] fix: trans mutex init --- source/dnode/mnode/impl/src/mndTrans.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 76d6e21c64..29a8ae1f29 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -472,6 +472,7 @@ void mndTransDropData(STrans *pTrans) { pTrans->param = NULL; pTrans->paramLen = 0; } + (void)taosThreadMutexDestroy(&pTrans->mutex); } static int32_t mndTransDelete(SSdb *pSdb, STrans *pTrans, bool callFunc) { @@ -579,6 +580,7 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnConflct conflict, pTrans->pRpcArray = taosArrayInit(1, sizeof(SRpcHandleInfo)); pTrans->mTraceId = pReq ? TRACE_GET_ROOTID(&pReq->info.traceId) : tGenIdPI64(); taosInitRWLatch(&pTrans->lockRpcArray); + taosThreadMutexInit(&pTrans->mutex, NULL); if (pTrans->redoActions == NULL || pTrans->undoActions == NULL || pTrans->commitActions == NULL || pTrans->pRpcArray == NULL) { From 720982374cc222c205de4c185a6ad6980240be13 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Fri, 20 Oct 2023 10:36:52 +0800 Subject: [PATCH 04/11] fix: union all cast func resType check error --- source/libs/parser/src/parAstCreater.c | 4 ++++ source/libs/parser/src/parTranslater.c | 5 +++-- source/libs/parser/src/parUtil.c | 2 +- tests/system-test/2-query/varchar.py | 6 ++++++ 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 12062e0d4a..e8614147f4 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -1379,6 +1379,10 @@ SNode* createColumnDefNode(SAstCreateContext* pCxt, SToken* pColName, SDataType if (!checkColumnName(pCxt, pColName) || !checkComment(pCxt, pComment, false)) { return NULL; } + if (IS_VAR_DATA_TYPE(dataType.type) && dataType.bytes == 0) { + pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN); + return NULL; + } SColumnDefNode* pCol = (SColumnDefNode*)nodesMakeNode(QUERY_NODE_COLUMN_DEF); CHECK_OUT_OF_MEM(pCol); COPY_STRING_FORM_ID_TOKEN(pCol->colName, pColName); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 42ab2f7a2f..756a1f867b 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2185,11 +2185,12 @@ static int32_t createCastFunc(STranslateContext* pCxt, SNode* pExpr, SDataType d nodesDestroyNode((SNode*)pFunc); return TSDB_CODE_OUT_OF_MEMORY; } - if (TSDB_CODE_SUCCESS != getFuncInfo(pCxt, pFunc)) { + int32_t code = getFuncInfo(pCxt, pFunc); + if (TSDB_CODE_SUCCESS != code) { nodesClearList(pFunc->pParameterList); pFunc->pParameterList = NULL; nodesDestroyNode((SNode*)pFunc); - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)pExpr)->aliasName); + return code; } *pCast = (SNode*)pFunc; return TSDB_CODE_SUCCESS; diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 1b8bac4cbc..0ab8927bd0 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -126,7 +126,7 @@ static char* getSyntaxErrFormat(int32_t errCode) { case TSDB_CODE_PAR_INVALID_FIRST_COLUMN: return "First column must be timestamp"; case TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN: - return "Invalid binary/nchar column length"; + return "Invalid column length for var length type"; case TSDB_CODE_PAR_INVALID_TAGS_NUM: return "Invalid number of tag columns"; case TSDB_CODE_PAR_INVALID_INTERNAL_PK: diff --git a/tests/system-test/2-query/varchar.py b/tests/system-test/2-query/varchar.py index f0849010c6..debcd1f95e 100644 --- a/tests/system-test/2-query/varchar.py +++ b/tests/system-test/2-query/varchar.py @@ -76,6 +76,12 @@ class TDTestCase: for i in range(tdSql.queryRows): tdSql.checkData(i,0, data_ct1_c8[i]) + tdSql.error("create stable empty_col_stable(ts timestamp, c2 varchar(0)) tags(tg1 int)") + tdSql.error("create stable empty_col_stable(ts timestamp, c2 varchar(10)) tags(tg1 varchar(0))") + tdSql.error("create stable empty_col_stable(ts timestamp, c2 varchar(10)) tags(tg1 nchar(0))") + tdSql.error("create stable empty_col_stable(ts timestamp, c2 varchar(10)) tags(tg1 binary(0))") + tdSql.error("create stable empty_col_stable(ts timestamp, c2 varchar(10)) tags(tg1 varbinary(0))") + # tdSql.query("select c8 from ct4") # data_ct4 = [tdSql.getData(i,0) for i in range(tdSql.queryRows)] From f8097979995e3620a44dd8106a7cbf540a4e4d08 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 23 Oct 2023 16:04:45 +0800 Subject: [PATCH 05/11] enh: taosd build info --- cmake/cmake.version | 14 +++++++++++++- source/util/src/version.c.in | 2 +- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/cmake/cmake.version b/cmake/cmake.version index fa6ec4df17..c0ddae569c 100644 --- a/cmake/cmake.version +++ b/cmake/cmake.version @@ -50,7 +50,19 @@ ENDIF () IF (DEFINED VERDATE) SET(TD_VER_DATE ${VERDATE}) ELSE () - STRING(TIMESTAMP TD_VER_DATE "%Y-%m-%d %H:%M:%S") + STRING(COMPARE GREATER_EQUAL "${CMAKE_VERSION}" "3.26" TD_CMAKE_SUPPORT_TZ) + + IF (TD_CMAKE_SUPPORT_TZ) + STRING(TIMESTAMP TD_VER_DATE "%Y-%m-%d %H:%M:%S %z") + ELSE () + IF (TD_WINDOWS) + STRING(TIMESTAMP TD_VER_DATE "%Y-%m-%d %H:%M:%S") + ELSE () + EXECUTE_PROCESS(COMMAND date +"%F %T %z" OUTPUT_VARIABLE TD_VER_DATE) + STRING(REPLACE "\"" "" TD_VER_DATE ${TD_VER_DATE}) + STRING(STRIP ${TD_VER_DATE} TD_VER_DATE) + ENDIF () + ENDIF () ENDIF () IF (DEFINED VERTYPE) diff --git a/source/util/src/version.c.in b/source/util/src/version.c.in index 71998e3321..ec6449a02f 100644 --- a/source/util/src/version.c.in +++ b/source/util/src/version.c.in @@ -2,6 +2,6 @@ char version[64] = "${TD_VER_NUMBER}"; char compatible_version[12] = "${TD_VER_COMPATIBLE}"; char gitinfo[48] = "${TD_VER_GIT}"; char gitinfoOfInternal[48] = "${TD_VER_GIT_INTERNAL}"; -char buildinfo[64] = "Built at ${TD_VER_DATE}"; +char buildinfo[64] = "Built ${TD_VER_OSTYPE}-${TD_VER_CPUTYPE} at ${TD_VER_DATE}"; void libtaos_${TD_LIB_VER_NUMBER}_${TD_VER_OSTYPE}_${TD_VER_CPUTYPE}_${TD_VER_VERTYPE}() {}; From fd87f43f8cdfbb3668bfe011b14d1ce2d3794567 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 24 Oct 2023 09:10:58 +0800 Subject: [PATCH 06/11] enh: set CPUTYPE in cmake --- cmake/cmake.platform | 6 ++++++ cmake/cmake.version | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/cmake/cmake.platform b/cmake/cmake.platform index 18fd17f018..ca2cbbec2e 100644 --- a/cmake/cmake.platform +++ b/cmake/cmake.platform @@ -96,33 +96,39 @@ IF ("${CPUTYPE}" STREQUAL "") IF (CMAKE_SYSTEM_PROCESSOR MATCHES "(amd64)|(AMD64)") MESSAGE(STATUS "Current platform is amd64") SET(PLATFORM_ARCH_STR "amd64") + SET(CPUTYPE "x64") SET(TD_INTEL_64 TRUE) ADD_DEFINITIONS("-D_TD_X86_") ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)") MESSAGE(STATUS "Current platform is x86") SET(PLATFORM_ARCH_STR "i386") + SET(CPUTYPE "x86") SET(TD_INTEL_32 TRUE) ADD_DEFINITIONS("-D_TD_X86_") ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "armv7l") MESSAGE(STATUS "Current platform is aarch32") SET(PLATFORM_ARCH_STR "arm") + SET(CPUTYPE "arm32") SET(TD_ARM_32 TRUE) ADD_DEFINITIONS("-D_TD_ARM_") ADD_DEFINITIONS("-D_TD_ARM_32") ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "(aarch64)|(arm64)") MESSAGE(STATUS "Current platform is aarch64") SET(PLATFORM_ARCH_STR "arm64") + SET(CPUTYPE "arm64") SET(TD_ARM_64 TRUE) ADD_DEFINITIONS("-D_TD_ARM_") ADD_DEFINITIONS("-D_TD_ARM_64") ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "loongarch64") MESSAGE(STATUS "The current platform is loongarch64") SET(PLATFORM_ARCH_STR "loongarch64") + SET(CPUTYPE "loongarch64") SET(TD_LOONGARCH_64 TRUE) ADD_DEFINITIONS("-D_TD_LOONGARCH_") ADD_DEFINITIONS("-D_TD_LOONGARCH_64") ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "mips64") SET(PLATFORM_ARCH_STR "mips") + SET(CPUTYPE "mips64") MESSAGE(STATUS "input cpuType: mips64") SET(TD_MIPS_64 TRUE) ADD_DEFINITIONS("-D_TD_MIPS_") diff --git a/cmake/cmake.version b/cmake/cmake.version index c0ddae569c..b8871dbce3 100644 --- a/cmake/cmake.version +++ b/cmake/cmake.version @@ -79,9 +79,9 @@ ELSE () ELSEIF (TD_LINUX_32) SET(TD_VER_CPUTYPE "x86") ELSEIF (TD_ARM_32) - SET(TD_VER_CPUTYPE "x86") + SET(TD_VER_CPUTYPE "arm32") ELSEIF (TD_MIPS_32) - SET(TD_VER_CPUTYPE "x86") + SET(TD_VER_CPUTYPE "mips32") ELSE () SET(TD_VER_CPUTYPE "x64") ENDIF () From be7791f9cd25b1362907b414773ce6b5daf4ff33 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Tue, 24 Oct 2023 13:58:30 +0800 Subject: [PATCH 07/11] fix: disable life extending of conn cache obj when show queries/conns --- include/util/tcache.h | 2 ++ source/dnode/mnode/impl/src/mndProfile.c | 13 +++++++------ source/util/src/tcache.c | 12 ++++++++++++ 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/include/util/tcache.h b/include/util/tcache.h index d8ab018570..474e5274de 100644 --- a/include/util/tcache.h +++ b/include/util/tcache.h @@ -151,6 +151,8 @@ void *taosCacheIterGetData(const SCacheIter *pIter, size_t *dataLen); void *taosCacheIterGetKey(const SCacheIter *pIter, size_t *keyLen); void taosCacheDestroyIter(SCacheIter *pIter); +void taosCacheTryExtendLifeSpan(SCacheObj *pCacheObj, void **data); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 1f8c3b161b..4366053237 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -59,7 +59,7 @@ static SConnObj *mndCreateConn(SMnode *pMnode, const char *user, int8_t connType int32_t pid, const char *app, int64_t startTime); static void mndFreeConn(SConnObj *pConn); static SConnObj *mndAcquireConn(SMnode *pMnode, uint32_t connId); -static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn); +static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn, bool extendLifespan); static void *mndGetNextConn(SMnode *pMnode, SCacheIter *pIter); static void mndCancelGetNextConn(SMnode *pMnode, void *pIter); static int32_t mndProcessHeartBeatReq(SRpcMsg *pReq); @@ -79,7 +79,7 @@ int32_t mndInitProfile(SMnode *pMnode) { // in ms int32_t checkTime = tsShellActivityTimer * 2 * 1000; - pMgmt->connCache = taosCacheInit(TSDB_DATA_TYPE_UINT, checkTime, true, (__cache_free_fn_t)mndFreeConn, "conn"); + pMgmt->connCache = taosCacheInit(TSDB_DATA_TYPE_UINT, checkTime, false, (__cache_free_fn_t)mndFreeConn, "conn"); if (pMgmt->connCache == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; mError("failed to alloc profile cache since %s", terrstr()); @@ -185,11 +185,12 @@ static SConnObj *mndAcquireConn(SMnode *pMnode, uint32_t connId) { return pConn; } -static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn) { +static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn, bool extendLifespan) { if (pConn == NULL) return; mTrace("conn:%u, released from cache, data:%p", pConn->id, pConn); SProfileMgmt *pMgmt = &pMnode->profileMgmt; + if (extendLifespan) taosCacheTryExtendLifeSpan(pMgmt->connCache, (void **)&pConn); taosCacheRelease(pMgmt->connCache, (void **)&pConn, false); } @@ -323,7 +324,7 @@ _OVER: mndReleaseUser(pMnode, pUser); mndReleaseDb(pMnode, pDb); - mndReleaseConn(pMnode, pConn); + mndReleaseConn(pMnode, pConn, true); return code; } @@ -485,7 +486,7 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb SQueryHbRspBasic *rspBasic = taosMemoryCalloc(1, sizeof(SQueryHbRspBasic)); if (rspBasic == NULL) { - mndReleaseConn(pMnode, pConn); + mndReleaseConn(pMnode, pConn, true); terrno = TSDB_CODE_OUT_OF_MEMORY; mError("user:%s, conn:%u failed to process hb while since %s", pConn->user, pBasic->connId, terrstr()); return -1; @@ -508,7 +509,7 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb mndCreateQnodeList(pMnode, &rspBasic->pQnodeList, -1); - mndReleaseConn(pMnode, pConn); + mndReleaseConn(pMnode, pConn, true); hbRsp.query = rspBasic; } else { diff --git a/source/util/src/tcache.c b/source/util/src/tcache.c index 392ac5d8b2..11f8df4c93 100644 --- a/source/util/src/tcache.c +++ b/source/util/src/tcache.c @@ -997,3 +997,15 @@ void taosCacheDestroyIter(SCacheIter *pIter) { taosMemoryFreeClear(pIter->pCurrent); taosMemoryFreeClear(pIter); } + +void taosCacheTryExtendLifeSpan(SCacheObj *pCacheObj, void **data) { + if (!pCacheObj || !(*data)) return; + + SCacheNode *pNode = (SCacheNode *)((char *)(*data) - sizeof(SCacheNode)); + if (pNode->signature != pNode) return; + + if (!pNode->inTrashcan) { + atomic_store_64(&pNode->expireTime, pNode->lifespan + taosGetTimestampMs()); + uDebug("cache:%s, data:%p extend expire time: %" PRId64, pCacheObj->name, pNode->data, pNode->expireTime); + } +} From f9ca8fe9620e57f0c64c3ebb62d4dca2647352d9 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 24 Oct 2023 16:10:46 +0800 Subject: [PATCH 08/11] cmake: new option: BUILD_S3 --- CMakeLists.txt | 4 +--- cmake/cmake.options | 28 +++++++++++++++++++++------- contrib/CMakeLists.txt | 23 +++++++---------------- source/common/CMakeLists.txt | 14 +++++--------- source/dnode/vnode/CMakeLists.txt | 6 +++--- 5 files changed, 37 insertions(+), 38 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5048287cf6..66a6fd328d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,8 +16,6 @@ set(TD_SUPPORT_DIR "${TD_SOURCE_DIR}/cmake") set(TD_CONTRIB_DIR "${TD_SOURCE_DIR}/contrib") - - include(${TD_SUPPORT_DIR}/cmake.platform) include(${TD_SUPPORT_DIR}/cmake.define) include(${TD_SUPPORT_DIR}/cmake.options) @@ -46,4 +44,4 @@ add_subdirectory(examples/c) include(${TD_SUPPORT_DIR}/cmake.install) # docs -add_subdirectory(docs/doxgen) \ No newline at end of file +add_subdirectory(docs/doxgen) diff --git a/cmake/cmake.options b/cmake/cmake.options index bacc5adfb7..d34c34dd89 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -128,12 +128,16 @@ option( IF(${TD_LINUX}) option( - BUILD_WITH_S3 + BUILD_S3 "If build with s3" ON ) -IF(${BUILD_WITH_S3}) +option( + BUILD_WITH_S3 + "If build with s3" + ON +) option( BUILD_WITH_COS @@ -141,16 +145,26 @@ option( OFF ) +ENDIF () + +IF(${BUILD_S3}) + +IF(${BUILD_WITH_S3}) + +option(BUILD_WITH_COS "If build with cos" OFF) + ELSE () -option( - BUILD_WITH_COS - "If build with cos" - ON -) +option(BUILD_WITH_COS "If build with cos" ON) ENDIF () +ELSE () + +option(BUILD_WITH_S3 "If build with s3" OFF) + +option(BUILD_WITH_COS "If build with cos" OFF) + ENDIF () option( diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index e2a5378f0d..d631ceff11 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -6,12 +6,10 @@ function(cat IN_FILE OUT_FILE) file(APPEND ${OUT_FILE} "${CONTENTS}") endfunction(cat IN_FILE OUT_FILE) -if(${TD_LINUX}) - if(${BUILD_WITH_S3}) file(MAKE_DIRECTORY $ENV{HOME}/.cos-local.2/) -else() +elseif(${BUILD_WITH_COS}) set(CONTRIB_TMP_FILE3 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in3") configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3}) @@ -42,9 +40,8 @@ execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" . execute_process(COMMAND "${CMAKE_COMMAND}" --build . WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download") -endif(${BUILD_WITH_S3}) +endif() -endif(${TD_LINUX}) set(CONTRIB_TMP_FILE "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in") configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) @@ -170,19 +167,16 @@ if(${BUILD_WITH_S3}) cat("${TD_SUPPORT_DIR}/libs3_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) add_definitions(-DUSE_S3) -else() - # cos -if(${BUILD_WITH_COS}) +elseif(${BUILD_WITH_COS}) #cat("${TD_SUPPORT_DIR}/mxml_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) #cat("${TD_SUPPORT_DIR}/apr_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) #cat("${TD_SUPPORT_DIR}/apr-util_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) #cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) cat("${TD_SUPPORT_DIR}/cos_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) add_definitions(-DUSE_COS) -endif(${BUILD_WITH_COS}) -endif(${BUILD_WITH_S3}) +endif() # lucene if(${BUILD_WITH_LUCENE}) @@ -269,7 +263,7 @@ unset(CMAKE_PROJECT_INCLUDE_BEFORE) # xml2 if(${BUILD_WITH_S3}) add_subdirectory(xml2 EXCLUDE_FROM_ALL) -endif(${BUILD_WITH_S3}) +endif() # lz4 add_subdirectory(lz4/build/cmake EXCLUDE_FROM_ALL) @@ -417,10 +411,8 @@ if(${BUILD_WITH_S3}) INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.2/include) MESSAGE("build with s3: ${BUILD_WITH_S3}") -else() - # cos -if(${BUILD_WITH_COS}) +elseif(${BUILD_WITH_COS}) if(${TD_LINUX}) set(CMAKE_PREFIX_PATH $ENV{HOME}/.cos-local.1) #ADD_DEFINITIONS(-DMINIXML_LIBRARY=${CMAKE_BINARY_DIR}/build/lib/libxml.a) @@ -442,9 +434,8 @@ if(${BUILD_WITH_COS}) else() endif(${TD_LINUX}) -endif(${BUILD_WITH_COS}) -endif(${BUILD_WITH_S3}) +endif() # lucene # To support build on ubuntu: sudo apt-get install libboost-all-dev diff --git a/source/common/CMakeLists.txt b/source/common/CMakeLists.txt index 6bdcdf7d1d..c35845f9df 100644 --- a/source/common/CMakeLists.txt +++ b/source/common/CMakeLists.txt @@ -17,15 +17,11 @@ IF (TD_STORAGE) ADD_DEFINITIONS(-D_STORAGE) TARGET_LINK_LIBRARIES(common PRIVATE storage) - IF(${TD_LINUX}) - IF(${BUILD_WITH_COS}) - add_definitions(-DUSE_COS) - ENDIF(${BUILD_WITH_COS}) - - IF(${BUILD_WITH_S3}) - add_definitions(-DUSE_S3) - ENDIF(${BUILD_WITH_S3}) - ENDIF(${TD_LINUX}) + IF(${BUILD_WITH_S3}) + add_definitions(-DUSE_S3) + ELSEIF(${BUILD_WITH_COS}) + add_definitions(-DUSE_COS) + ENDIF() ENDIF () diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index b83d64c569..a10b38a1bf 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -161,7 +161,7 @@ target_link_libraries( PUBLIC index ) -if(${TD_LINUX}) +if(${BUILD_S3}) if(${BUILD_WITH_S3}) target_include_directories( @@ -188,7 +188,7 @@ if(${BUILD_WITH_S3}) ) add_definitions(-DUSE_S3) -endif(${BUILD_WITH_S3}) +endif() if(${BUILD_WITH_COS}) @@ -227,7 +227,7 @@ target_include_directories( add_definitions(-DUSE_COS) endif(${BUILD_WITH_COS}) -endif(${TD_LINUX}) +endif() IF (TD_GRANT) TARGET_LINK_LIBRARIES(vnode PUBLIC grant) From aa7fd64b9b964781493da67985b591f713605be4 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 24 Oct 2023 18:04:50 +0800 Subject: [PATCH 09/11] enh: cmake system process support x86_64 --- cmake/cmake.platform | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/cmake.platform b/cmake/cmake.platform index ca2cbbec2e..9b96ebe7cb 100644 --- a/cmake/cmake.platform +++ b/cmake/cmake.platform @@ -93,7 +93,7 @@ ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "Windows") ENDIF() IF ("${CPUTYPE}" STREQUAL "") - IF (CMAKE_SYSTEM_PROCESSOR MATCHES "(amd64)|(AMD64)") + IF (CMAKE_SYSTEM_PROCESSOR MATCHES "(amd64)|(AMD64)|(x86_64)|(X86_64)") MESSAGE(STATUS "Current platform is amd64") SET(PLATFORM_ARCH_STR "amd64") SET(CPUTYPE "x64") From c67853888f80030a29878149e608e20324619903 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 24 Oct 2023 18:06:19 +0800 Subject: [PATCH 10/11] cmake/xml2: build xml2 before s3 --- cmake/xml2_CMakeLists.txt.in | 16 ++++++++++------ contrib/CMakeLists.txt | 6 +++--- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/cmake/xml2_CMakeLists.txt.in b/cmake/xml2_CMakeLists.txt.in index 4c234d7a37..25eb5a69ac 100644 --- a/cmake/xml2_CMakeLists.txt.in +++ b/cmake/xml2_CMakeLists.txt.in @@ -1,12 +1,16 @@ # xml2 ExternalProject_Add(xml2 - GIT_REPOSITORY https://github.com/GNOME/libxml2 - GIT_TAG v2.11.5 + URL https://github.com/GNOME/libxml2/archive/refs/tags/v2.11.5.tar.gz + #GIT_REPOSITORY https://github.com/GNOME/libxml2 + #GIT_TAG v2.11.5 + DOWNLOAD_NO_PROGRESS 1 + DOWNLOAD_DIR "${TD_CONTRIB_DIR}/deps-download" SOURCE_DIR "${TD_CONTRIB_DIR}/xml2" - BINARY_DIR "" - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND "" + #BINARY_DIR "" + BUILD_IN_SOURCE TRUE + CONFIGURE_COMMAND ./autogen.sh && ./configure --prefix=$ENV{HOME}/.cos-local.2 --enable-shared=no --enable-static=yes --without-python + BUILD_COMMAND make -j + INSTALL_COMMAND make install && ln -s $ENV{HOME}/.cos-local.2/include/libxml2/libxml $ENV{HOME}/.cos-local.2/include/libxml TEST_COMMAND "" ) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index d631ceff11..c5715bd53f 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -261,9 +261,9 @@ target_include_directories( unset(CMAKE_PROJECT_INCLUDE_BEFORE) # xml2 -if(${BUILD_WITH_S3}) - add_subdirectory(xml2 EXCLUDE_FROM_ALL) -endif() +#if(${BUILD_WITH_S3}) +# add_subdirectory(xml2 EXCLUDE_FROM_ALL) +#endif() # lz4 add_subdirectory(lz4/build/cmake EXCLUDE_FROM_ALL) From d77ecbbb1b150611e5ccb9a6314213499ae0769c Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 25 Oct 2023 17:41:55 +0800 Subject: [PATCH 11/11] cmake/xml2: use sha256 to verify distribution release --- cmake/xml2_CMakeLists.txt.in | 6 ++++-- source/dnode/vnode/CMakeLists.txt | 5 +++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/cmake/xml2_CMakeLists.txt.in b/cmake/xml2_CMakeLists.txt.in index 25eb5a69ac..ad0704cdb9 100644 --- a/cmake/xml2_CMakeLists.txt.in +++ b/cmake/xml2_CMakeLists.txt.in @@ -1,7 +1,9 @@ # xml2 ExternalProject_Add(xml2 - URL https://github.com/GNOME/libxml2/archive/refs/tags/v2.11.5.tar.gz + URL https://download.gnome.org/sources/libxml2/2.11/libxml2-2.11.5.tar.xz + URL_HASH SHA256=3727b078c360ec69fa869de14bd6f75d7ee8d36987b071e6928d4720a28df3a6 + #https://github.com/GNOME/libxml2/archive/refs/tags/v2.11.5.tar.gz #GIT_REPOSITORY https://github.com/GNOME/libxml2 #GIT_TAG v2.11.5 DOWNLOAD_NO_PROGRESS 1 @@ -9,7 +11,7 @@ ExternalProject_Add(xml2 SOURCE_DIR "${TD_CONTRIB_DIR}/xml2" #BINARY_DIR "" BUILD_IN_SOURCE TRUE - CONFIGURE_COMMAND ./autogen.sh && ./configure --prefix=$ENV{HOME}/.cos-local.2 --enable-shared=no --enable-static=yes --without-python + CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.2 --enable-shared=no --enable-static=yes --without-python --without-lzma BUILD_COMMAND make -j INSTALL_COMMAND make install && ln -s $ENV{HOME}/.cos-local.2/include/libxml2/libxml $ENV{HOME}/.cos-local.2/include/libxml TEST_COMMAND "" diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index a10b38a1bf..dcc9f9a115 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -173,7 +173,8 @@ if(${BUILD_WITH_S3}) set(CMAKE_FIND_LIBRARY_SUFFIXES ".a") set(CMAKE_PREFIX_PATH $ENV{HOME}/.cos-local.2) find_library(S3_LIBRARY s3) - find_library(CURL_LIBRARY curl) + find_library(CURL_LIBRARY curl $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH) + find_library(XML2_LIBRARY xml2) find_library(SSL_LIBRARY ssl $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH) find_library(CRYPTO_LIBRARY crypto $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH) target_link_libraries( @@ -184,7 +185,7 @@ if(${BUILD_WITH_S3}) PUBLIC ${CURL_LIBRARY} PUBLIC ${SSL_LIBRARY} PUBLIC ${CRYPTO_LIBRARY} - PUBLIC xml2 + PUBLIC ${XML2_LIBRARY} ) add_definitions(-DUSE_S3)