From 956aa1742cb640a60b4b88a7894ce3f61de08f37 Mon Sep 17 00:00:00 2001 From: dmchen Date: Tue, 24 Oct 2023 05:32:48 +0000 Subject: [PATCH 1/6] TD-26884 --- source/common/src/tglobal.c | 2 +- source/dnode/mnode/impl/src/mndProfile.c | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index d9b1b2e0ed..f8feb949c7 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -98,7 +98,7 @@ int32_t tsMonitorMaxLogs = 100; bool tsMonitorComp = false; // audit -bool tsEnableAudit = false; +bool tsEnableAudit = true; char tsAuditFqdn[TSDB_FQDN_LEN] = {0}; uint16_t tsAuditPort = 6043; diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 1f8c3b161b..159774efdc 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -314,8 +314,7 @@ _CONNECT: sprintf(obj, "%s:%d", ip, pConn->port); char detail[1000] = {0}; - sprintf(detail, "connType:%d, db:%s, pid:%d, startTime:%" PRId64 ", sVer:%s, app:%s", - connReq.connType, connReq.db, connReq.pid, connReq.startTime, connReq.sVer, connReq.app); + sprintf(detail, "app:%s", connReq.app); auditRecord(pReq, pMnode->clusterId, "login", connReq.user, obj, detail, strlen(detail)); From 162066235b29533b05806425d1b6c4870271f8b4 Mon Sep 17 00:00:00 2001 From: dmchen Date: Tue, 24 Oct 2023 06:17:13 +0000 Subject: [PATCH 2/6] create table --- include/common/tglobal.h | 1 + packaging/cfg/taos.cfg | 3 +++ source/common/src/tglobal.c | 3 +++ source/dnode/vnode/src/vnd/vnodeSvr.c | 4 ++-- 4 files changed, 9 insertions(+), 2 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index c4037ed2ea..3e29703070 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -107,6 +107,7 @@ extern bool tsMonitorComp; extern bool tsEnableAudit; extern char tsAuditFqdn[]; extern uint16_t tsAuditPort; +extern bool tsEnableAuditCreateTable; // telem extern bool tsEnableTelem; diff --git a/packaging/cfg/taos.cfg b/packaging/cfg/taos.cfg index 743edb6ca3..6d25db843d 100644 --- a/packaging/cfg/taos.cfg +++ b/packaging/cfg/taos.cfg @@ -101,6 +101,9 @@ # enable/disable audit log # audit 1 +# enable/disable audit create table +# auditCreateTable 1 + # The following parameter is used to limit the maximum number of lines in log files. # max number of lines per log filters # numOfLogLines 10000000 diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index f8feb949c7..0a155f4ea1 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -101,6 +101,7 @@ bool tsMonitorComp = false; bool tsEnableAudit = true; char tsAuditFqdn[TSDB_FQDN_LEN] = {0}; uint16_t tsAuditPort = 6043; +bool tsEnableAuditCreateTable = true; // telem bool tsEnableTelem = true; @@ -632,6 +633,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddBool(pCfg, "audit", tsEnableAudit, CFG_SCOPE_SERVER) != 0) return -1; if (cfgAddString(pCfg, "auditFqdn", tsAuditFqdn, CFG_SCOPE_SERVER) != 0) return -1; if (cfgAddInt32(pCfg, "auditPort", tsAuditPort, 1, 65056, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddBool(pCfg, "auditCreateTable", tsEnableAuditCreateTable, CFG_SCOPE_SERVER) != 0) return -1; if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, CFG_SCOPE_BOTH) != 0) return -1; if (cfgAddBool(pCfg, "telemetryReporting", tsEnableTelem, CFG_SCOPE_BOTH) != 0) return -1; @@ -1050,6 +1052,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsEnableAudit = cfgGetItem(pCfg, "audit")->bval; tstrncpy(tsAuditFqdn, cfgGetItem(pCfg, "auditFqdn")->str, TSDB_FQDN_LEN); tsAuditPort = (uint16_t)cfgGetItem(pCfg, "auditPort")->i32; + tsEnableAuditCreateTable = cfgGetItem(pCfg, "auditCreateTable")->bval; tsEnableTelem = cfgGetItem(pCfg, "telemetryReporting")->bval; tsEnableCrashReport = cfgGetItem(pCfg, "crashReporting")->bval; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index efa722d41a..e3aa459e1b 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -947,13 +947,13 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, taosArrayPush(rsp.pArray, &cRsp); - if(pCreateReq->sqlLen > 0){ //skip auto create table, not set sql when auto create table + if(tsEnableAuditCreateTable){ int32_t clusterId = pVnode->config.syncCfg.nodeInfo[0].clusterId; SName name = {0}; tNameFromString(&name, pVnode->config.dbname, T_NAME_ACCT | T_NAME_DB); - auditRecord(pReq, clusterId, "createTable", name.dbname, pCreateReq->name, pCreateReq->sql, pCreateReq->sqlLen); + auditRecord(pReq, clusterId, "createTable", name.dbname, "", pCreateReq->name, strlen(pCreateReq->name)); } } From 14be9c81e268ee9665c138869c24fd2544a5feed Mon Sep 17 00:00:00 2001 From: dmchen Date: Wed, 25 Oct 2023 06:49:23 +0000 Subject: [PATCH 3/6] create table user and clusterid --- source/dnode/vnode/src/vnd/vnodeSvr.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index e3aa459e1b..c46ea15111 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -948,12 +948,12 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, taosArrayPush(rsp.pArray, &cRsp); if(tsEnableAuditCreateTable){ - int32_t clusterId = pVnode->config.syncCfg.nodeInfo[0].clusterId; + int64_t clusterId = pVnode->config.syncCfg.nodeInfo[0].clusterId; SName name = {0}; tNameFromString(&name, pVnode->config.dbname, T_NAME_ACCT | T_NAME_DB); - auditRecord(pReq, clusterId, "createTable", name.dbname, "", pCreateReq->name, strlen(pCreateReq->name)); + auditRecord(NULL, clusterId, "createTable", name.dbname, "", pCreateReq->name, strlen(pCreateReq->name)); } } From d77ecbbb1b150611e5ccb9a6314213499ae0769c Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 25 Oct 2023 17:41:55 +0800 Subject: [PATCH 4/6] cmake/xml2: use sha256 to verify distribution release --- cmake/xml2_CMakeLists.txt.in | 6 ++++-- source/dnode/vnode/CMakeLists.txt | 5 +++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/cmake/xml2_CMakeLists.txt.in b/cmake/xml2_CMakeLists.txt.in index 25eb5a69ac..ad0704cdb9 100644 --- a/cmake/xml2_CMakeLists.txt.in +++ b/cmake/xml2_CMakeLists.txt.in @@ -1,7 +1,9 @@ # xml2 ExternalProject_Add(xml2 - URL https://github.com/GNOME/libxml2/archive/refs/tags/v2.11.5.tar.gz + URL https://download.gnome.org/sources/libxml2/2.11/libxml2-2.11.5.tar.xz + URL_HASH SHA256=3727b078c360ec69fa869de14bd6f75d7ee8d36987b071e6928d4720a28df3a6 + #https://github.com/GNOME/libxml2/archive/refs/tags/v2.11.5.tar.gz #GIT_REPOSITORY https://github.com/GNOME/libxml2 #GIT_TAG v2.11.5 DOWNLOAD_NO_PROGRESS 1 @@ -9,7 +11,7 @@ ExternalProject_Add(xml2 SOURCE_DIR "${TD_CONTRIB_DIR}/xml2" #BINARY_DIR "" BUILD_IN_SOURCE TRUE - CONFIGURE_COMMAND ./autogen.sh && ./configure --prefix=$ENV{HOME}/.cos-local.2 --enable-shared=no --enable-static=yes --without-python + CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.2 --enable-shared=no --enable-static=yes --without-python --without-lzma BUILD_COMMAND make -j INSTALL_COMMAND make install && ln -s $ENV{HOME}/.cos-local.2/include/libxml2/libxml $ENV{HOME}/.cos-local.2/include/libxml TEST_COMMAND "" diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index a10b38a1bf..dcc9f9a115 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -173,7 +173,8 @@ if(${BUILD_WITH_S3}) set(CMAKE_FIND_LIBRARY_SUFFIXES ".a") set(CMAKE_PREFIX_PATH $ENV{HOME}/.cos-local.2) find_library(S3_LIBRARY s3) - find_library(CURL_LIBRARY curl) + find_library(CURL_LIBRARY curl $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH) + find_library(XML2_LIBRARY xml2) find_library(SSL_LIBRARY ssl $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH) find_library(CRYPTO_LIBRARY crypto $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH) target_link_libraries( @@ -184,7 +185,7 @@ if(${BUILD_WITH_S3}) PUBLIC ${CURL_LIBRARY} PUBLIC ${SSL_LIBRARY} PUBLIC ${CRYPTO_LIBRARY} - PUBLIC xml2 + PUBLIC ${XML2_LIBRARY} ) add_definitions(-DUSE_S3) From 6c9c06552d8679e7f65da4008b856ad3e9caa2ac Mon Sep 17 00:00:00 2001 From: dmchen Date: Thu, 26 Oct 2023 06:10:09 +0000 Subject: [PATCH 5/6] fix case --- tests/system-test/0-others/taosdMonitor.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/system-test/0-others/taosdMonitor.py b/tests/system-test/0-others/taosdMonitor.py index a07d7f411e..169c70e675 100644 --- a/tests/system-test/0-others/taosdMonitor.py +++ b/tests/system-test/0-others/taosdMonitor.py @@ -268,6 +268,8 @@ class TDTestCase: updatecfgDict["monitorMaxLogs"] = "10" updatecfgDict["monitorComp"] = "1" + updatecfgDict["audit"] = '0' + print ("===================: ", updatecfgDict) def init(self, conn, logSql, replicaVar=1): From b3147581086a8bd0adfb540b3ea8abe71039d547 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 26 Oct 2023 16:24:06 +0800 Subject: [PATCH 6/6] tsdb/commit: make ts big enough to skip row --- source/dnode/vnode/src/tsdb/tsdbCommit2.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c index a2de5a81ca..6dc492f420 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c @@ -137,7 +137,12 @@ static int32_t tsdbCommitTSData(SCommitter2 *committer) { } int64_t ts = TSDBROW_TS(&row->row); - if (ts > committer->ctx->maxKey || skipRow) { + + if (skipRow && ts <= committer->ctx->maxKey) { + ts = committer->ctx->maxKey + 1; + } + + if (ts > committer->ctx->maxKey) { committer->ctx->nextKey = TMIN(committer->ctx->nextKey, ts); code = tsdbIterMergerSkipTableData(committer->dataIterMerger, committer->ctx->tbid); TSDB_CHECK_CODE(code, lino, _exit);