Merge branch '3.0' of https://github.com/taosdata/TDengine into feat/TD-21898

cmake/xml2_CMakeLists.txt.in

@@ -1,7 +1,9 @@
 
 # xml2
 ExternalProject_Add(xml2
-        URL https://github.com/GNOME/libxml2/archive/refs/tags/v2.11.5.tar.gz
+        URL https://download.gnome.org/sources/libxml2/2.11/libxml2-2.11.5.tar.xz
+        URL_HASH SHA256=3727b078c360ec69fa869de14bd6f75d7ee8d36987b071e6928d4720a28df3a6
+        #https://github.com/GNOME/libxml2/archive/refs/tags/v2.11.5.tar.gz
         #GIT_REPOSITORY https://github.com/GNOME/libxml2
         #GIT_TAG v2.11.5
         DOWNLOAD_NO_PROGRESS 1
@@ -9,7 +11,7 @@ ExternalProject_Add(xml2
         SOURCE_DIR "${TD_CONTRIB_DIR}/xml2"
         #BINARY_DIR ""
         BUILD_IN_SOURCE TRUE
-        CONFIGURE_COMMAND ./autogen.sh && ./configure --prefix=$ENV{HOME}/.cos-local.2 --enable-shared=no --enable-static=yes --without-python
+        CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.2 --enable-shared=no --enable-static=yes --without-python --without-lzma
         BUILD_COMMAND make -j
         INSTALL_COMMAND make install && ln -s $ENV{HOME}/.cos-local.2/include/libxml2/libxml $ENV{HOME}/.cos-local.2/include/libxml
         TEST_COMMAND ""

include/common/tglobal.h

@@ -107,6 +107,7 @@ extern bool     tsMonitorComp;
 extern bool     tsEnableAudit;
 extern char     tsAuditFqdn[];
 extern uint16_t tsAuditPort;
+extern bool     tsEnableAuditCreateTable;
 
 // telem
 extern bool     tsEnableTelem;

packaging/cfg/taos.cfg

@@ -101,6 +101,9 @@
 # enable/disable audit log
 # audit                     1
 
+# enable/disable audit create table
+# auditCreateTable                     1
+
 # The following parameter is used to limit the maximum number of lines in log files.
 # max number of lines per log filters
 # numOfLogLines             10000000

source/common/src/tglobal.c

@@ -98,9 +98,10 @@ int32_t  tsMonitorMaxLogs = 100;
 bool     tsMonitorComp = false;
 
 // audit
-bool     tsEnableAudit = false;
+bool     tsEnableAudit = true;
 char     tsAuditFqdn[TSDB_FQDN_LEN] = {0};
 uint16_t tsAuditPort = 6043;
+bool     tsEnableAuditCreateTable = true;
 
 // telem
 bool     tsEnableTelem = true;
@@ -632,6 +633,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
   if (cfgAddBool(pCfg, "audit", tsEnableAudit, CFG_SCOPE_SERVER) != 0) return -1;
   if (cfgAddString(pCfg, "auditFqdn", tsAuditFqdn, CFG_SCOPE_SERVER) != 0) return -1;
   if (cfgAddInt32(pCfg, "auditPort", tsAuditPort, 1, 65056, CFG_SCOPE_SERVER) != 0) return -1;
+  if (cfgAddBool(pCfg, "auditCreateTable", tsEnableAuditCreateTable, CFG_SCOPE_SERVER) != 0) return -1;
 
   if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, CFG_SCOPE_BOTH) != 0) return -1;
   if (cfgAddBool(pCfg, "telemetryReporting", tsEnableTelem, CFG_SCOPE_BOTH) != 0) return -1;
@@ -1050,6 +1052,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
   tsEnableAudit = cfgGetItem(pCfg, "audit")->bval;
   tstrncpy(tsAuditFqdn, cfgGetItem(pCfg, "auditFqdn")->str, TSDB_FQDN_LEN);
   tsAuditPort = (uint16_t)cfgGetItem(pCfg, "auditPort")->i32;
+  tsEnableAuditCreateTable = cfgGetItem(pCfg, "auditCreateTable")->bval;
 
   tsEnableTelem = cfgGetItem(pCfg, "telemetryReporting")->bval;
   tsEnableCrashReport = cfgGetItem(pCfg, "crashReporting")->bval;

source/dnode/mnode/impl/src/mndProfile.c

@@ -315,8 +315,7 @@ _CONNECT:
   sprintf(obj, "%s:%d", ip, pConn->port);
 
   char detail[1000] = {0};
-  sprintf(detail, "connType:%d, db:%s, pid:%d, startTime:%" PRId64 ", sVer:%s, app:%s",
+  sprintf(detail, "app:%s", connReq.app);
-          connReq.connType, connReq.db, connReq.pid, connReq.startTime, connReq.sVer, connReq.app);
 
   auditRecord(pReq, pMnode->clusterId, "login", connReq.user, obj, detail, strlen(detail));
 

source/dnode/vnode/CMakeLists.txt

@@ -173,7 +173,8 @@ if(${BUILD_WITH_S3})
   set(CMAKE_FIND_LIBRARY_SUFFIXES ".a")
   set(CMAKE_PREFIX_PATH $ENV{HOME}/.cos-local.2)
   find_library(S3_LIBRARY s3)
-  find_library(CURL_LIBRARY curl)
+  find_library(CURL_LIBRARY curl $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH)
+  find_library(XML2_LIBRARY xml2)
   find_library(SSL_LIBRARY ssl $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH)
   find_library(CRYPTO_LIBRARY crypto $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH)
   target_link_libraries(
@@ -184,7 +185,7 @@ if(${BUILD_WITH_S3})
     PUBLIC ${CURL_LIBRARY}
     PUBLIC ${SSL_LIBRARY}
     PUBLIC ${CRYPTO_LIBRARY}
-    PUBLIC xml2
+    PUBLIC ${XML2_LIBRARY}
   )
 
   add_definitions(-DUSE_S3)

source/dnode/vnode/src/tsdb/tsdbCommit2.c

@@ -137,7 +137,12 @@ static int32_t tsdbCommitTSData(SCommitter2 *committer) {
     }
 
     int64_t ts = TSDBROW_TS(&row->row);
-    if (ts > committer->ctx->maxKey || skipRow) {
+
+    if (skipRow && ts <= committer->ctx->maxKey) {
+      ts = committer->ctx->maxKey + 1;
+    }
+
+    if (ts > committer->ctx->maxKey) {
       committer->ctx->nextKey = TMIN(committer->ctx->nextKey, ts);
       code = tsdbIterMergerSkipTableData(committer->dataIterMerger, committer->ctx->tbid);
       TSDB_CHECK_CODE(code, lino, _exit);

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -947,13 +947,13 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq,
 
     taosArrayPush(rsp.pArray, &cRsp);
 
-    if(pCreateReq->sqlLen > 0){ //skip auto create table, not set sql when auto create table
+    if(tsEnableAuditCreateTable){
-      int32_t clusterId = pVnode->config.syncCfg.nodeInfo[0].clusterId;
+      int64_t clusterId = pVnode->config.syncCfg.nodeInfo[0].clusterId;
 
       SName name = {0};
       tNameFromString(&name, pVnode->config.dbname, T_NAME_ACCT | T_NAME_DB);
 
-      auditRecord(pReq, clusterId, "createTable", name.dbname, pCreateReq->name, pCreateReq->sql, pCreateReq->sqlLen);
+      auditRecord(NULL, clusterId, "createTable", name.dbname, "", pCreateReq->name, strlen(pCreateReq->name));
     }
   }
 

tests/parallel_test/container_build.sh

@@ -68,9 +68,9 @@ docker run \
     -v ${REP_REAL_PATH}/community/contrib/cpp-stub/:${REP_DIR}/community/contrib/cpp-stub \
     -v ${REP_REAL_PATH}/community/contrib/libuv/:${REP_DIR}/community/contrib/libuv \
     -v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \
-    -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \
     --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_TAOSX=true -DJEMALLOC_ENABLED=0;make -j 10|| exit 1"
  # -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \
+# -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \
 
 if [[ -d ${WORKDIR}/debugNoSan  ]] ;then
     echo "delete  ${WORKDIR}/debugNoSan"
@@ -97,10 +97,10 @@ docker run \
     -v ${REP_REAL_PATH}/community/contrib/cpp-stub/:${REP_DIR}/community/contrib/cpp-stub \
     -v ${REP_REAL_PATH}/community/contrib/libuv/:${REP_DIR}/community/contrib/libuv \
     -v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \
-    -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \
     -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \
     --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true   -DBUILD_SANITIZER=1  -DTOOLS_SANITIZE=true -DTOOLS_BUILD_TYPE=Debug -DBUILD_TAOSX=true -DJEMALLOC_ENABLED=0;make -j 10|| exit 1 "
 
+    # -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \
 mv  ${REP_REAL_PATH}/debug  ${WORKDIR}/debugSan
 
 ret=$?

tests/system-test/0-others/taosdMonitor.py

@@ -268,6 +268,8 @@ class TDTestCase:
     updatecfgDict["monitorMaxLogs"]        = "10"
     updatecfgDict["monitorComp"]        = "1"
 
+    updatecfgDict["audit"]            = '0'
+
     print ("===================: ", updatecfgDict)
 
     def init(self, conn, logSql, replicaVar=1):

tests/system-test/7-tmq/tmqVnodeTransform-stb.py

@@ -140,7 +140,7 @@ class TDTestCase:
                     'rowsPerTbl': 10000,
                     'batchNum':   10,
                     'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
-                    'pollDelay':  30,
+                    'pollDelay':  60,
                     'showMsg':    1,
                     'showRow':    1,
                     'snapshot':   0}