From 29458b55e50301b4e077e4b0969af2d2df9119f1 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Mon, 25 Sep 2023 17:16:19 +0800 Subject: [PATCH 01/59] test: increase numbers of cases in windows --- tests/system-test/simpletest.bat | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/system-test/simpletest.bat b/tests/system-test/simpletest.bat index 5ae2d3feb3..31b76cad4a 100644 --- a/tests/system-test/simpletest.bat +++ b/tests/system-test/simpletest.bat @@ -18,7 +18,7 @@ python3 .\test.py -f 1-insert\influxdb_line_taosc_insert.py @REM #python3 .\test.py -f 1-insert\test_stmt_muti_insert_query.py @REM python3 .\test.py -f 1-insert\alter_stable.py @REM python3 .\test.py -f 1-insert\alter_table.py -@REM python3 .\test.py -f 2-query\between.py +python3 .\test.py -f 2-query\between.py @REM python3 .\test.py -f 2-query\distinct.py @REM python3 .\test.py -f 2-query\varchar.py @REM python3 .\test.py -f 2-query\ltrim.py @@ -101,3 +101,4 @@ python3 .\test.py -f 7-tmq\subscribeStb.py @REM python3 .\test.py -f 7-tmq\subscribeStb3.py @REM python3 .\test.py -f 7-tmq\subscribeStb4.py @REM python3 .\test.py -f 7-tmq\db.py +python3 .\test.py -f 6-cluster\5dnode3mnodeSep1VnodeStopDnodeModifyMeta.py -N 6 -M 3 \ No newline at end of file From b1014f31bca665cf17bddc1881fb8c4e2f5b3e5a Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Tue, 26 Sep 2023 20:45:40 +0800 Subject: [PATCH 02/59] test: add testcase of TS-4074 --- tests/system-test/0-others/compatibility.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/system-test/0-others/compatibility.py b/tests/system-test/0-others/compatibility.py index cb804aad0c..83bfb2bed7 100644 --- a/tests/system-test/0-others/compatibility.py +++ b/tests/system-test/0-others/compatibility.py @@ -30,7 +30,7 @@ class TDTestCase: self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) - self.deletedDataSql= '''drop database if exists deldata;create database deldata duration 300;use deldata; + self.deletedDataSql= '''drop database if exists deldata;create database deldata duration 300 stt_trigger 4; ;use deldata; create table deldata.stb1 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) tags (t1 int); create table deldata.ct1 using deldata.stb1 tags ( 1 ); insert into deldata.ct1 values ( now()-0s, 0, 0, 0, 0, 0.0, 0.0, 0, 'binary0', 'nchar0', now()+0a ) ( now()-10s, 1, 11111, 111, 11, 1.11, 11.11, 1, 'binary1', 'nchar1', now()+1a ) ( now()-20s, 2, 22222, 222, 22, 2.22, 22.22, 0, 'binary2', 'nchar2', now()+2a ) ( now()-30s, 3, 33333, 333, 33, 3.33, 33.33, 1, 'binary3', 'nchar3', now()+3a ); @@ -38,7 +38,9 @@ class TDTestCase: delete from deldata.stb1; flush database deldata; insert into deldata.ct1 values ( now()-0s, 0, 0, 0, 0, 0.0, 0.0, 0, 'binary0', 'nchar0', now()+0a ) ( now()-10s, 1, 11111, 111, 11, 1.11, 11.11, 1, 'binary1', 'nchar1', now()+1a ) ( now()-20s, 2, 22222, 222, 22, 2.22, 22.22, 0, 'binary2', 'nchar2', now()+2a ) ( now()-30s, 3, 33333, 333, 33, 3.33, 33.33, 1, 'binary3', 'nchar3', now()+3a ); - delete from deldata.ct1;''' + delete from deldata.ct1; + insert into deldata.ct1 values ( now()-0s, 0, 0, 0, 0, 0.0, 0.0, 0, 'binary0', 'nchar0', now()+0a ); + flush database deldata;''' def checkProcessPid(self,processName): i=0 while i<60: @@ -262,7 +264,7 @@ class TDTestCase: if self.is_list_same_as_ordered_list(resultList,expectList): print("The unordered list is the same as the ordered list.") else: - tdlog.error("The unordered list is not the same as the ordered list.") + tdLog.exit("The unordered list is not the same as the ordered list.") tdsql.execute("insert into test.d80 values (now+1s, 11, 103, 0.21);") tdsql.execute("insert into test.d9 values (now+5s, 4.3, 104, 0.4);") From 65448e20269039b4d2ab165d09e7c8c6c9516f79 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Wed, 27 Sep 2023 10:56:54 +0800 Subject: [PATCH 03/59] test: support replica3 in splited/transform tmqvnode --- tests/parallel_test/cases.task | 2 + tests/system-test/7-tmq/tmqVnodeSplit.py | 1 - tests/system-test/7-tmq/tmqVnodeTransform.py | 44 ++++++++++++-------- 3 files changed, 28 insertions(+), 19 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 9995c2970f..cf161ab004 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -168,7 +168,9 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -i True ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -n 3 -i True ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform.py -N 2 -n 1 +,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform.py -N 6 -n 3 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit.py -N 2 -n 1 +,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit.py -N 3 -n 3 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeReplicate.py -M 3 -N 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-19201.py ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-21561.py diff --git a/tests/system-test/7-tmq/tmqVnodeSplit.py b/tests/system-test/7-tmq/tmqVnodeSplit.py index c6cdc2bf83..67deb84620 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit.py @@ -194,7 +194,6 @@ class TDTestCase: tdLog.printNoPrefix("======== test case 1 end ...... ") def run(self): - tdSql.prepare() self.prepareTestEnv() self.tmqCase1(True) self.prepareTestEnv() diff --git a/tests/system-test/7-tmq/tmqVnodeTransform.py b/tests/system-test/7-tmq/tmqVnodeTransform.py index fa50e46853..3ebf48b93a 100644 --- a/tests/system-test/7-tmq/tmqVnodeTransform.py +++ b/tests/system-test/7-tmq/tmqVnodeTransform.py @@ -58,7 +58,7 @@ class TDTestCase: paraDict['ctbNum'] = self.ctbNum paraDict['rowsPerTbl'] = self.rowsPerTbl - tdCom.drop_all_db(); + tdCom.drop_all_db() tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], wal_retention_period=36000,vgroups=paraDict["vgroups"],replica=self.replicaVar) tdLog.info("create stb") @@ -102,7 +102,7 @@ class TDTestCase: tdSql.query("show dnodes") for result in tdSql.queryResult: dnodesList.append(result[0]) - + print("dnodeList:",dnodesList) tdSql.query("select * from information_schema.ins_vnodes") vnodeId = 0 for result in tdSql.queryResult: @@ -110,9 +110,16 @@ class TDTestCase: tdLog.debug("dnode is %d"%(result[0])) dnodesList.remove(result[0]) vnodeId = result[1] - break - redistributeSql = "redistribute vgroup %d dnode %d" %(vnodeId, dnodesList[0]) - tdLog.debug("redistributeSql:%s"%(redistributeSql)) + print("its all data",dnodesList) + # if self.replicaVar == 1: + # redistributeSql = "redistribute vgroup %d dnode %d" %(vnodeId, dnodesList[0]) + # else: + redistributeSql = f"redistribute vgroup {vnodeId} " + for vgdnode in dnodesList: + redistributeSql += f"dnode {vgdnode} " + print(redistributeSql) + + tdLog.debug(f"redistributeSql:{redistributeSql}") tdSql.query(redistributeSql) tdLog.debug("redistributeSql ok") @@ -179,7 +186,7 @@ class TDTestCase: tmqCom.getStartCommitNotifyFromTmqsim() #restart dnode & remove wal - self.restartAndRemoveWal() + # self.restartAndRemoveWal() # redistribute vgroup self.redistributeVgroups(); @@ -228,7 +235,7 @@ class TDTestCase: tdSql.execute(sqlString) tdSql.query("flush database %s"%(paraDict['dbName'])) #restart dnode & remove wal - self.restartAndRemoveWal() + # self.restartAndRemoveWal() # redistribute vgroup self.redistributeVgroups(); @@ -236,7 +243,8 @@ class TDTestCase: sqlString = "alter table %s.%s modify column i nchar(16)" %(paraDict['dbName'], ntbName) tdLog.info("alter table sql: %s"%sqlString) tdSql.error(sqlString) - + expectRows = 0 + resultList = tmqCom.selectConsumeResult(expectRows) time.sleep(1) for i in range(len(topicNameList)): tdSql.query("drop topic %s"%topicNameList[i]) @@ -284,7 +292,7 @@ class TDTestCase: startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) tdLog.info("create topics from stb with filter") - queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName']) + queryString = "select * from %s.%s where c2 > 0 "%(paraDict['dbName'], paraDict['stbName']) sqlString = "create topic %s as %s" %(topicNameList[0], queryString) tdLog.info("create topic sql: %s"%sqlString) tdSql.execute(sqlString) @@ -305,27 +313,27 @@ class TDTestCase: time.sleep(5) #restart dnode & remove wal - self.restartAndRemoveWal() + # self.restartAndRemoveWal() # redistribute vgroup - self.redistributeVgroups(); + self.redistributeVgroups() tdLog.info("start consume processor") tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) tdLog.info("wait the consume result") - - time.sleep(10) + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + time.sleep(20) for i in range(len(topicNameList)): tdSql.query("drop topic %s"%topicNameList[i]) tdLog.printNoPrefix("======== test case 3 end ...... ") def run(self): - - tdSql.prepare() - self.prepareTestEnv() - self.tmqCase1() - self.tmqCase2() + # self.prepareTestEnv() + # self.tmqCase1() + # self.tmqCase2() self.prepareTestEnv() self.tmqCase3() From a30beeee6de93bb4bdf69940465af678ae8e3cf7 Mon Sep 17 00:00:00 2001 From: haoranchen Date: Sun, 8 Oct 2023 16:46:03 +0800 Subject: [PATCH 04/59] Update tmqVnodeTransform.py --- tests/system-test/7-tmq/tmqVnodeTransform.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/system-test/7-tmq/tmqVnodeTransform.py b/tests/system-test/7-tmq/tmqVnodeTransform.py index 3ebf48b93a..aab94bc7a2 100644 --- a/tests/system-test/7-tmq/tmqVnodeTransform.py +++ b/tests/system-test/7-tmq/tmqVnodeTransform.py @@ -331,9 +331,9 @@ class TDTestCase: tdLog.printNoPrefix("======== test case 3 end ...... ") def run(self): - # self.prepareTestEnv() - # self.tmqCase1() - # self.tmqCase2() + self.prepareTestEnv() + self.tmqCase1() + self.tmqCase2() self.prepareTestEnv() self.tmqCase3() From 8253874398cc78fa2d76df35963e0e290154c8a4 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 11 Oct 2023 14:47:26 +0800 Subject: [PATCH 05/59] enh(tsdb/cos): new sdk for s3 --- cmake/cmake.options | 18 ++ cmake/libs3.GNUmakefile | 430 ++++++++++++++++++++++++++++++ cmake/libs3_CMakeLists.txt.in | 28 ++ cmake/ssl_CMakeLists.txt.in | 15 ++ contrib/CMakeLists.txt | 54 ++++ source/dnode/vnode/CMakeLists.txt | 7 +- 6 files changed, 551 insertions(+), 1 deletion(-) create mode 100644 cmake/libs3.GNUmakefile create mode 100644 cmake/libs3_CMakeLists.txt.in create mode 100644 cmake/ssl_CMakeLists.txt.in diff --git a/cmake/cmake.options b/cmake/cmake.options index 1d4e9ba515..bacc5adfb7 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -127,6 +127,22 @@ option( IF(${TD_LINUX}) +option( + BUILD_WITH_S3 + "If build with s3" + ON +) + +IF(${BUILD_WITH_S3}) + +option( + BUILD_WITH_COS + "If build with cos" + OFF +) + +ELSE () + option( BUILD_WITH_COS "If build with cos" @@ -135,6 +151,8 @@ option( ENDIF () +ENDIF () + option( BUILD_WITH_SQLITE "If build with sqlite" diff --git a/cmake/libs3.GNUmakefile b/cmake/libs3.GNUmakefile new file mode 100644 index 0000000000..abf954f5c1 --- /dev/null +++ b/cmake/libs3.GNUmakefile @@ -0,0 +1,430 @@ +# GNUmakefile +# +# Copyright 2008 Bryan Ischo +# +# This file is part of libs3. +# +# libs3 is free software: you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation, version 3 or above of the License. You can also +# redistribute and/or modify it under the terms of the GNU General Public +# License, version 2 or above of the License. +# +# In addition, as a special exception, the copyright holders give +# permission to link the code of this library and its programs with the +# OpenSSL library, and distribute linked combinations including the two. +# +# libs3 is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# version 3 along with libs3, in a file named COPYING. If not, see +# . +# +# You should also have received a copy of the GNU General Public License +# version 2 along with libs3, in a file named COPYING-GPLv2. If not, see +# . + +# I tried to use the autoconf/automake/autolocal/etc (i.e. autohell) tools +# but I just couldn't stomach them. Since this is a Makefile for POSIX +# systems, I will simply do away with autohell completely and use a GNU +# Makefile. GNU make ought to be available pretty much everywhere, so I +# don't see this being a significant issue for portability. + +# All commands assume a GNU compiler. For systems which do not use a GNU +# compiler, write scripts with the same names as these commands, and taking +# the same arguments, and translate the arguments and commands into the +# appropriate non-POSIX ones as needed. libs3 assumes a GNU toolchain as +# the most portable way to build software possible. Non-POSIX, non-GNU +# systems can do the work of supporting this build infrastructure. + + +# -------------------------------------------------------------------------- +# Set libs3 version number, unless it is already set. + +LIBS3_VER_MAJOR ?= 4 +LIBS3_VER_MINOR ?= 1 +LIBS3_VER := $(LIBS3_VER_MAJOR).$(LIBS3_VER_MINOR) + + +# ----------------------------------------------------------------------------- +# Determine verbosity. VERBOSE_SHOW should be prepended to every command which +# should only be displayed if VERBOSE is set. QUIET_ECHO may be used to +# echo text only if VERBOSE is not set. Typically, a VERBOSE_SHOW command will +# be paired with a QUIET_ECHO command, to provide a command which is displayed +# in VERBOSE mode, along with text which is displayed in non-VERBOSE mode to +# describe the command. +# +# No matter what VERBOSE is defined to, it ends up as true if it's defined. +# This will be weird if you defined VERBOSE=false in the environment, and we +# switch it to true here; but the meaning of VERBOSE is, "if it's defined to +# any value, then verbosity is turned on". So don't define VERBOSE if you +# don't want verbosity in the build process. +# ----------------------------------------------------------------------------- + +ifdef VERBOSE + VERBOSE = true + VERBOSE_ECHO = @ echo + VERBOSE_SHOW = + QUIET_ECHO = @ echo > /dev/null +else + VERBOSE = false + VERBOSE_ECHO = @ echo > /dev/null + VERBOSE_SHOW = @ + QUIET_ECHO = @ echo +endif + + +# -------------------------------------------------------------------------- +# BUILD directory +ifndef BUILD + ifdef DEBUG + BUILD := build-debug + else + BUILD := build + endif +endif + + +# -------------------------------------------------------------------------- +# DESTDIR directory +ifndef DESTDIR + DESTDIR := ${HOME}/.cos-local.1 +endif + +# -------------------------------------------------------------------------- +# LIBDIR directory +ifndef LIBDIR + LIBDIR := ${DESTDIR}/lib +endif + +# -------------------------------------------------------------------------- +# Compiler CC handling +ifndef CC + CC := gcc +endif + +# -------------------------------------------------------------------------- +# Acquire configuration information for libraries that libs3 depends upon + +ifndef CURL_LIBS + CURL_LIBS := $(shell curl-config --libs) +endif + +ifndef CURL_CFLAGS + CURL_CFLAGS := $(shell curl-config --cflags) +endif + +ifndef LIBXML2_LIBS + LIBXML2_LIBS := $(shell xml2-config --libs) +endif + +ifndef LIBXML2_CFLAGS + LIBXML2_CFLAGS := $(shell xml2-config --cflags) +endif + +ifndef OPENSSL_LIBS + OPENSSL_LIBS := -lssl -lcrypto +endif + +# -------------------------------------------------------------------------- +# These CFLAGS assume a GNU compiler. For other compilers, write a script +# which converts these arguments into their equivalent for that particular +# compiler. + +ifndef CFLAGS + ifdef DEBUG + CFLAGS := -g + else + CFLAGS := -O3 + endif +endif + +CFLAGS += -Wall -Werror -Wshadow -Wextra -Iinc \ + $(CURL_CFLAGS) $(LIBXML2_CFLAGS) \ + -DLIBS3_VER_MAJOR=\"$(LIBS3_VER_MAJOR)\" \ + -DLIBS3_VER_MINOR=\"$(LIBS3_VER_MINOR)\" \ + -DLIBS3_VER=\"$(LIBS3_VER)\" \ + -D__STRICT_ANSI__ \ + -D_ISOC99_SOURCE \ + -D_POSIX_C_SOURCE=200112L + +LDFLAGS = $(CURL_LIBS) $(LIBXML2_LIBS) $(OPENSSL_LIBS) -lpthread + +STRIP ?= strip +INSTALL := install --strip-program=$(STRIP) + + +# -------------------------------------------------------------------------- +# Default targets are everything + +.PHONY: all +all: exported test + + +# -------------------------------------------------------------------------- +# Exported targets are the library and driver program + +.PHONY: exported +exported: libs3 s3 headers +exported_static: $(LIBS3_STATIC) + +# -------------------------------------------------------------------------- +# Install target + +.PHONY: install install_static +install_static: exported_static + $(QUIET_ECHO) $(LIBDIR)/libs3.a: Installing static library + $(VERBOSE_SHOW) $(INSTALL) -Dp -m u+rw,go+r $(BUILD)/lib/libs3.a \ + $(LIBDIR)/libs3.a + $(QUIET_ECHO) $(DESTDIR)/include/libs3.h: Installing header + $(VERBOSE_SHOW) $(INSTALL) -Dp -m u+rw,go+r inc/libs3.h \ + $(DESTDIR)/include/libs3.h + +install: exported + $(QUIET_ECHO) $(DESTDIR)/bin/s3: Installing executable + $(VERBOSE_SHOW) $(INSTALL) -Dps -m u+rwx,go+rx $(BUILD)/bin/s3 \ + $(DESTDIR)/bin/s3 + $(QUIET_ECHO) \ + $(LIBDIR)/libs3.so.$(LIBS3_VER): Installing shared library + $(VERBOSE_SHOW) $(INSTALL) -Dps -m u+rw,go+r \ + $(BUILD)/lib/libs3.so.$(LIBS3_VER_MAJOR) \ + $(LIBDIR)/libs3.so.$(LIBS3_VER) + $(QUIET_ECHO) \ + $(LIBDIR)/libs3.so.$(LIBS3_VER_MAJOR): Linking shared library + $(VERBOSE_SHOW) ln -sf libs3.so.$(LIBS3_VER) \ + $(LIBDIR)/libs3.so.$(LIBS3_VER_MAJOR) + $(QUIET_ECHO) $(LIBDIR)/libs3.so: Linking shared library + $(VERBOSE_SHOW) ln -sf libs3.so.$(LIBS3_VER_MAJOR) $(LIBDIR)/libs3.so + $(QUIET_ECHO) $(LIBDIR)/libs3.a: Installing static library + $(VERBOSE_SHOW) $(INSTALL) -Dp -m u+rw,go+r $(BUILD)/lib/libs3.a \ + $(LIBDIR)/libs3.a + $(QUIET_ECHO) $(DESTDIR)/include/libs3.h: Installing header + $(VERBOSE_SHOW) $(INSTALL) -Dp -m u+rw,go+r $(BUILD)/include/libs3.h \ + $(DESTDIR)/include/libs3.h + + +# -------------------------------------------------------------------------- +# Uninstall target + +.PHONY: uninstall +uninstall: + $(QUIET_ECHO) Installed files: Uninstalling + $(VERBOSE_SHOW) \ + rm -f $(DESTDIR)/bin/s3 \ + $(DESTDIR)/include/libs3.h \ + $(DESTDIR)/lib/libs3.a \ + $(DESTDIR)/lib/libs3.so \ + $(DESTDIR)/lib/libs3.so.$(LIBS3_VER_MAJOR) \ + $(DESTDIR)/lib/libs3.so.$(LIBS3_VER) + + +# -------------------------------------------------------------------------- +# Compile target patterns + +$(BUILD)/obj/%.o: src/%.c + $(QUIET_ECHO) $@: Compiling object + @ mkdir -p $(dir $(BUILD)/dep/$<) + @ $(CC) $(CFLAGS) -M -MG -MQ $@ -DCOMPILINGDEPENDENCIES \ + -o $(BUILD)/dep/$(<:%.c=%.d) -c $< + @ mkdir -p $(dir $@) + $(VERBOSE_SHOW) $(CC) $(CFLAGS) -o $@ -c $< + +$(BUILD)/obj/%.do: src/%.c + $(QUIET_ECHO) $@: Compiling dynamic object + $(QUIET_ECHO) cflags:${CFLAGS} + @ mkdir -p $(dir $(BUILD)/dep/$<) + @ $(CC) $(CFLAGS) -M -MG -MQ $@ -DCOMPILINGDEPENDENCIES \ + -o $(BUILD)/dep/$(<:%.c=%.dd) -c $< + @ mkdir -p $(dir $@) + $(VERBOSE_SHOW) $(CC) $(CFLAGS) -fpic -fPIC -o $@ -c $< + + +# -------------------------------------------------------------------------- +# libs3 library targets + +LIBS3_SHARED = $(BUILD)/lib/libs3.so.$(LIBS3_VER_MAJOR) +LIBS3_STATIC = $(BUILD)/lib/libs3.a + +.PHONY: libs3 +libs3: $(LIBS3_SHARED) $(LIBS3_STATIC) + +LIBS3_SOURCES := bucket.c bucket_metadata.c error_parser.c general.c \ + object.c request.c request_context.c \ + response_headers_handler.c service_access_logging.c \ + service.c simplexml.c util.c multipart.c + +$(LIBS3_SHARED): $(LIBS3_SOURCES:%.c=$(BUILD)/obj/%.do) + $(QUIET_ECHO) $@: Building shared library + @ mkdir -p $(dir $@) + $(VERBOSE_SHOW) $(CC) -shared -Wl,-soname,libs3.so.$(LIBS3_VER_MAJOR) \ + -o $@ $^ $(LDFLAGS) + +$(LIBS3_STATIC): $(LIBS3_SOURCES:%.c=$(BUILD)/obj/%.o) + $(QUIET_ECHO) $@: Building static library + @ mkdir -p $(dir $@) + $(VERBOSE_SHOW) $(AR) cr $@ $^ + + +# -------------------------------------------------------------------------- +# Driver program targets + +.PHONY: s3 +s3: $(BUILD)/bin/s3 + +$(BUILD)/bin/s3: $(BUILD)/obj/s3.o $(LIBS3_SHARED) + $(QUIET_ECHO) $@: Building executable + @ mkdir -p $(dir $@) + $(VERBOSE_SHOW) $(CC) -o $@ $^ $(LDFLAGS) + + +# -------------------------------------------------------------------------- +# libs3 header targets + +.PHONY: headers +headers: $(BUILD)/include/libs3.h + +$(BUILD)/include/libs3.h: inc/libs3.h + $(QUIET_ECHO) $@: Linking header + @ mkdir -p $(dir $@) + $(VERBOSE_SHOW) ln -sf $(abspath $<) $@ + + +# -------------------------------------------------------------------------- +# Test targets + +.PHONY: test +test: $(BUILD)/bin/testsimplexml + +$(BUILD)/bin/testsimplexml: $(BUILD)/obj/testsimplexml.o $(LIBS3_STATIC) + $(QUIET_ECHO) $@: Building executable + @ mkdir -p $(dir $@) + $(VERBOSE_SHOW) $(CC) -o $@ $^ $(LIBXML2_LIBS) + + +# -------------------------------------------------------------------------- +# Clean target + +.PHONY: clean +clean: + $(QUIET_ECHO) $(BUILD): Cleaning + $(VERBOSE_SHOW) rm -rf $(BUILD) + +.PHONY: distclean +distclean: + $(QUIET_ECHO) $(BUILD): Cleaning + $(VERBOSE_SHOW) rm -rf $(BUILD) + + +# -------------------------------------------------------------------------- +# Clean dependencies target + +.PHONY: cleandeps +cleandeps: + $(QUIET_ECHO) $(BUILD)/dep: Cleaning dependencies + $(VERBOSE_SHOW) rm -rf $(BUILD)/dep + + +# -------------------------------------------------------------------------- +# Dependencies + +ALL_SOURCES := $(LIBS3_SOURCES) s3.c testsimplexml.c + +$(foreach i, $(ALL_SOURCES), $(eval -include $(BUILD)/dep/src/$(i:%.c=%.d))) +$(foreach i, $(ALL_SOURCES), $(eval -include $(BUILD)/dep/src/$(i:%.c=%.dd))) + + +# -------------------------------------------------------------------------- +# Debian package target + +DEBPKG = $(BUILD)/pkg/libs3_$(LIBS3_VER).deb +DEBDEVPKG = $(BUILD)/pkg/libs3-dev_$(LIBS3_VER).deb + +.PHONY: deb +deb: $(DEBPKG) $(DEBDEVPKG) + +$(DEBPKG): DEBARCH = $(shell dpkg-architecture | grep ^DEB_BUILD_ARCH= | \ + cut -d '=' -f 2) +$(DEBPKG): exported $(BUILD)/deb/DEBIAN/control $(BUILD)/deb/DEBIAN/shlibs \ + $(BUILD)/deb/DEBIAN/postinst \ + $(BUILD)/deb/usr/share/doc/libs3/changelog.gz \ + $(BUILD)/deb/usr/share/doc/libs3/changelog.Debian.gz \ + $(BUILD)/deb/usr/share/doc/libs3/copyright + DESTDIR=$(BUILD)/deb/usr $(MAKE) install + rm -rf $(BUILD)/deb/usr/include + rm -f $(BUILD)/deb/usr/lib/libs3.a + @mkdir -p $(dir $@) + fakeroot dpkg-deb -b $(BUILD)/deb $@ + mv $@ $(BUILD)/pkg/libs3_$(LIBS3_VER)_$(DEBARCH).deb + +$(DEBDEVPKG): DEBARCH = $(shell dpkg-architecture | grep ^DEB_BUILD_ARCH= | \ + cut -d '=' -f 2) +$(DEBDEVPKG): exported $(BUILD)/deb-dev/DEBIAN/control \ + $(BUILD)/deb-dev/usr/share/doc/libs3-dev/changelog.gz \ + $(BUILD)/deb-dev/usr/share/doc/libs3-dev/changelog.Debian.gz \ + $(BUILD)/deb-dev/usr/share/doc/libs3-dev/copyright + DESTDIR=$(BUILD)/deb-dev/usr $(MAKE) install + rm -rf $(BUILD)/deb-dev/usr/bin + rm -f $(BUILD)/deb-dev/usr/lib/libs3.so* + @mkdir -p $(dir $@) + fakeroot dpkg-deb -b $(BUILD)/deb-dev $@ + mv $@ $(BUILD)/pkg/libs3-dev_$(LIBS3_VER)_$(DEBARCH).deb + +$(BUILD)/deb/DEBIAN/control: debian/control + @mkdir -p $(dir $@) + echo -n "Depends: " > $@ + dpkg-shlibdeps -Sbuild -O $(BUILD)/lib/libs3.so.$(LIBS3_VER_MAJOR) | \ + cut -d '=' -f 2- >> $@ + sed -e 's/LIBS3_VERSION/$(LIBS3_VER)/' \ + < $< | sed -e 's/DEBIAN_ARCHITECTURE/$(DEBARCH)/' | \ + grep -v ^Source: >> $@ + +$(BUILD)/deb-dev/DEBIAN/control: debian/control.dev + @mkdir -p $(dir $@) + sed -e 's/LIBS3_VERSION/$(LIBS3_VER)/' \ + < $< | sed -e 's/DEBIAN_ARCHITECTURE/$(DEBARCH)/' > $@ + +$(BUILD)/deb/DEBIAN/shlibs: + echo -n "libs3 $(LIBS3_VER_MAJOR) libs3 " > $@ + echo "(>= $(LIBS3_VER))" >> $@ + +$(BUILD)/deb/DEBIAN/postinst: debian/postinst + @mkdir -p $(dir $@) + cp $< $@ + +$(BUILD)/deb/usr/share/doc/libs3/copyright: LICENSE + @mkdir -p $(dir $@) + cp $< $@ + @echo >> $@ + @echo -n "An alternate location for the GNU General Public " >> $@ + @echo "License version 3 on Debian" >> $@ + @echo "systems is /usr/share/common-licenses/GPL-3." >> $@ + +$(BUILD)/deb-dev/usr/share/doc/libs3-dev/copyright: LICENSE + @mkdir -p $(dir $@) + cp $< $@ + @echo >> $@ + @echo -n "An alternate location for the GNU General Public " >> $@ + @echo "License version 3 on Debian" >> $@ + @echo "systems is /usr/share/common-licenses/GPL-3." >> $@ + +$(BUILD)/deb/usr/share/doc/libs3/changelog.gz: debian/changelog + @mkdir -p $(dir $@) + gzip --best -c $< > $@ + +$(BUILD)/deb-dev/usr/share/doc/libs3-dev/changelog.gz: debian/changelog + @mkdir -p $(dir $@) + gzip --best -c $< > $@ + +$(BUILD)/deb/usr/share/doc/libs3/changelog.Debian.gz: debian/changelog.Debian + @mkdir -p $(dir $@) + gzip --best -c $< > $@ + +$(BUILD)/deb-dev/usr/share/doc/libs3-dev/changelog.Debian.gz: \ + debian/changelog.Debian + @mkdir -p $(dir $@) + gzip --best -c $< > $@ + + diff --git a/cmake/libs3_CMakeLists.txt.in b/cmake/libs3_CMakeLists.txt.in new file mode 100644 index 0000000000..b77299baf3 --- /dev/null +++ b/cmake/libs3_CMakeLists.txt.in @@ -0,0 +1,28 @@ +# libs3 + +set(s3_flags "${CMAKE_C_FLAGS} -O3 -Iinc -I$ENV{HOME}/.cos-local.1/include -Iinc ") + +set(SED_CMD "s/CFLAGS += -Wall -Werror/CFLAGS += -I$ENV{HOME}/.cos-local.1/include /") + +function(update_cflags) + file(READ "GNUmakefile" S3_CONTENT) + string(REPLACE "CFLAGS += -Wall -Werror" "CFLAGS += -I$ENV{HOME}/.cos-local.1/include " S3_CONTENT "${S3_CONTENT}") + file(WRITE "GNUmakefile" "${S3_CONTENT}") +endfunction(update_cflags) + +ExternalProject_Add(libs3 + GIT_REPOSITORY https://github.com/bji/libs3 + #GIT_TAG v5.0.16 + SOURCE_DIR "${TD_CONTRIB_DIR}/libs3" + #BINARY_DIR "" + BUILD_IN_SOURCE TRUE + BUILD_ALWAYS 1 + UPDATE_COMMAND "" + #sed_cmd "s/CFLAGS += -Wall -Werror/CFLAGS += -I$ENV{HOME}/.cos-local.1/include /" + CONFIGURE_COMMAND cp ${TD_SUPPORT_DIR}/libs3.GNUmakefile GNUmakefile && sed -i "s|CFLAGS += -Wall -Werror|CFLAGS += -I'$ENV{HOME}/.cos-local.1/include' -L'$ENV{HOME}/.cos-local.1/lib' |" ./GNUmakefile + #BUILD_COMMAND make CFLAGS=${s3_flags} DESTDIR=$ENV{HOME}/.cos-local.1 build/lib/libs3.a + #BUILD_COMMAND make DESTDIR="$ENV{HOME}/.cos-local.1" build/lib/libs3.a + BUILD_COMMAND make build/lib/libs3.a + INSTALL_COMMAND make install_static + TEST_COMMAND "" +) diff --git a/cmake/ssl_CMakeLists.txt.in b/cmake/ssl_CMakeLists.txt.in new file mode 100644 index 0000000000..7821848618 --- /dev/null +++ b/cmake/ssl_CMakeLists.txt.in @@ -0,0 +1,15 @@ +# openssl +ExternalProject_Add(openssl + URL https://www.openssl.org/source/openssl-3.1.3.tar.gz + URL_HASH SHA256=f0316a2ebd89e7f2352976445458689f80302093788c466692fb2a188b2eacf6 + DOWNLOAD_NO_PROGRESS 1 + DOWNLOAD_DIR "${TD_CONTRIB_DIR}/deps-download" + SOURCE_DIR "${TD_CONTRIB_DIR}/openssl" + BUILD_IN_SOURCE TRUE + #BUILD_ALWAYS 1 + #UPDATE_COMMAND "" + CONFIGURE_COMMAND ./Configure --prefix=$ENV{HOME}/.cos-local.1 -static #--no-shared + BUILD_COMMAND make -j + INSTALL_COMMAND make install + TEST_COMMAND "" +) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index e3e48ac3a1..39bf85df8f 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -8,6 +8,37 @@ endfunction(cat IN_FILE OUT_FILE) if(${TD_LINUX}) + if(${BUILD_WITH_S3}) + +set(CONTRIB_TMP_FILE3 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in3") +configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3}) + +if(${BUILD_WITH_COS}) + file(MAKE_DIRECTORY $ENV{HOME}/.cos-local.1/) + cat("${TD_SUPPORT_DIR}/ssl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3}) +endif(${BUILD_WITH_COS}) + +configure_file(${CONTRIB_TMP_FILE3} "${TD_CONTRIB_DIR}/deps-download/CMakeLists.txt") +execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" . + WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download") +execute_process(COMMAND "${CMAKE_COMMAND}" --build . + WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download") + +set(CONTRIB_TMP_FILE2 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in2") +configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE2}) + +if(${BUILD_WITH_COS}) + cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE2}) +endif(${BUILD_WITH_COS}) + +configure_file(${CONTRIB_TMP_FILE2} "${TD_CONTRIB_DIR}/deps-download/CMakeLists.txt") +execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" . + WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download") +execute_process(COMMAND "${CMAKE_COMMAND}" --build . + WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download") + +else() + set(CONTRIB_TMP_FILE3 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in3") configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3}) @@ -37,6 +68,8 @@ execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" . execute_process(COMMAND "${CMAKE_COMMAND}" --build . WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download") +endif(${BUILD_WITH_S3}) + endif(${TD_LINUX}) set(CONTRIB_TMP_FILE "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in") @@ -155,6 +188,18 @@ if(${BUILD_WITH_SQLITE}) cat("${TD_SUPPORT_DIR}/sqlite_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) endif(${BUILD_WITH_SQLITE}) +# s3 +if(${BUILD_WITH_S3}) + #cat("${TD_SUPPORT_DIR}/mxml_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) + #cat("${TD_SUPPORT_DIR}/apr_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) + #cat("${TD_SUPPORT_DIR}/apr-util_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) + #cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) + #INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.1/include) + cat("${TD_SUPPORT_DIR}/libs3_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) + add_definitions(-DUSE_S3) + +else() + # cos if(${BUILD_WITH_COS}) #cat("${TD_SUPPORT_DIR}/mxml_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) @@ -165,6 +210,8 @@ if(${BUILD_WITH_COS}) add_definitions(-DUSE_COS) endif(${BUILD_WITH_COS}) +endif(${BUILD_WITH_S3}) + # lucene if(${BUILD_WITH_LUCENE}) cat("${TD_SUPPORT_DIR}/lucene_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) @@ -390,6 +437,11 @@ if (${BUILD_WITH_ROCKSDB}) endif() endif() +if(${BUILD_WITH_S3}) + INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.1/include) + +else() + # cos if(${BUILD_WITH_COS}) if(${TD_LINUX}) @@ -415,6 +467,8 @@ if(${BUILD_WITH_COS}) endif(${TD_LINUX}) endif(${BUILD_WITH_COS}) +endif(${BUILD_WITH_S3}) + # lucene # To support build on ubuntu: sudo apt-get install libboost-all-dev if(${BUILD_WITH_LUCENE}) diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 84d54f3350..516c8a8b69 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -162,6 +162,12 @@ target_link_libraries( ) if(${TD_LINUX}) + if(${BUILD_WITH_S3}) + + endif(${BUILD_WITH_S3}) + +if(${BUILD_WITH_COS}) + set(CMAKE_FIND_LIBRARY_SUFFIXES ".a") find_library(APR_LIBRARY apr-1 PATHS /usr/local/apr/lib/) find_library(APR_UTIL_LIBRARY aprutil-1 PATHS /usr/local/apr/lib/) @@ -194,7 +200,6 @@ target_include_directories( PUBLIC "$ENV{HOME}/.cos-local.1/include" ) -if(${BUILD_WITH_COS}) add_definitions(-DUSE_COS) endif(${BUILD_WITH_COS}) From 0e31231212417690974c3d641b5b29ffa25a34d8 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Fri, 13 Oct 2023 16:22:32 +0800 Subject: [PATCH 06/59] test: support replica3 in splited/transform tmqvnode --- .../5dnode3mnodeSep1VnodeStopDnodeCreateDb.py | 4 +- .../system-test/7-tmq/tmqVnodeSplit-column.py | 213 +++++++++ tests/system-test/7-tmq/tmqVnodeSplit-db.py | 214 +++++++++ ...tmqVnodeSplit-stb-select-duplicatedata.py} | 23 +- .../7-tmq/tmqVnodeSplit-stb-select.py | 213 +++++++++ tests/system-test/7-tmq/tmqVnodeSplit-stb.py | 213 +++++++++ .../7-tmq/tmqVnodeTransform-stb.py | 433 ++++++++++++++++++ 7 files changed, 1301 insertions(+), 12 deletions(-) create mode 100644 tests/system-test/7-tmq/tmqVnodeSplit-column.py create mode 100644 tests/system-test/7-tmq/tmqVnodeSplit-db.py rename tests/system-test/7-tmq/{tmqVnodeSplit.py => tmqVnodeSplit-stb-select-duplicatedata.py} (93%) create mode 100644 tests/system-test/7-tmq/tmqVnodeSplit-stb-select.py create mode 100644 tests/system-test/7-tmq/tmqVnodeSplit-stb.py create mode 100644 tests/system-test/7-tmq/tmqVnodeTransform-stb.py diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py index 3a972ff4e9..fb62110b14 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py @@ -146,9 +146,9 @@ class TDTestCase: # a11111=paraDict["dbNumbers"] # print(f"==================={dbNameIndex},{a11111}") threads.append(threading.Thread(target=clusterComCreate.createDeltedatabases, args=(newTdSql, dbNameIndex,repeatNumber,paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica']))) - + newTdSql2=tdCom.newTdSql() redbNameIndex = '%s%d'%(paraDict["dbName"],i+100) - threads.append(threading.Thread(target=clusterComCreate.createDeltedatabases, args=(newTdSql, redbNameIndex,1,paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica']))) + threads.append(threading.Thread(target=clusterComCreate.createDeltedatabases, args=(newTdSql2, redbNameIndex,1,paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica']))) for tr in threads: tr.start() diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-column.py b/tests/system-test/7-tmq/tmqVnodeSplit-column.py new file mode 100644 index 0000000000..95c35363a8 --- /dev/null +++ b/tests/system-test/7-tmq/tmqVnodeSplit-column.py @@ -0,0 +1,213 @@ + +import taos +import sys +import time +import socket +import os +import threading +import math + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +from util.cluster import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def __init__(self): + self.vgroups = 1 + self.ctbNum = 10 + self.rowsPerTbl = 10000 + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def getDataPath(self): + selfPath = tdCom.getBuildPath() + + return selfPath + '/../sim/dnode%d/data/vnode/vnode%d/wal/*'; + + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 60, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tdCom.drop_all_db() + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], wal_retention_period=36000,vgroups=paraDict["vgroups"],replica=self.replicaVar) + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + return + + def restartAndRemoveWal(self, deleteWal): + tdDnodes = cluster.dnodes + tdSql.query("select * from information_schema.ins_vnodes") + for result in tdSql.queryResult: + if result[2] == 'dbt': + tdLog.debug("dnode is %d"%(result[0])) + dnodeId = result[0] + vnodeId = result[1] + + tdDnodes[dnodeId - 1].stoptaosd() + time.sleep(1) + dataPath = self.getDataPath() + dataPath = dataPath%(dnodeId,vnodeId) + tdLog.debug("dataPath:%s"%dataPath) + if deleteWal: + if os.system('rm -rf ' + dataPath) != 0: + tdLog.exit("rm error") + + tdDnodes[dnodeId - 1].starttaosd() + time.sleep(1) + break + tdLog.debug("restart dnode ok") + + def splitVgroups(self): + tdSql.query("select * from information_schema.ins_vnodes") + vnodeId = 0 + for result in tdSql.queryResult: + if result[2] == 'dbt': + vnodeId = result[1] + tdLog.debug("vnode is %d"%(vnodeId)) + break + splitSql = "split vgroup %d" %(vnodeId) + tdLog.debug("splitSql:%s"%(splitSql)) + tdSql.query(splitSql) + tdLog.debug("splitSql ok") + + def tmqCase1(self, deleteWal=False): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb1', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 60, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + topicNameList = ['topic1'] + # expectRowsList = [] + tmqCom.initConsumerTable() + + tdLog.info("create topics from stb with filter") + queryString = "select * from %s.%s where c2 >= 0 "%(paraDict['dbName'], paraDict['stbName']) + # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + # tdSql.query(queryString) + # expectRowsList.append(tdSql.getRows()) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2 + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + + tdLog.info("create ctb1") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("create ctb2") + paraDict2 = paraDict.copy() + paraDict2['ctbPrefix'] = "ctb2" + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict2['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("insert ctb1 data") + pInsertThread = tmqCom.asyncInsertDataByInterlace(paraDict) + + tmqCom.getStartConsumeNotifyFromTmqsim() + tmqCom.getStartCommitNotifyFromTmqsim() + + #restart dnode & remove wal + self.restartAndRemoveWal(deleteWal) + + # split vgroup + self.splitVgroups() + + + tdLog.info("insert ctb2 data") + pInsertThread1 = tmqCom.asyncInsertDataByInterlace(paraDict2) + pInsertThread.join() + pInsertThread1.join() + + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + if expectrowcnt / 2 >= resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0])) + tdLog.exit("%d tmq consume rows error!"%consumerId) + + # tmqCom.checkFileContent(consumerId, queryString) + + time.sleep(2) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def run(self): + self.prepareTestEnv() + self.tmqCase1(True) + self.prepareTestEnv() + self.tmqCase1(False) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-db.py b/tests/system-test/7-tmq/tmqVnodeSplit-db.py new file mode 100644 index 0000000000..1c2e867758 --- /dev/null +++ b/tests/system-test/7-tmq/tmqVnodeSplit-db.py @@ -0,0 +1,214 @@ + +import taos +import sys +import time +import socket +import os +import threading +import math + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +from util.cluster import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def __init__(self): + self.vgroups = 1 + self.ctbNum = 10 + self.rowsPerTbl = 10000 + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def getDataPath(self): + selfPath = tdCom.getBuildPath() + + return selfPath + '/../sim/dnode%d/data/vnode/vnode%d/wal/*'; + + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 60, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tdCom.drop_all_db() + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], wal_retention_period=36000,vgroups=paraDict["vgroups"],replica=self.replicaVar) + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + return + + def restartAndRemoveWal(self, deleteWal): + tdDnodes = cluster.dnodes + tdSql.query("select * from information_schema.ins_vnodes") + for result in tdSql.queryResult: + if result[2] == 'dbt': + tdLog.debug("dnode is %d"%(result[0])) + dnodeId = result[0] + vnodeId = result[1] + + tdDnodes[dnodeId - 1].stoptaosd() + time.sleep(1) + dataPath = self.getDataPath() + dataPath = dataPath%(dnodeId,vnodeId) + tdLog.debug("dataPath:%s"%dataPath) + if deleteWal: + if os.system('rm -rf ' + dataPath) != 0: + tdLog.exit("rm error") + + tdDnodes[dnodeId - 1].starttaosd() + time.sleep(1) + break + tdLog.debug("restart dnode ok") + + def splitVgroups(self): + tdSql.query("select * from information_schema.ins_vnodes") + vnodeId = 0 + for result in tdSql.queryResult: + if result[2] == 'dbt': + vnodeId = result[1] + tdLog.debug("vnode is %d"%(vnodeId)) + break + splitSql = "split vgroup %d" %(vnodeId) + tdLog.debug("splitSql:%s"%(splitSql)) + tdSql.query(splitSql) + tdLog.debug("splitSql ok") + + def tmqCase1(self, deleteWal=False): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb1', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 60, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + topicNameList = ['topic1'] + # expectRowsList = [] + tmqCom.initConsumerTable() + + tdLog.info("create topics from db") + queryString = "database %s"%(paraDict['dbName']) + # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + # tdSql.query(queryString) + # expectRowsList.append(tdSql.getRows()) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2 + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + + tdLog.info("create ctb1") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("create ctb2") + paraDict2 = paraDict.copy() + + paraDict2['ctbPrefix'] = "ctb2" + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict2['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("insert ctb1 data") + pInsertThread = tmqCom.asyncInsertDataByInterlace(paraDict) + + tmqCom.getStartConsumeNotifyFromTmqsim() + tmqCom.getStartCommitNotifyFromTmqsim() + + #restart dnode & remove wal + self.restartAndRemoveWal(deleteWal) + + # split vgroup + self.splitVgroups() + + + tdLog.info("insert ctb2 data") + pInsertThread1 = tmqCom.asyncInsertDataByInterlace(paraDict2) + pInsertThread.join() + pInsertThread1.join() + + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + if expectrowcnt / 2 >= resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0])) + tdLog.exit("%d tmq consume rows error!"%consumerId) + + # tmqCom.checkFileContent(consumerId, queryString) + + time.sleep(2) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def run(self): + self.prepareTestEnv() + self.tmqCase1(True) + self.prepareTestEnv() + self.tmqCase1(False) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/tmqVnodeSplit.py b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py similarity index 93% rename from tests/system-test/7-tmq/tmqVnodeSplit.py rename to tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py index 67deb84620..086fe6e4c5 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py @@ -58,7 +58,7 @@ class TDTestCase: paraDict['ctbNum'] = self.ctbNum paraDict['rowsPerTbl'] = self.rowsPerTbl - tdCom.drop_all_db(); + tdCom.drop_all_db() tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], wal_retention_period=36000,vgroups=paraDict["vgroups"],replica=self.replicaVar) tdLog.info("create stb") @@ -112,13 +112,13 @@ class TDTestCase: 'tagPrefix': 't', 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], - 'ctbPrefix': 'ctb', + 'ctbPrefix': 'ctb1', 'ctbStartIdx': 0, 'ctbNum': 10, 'rowsPerTbl': 10000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 60, + 'pollDelay': 120, 'showMsg': 1, 'showRow': 1, 'snapshot': 0} @@ -157,7 +157,13 @@ class TDTestCase: tdLog.info("create ctb1") tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) - tdLog.info("insert data") + + tdLog.info("create ctb2") + paraDict['ctbPrefix'] = "ctb2" + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("insert ctb1 data") pInsertThread = tmqCom.asyncInsertDataByInterlace(paraDict) tmqCom.getStartConsumeNotifyFromTmqsim() @@ -169,11 +175,8 @@ class TDTestCase: # split vgroup self.splitVgroups() - tdLog.info("create ctb2") - paraDict['ctbPrefix'] = "ctbn" - tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], - ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) - tdLog.info("insert data") + + tdLog.info("insert ctb2 data") pInsertThread1 = tmqCom.asyncInsertDataByInterlace(paraDict) pInsertThread.join() pInsertThread1.join() @@ -181,7 +184,7 @@ class TDTestCase: expectRows = 1 resultList = tmqCom.selectConsumeResult(expectRows) - if expectrowcnt / 2 >= resultList[0]: + if expectrowcnt / 2 >= resultList[0] or expectrowcnt <= resultList[0]: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0])) tdLog.exit("%d tmq consume rows error!"%consumerId) diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select.py b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select.py new file mode 100644 index 0000000000..3214c2f5c4 --- /dev/null +++ b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select.py @@ -0,0 +1,213 @@ + +import taos +import sys +import time +import socket +import os +import threading +import math + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +from util.cluster import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def __init__(self): + self.vgroups = 1 + self.ctbNum = 10 + self.rowsPerTbl = 10000 + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def getDataPath(self): + selfPath = tdCom.getBuildPath() + + return selfPath + '/../sim/dnode%d/data/vnode/vnode%d/wal/*'; + + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 60, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tdCom.drop_all_db() + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], wal_retention_period=36000,vgroups=paraDict["vgroups"],replica=self.replicaVar) + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + return + + def restartAndRemoveWal(self, deleteWal): + tdDnodes = cluster.dnodes + tdSql.query("select * from information_schema.ins_vnodes") + for result in tdSql.queryResult: + if result[2] == 'dbt': + tdLog.debug("dnode is %d"%(result[0])) + dnodeId = result[0] + vnodeId = result[1] + + tdDnodes[dnodeId - 1].stoptaosd() + time.sleep(1) + dataPath = self.getDataPath() + dataPath = dataPath%(dnodeId,vnodeId) + tdLog.debug("dataPath:%s"%dataPath) + if deleteWal: + if os.system('rm -rf ' + dataPath) != 0: + tdLog.exit("rm error") + + tdDnodes[dnodeId - 1].starttaosd() + time.sleep(1) + break + tdLog.debug("restart dnode ok") + + def splitVgroups(self): + tdSql.query("select * from information_schema.ins_vnodes") + vnodeId = 0 + for result in tdSql.queryResult: + if result[2] == 'dbt': + vnodeId = result[1] + tdLog.debug("vnode is %d"%(vnodeId)) + break + splitSql = "split vgroup %d" %(vnodeId) + tdLog.debug("splitSql:%s"%(splitSql)) + tdSql.query(splitSql) + tdLog.debug("splitSql ok") + + def tmqCase1(self, deleteWal=False): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb1', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 120, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + topicNameList = ['topic1'] + # expectRowsList = [] + tmqCom.initConsumerTable() + + tdLog.info("create topics from stb with filter") + queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName']) + # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + # tdSql.query(queryString) + # expectRowsList.append(tdSql.getRows()) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2 + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + + tdLog.info("create ctb1") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("create ctb2") + paraDict2 = paraDict.copy() + paraDict2['ctbPrefix'] = "ctb2" + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict2['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("insert ctb1 data") + pInsertThread = tmqCom.asyncInsertDataByInterlace(paraDict) + + tmqCom.getStartConsumeNotifyFromTmqsim() + tmqCom.getStartCommitNotifyFromTmqsim() + + #restart dnode & remove wal + self.restartAndRemoveWal(deleteWal) + + # split vgroup + self.splitVgroups() + + + tdLog.info("insert ctb2 data") + pInsertThread1 = tmqCom.asyncInsertDataByInterlace(paraDict2) + pInsertThread.join() + pInsertThread1.join() + + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + if expectrowcnt / 2 >= resultList[0] or expectrowcnt < resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0])) + tdLog.exit("%d tmq consume rows error!"%consumerId) + + # tmqCom.checkFileContent(consumerId, queryString) + + time.sleep(2) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def run(self): + self.prepareTestEnv() + self.tmqCase1(True) + self.prepareTestEnv() + self.tmqCase1(False) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-stb.py b/tests/system-test/7-tmq/tmqVnodeSplit-stb.py new file mode 100644 index 0000000000..27d296ed0e --- /dev/null +++ b/tests/system-test/7-tmq/tmqVnodeSplit-stb.py @@ -0,0 +1,213 @@ + +import taos +import sys +import time +import socket +import os +import threading +import math + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +from util.cluster import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def __init__(self): + self.vgroups = 1 + self.ctbNum = 10 + self.rowsPerTbl = 10000 + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def getDataPath(self): + selfPath = tdCom.getBuildPath() + + return selfPath + '/../sim/dnode%d/data/vnode/vnode%d/wal/*'; + + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 60, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tdCom.drop_all_db() + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], wal_retention_period=36000,vgroups=paraDict["vgroups"],replica=self.replicaVar) + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + return + + def restartAndRemoveWal(self, deleteWal): + tdDnodes = cluster.dnodes + tdSql.query("select * from information_schema.ins_vnodes") + for result in tdSql.queryResult: + if result[2] == 'dbt': + tdLog.debug("dnode is %d"%(result[0])) + dnodeId = result[0] + vnodeId = result[1] + + tdDnodes[dnodeId - 1].stoptaosd() + time.sleep(1) + dataPath = self.getDataPath() + dataPath = dataPath%(dnodeId,vnodeId) + tdLog.debug("dataPath:%s"%dataPath) + if deleteWal: + if os.system('rm -rf ' + dataPath) != 0: + tdLog.exit("rm error") + + tdDnodes[dnodeId - 1].starttaosd() + time.sleep(1) + break + tdLog.debug("restart dnode ok") + + def splitVgroups(self): + tdSql.query("select * from information_schema.ins_vnodes") + vnodeId = 0 + for result in tdSql.queryResult: + if result[2] == 'dbt': + vnodeId = result[1] + tdLog.debug("vnode is %d"%(vnodeId)) + break + splitSql = "split vgroup %d" %(vnodeId) + tdLog.debug("splitSql:%s"%(splitSql)) + tdSql.query(splitSql) + tdLog.debug("splitSql ok") + + def tmqCase1(self, deleteWal=False): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb1', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 60, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + topicNameList = ['topic1'] + # expectRowsList = [] + tmqCom.initConsumerTable() + + tdLog.info("create topics from stb ") + queryString = "stable %s.%s"%(paraDict['dbName'], paraDict['stbName']) + # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + # tdSql.query(queryString) + # expectRowsList.append(tdSql.getRows()) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2 + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + + tdLog.info("create ctb1") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("create ctb2") + paraDict2 = paraDict.copy() + paraDict2['ctbPrefix'] = "ctb2" + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict2['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("insert ctb1 data") + pInsertThread = tmqCom.asyncInsertDataByInterlace(paraDict) + + tmqCom.getStartConsumeNotifyFromTmqsim() + tmqCom.getStartCommitNotifyFromTmqsim() + + #restart dnode & remove wal + self.restartAndRemoveWal(deleteWal) + + # split vgroup + self.splitVgroups() + + + tdLog.info("insert ctb2 data") + pInsertThread1 = tmqCom.asyncInsertDataByInterlace(paraDict2) + pInsertThread.join() + pInsertThread1.join() + + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + if expectrowcnt / 2 >= resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0])) + tdLog.exit("%d tmq consume rows error!"%consumerId) + + # tmqCom.checkFileContent(consumerId, queryString) + + time.sleep(2) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def run(self): + self.prepareTestEnv() + self.tmqCase1(True) + self.prepareTestEnv() + self.tmqCase1(False) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/tmqVnodeTransform-stb.py b/tests/system-test/7-tmq/tmqVnodeTransform-stb.py new file mode 100644 index 0000000000..18f9fafe9a --- /dev/null +++ b/tests/system-test/7-tmq/tmqVnodeTransform-stb.py @@ -0,0 +1,433 @@ + +import taos +import sys +import time +import socket +import os +import threading +import math + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +from util.cluster import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def __init__(self): + self.vgroups = 1 + self.ctbNum = 10 + self.rowsPerTbl = 10000 + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def getDataPath(self): + selfPath = tdCom.getBuildPath() + + return selfPath + '/../sim/dnode%d/data/vnode/vnode%d/wal/*'; + + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 60, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tdCom.drop_all_db() + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], wal_retention_period=36000,vgroups=paraDict["vgroups"],replica=self.replicaVar) + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + # tdLog.info("create ctb") + # tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + # ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + # tdLog.info("insert data") + # tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + # ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + # startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + + # tdLog.info("restart taosd to ensure that the data falls into the disk") + # tdDnodes.stop(1) + # tdDnodes.start(1) + # tdSql.query("flush database %s"%(paraDict['dbName'])) + return + + def restartAndRemoveWal(self): + tdDnodes = cluster.dnodes + tdSql.query("select * from information_schema.ins_vnodes") + for result in tdSql.queryResult: + if result[2] == 'dbt': + tdLog.debug("dnode is %d"%(result[0])) + dnodeId = result[0] + vnodeId = result[1] + + tdDnodes[dnodeId - 1].stoptaosd() + time.sleep(1) + dataPath = self.getDataPath() + dataPath = dataPath%(dnodeId,vnodeId) + os.system('rm -rf ' + dataPath) + tdLog.debug("dataPath:%s"%dataPath) + tdDnodes[dnodeId - 1].starttaosd() + time.sleep(1) + break + tdLog.debug("restart dnode ok") + + def redistributeVgroups(self): + dnodesList = [] + tdSql.query("show dnodes") + for result in tdSql.queryResult: + dnodesList.append(result[0]) + print("dnodeList:",dnodesList) + tdSql.query("select * from information_schema.ins_vnodes") + vnodeId = 0 + for result in tdSql.queryResult: + if result[2] == 'dbt': + tdLog.debug("dnode is %d"%(result[0])) + dnodesList.remove(result[0]) + vnodeId = result[1] + print("its all data",dnodesList) + # if self.replicaVar == 1: + # redistributeSql = "redistribute vgroup %d dnode %d" %(vnodeId, dnodesList[0]) + # else: + redistributeSql = f"redistribute vgroup {vnodeId} " + for vgdnode in dnodesList: + redistributeSql += f"dnode {vgdnode} " + print(redistributeSql) + + tdLog.debug(f"redistributeSql:{redistributeSql}") + tdSql.query(redistributeSql) + tdLog.debug("redistributeSql ok") + + def tmqCaseStable(self): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb1', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 60, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + topicNameList = ['topic1'] + # expectRowsList = [] + tmqCom.initConsumerTable() + + tdLog.info("create topics from stb with filter") + queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName']) + # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + # tdSql.query(queryString) + # expectRowsList.append(tdSql.getRows()) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2 + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + + tdLog.info("create ctb1") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("create ctb2") + paraDict2 = paraDict.copy() + paraDict2['ctbPrefix'] = "ctb2" + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict2['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("insert data") + pInsertThread = tmqCom.asyncInsertDataByInterlace(paraDict) + + tmqCom.getStartConsumeNotifyFromTmqsim() + tmqCom.getStartCommitNotifyFromTmqsim() + + #restart dnode & remove wal + # self.restartAndRemoveWal() + + # redistribute vgroup + self.redistributeVgroups() + + + tdLog.info("insert data") + pInsertThread1 = tmqCom.asyncInsertDataByInterlace(paraDict2) + pInsertThread.join() + pInsertThread1.join() + + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + if expectrowcnt / 2 >= resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0])) + tdLog.exit("%d tmq consume rows error!"%consumerId) + + # tmqCom.checkFileContent(consumerId, queryString) + + time.sleep(5) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def tmqCaseNtable(self): + tdLog.printNoPrefix("======== test case 2: ") + paraDict = {'dbName':'dbt'} + + ntbName = "ntb" + + topicNameList = ['topic2'] + tmqCom.initConsumerTable() + + sqlString = "create table %s.%s(ts timestamp, i nchar(8))" %(paraDict['dbName'], ntbName) + tdLog.info("create nomal table sql: %s"%sqlString) + tdSql.execute(sqlString) + + tdLog.info("create topics from nomal table") + queryString = "select * from %s.%s"%(paraDict['dbName'], ntbName) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + tdSql.query("flush database %s"%(paraDict['dbName'])) + #restart dnode & remove wal + # self.restartAndRemoveWal() + + # redistribute vgroup + self.redistributeVgroups() + + sqlString = "alter table %s.%s modify column i nchar(16)" %(paraDict['dbName'], ntbName) + tdLog.info("alter table sql: %s"%sqlString) + tdSql.error(sqlString) + expectRows = 0 + resultList = tmqCom.selectConsumeResult(expectRows) + time.sleep(5) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 2 end ...... ") + + def tmqCaseStableSelect(self): + tdLog.printNoPrefix("======== test case subscrib column start : ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stbn', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 2, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + topicNameList = ['topic3'] + tmqCom.initConsumerTable() + + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + + tdLog.info("create ctb") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + tdLog.info("insert data") + tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("create topics from stb with filter") + queryString = "select * from %s.%s where c2 > 0 "%(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + + time.sleep(5) + #restart dnode & remove wal + # self.restartAndRemoveWal() + + # redistribute vgroup + self.redistributeVgroups() + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + time.sleep(5) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case subscrib column end ...... ") + + def tmqCaseDbname(self): + tdLog.printNoPrefix("======== test case subscrib Dbname start: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stbn', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 2, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + topicNameList = ['topic4'] + tmqCom.initConsumerTable() + + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + + tdLog.info("create ctb") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + tdLog.info("insert data") + tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("create topics from database ") + queryString = "database %s "%(paraDict['dbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + + time.sleep(5) + #restart dnode & remove wal + # self.restartAndRemoveWal() + + # redistribute vgroup + self.redistributeVgroups() + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + time.sleep(5) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case subscrib Dbname end ...... ") + + def run(self): + self.prepareTestEnv() + self.tmqCaseStable() + self.prepareTestEnv() + self.tmqCaseNtable() + self.prepareTestEnv() + self.tmqCaseStableSelect() + self.prepareTestEnv() + self.tmqCaseDbname() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From 0a1ff3283e50db9bde8cfe62a8a99166bd04504d Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 13 Oct 2023 16:59:05 +0800 Subject: [PATCH 07/59] vnode/cos: first round size delte and get --- cmake/curl_CMakeLists.txt.in | 3 +- cmake/libs3_CMakeLists.txt.in | 1 + contrib/CMakeLists.txt | 14 +- source/common/src/tglobal.c | 16 +- source/dnode/vnode/CMakeLists.txt | 29 +++- source/dnode/vnode/src/vnd/vnodeCos.c | 205 +++++++++++++++++++++++++- 6 files changed, 251 insertions(+), 17 deletions(-) diff --git a/cmake/curl_CMakeLists.txt.in b/cmake/curl_CMakeLists.txt.in index 1f2291c519..24a1210422 100644 --- a/cmake/curl_CMakeLists.txt.in +++ b/cmake/curl_CMakeLists.txt.in @@ -10,7 +10,8 @@ ExternalProject_Add(curl BUILD_IN_SOURCE TRUE BUILD_ALWAYS 1 #UPDATE_COMMAND "" - CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --without-ssl --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd + #CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --without-ssl --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd + CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --with-ssl=$ENV{HOME}/.cos-local.1 --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd #CONFIGURE_COMMAND ./configure --without-ssl BUILD_COMMAND make INSTALL_COMMAND make install diff --git a/cmake/libs3_CMakeLists.txt.in b/cmake/libs3_CMakeLists.txt.in index b77299baf3..b54a084f66 100644 --- a/cmake/libs3_CMakeLists.txt.in +++ b/cmake/libs3_CMakeLists.txt.in @@ -13,6 +13,7 @@ endfunction(update_cflags) ExternalProject_Add(libs3 GIT_REPOSITORY https://github.com/bji/libs3 #GIT_TAG v5.0.16 + DEPENDS curl SOURCE_DIR "${TD_CONTRIB_DIR}/libs3" #BINARY_DIR "" BUILD_IN_SOURCE TRUE diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 39bf85df8f..1588eb4025 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -13,10 +13,8 @@ if(${TD_LINUX}) set(CONTRIB_TMP_FILE3 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in3") configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3}) -if(${BUILD_WITH_COS}) file(MAKE_DIRECTORY $ENV{HOME}/.cos-local.1/) cat("${TD_SUPPORT_DIR}/ssl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3}) -endif(${BUILD_WITH_COS}) configure_file(${CONTRIB_TMP_FILE3} "${TD_CONTRIB_DIR}/deps-download/CMakeLists.txt") execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" . @@ -27,9 +25,7 @@ execute_process(COMMAND "${CMAKE_COMMAND}" --build . set(CONTRIB_TMP_FILE2 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in2") configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE2}) -if(${BUILD_WITH_COS}) cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE2}) -endif(${BUILD_WITH_COS}) configure_file(${CONTRIB_TMP_FILE2} "${TD_CONTRIB_DIR}/deps-download/CMakeLists.txt") execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" . @@ -129,6 +125,9 @@ if(${BUILD_TEST}) cat("${TD_SUPPORT_DIR}/stub_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) endif(${BUILD_TEST}) +# xml2 +cat("${TD_SUPPORT_DIR}/xml2_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) + # lz4 cat("${TD_SUPPORT_DIR}/lz4_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) @@ -195,6 +194,7 @@ if(${BUILD_WITH_S3}) #cat("${TD_SUPPORT_DIR}/apr-util_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) #cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) #INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.1/include) + cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE2}) cat("${TD_SUPPORT_DIR}/libs3_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) add_definitions(-DUSE_S3) @@ -295,6 +295,9 @@ target_include_directories( ) unset(CMAKE_PROJECT_INCLUDE_BEFORE) +# xml2 +add_subdirectory(xml2 EXCLUDE_FROM_ALL) + # lz4 add_subdirectory(lz4/build/cmake EXCLUDE_FROM_ALL) target_include_directories( @@ -439,6 +442,7 @@ endif() if(${BUILD_WITH_S3}) INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.1/include) + MESSAGE("build with s3: ${BUILD_WITH_S3}") else() @@ -449,7 +453,7 @@ if(${BUILD_WITH_COS}) #ADD_DEFINITIONS(-DMINIXML_LIBRARY=${CMAKE_BINARY_DIR}/build/lib/libxml.a) option(ENABLE_TEST "Enable the tests" OFF) INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.1/include) - MESSAGE("$ENV{HOME}/.cos-local.1/include") + #MESSAGE("$ENV{HOME}/.cos-local.1/include") set(CMAKE_BUILD_TYPE debug) set(ORIG_CMAKE_PROJECT_NAME ${CMAKE_PROJECT_NAME}) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 92b5ff2828..8ea5528fbb 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -15,12 +15,12 @@ #define _DEFAULT_SOURCE #include "tglobal.h" +#include "defines.h" #include "os.h" #include "tconfig.h" #include "tgrant.h" #include "tlog.h" #include "tmisce.h" -#include "defines.h" #if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL) #include "cus_name.h" @@ -221,7 +221,7 @@ float tsFPrecision = 1E-8; // float column precision double tsDPrecision = 1E-16; // double column precision uint32_t tsMaxRange = 500; // max quantization intervals uint32_t tsCurRange = 100; // current quantization intervals -bool tsIfAdtFse = false; // ADT-FSE algorithom or original huffman algorithom +bool tsIfAdtFse = false; // ADT-FSE algorithom or original huffman algorithom char tsCompressor[32] = "ZSTD_COMPRESSOR"; // ZSTD_COMPRESSOR or GZIP_COMPRESSOR // udf @@ -266,6 +266,9 @@ char tsS3BucketName[TSDB_FQDN_LEN] = ""; char tsS3AppId[TSDB_FQDN_LEN] = ""; int8_t tsS3Enabled = false; +int8_t tsS3Https = true; +char tsS3Hostname[TSDB_FQDN_LEN] = ""; + int32_t tsS3BlockSize = 4096; // number of tsdb pages int32_t tsS3BlockCacheSize = 16; // number of blocks @@ -307,6 +310,14 @@ int32_t taosSetS3Cfg(SConfig *pCfg) { tstrncpy(tsS3AccessKeySecret, colon + 1, TSDB_FQDN_LEN); tstrncpy(tsS3Endpoint, cfgGetItem(pCfg, "s3Endpoint")->str, TSDB_FQDN_LEN); tstrncpy(tsS3BucketName, cfgGetItem(pCfg, "s3BucketName")->str, TSDB_FQDN_LEN); + char *proto = strstr(tsS3Endpoint, "https://"); + if (!proto) { + tsS3Https = false; + tstrncpy(tsS3Hostname, tsS3Endpoint + 7, TSDB_FQDN_LEN); + } else { + tstrncpy(tsS3Hostname, tsS3Endpoint + 8, TSDB_FQDN_LEN); + } + char *cos = strstr(tsS3Endpoint, "cos."); if (cos) { char *appid = strrchr(tsS3BucketName, '-'); @@ -1086,7 +1097,6 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsIfAdtFse = cfgGetItem(pCfg, "IfAdtFse")->bval; tstrncpy(tsCompressor, cfgGetItem(pCfg, "Compressor")->str, sizeof(tsCompressor)); - tsDisableStream = cfgGetItem(pCfg, "disableStream")->bval; tsStreamBufferSize = cfgGetItem(pCfg, "streamBufferSize")->i64; diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 516c8a8b69..e554d94e4d 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -162,9 +162,34 @@ target_link_libraries( ) if(${TD_LINUX}) - if(${BUILD_WITH_S3}) - endif(${BUILD_WITH_S3}) +if(${BUILD_WITH_S3}) + MESSAGE("build with s3: ${BUILD_WITH_S3}") + target_include_directories( + vnode + + PUBLIC "$ENV{HOME}/.cos-local.1/include" + ) + + set(CMAKE_FIND_LIBRARY_SUFFIXES ".a") + set(CMAKE_PREFIX_PATH $ENV{HOME}/.cos-local.1) + find_library(S3_LIBRARY s3) + find_library(CURL_LIBRARY curl) + find_library(SSL_LIBRARY ssl PATHS $ENV{HOME}/.cos-local.1/lib64) + find_library(CRYPTO_LIBRARY crypto PATHS $ENV{HOME}/.cos-local.1/lib64) + target_link_libraries( + vnode + + # s3 + PUBLIC ${S3_LIBRARY} + PUBLIC ${CURL_LIBRARY} + PUBLIC ${SSL_LIBRARY} + PUBLIC ${CRYPTO_LIBRARY} + PUBLIC xml2 + ) + + add_definitions(-DUSE_S3) +endif(${BUILD_WITH_S3}) if(${BUILD_WITH_COS}) diff --git a/source/dnode/vnode/src/vnd/vnodeCos.c b/source/dnode/vnode/src/vnd/vnodeCos.c index 5f650be97d..8984d33ad8 100644 --- a/source/dnode/vnode/src/vnd/vnodeCos.c +++ b/source/dnode/vnode/src/vnd/vnodeCos.c @@ -2,13 +2,206 @@ #include "vndCos.h" -extern char tsS3Endpoint[]; -extern char tsS3AccessKeyId[]; -extern char tsS3AccessKeySecret[]; -extern char tsS3BucketName[]; -extern char tsS3AppId[]; +extern char tsS3Endpoint[]; +extern char tsS3AccessKeyId[]; +extern char tsS3AccessKeySecret[]; +extern char tsS3BucketName[]; +extern char tsS3AppId[]; +extern char tsS3Hostname[]; +extern int8_t tsS3Https; + +#if defined(USE_S3) + +#include "libs3.h" + +static int verifyPeerG = 0; +static const char *awsRegionG = NULL; +static int forceG = 0; +static int showResponsePropertiesG = 0; +static S3Protocol protocolG = S3ProtocolHTTPS; +// static S3Protocol protocolG = S3ProtocolHTTP; +static S3UriStyle uriStyleG = S3UriStylePath; +static int retriesG = 5; +static int timeoutMsG = 0; + +static int32_t s3Begin() { + S3Status status; + const char *hostname = tsS3Hostname; + const char *env_hn = getenv("S3_HOSTNAME"); + + if (env_hn) { + hostname = env_hn; + } + + if ((status = S3_initialize("s3", verifyPeerG | S3_INIT_ALL, hostname)) != S3StatusOK) { + vError("Failed to initialize libs3: %s\n", S3_get_status_name(status)); + return -1; + } + + protocolG = !tsS3Https; + + return 0; +} + +static void s3End() { S3_deinitialize(); } +int32_t s3Init() { return s3Begin(); } + +void s3CleanUp() { s3End(); } + +static int should_retry() { + /* + if (retriesG--) { + // Sleep before next retry; start out with a 1 second sleep + static int retrySleepInterval = 1 * SLEEP_UNITS_PER_SECOND; + sleep(retrySleepInterval); + // Next sleep 1 second longer + retrySleepInterval++; + return 1; + } + */ + + return 0; +} + +typedef struct { + uint64_t content_length; + int status; + char *buf; + char err_msg[4096]; +} TS3SizeCBD; + +static S3Status responsePropertiesCallback(const S3ResponseProperties *properties, void *callbackData) { + //(void)callbackData; + TS3SizeCBD *cbd = callbackData; + if (properties->contentLength > 0) { + cbd->content_length = properties->contentLength; + } else { + cbd->content_length = 0; + } + + return S3StatusOK; +} + +static void responseCompleteCallback(S3Status status, const S3ErrorDetails *error, void *callbackData) { + TS3SizeCBD *cbd = callbackData; + cbd->status = status; + + int len = 0; + const int elen = sizeof(cbd->err_msg); + if (error) { + if (error->message) { + len += snprintf(&(cbd->err_msg[len]), elen - len, " Message: %s\n", error->message); + } + if (error->resource) { + len += snprintf(&(cbd->err_msg[len]), elen - len, " Resource: %s\n", error->resource); + } + if (error->furtherDetails) { + len += snprintf(&(cbd->err_msg[len]), elen - len, " Further Details: %s\n", error->furtherDetails); + } + if (error->extraDetailsCount) { + len += snprintf(&(cbd->err_msg[len]), elen - len, "%s", " Extra Details:\n"); + for (int i = 0; i < error->extraDetailsCount; i++) { + len += snprintf(&(cbd->err_msg[len]), elen - len, " %s: %s\n", error->extraDetails[i].name, + error->extraDetails[i].value); + } + } + } +} + +int32_t s3PutObjectFromFile2(const char *file, const char *object) { return 0; } +void s3DeleteObjectsByPrefix(const char *prefix) {} + +void s3DeleteObjects(const char *object_name[], int nobject) { + int status = 0; + S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, + 0, awsRegionG}; + S3ResponseHandler responseHandler = {0, &responseCompleteCallback}; + + for (int i = 0; i < nobject; ++i) { + TS3SizeCBD cbd = {0}; + do { + S3_delete_object(&bucketContext, object_name[i], 0, timeoutMsG, &responseHandler, &cbd); + } while (S3_status_is_retryable(cbd.status) && should_retry()); + + if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) { + vError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg); + } + } +} + +static S3Status getObjectDataCallback(int bufferSize, const char *buffer, void *callbackData) { + TS3SizeCBD *cbd = callbackData; + if (cbd->content_length != bufferSize) { + cbd->status = S3StatusAbortedByCallback; + return S3StatusAbortedByCallback; + } + + char *buf = taosMemoryCalloc(1, bufferSize); + if (buf) { + memcpy(buf, buffer, bufferSize); + + cbd->status = S3StatusOK; + return S3StatusOK; + } else { + cbd->status = S3StatusAbortedByCallback; + return S3StatusAbortedByCallback; + } +} + +int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size, uint8_t **ppBlock) { + int status = 0; + int64_t ifModifiedSince = -1, ifNotModifiedSince = -1; + const char *ifMatch = 0, *ifNotMatch = 0; + + S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, + 0, awsRegionG}; + S3GetConditions getConditions = {ifModifiedSince, ifNotModifiedSince, ifMatch, ifNotMatch}; + S3GetObjectHandler getObjectHandler = {{&responsePropertiesCallback, &responseCompleteCallback}, + &getObjectDataCallback}; + + TS3SizeCBD cbd = {0}; + cbd.content_length = size; + do { + S3_get_object(&bucketContext, object_name, &getConditions, offset, size, 0, 0, &getObjectHandler, &cbd); + } while (S3_status_is_retryable(cbd.status) && should_retry()); + + if (cbd.status != S3StatusOK) { + vError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg); + return TAOS_SYSTEM_ERROR(EIO); + } + + *ppBlock = cbd.buf; + + return 0; +} + +long s3Size(const char *object_name) { + long size = 0; + int status = 0; + + S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, + 0, awsRegionG}; + + S3ResponseHandler responseHandler = {&responsePropertiesCallback, &responseCompleteCallback}; + + TS3SizeCBD cbd = {0}; + do { + S3_head_object(&bucketContext, object_name, 0, 0, &responseHandler, &cbd); + } while (S3_status_is_retryable(cbd.status) && should_retry()); + + if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) { + vError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg); + } + + size = cbd.content_length; + + return size; +} + +void s3EvictCache(const char *path, long object_size) {} + +#elif defined(USE_COS) -#ifdef USE_COS #include "cos_api.h" #include "cos_http_io.h" #include "cos_log.h" From 11f7cdcf9d3b6faea3db55cbc4227ecbee29b4cc Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 13 Oct 2023 17:16:19 +0800 Subject: [PATCH 08/59] cos/xml2: cmake file for xml2 --- cmake/xml2_CMakeLists.txt.in | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 cmake/xml2_CMakeLists.txt.in diff --git a/cmake/xml2_CMakeLists.txt.in b/cmake/xml2_CMakeLists.txt.in new file mode 100644 index 0000000000..4c234d7a37 --- /dev/null +++ b/cmake/xml2_CMakeLists.txt.in @@ -0,0 +1,12 @@ + +# xml2 +ExternalProject_Add(xml2 + GIT_REPOSITORY https://github.com/GNOME/libxml2 + GIT_TAG v2.11.5 + SOURCE_DIR "${TD_CONTRIB_DIR}/xml2" + BINARY_DIR "" + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + TEST_COMMAND "" + ) From c538478d0f91c26c34f998a31b0461016a769e9a Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 13 Oct 2023 17:30:09 +0800 Subject: [PATCH 09/59] contrib/cmake: fix curl file --- contrib/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 1588eb4025..1291506136 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -194,7 +194,7 @@ if(${BUILD_WITH_S3}) #cat("${TD_SUPPORT_DIR}/apr-util_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) #cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) #INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.1/include) - cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE2}) + cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) cat("${TD_SUPPORT_DIR}/libs3_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) add_definitions(-DUSE_S3) From f66e57bf1f650139e5d40ba939387da72edd7d2c Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 13 Oct 2023 17:47:42 +0800 Subject: [PATCH 10/59] contrib/cos: make libs3 depends xml2 --- cmake/libs3_CMakeLists.txt.in | 2 +- contrib/CMakeLists.txt | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/cmake/libs3_CMakeLists.txt.in b/cmake/libs3_CMakeLists.txt.in index b54a084f66..93d0eff41a 100644 --- a/cmake/libs3_CMakeLists.txt.in +++ b/cmake/libs3_CMakeLists.txt.in @@ -13,7 +13,7 @@ endfunction(update_cflags) ExternalProject_Add(libs3 GIT_REPOSITORY https://github.com/bji/libs3 #GIT_TAG v5.0.16 - DEPENDS curl + DEPENDS curl xml2 SOURCE_DIR "${TD_CONTRIB_DIR}/libs3" #BINARY_DIR "" BUILD_IN_SOURCE TRUE diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 1291506136..24ad63db13 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -125,9 +125,6 @@ if(${BUILD_TEST}) cat("${TD_SUPPORT_DIR}/stub_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) endif(${BUILD_TEST}) -# xml2 -cat("${TD_SUPPORT_DIR}/xml2_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) - # lz4 cat("${TD_SUPPORT_DIR}/lz4_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) @@ -194,6 +191,8 @@ if(${BUILD_WITH_S3}) #cat("${TD_SUPPORT_DIR}/apr-util_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) #cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) #INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.1/include) + + cat("${TD_SUPPORT_DIR}/xml2_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) cat("${TD_SUPPORT_DIR}/libs3_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) add_definitions(-DUSE_S3) From 4a77a49a3e3c38d6519390eb6d00df15496a9d9b Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 16 Oct 2023 15:25:48 +0800 Subject: [PATCH 11/59] prefix/list: list bucket with prefix --- cmake/libs3_CMakeLists.txt.in | 2 +- contrib/CMakeLists.txt | 2 +- source/dnode/vnode/CMakeLists.txt | 10 +-- source/dnode/vnode/src/vnd/vnodeCos.c | 93 ++++++++++++++++++++++++++- 4 files changed, 97 insertions(+), 10 deletions(-) diff --git a/cmake/libs3_CMakeLists.txt.in b/cmake/libs3_CMakeLists.txt.in index 93d0eff41a..790a6fe312 100644 --- a/cmake/libs3_CMakeLists.txt.in +++ b/cmake/libs3_CMakeLists.txt.in @@ -13,7 +13,7 @@ endfunction(update_cflags) ExternalProject_Add(libs3 GIT_REPOSITORY https://github.com/bji/libs3 #GIT_TAG v5.0.16 - DEPENDS curl xml2 + DEPENDS curl xml2 openssl SOURCE_DIR "${TD_CONTRIB_DIR}/libs3" #BINARY_DIR "" BUILD_IN_SOURCE TRUE diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 24ad63db13..63a60fa59a 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -14,7 +14,6 @@ set(CONTRIB_TMP_FILE3 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in3") configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3}) file(MAKE_DIRECTORY $ENV{HOME}/.cos-local.1/) - cat("${TD_SUPPORT_DIR}/ssl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3}) configure_file(${CONTRIB_TMP_FILE3} "${TD_CONTRIB_DIR}/deps-download/CMakeLists.txt") execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" . @@ -192,6 +191,7 @@ if(${BUILD_WITH_S3}) #cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) #INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.1/include) + cat("${TD_SUPPORT_DIR}/ssl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3}) cat("${TD_SUPPORT_DIR}/xml2_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) cat("${TD_SUPPORT_DIR}/libs3_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index e554d94e4d..d4ea90f48a 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -181,11 +181,11 @@ if(${BUILD_WITH_S3}) vnode # s3 - PUBLIC ${S3_LIBRARY} - PUBLIC ${CURL_LIBRARY} - PUBLIC ${SSL_LIBRARY} - PUBLIC ${CRYPTO_LIBRARY} - PUBLIC xml2 + PRIVATE ${S3_LIBRARY} + PRIVATE ${CURL_LIBRARY} + PRIVATE ${SSL_LIBRARY} + PRIVATE ${CRYPTO_LIBRARY} + PRIVATE xml2 ) add_definitions(-DUSE_S3) diff --git a/source/dnode/vnode/src/vnd/vnodeCos.c b/source/dnode/vnode/src/vnd/vnodeCos.c index 8984d33ad8..cf19a08813 100644 --- a/source/dnode/vnode/src/vnd/vnodeCos.c +++ b/source/dnode/vnode/src/vnd/vnodeCos.c @@ -63,9 +63,17 @@ static int should_retry() { return 0; } +static void s3PrintError(const char *func, S3Status status, char error_details[]) { + if (status < S3StatusErrorAccessDenied) { + vError("%s: %s", __func__, S3_get_status_name(status)); + } else { + vError("%s: %s, %s", __func__, S3_get_status_name(status), error_details); + } +} + typedef struct { uint64_t content_length; - int status; + S3Status status; char *buf; char err_msg[4096]; } TS3SizeCBD; @@ -109,7 +117,86 @@ static void responseCompleteCallback(S3Status status, const S3ErrorDetails *erro } int32_t s3PutObjectFromFile2(const char *file, const char *object) { return 0; } -void s3DeleteObjectsByPrefix(const char *prefix) {} + +typedef struct list_bucket_callback_data { + int isTruncated; + char nextMarker[1024]; + int keyCount; + int allDetails; + S3Status status; + char err_msg[4096]; +} list_bucket_callback_data; + +static S3Status listBucketCallback(int isTruncated, const char *nextMarker, int contentsCount, + const S3ListBucketContent *contents, int commonPrefixesCount, + const char **commonPrefixes, void *callbackData) { + list_bucket_callback_data *data = (list_bucket_callback_data *)callbackData; + + data->isTruncated = isTruncated; + if ((!nextMarker || !nextMarker[0]) && contentsCount) { + nextMarker = contents[contentsCount - 1].key; + } + if (nextMarker) { + snprintf(data->nextMarker, sizeof(data->nextMarker), "%s", nextMarker); + } else { + data->nextMarker[0] = 0; + } + + if (contentsCount && !data->keyCount) { + // printListBucketHeader(data->allDetails); + } + + int i; + for (i = 0; i < contentsCount; ++i) { + const S3ListBucketContent *content = &(contents[i]); + printf("%-50s", content->key); + } + data->keyCount += contentsCount; + + for (i = 0; i < commonPrefixesCount; i++) { + // printf("\nCommon Prefix: %s\n", commonPrefixes[i]); + } + + return S3StatusOK; +} + +void s3DeleteObjectsByPrefix(const char *prefix) { + S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, + 0, awsRegionG}; + S3ListBucketHandler listBucketHandler = {{&responsePropertiesCallback, &responseCompleteCallback}, + &listBucketCallback}; + + const char *marker = 0, *delimiter = 0; + int maxkeys = 0, allDetails = 0; + list_bucket_callback_data data; + + if (marker) { + snprintf(data.nextMarker, sizeof(data.nextMarker), "%s", marker); + } else { + data.nextMarker[0] = 0; + } + data.keyCount = 0; + data.allDetails = allDetails; + + do { + data.isTruncated = 0; + do { + S3_list_bucket(&bucketContext, prefix, data.nextMarker, delimiter, maxkeys, 0, timeoutMsG, &listBucketHandler, + &data); + } while (S3_status_is_retryable(data.status) && should_retry()); + if (data.status != S3StatusOK) { + break; + } + } while (data.isTruncated && (!maxkeys || (data.keyCount < maxkeys))); + + if (data.status == S3StatusOK) { + if (!data.keyCount) { + // printListBucketHeader(allDetails); + } + } else { + s3PrintError(__func__, data.status, data.err_msg); + } +} void s3DeleteObjects(const char *object_name[], int nobject) { int status = 0; @@ -124,7 +211,7 @@ void s3DeleteObjects(const char *object_name[], int nobject) { } while (S3_status_is_retryable(cbd.status) && should_retry()); if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) { - vError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg); + s3PrintError(__func__, cbd.status, cbd.err_msg); } } } From 2479bb343e56f98f37ad749e6607d2449b8892a7 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 16 Oct 2023 15:54:51 +0800 Subject: [PATCH 12/59] prefix/delete: delete with prefix --- contrib/CMakeLists.txt | 2 +- source/dnode/vnode/src/vnd/vnodeCos.c | 19 +++++++++++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 63a60fa59a..b40a7101c3 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -191,7 +191,7 @@ if(${BUILD_WITH_S3}) #cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) #INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.1/include) - cat("${TD_SUPPORT_DIR}/ssl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3}) + cat("${TD_SUPPORT_DIR}/ssl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) cat("${TD_SUPPORT_DIR}/xml2_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) cat("${TD_SUPPORT_DIR}/libs3_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) diff --git a/source/dnode/vnode/src/vnd/vnodeCos.c b/source/dnode/vnode/src/vnd/vnodeCos.c index cf19a08813..726d78e167 100644 --- a/source/dnode/vnode/src/vnd/vnodeCos.c +++ b/source/dnode/vnode/src/vnd/vnodeCos.c @@ -123,6 +123,7 @@ typedef struct list_bucket_callback_data { char nextMarker[1024]; int keyCount; int allDetails; + SArray *objectArray; S3Status status; char err_msg[4096]; } list_bucket_callback_data; @@ -149,7 +150,9 @@ static S3Status listBucketCallback(int isTruncated, const char *nextMarker, int int i; for (i = 0; i < contentsCount; ++i) { const S3ListBucketContent *content = &(contents[i]); - printf("%-50s", content->key); + // printf("%-50s", content->key); + char *object_key = strdup(content->key); + taosArrayPush(data->objectArray, &object_key); } data->keyCount += contentsCount; @@ -160,6 +163,11 @@ static S3Status listBucketCallback(int isTruncated, const char *nextMarker, int return S3StatusOK; } +static void s3FreeObjectKey(void *pItem) { + char *key = (char *)pItem; + taosMemoryFree(key); +} + void s3DeleteObjectsByPrefix(const char *prefix) { S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, 0, awsRegionG}; @@ -169,7 +177,11 @@ void s3DeleteObjectsByPrefix(const char *prefix) { const char *marker = 0, *delimiter = 0; int maxkeys = 0, allDetails = 0; list_bucket_callback_data data; - + data.objectArray = taosArrayInit(32, POINTER_BYTES); + if (!data.objectArray) { + vError("%s: %s", __func__, "out of memoty"); + return; + } if (marker) { snprintf(data.nextMarker, sizeof(data.nextMarker), "%s", marker); } else { @@ -192,10 +204,13 @@ void s3DeleteObjectsByPrefix(const char *prefix) { if (data.status == S3StatusOK) { if (!data.keyCount) { // printListBucketHeader(allDetails); + s3DeleteObjects(TARRAY_DATA(data.objectArray), TARRAY_SIZE(data.objectArray)); } } else { s3PrintError(__func__, data.status, data.err_msg); } + + taosArrayDestroyEx(data.objectArray, s3FreeObjectKey); } void s3DeleteObjects(const char *object_name[], int nobject) { From 93b99e4573e315f1a5bd5d0133738c44d03a10f9 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Mon, 16 Oct 2023 18:21:19 +0800 Subject: [PATCH 13/59] test: support replica3 in splited/transform tmqvnode --- tests/parallel_test/cases.task | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 9a888c1225..b221d99666 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -172,9 +172,14 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -i True ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -n 3 -i True ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform.py -N 2 -n 1 -,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform.py -N 6 -n 3 -,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit.py -N 2 -n 1 -,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit.py -N 3 -n 3 +,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-stb.py -N 6 -n 3 +,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select.py -N 2 -n 1 +,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py -N 3 -n 3 +,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select.py -N 3 -n 3 +,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-stb.py -N 3 -n 3 +,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-column.py.py -N 3 -n 3 +,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-db.py -N 3 -n 3 +e ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeReplicate.py -M 3 -N 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-19201.py ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-21561.py From 38c1f7d48d0b692e29a32c657f948b9901f8ead3 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 16 Oct 2023 20:20:04 +0800 Subject: [PATCH 14/59] fix: add lock for trans --- source/dnode/mnode/impl/inc/mndDef.h | 2 +- source/dnode/mnode/impl/src/mndTrans.c | 12 +++--------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index f4236964ca..e5f46c03cd 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -181,7 +181,7 @@ typedef struct { SArray* pRpcArray; SRWLatch lockRpcArray; int64_t mTraceId; - TdThreadMutex mutex; + int8_t lock; } STrans; typedef struct { diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 1d8dd5e345..f8d6cba171 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -470,7 +470,6 @@ void mndTransDropData(STrans *pTrans) { pTrans->param = NULL; pTrans->paramLen = 0; } - (void)taosThreadMutexDestroy(&pTrans->mutex); } static int32_t mndTransDelete(SSdb *pSdb, STrans *pTrans, bool callFunc) { @@ -543,10 +542,6 @@ STrans *mndAcquireTrans(SMnode *pMnode, int32_t transId) { STrans *pTrans = sdbAcquire(pMnode->pSdb, SDB_TRANS, &transId); if (pTrans == NULL) { terrno = TSDB_CODE_MND_TRANS_NOT_EXIST; - } else { - #ifdef WINDOWS - taosThreadMutexInit(&pTrans->mutex, NULL); - #endif } return pTrans; } @@ -582,7 +577,6 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnConflct conflict, pTrans->pRpcArray = taosArrayInit(1, sizeof(SRpcHandleInfo)); pTrans->mTraceId = pReq ? TRACE_GET_ROOTID(&pReq->info.traceId) : tGenIdPI64(); taosInitRWLatch(&pTrans->lockRpcArray); - taosThreadMutexInit(&pTrans->mutex, NULL); if (pTrans->redoActions == NULL || pTrans->undoActions == NULL || pTrans->commitActions == NULL || pTrans->pRpcArray == NULL) { @@ -1264,10 +1258,10 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans) int32_t numOfActions = taosArrayGetSize(pTrans->redoActions); if (numOfActions == 0) return code; - taosThreadMutexLock(&pTrans->mutex); + if (atomic_val_compare_exchange_8(&pTrans->lock, 0, 1) != 0) return code; if (pTrans->redoActionPos >= numOfActions) { - taosThreadMutexUnlock(&pTrans->mutex); + atomic_store_8(&pTrans->lock, 0); return code; } @@ -1339,7 +1333,7 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans) } } - taosThreadMutexUnlock(&pTrans->mutex); + atomic_store_8(&pTrans->lock, 0); return code; } From a30eedafecc8a320afb0fac4dbb25a115c532ecd Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 17 Oct 2023 11:06:06 +0800 Subject: [PATCH 15/59] fix: trans mutex init --- source/dnode/mnode/impl/inc/mndDef.h | 2 +- source/dnode/mnode/impl/src/mndTrans.c | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index e5f46c03cd..f4236964ca 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -181,7 +181,7 @@ typedef struct { SArray* pRpcArray; SRWLatch lockRpcArray; int64_t mTraceId; - int8_t lock; + TdThreadMutex mutex; } STrans; typedef struct { diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index f8d6cba171..76d6e21c64 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -428,6 +428,8 @@ static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans) { mInfo("trans:%d, perform insert action, row:%p stage:%s, callfunc:1, startFunc:%d", pTrans->id, pTrans, mndTransStr(pTrans->stage), pTrans->startFunc); + taosThreadMutexInit(&pTrans->mutex, NULL); + if (pTrans->startFunc > 0) { TransCbFp fp = mndTransGetCbFp(pTrans->startFunc); if (fp) { @@ -1258,10 +1260,10 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans) int32_t numOfActions = taosArrayGetSize(pTrans->redoActions); if (numOfActions == 0) return code; - if (atomic_val_compare_exchange_8(&pTrans->lock, 0, 1) != 0) return code; + taosThreadMutexLock(&pTrans->mutex); if (pTrans->redoActionPos >= numOfActions) { - atomic_store_8(&pTrans->lock, 0); + taosThreadMutexUnlock(&pTrans->mutex); return code; } @@ -1333,7 +1335,7 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans) } } - atomic_store_8(&pTrans->lock, 0); + taosThreadMutexUnlock(&pTrans->mutex); return code; } From 29c6daaff5efd137cc9d16f5aaa5a5869a766e45 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 17 Oct 2023 11:12:52 +0800 Subject: [PATCH 16/59] fix: trans mutex init --- source/dnode/mnode/impl/src/mndTrans.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 76d6e21c64..29a8ae1f29 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -472,6 +472,7 @@ void mndTransDropData(STrans *pTrans) { pTrans->param = NULL; pTrans->paramLen = 0; } + (void)taosThreadMutexDestroy(&pTrans->mutex); } static int32_t mndTransDelete(SSdb *pSdb, STrans *pTrans, bool callFunc) { @@ -579,6 +580,7 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnConflct conflict, pTrans->pRpcArray = taosArrayInit(1, sizeof(SRpcHandleInfo)); pTrans->mTraceId = pReq ? TRACE_GET_ROOTID(&pReq->info.traceId) : tGenIdPI64(); taosInitRWLatch(&pTrans->lockRpcArray); + taosThreadMutexInit(&pTrans->mutex, NULL); if (pTrans->redoActions == NULL || pTrans->undoActions == NULL || pTrans->commitActions == NULL || pTrans->pRpcArray == NULL) { From 1459c82d75ef5d7cd43e525c7c55dcd0885e4513 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Tue, 17 Oct 2023 11:23:21 +0800 Subject: [PATCH 17/59] test: support replica3 in splited/transform tmqvnode --- tests/parallel_test/cases.task | 4 ++-- .../7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index b221d99666..66ffc7f6dd 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -171,13 +171,13 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqSubscribeStb-r3.py -N 5 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -i True ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -n 3 -i True -,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform.py -N 2 -n 1 +,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-stb.py -N 2 -n 1 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-stb.py -N 6 -n 3 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select.py -N 2 -n 1 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py -N 3 -n 3 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select.py -N 3 -n 3 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-stb.py -N 3 -n 3 -,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-column.py.py -N 3 -n 3 +,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-column.py -N 3 -n 3 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-db.py -N 3 -n 3 e ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeReplicate.py -M 3 -N 3 -n 3 diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py index 086fe6e4c5..c76188eee2 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py @@ -184,7 +184,7 @@ class TDTestCase: expectRows = 1 resultList = tmqCom.selectConsumeResult(expectRows) - if expectrowcnt / 2 >= resultList[0] or expectrowcnt <= resultList[0]: + if expectrowcnt / 2 >= resultList[0]: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0])) tdLog.exit("%d tmq consume rows error!"%consumerId) From 8fa8c40391fbea4dbb668202dabb9160a281d289 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 17 Oct 2023 14:59:03 +0800 Subject: [PATCH 18/59] cos/put: simple object putting(< 5 gigabytes) --- source/dnode/vnode/src/vnd/vnodeCos.c | 250 +++++++++++++++++++++++++- 1 file changed, 249 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/vnd/vnodeCos.c b/source/dnode/vnode/src/vnd/vnodeCos.c index 726d78e167..2a36a898e0 100644 --- a/source/dnode/vnode/src/vnd/vnodeCos.c +++ b/source/dnode/vnode/src/vnd/vnodeCos.c @@ -116,7 +116,255 @@ static void responseCompleteCallback(S3Status status, const S3ErrorDetails *erro } } -int32_t s3PutObjectFromFile2(const char *file, const char *object) { return 0; } +typedef struct growbuffer { + // The total number of bytes, and the start byte + int size; + // The start byte + int start; + // The blocks + char data[64 * 1024]; + struct growbuffer *prev, *next; +} growbuffer; + +// returns nonzero on success, zero on out of memory +static int growbuffer_append(growbuffer **gb, const char *data, int dataLen) { + int origDataLen = dataLen; + while (dataLen) { + growbuffer *buf = *gb ? (*gb)->prev : 0; + if (!buf || (buf->size == sizeof(buf->data))) { + buf = (growbuffer *)malloc(sizeof(growbuffer)); + if (!buf) { + return 0; + } + buf->size = 0; + buf->start = 0; + if (*gb && (*gb)->prev) { + buf->prev = (*gb)->prev; + buf->next = *gb; + (*gb)->prev->next = buf; + (*gb)->prev = buf; + } else { + buf->prev = buf->next = buf; + *gb = buf; + } + } + + int toCopy = (sizeof(buf->data) - buf->size); + if (toCopy > dataLen) { + toCopy = dataLen; + } + + memcpy(&(buf->data[buf->size]), data, toCopy); + + buf->size += toCopy, data += toCopy, dataLen -= toCopy; + } + + return origDataLen; +} + +static void growbuffer_read(growbuffer **gb, int amt, int *amtReturn, char *buffer) { + *amtReturn = 0; + + growbuffer *buf = *gb; + + if (!buf) { + return; + } + + *amtReturn = (buf->size > amt) ? amt : buf->size; + + memcpy(buffer, &(buf->data[buf->start]), *amtReturn); + + buf->start += *amtReturn, buf->size -= *amtReturn; + + if (buf->size == 0) { + if (buf->next == buf) { + *gb = 0; + } else { + *gb = buf->next; + buf->prev->next = buf->next; + buf->next->prev = buf->prev; + } + free(buf); + buf = NULL; + } +} + +static void growbuffer_destroy(growbuffer *gb) { + growbuffer *start = gb; + + while (gb) { + growbuffer *next = gb->next; + free(gb); + gb = (next == start) ? 0 : next; + } +} + +typedef struct put_object_callback_data { + // FILE *infile; + TdFilePtr infileFD; + growbuffer *gb; + uint64_t contentLength, originalContentLength; + uint64_t totalContentLength, totalOriginalContentLength; + int noStatus; + S3Status status; + char err_msg[4096]; +} put_object_callback_data; + +static int putObjectDataCallback(int bufferSize, char *buffer, void *callbackData) { + put_object_callback_data *data = (put_object_callback_data *)callbackData; + + int ret = 0; + + if (data->contentLength) { + int toRead = ((data->contentLength > (unsigned)bufferSize) ? (unsigned)bufferSize : data->contentLength); + if (data->gb) { + growbuffer_read(&(data->gb), toRead, &ret, buffer); + } else if (data->infileFD) { + // ret = fread(buffer, 1, toRead, data->infile); + ret = taosReadFile(data->infileFD, buffer, toRead); + } + } + + data->contentLength -= ret; + data->totalContentLength -= ret; + + if (data->contentLength && !data->noStatus) { + vTrace("%llu bytes remaining ", (unsigned long long)data->totalContentLength); + vTrace("(%d%% complete) ...\n", (int)(((data->totalOriginalContentLength - data->totalContentLength) * 100) / + data->totalOriginalContentLength)); + } + + return ret; +} + +#define MULTIPART_CHUNK_SIZE (768 << 20) // multipart is 768M + +typedef struct UploadManager { + // used for initial multipart + char *upload_id; + + // used for upload part object + char **etags; + int next_etags_pos; + + // used for commit Upload + growbuffer *gb; + int remaining; +} UploadManager; + +typedef struct MultipartPartData { + put_object_callback_data put_object_data; + int seq; + UploadManager *manager; +} MultipartPartData; + +S3Status initial_multipart_callback(const char *upload_id, void *callbackData) { + UploadManager *manager = (UploadManager *)callbackData; + manager->upload_id = strdup(upload_id); + return S3StatusOK; +} + +S3Status MultipartResponseProperiesCallback(const S3ResponseProperties *properties, void *callbackData) { + responsePropertiesCallback(properties, callbackData); + + MultipartPartData *data = (MultipartPartData *)callbackData; + int seq = data->seq; + const char *etag = properties->eTag; + data->manager->etags[seq - 1] = strdup(etag); + data->manager->next_etags_pos = seq; + return S3StatusOK; +} + +static int multipartPutXmlCallback(int bufferSize, char *buffer, void *callbackData) { + UploadManager *manager = (UploadManager *)callbackData; + int ret = 0; + + if (manager->remaining) { + int toRead = ((manager->remaining > bufferSize) ? bufferSize : manager->remaining); + growbuffer_read(&(manager->gb), toRead, &ret, buffer); + } + manager->remaining -= ret; + return ret; +} + +static int try_get_parts_info(const char *bucketName, const char *key, UploadManager *manager) { + // + + return 0; +} + +int32_t s3PutObjectFromFile2(const char *file, const char *object) { + int32_t code = 0; + const char *key = object; + const char *uploadId = 0; + const char *filename = 0; + uint64_t contentLength = 0; + const char *cacheControl = 0, *contentType = 0, *md5 = 0; + const char *contentDispositionFilename = 0, *contentEncoding = 0; + int64_t expires = -1; + S3CannedAcl cannedAcl = S3CannedAclPrivate; + int metaPropertiesCount = 0; + S3NameValue metaProperties[S3_MAX_METADATA_COUNT]; + char useServerSideEncryption = 0; + int noStatus = 0; + put_object_callback_data data; + + // data.infile = 0; + data.infileFD = NULL; + data.gb = 0; + data.noStatus = noStatus; + + if (taosStatFile(file, &contentLength, NULL, NULL) < 0) { + vError("ERROR: %s Failed to stat file %s: ", __func__, file); + code = TAOS_SYSTEM_ERROR(errno); + return code; + } + + if (!(data.infileFD = taosOpenFile(file, TD_FILE_READ))) { + vError("ERROR: %s Failed to open file %s: ", __func__, file); + code = TAOS_SYSTEM_ERROR(errno); + return code; + } + + data.totalContentLength = data.totalOriginalContentLength = data.contentLength = data.originalContentLength = + contentLength; + + S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, + 0, awsRegionG}; + + S3PutProperties putProperties = {contentType, md5, + cacheControl, contentDispositionFilename, + contentEncoding, expires, + cannedAcl, metaPropertiesCount, + metaProperties, useServerSideEncryption}; + + if (contentLength <= MULTIPART_CHUNK_SIZE) { + S3PutObjectHandler putObjectHandler = {{&responsePropertiesCallback, &responseCompleteCallback}, + &putObjectDataCallback}; + + do { + S3_put_object(&bucketContext, key, contentLength, &putProperties, 0, 0, &putObjectHandler, &data); + } while (S3_status_is_retryable(data.status) && should_retry()); + + if (data.infileFD) { + taosCloseFile(&data.infileFD); + } else if (data.gb) { + growbuffer_destroy(data.gb); + } + + if (data.status != S3StatusOK) { + s3PrintError(__func__, data.status, data.err_msg); + } else if (data.contentLength) { + vError("ERROR: %s Failed to read remaining %llu bytes from input", __func__, + (unsigned long long)data.contentLength); + } + } else { + uint64_t totalContentLength = contentLength; + uint64_t todoContentLength = contentLength; + } + return 0; +} typedef struct list_bucket_callback_data { int isTruncated; From d8b0d8085be4cc51903d489c915a2a8a5331b9d4 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 17 Oct 2023 15:16:20 +0800 Subject: [PATCH 19/59] cmake/ssl: make curl depends openssl --- cmake/curl_CMakeLists.txt.in | 1 + cmake/libs3_CMakeLists.txt.in | 4 ++-- contrib/CMakeLists.txt | 1 - 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/curl_CMakeLists.txt.in b/cmake/curl_CMakeLists.txt.in index 24a1210422..fed98c5f0e 100644 --- a/cmake/curl_CMakeLists.txt.in +++ b/cmake/curl_CMakeLists.txt.in @@ -7,6 +7,7 @@ ExternalProject_Add(curl #GIT_REPOSITORY https://github.com/curl/curl.git #GIT_TAG curl-7_88_1 SOURCE_DIR "${TD_CONTRIB_DIR}/curl" + DEPENDS openssl BUILD_IN_SOURCE TRUE BUILD_ALWAYS 1 #UPDATE_COMMAND "" diff --git a/cmake/libs3_CMakeLists.txt.in b/cmake/libs3_CMakeLists.txt.in index 790a6fe312..b2d2d19059 100644 --- a/cmake/libs3_CMakeLists.txt.in +++ b/cmake/libs3_CMakeLists.txt.in @@ -12,8 +12,8 @@ endfunction(update_cflags) ExternalProject_Add(libs3 GIT_REPOSITORY https://github.com/bji/libs3 - #GIT_TAG v5.0.16 - DEPENDS curl xml2 openssl + #GIT_TAG v5.0.16 + DEPENDS curl xml2 SOURCE_DIR "${TD_CONTRIB_DIR}/libs3" #BINARY_DIR "" BUILD_IN_SOURCE TRUE diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index b40a7101c3..07201f8713 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -277,7 +277,6 @@ if(${BUILD_TEST}) ) endif(${TD_DARWIN}) - endif(${BUILD_TEST}) # cJson From ddfdbb3207415134c8b5c684864debe6da716848 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 17 Oct 2023 15:28:36 +0800 Subject: [PATCH 20/59] xml2: add sub dir if building libs3 --- contrib/CMakeLists.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 07201f8713..de3d9cfd37 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -24,7 +24,7 @@ execute_process(COMMAND "${CMAKE_COMMAND}" --build . set(CONTRIB_TMP_FILE2 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in2") configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE2}) - cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE2}) + #cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE2}) configure_file(${CONTRIB_TMP_FILE2} "${TD_CONTRIB_DIR}/deps-download/CMakeLists.txt") execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" . @@ -294,7 +294,9 @@ target_include_directories( unset(CMAKE_PROJECT_INCLUDE_BEFORE) # xml2 -add_subdirectory(xml2 EXCLUDE_FROM_ALL) +if(${BUILD_WITH_S3}) + add_subdirectory(xml2 EXCLUDE_FROM_ALL) +endif(${BUILD_WITH_S3}) # lz4 add_subdirectory(lz4/build/cmake EXCLUDE_FROM_ALL) From dd21d8e3ade60225d74d2cf37ea49fe4027bf9ba Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 17 Oct 2023 17:24:50 +0800 Subject: [PATCH 21/59] cos/multipart: first round multipart uploading --- source/dnode/vnode/src/vnd/vnodeCos.c | 256 ++++++++++++++++++++++++-- 1 file changed, 244 insertions(+), 12 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeCos.c b/source/dnode/vnode/src/vnd/vnodeCos.c index 2a36a898e0..b52d4809b2 100644 --- a/source/dnode/vnode/src/vnd/vnodeCos.c +++ b/source/dnode/vnode/src/vnd/vnodeCos.c @@ -72,10 +72,10 @@ static void s3PrintError(const char *func, S3Status status, char error_details[] } typedef struct { - uint64_t content_length; + char err_msg[128]; S3Status status; + uint64_t content_length; char *buf; - char err_msg[4096]; } TS3SizeCBD; static S3Status responsePropertiesCallback(const S3ResponseProperties *properties, void *callbackData) { @@ -201,14 +201,14 @@ static void growbuffer_destroy(growbuffer *gb) { } typedef struct put_object_callback_data { + char err_msg[128]; + S3Status status; // FILE *infile; TdFilePtr infileFD; growbuffer *gb; uint64_t contentLength, originalContentLength; uint64_t totalContentLength, totalOriginalContentLength; int noStatus; - S3Status status; - char err_msg[4096]; } put_object_callback_data; static int putObjectDataCallback(int bufferSize, char *buffer, void *callbackData) { @@ -241,6 +241,8 @@ static int putObjectDataCallback(int bufferSize, char *buffer, void *callbackDat #define MULTIPART_CHUNK_SIZE (768 << 20) // multipart is 768M typedef struct UploadManager { + char err_msg[128]; + S3Status status; // used for initial multipart char *upload_id; @@ -253,7 +255,26 @@ typedef struct UploadManager { int remaining; } UploadManager; +typedef struct list_parts_callback_data { + char err_msg[128]; + S3Status status; + int isTruncated; + char nextPartNumberMarker[24]; + char initiatorId[256]; + char initiatorDisplayName[256]; + char ownerId[256]; + char ownerDisplayName[256]; + char storageClass[256]; + int partsCount; + int handlePartsStart; + int allDetails; + int noPrint; + UploadManager *manager; +} list_parts_callback_data; + typedef struct MultipartPartData { + char err_msg[128]; + S3Status status; put_object_callback_data put_object_data; int seq; UploadManager *manager; @@ -288,16 +309,120 @@ static int multipartPutXmlCallback(int bufferSize, char *buffer, void *callbackD return ret; } +static S3Status listPartsCallback(int isTruncated, const char *nextPartNumberMarker, const char *initiatorId, + const char *initiatorDisplayName, const char *ownerId, const char *ownerDisplayName, + const char *storageClass, int partsCount, int handlePartsStart, + const S3ListPart *parts, void *callbackData) { + list_parts_callback_data *data = (list_parts_callback_data *)callbackData; + + data->isTruncated = isTruncated; + data->handlePartsStart = handlePartsStart; + UploadManager *manager = data->manager; + + if (nextPartNumberMarker) { + snprintf(data->nextPartNumberMarker, sizeof(data->nextPartNumberMarker), "%s", nextPartNumberMarker); + } else { + data->nextPartNumberMarker[0] = 0; + } + + if (initiatorId) { + snprintf(data->initiatorId, sizeof(data->initiatorId), "%s", initiatorId); + } else { + data->initiatorId[0] = 0; + } + + if (initiatorDisplayName) { + snprintf(data->initiatorDisplayName, sizeof(data->initiatorDisplayName), "%s", initiatorDisplayName); + } else { + data->initiatorDisplayName[0] = 0; + } + + if (ownerId) { + snprintf(data->ownerId, sizeof(data->ownerId), "%s", ownerId); + } else { + data->ownerId[0] = 0; + } + + if (ownerDisplayName) { + snprintf(data->ownerDisplayName, sizeof(data->ownerDisplayName), "%s", ownerDisplayName); + } else { + data->ownerDisplayName[0] = 0; + } + + if (storageClass) { + snprintf(data->storageClass, sizeof(data->storageClass), "%s", storageClass); + } else { + data->storageClass[0] = 0; + } + + if (partsCount && !data->partsCount && !data->noPrint) { + // printListPartsHeader(); + } + + int i; + for (i = 0; i < partsCount; i++) { + const S3ListPart *part = &(parts[i]); + char timebuf[256]; + if (data->noPrint) { + manager->etags[handlePartsStart + i] = strdup(part->eTag); + manager->next_etags_pos++; + manager->remaining = manager->remaining - part->size; + } else { + time_t t = (time_t)part->lastModified; + strftime(timebuf, sizeof(timebuf), "%Y-%m-%dT%H:%M:%SZ", gmtime(&t)); + printf("%-30s", timebuf); + printf("%-15llu", (unsigned long long)part->partNumber); + printf("%-45s", part->eTag); + printf("%-15llu\n", (unsigned long long)part->size); + } + } + + data->partsCount += partsCount; + + return S3StatusOK; +} + static int try_get_parts_info(const char *bucketName, const char *key, UploadManager *manager) { - // + S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, + 0, awsRegionG}; + + S3ListPartsHandler listPartsHandler = {{&responsePropertiesCallback, &responseCompleteCallback}, &listPartsCallback}; + + list_parts_callback_data data; + + memset(&data, 0, sizeof(list_parts_callback_data)); + + data.partsCount = 0; + data.allDetails = 0; + data.manager = manager; + data.noPrint = 1; + do { + data.isTruncated = 0; + do { + S3_list_parts(&bucketContext, key, data.nextPartNumberMarker, manager->upload_id, 0, 0, 0, timeoutMsG, + &listPartsHandler, &data); + } while (S3_status_is_retryable(data.status) && should_retry()); + if (data.status != S3StatusOK) { + break; + } + } while (data.isTruncated); + + if (data.status == S3StatusOK) { + if (!data.partsCount) { + // printListMultipartHeader(data.allDetails); + } + } else { + s3PrintError(__func__, data.status, data.err_msg); + return -1; + } return 0; } int32_t s3PutObjectFromFile2(const char *file, const char *object) { - int32_t code = 0; - const char *key = object; - const char *uploadId = 0; + int32_t code = 0; + const char *key = object; + // const char *uploadId = 0; const char *filename = 0; uint64_t contentLength = 0; const char *cacheControl = 0, *contentType = 0, *md5 = 0; @@ -360,20 +485,127 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { (unsigned long long)data.contentLength); } } else { - uint64_t totalContentLength = contentLength; - uint64_t todoContentLength = contentLength; + uint64_t totalContentLength = contentLength; + uint64_t todoContentLength = contentLength; + UploadManager manager; + manager.upload_id = 0; + manager.gb = 0; + + // div round up + int seq; + int totalSeq = ((contentLength + MULTIPART_CHUNK_SIZE - 1) / MULTIPART_CHUNK_SIZE); + + MultipartPartData partData; + memset(&partData, 0, sizeof(MultipartPartData)); + int partContentLength = 0; + + S3MultipartInitialHandler handler = {{&responsePropertiesCallback, &responseCompleteCallback}, + &initial_multipart_callback}; + + S3PutObjectHandler putObjectHandler = {{&MultipartResponseProperiesCallback, &responseCompleteCallback}, + &putObjectDataCallback}; + + S3MultipartCommitHandler commit_handler = { + {&responsePropertiesCallback, &responseCompleteCallback}, &multipartPutXmlCallback, 0}; + + manager.etags = (char **)taosMemoryMalloc(sizeof(char *) * totalSeq); + manager.next_etags_pos = 0; + /* + if (uploadId) { + manager.upload_id = strdup(uploadId); + manager.remaining = contentLength; + if (!try_get_parts_info(tsS3BucketName, key, &manager)) { + fseek(data.infile, -(manager.remaining), 2); + taosLSeekFile(data.infileFD, -(manager.remaining), SEEK_END); + contentLength = manager.remaining; + goto upload; + } else { + goto clean; + } + } + */ + do { + S3_initiate_multipart(&bucketContext, key, 0, &handler, 0, timeoutMsG, &manager); + } while (S3_status_is_retryable(manager.status) && should_retry()); + + if (manager.upload_id == 0 || manager.status != S3StatusOK) { + s3PrintError(__func__, manager.status, manager.err_msg); + goto clean; + } + + upload: + todoContentLength -= MULTIPART_CHUNK_SIZE * manager.next_etags_pos; + for (seq = manager.next_etags_pos + 1; seq <= totalSeq; seq++) { + partData.manager = &manager; + partData.seq = seq; + if (partData.put_object_data.gb == NULL) { + partData.put_object_data = data; + } + partContentLength = ((contentLength > MULTIPART_CHUNK_SIZE) ? MULTIPART_CHUNK_SIZE : contentLength); + // printf("%s Part Seq %d, length=%d\n", srcSize ? "Copying" : "Sending", seq, partContentLength); + partData.put_object_data.contentLength = partContentLength; + partData.put_object_data.originalContentLength = partContentLength; + partData.put_object_data.totalContentLength = todoContentLength; + partData.put_object_data.totalOriginalContentLength = totalContentLength; + putProperties.md5 = 0; + do { + S3_upload_part(&bucketContext, key, &putProperties, &putObjectHandler, seq, manager.upload_id, + partContentLength, 0, timeoutMsG, &partData); + } while (S3_status_is_retryable(partData.status) && should_retry()); + if (partData.status != S3StatusOK) { + s3PrintError(__func__, partData.status, partData.err_msg); + goto clean; + } + contentLength -= MULTIPART_CHUNK_SIZE; + todoContentLength -= MULTIPART_CHUNK_SIZE; + } + + int i; + int size = 0; + size += growbuffer_append(&(manager.gb), "", strlen("")); + char buf[256]; + int n; + for (i = 0; i < totalSeq; i++) { + n = snprintf(buf, sizeof(buf), + "%d" + "%s", + i + 1, manager.etags[i]); + size += growbuffer_append(&(manager.gb), buf, n); + } + size += growbuffer_append(&(manager.gb), "", strlen("")); + manager.remaining = size; + + do { + S3_complete_multipart_upload(&bucketContext, key, &commit_handler, manager.upload_id, manager.remaining, 0, + timeoutMsG, &manager); + } while (S3_status_is_retryable(manager.status) && should_retry()); + if (manager.status != S3StatusOK) { + s3PrintError(__func__, manager.status, manager.err_msg); + goto clean; + } + + clean: + if (manager.upload_id) { + taosMemoryFree(manager.upload_id); + } + for (i = 0; i < manager.next_etags_pos; i++) { + taosMemoryFree(manager.etags[i]); + } + growbuffer_destroy(manager.gb); + taosMemoryFree(manager.etags); } + return 0; } typedef struct list_bucket_callback_data { + char err_msg[128]; + S3Status status; int isTruncated; char nextMarker[1024]; int keyCount; int allDetails; SArray *objectArray; - S3Status status; - char err_msg[4096]; } list_bucket_callback_data; static S3Status listBucketCallback(int isTruncated, const char *nextMarker, int contentsCount, From 4bc3c22e0c92e8831d1cf6ee55bdb4be4ef6385b Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 17 Oct 2023 17:46:11 +0800 Subject: [PATCH 22/59] cmake/ssl: install install_sw only --- cmake/ssl_CMakeLists.txt.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/ssl_CMakeLists.txt.in b/cmake/ssl_CMakeLists.txt.in index 7821848618..65a02d7fd6 100644 --- a/cmake/ssl_CMakeLists.txt.in +++ b/cmake/ssl_CMakeLists.txt.in @@ -9,7 +9,7 @@ ExternalProject_Add(openssl #BUILD_ALWAYS 1 #UPDATE_COMMAND "" CONFIGURE_COMMAND ./Configure --prefix=$ENV{HOME}/.cos-local.1 -static #--no-shared - BUILD_COMMAND make -j - INSTALL_COMMAND make install + BUILD_COMMAND "" + INSTALL_COMMAND make install_sw -j TEST_COMMAND "" ) From 11b65e277925323b65efe49095a8c2cff3e72278 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 17 Oct 2023 18:11:13 +0800 Subject: [PATCH 23/59] vnode/cos: make dependent lib public --- cmake/curl_CMakeLists.txt.in | 2 +- source/dnode/vnode/CMakeLists.txt | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cmake/curl_CMakeLists.txt.in b/cmake/curl_CMakeLists.txt.in index fed98c5f0e..3f7b9ee1fe 100644 --- a/cmake/curl_CMakeLists.txt.in +++ b/cmake/curl_CMakeLists.txt.in @@ -12,7 +12,7 @@ ExternalProject_Add(curl BUILD_ALWAYS 1 #UPDATE_COMMAND "" #CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --without-ssl --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd - CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --with-ssl=$ENV{HOME}/.cos-local.1 --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd + CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --with-ssl=$ENV{HOME}/.cos-local.1 --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd --without-libidn2 #CONFIGURE_COMMAND ./configure --without-ssl BUILD_COMMAND make INSTALL_COMMAND make install diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index d4ea90f48a..e554d94e4d 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -181,11 +181,11 @@ if(${BUILD_WITH_S3}) vnode # s3 - PRIVATE ${S3_LIBRARY} - PRIVATE ${CURL_LIBRARY} - PRIVATE ${SSL_LIBRARY} - PRIVATE ${CRYPTO_LIBRARY} - PRIVATE xml2 + PUBLIC ${S3_LIBRARY} + PUBLIC ${CURL_LIBRARY} + PUBLIC ${SSL_LIBRARY} + PUBLIC ${CRYPTO_LIBRARY} + PUBLIC xml2 ) add_definitions(-DUSE_S3) From 6d9732e87a850b6aee3be6e6e77a16240861cd68 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Tue, 17 Oct 2023 18:26:04 +0800 Subject: [PATCH 24/59] test: set taosd log leavel 135 --- tests/pytest/util/dnodes.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index c4fc1ce654..d0ed1d874d 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -36,12 +36,12 @@ class TDSimClient: "locale": "en_US.UTF-8", "charset": "UTF-8", "asyncLog": "0", - "rpcDebugFlag": "143", + "rpcDebugFlag": "135", "tmrDebugFlag": "131", - "cDebugFlag": "143", - "uDebugFlag": "143", - "jniDebugFlag": "143", - "qDebugFlag": "143", + "cDebugFlag": "135", + "uDebugFlag": "135", + "jniDebugFlag": "135", + "qDebugFlag": "135", "supportVnodes": "1024", "enableQueryHb": "1", "telemetryReporting": "0", @@ -130,18 +130,18 @@ class TDDnode: "locale": "en_US.UTF-8", "charset": "UTF-8", "asyncLog": "0", - "mDebugFlag": "143", - "dDebugFlag": "143", - "vDebugFlag": "143", - "tqDebugFlag": "143", - "cDebugFlag": "143", - "jniDebugFlag": "143", - "qDebugFlag": "143", - "rpcDebugFlag": "143", + "mDebugFlag": "135", + "dDebugFlag": "135", + "vDebugFlag": "135", + "tqDebugFlag": "135", + "cDebugFlag": "135", + "jniDebugFlag": "135", + "qDebugFlag": "135", + "rpcDebugFlag": "135", "tmrDebugFlag": "131", - "uDebugFlag": "143", - "sDebugFlag": "143", - "wDebugFlag": "143", + "uDebugFlag": "135", + "sDebugFlag": "135", + "wDebugFlag": "135", "numOfLogLines": "100000000", "statusInterval": "1", "enableQueryHb": "1", From d6579d0498bbfd4728c29196dd42c303e897f48c Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Tue, 17 Oct 2023 18:30:47 +0800 Subject: [PATCH 25/59] test: support replica3 in splited/transform tmqvnode --- tests/system-test/7-tmq/tmqVnodeTransform-stb.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/system-test/7-tmq/tmqVnodeTransform-stb.py b/tests/system-test/7-tmq/tmqVnodeTransform-stb.py index 18f9fafe9a..006498d514 100644 --- a/tests/system-test/7-tmq/tmqVnodeTransform-stb.py +++ b/tests/system-test/7-tmq/tmqVnodeTransform-stb.py @@ -256,7 +256,7 @@ class TDTestCase: tdLog.printNoPrefix("======== test case 2 end ...... ") def tmqCaseStableSelect(self): - tdLog.printNoPrefix("======== test case subscrib column start : ") + tdLog.printNoPrefix("======== test case 3 subscrib column start : ") paraDict = {'dbName': 'dbt', 'dropFlag': 1, 'event': '', @@ -315,7 +315,7 @@ class TDTestCase: tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) tdLog.info("wait the consume result") - time.sleep(5) + time.sleep(1) #restart dnode & remove wal # self.restartAndRemoveWal() @@ -332,10 +332,10 @@ class TDTestCase: for i in range(len(topicNameList)): tdSql.query("drop topic %s"%topicNameList[i]) - tdLog.printNoPrefix("======== test case subscrib column end ...... ") + tdLog.printNoPrefix("======== test case 3 subscrib column end ...... ") def tmqCaseDbname(self): - tdLog.printNoPrefix("======== test case subscrib Dbname start: ") + tdLog.printNoPrefix("======== test case 4 subscrib Dbname start: ") paraDict = {'dbName': 'dbt', 'dropFlag': 1, 'event': '', @@ -394,7 +394,7 @@ class TDTestCase: tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) tdLog.info("wait the consume result") - time.sleep(5) + time.sleep(1) #restart dnode & remove wal # self.restartAndRemoveWal() @@ -411,7 +411,7 @@ class TDTestCase: for i in range(len(topicNameList)): tdSql.query("drop topic %s"%topicNameList[i]) - tdLog.printNoPrefix("======== test case subscrib Dbname end ...... ") + tdLog.printNoPrefix("======== test case 4 subscrib Dbname end ...... ") def run(self): self.prepareTestEnv() From a9dfb400691ee59d8e4d35fe3c97e649a5480250 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 18 Oct 2023 10:51:47 +0800 Subject: [PATCH 26/59] contrib/cmake: remove unused steps --- contrib/CMakeLists.txt | 29 +---------------------------- 1 file changed, 1 insertion(+), 28 deletions(-) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index de3d9cfd37..30853660ed 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -8,30 +8,9 @@ endfunction(cat IN_FILE OUT_FILE) if(${TD_LINUX}) - if(${BUILD_WITH_S3}) - -set(CONTRIB_TMP_FILE3 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in3") -configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3}) - +if(${BUILD_WITH_S3}) file(MAKE_DIRECTORY $ENV{HOME}/.cos-local.1/) -configure_file(${CONTRIB_TMP_FILE3} "${TD_CONTRIB_DIR}/deps-download/CMakeLists.txt") -execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" . - WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download") -execute_process(COMMAND "${CMAKE_COMMAND}" --build . - WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download") - -set(CONTRIB_TMP_FILE2 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in2") -configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE2}) - - #cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE2}) - -configure_file(${CONTRIB_TMP_FILE2} "${TD_CONTRIB_DIR}/deps-download/CMakeLists.txt") -execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" . - WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download") -execute_process(COMMAND "${CMAKE_COMMAND}" --build . - WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download") - else() set(CONTRIB_TMP_FILE3 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in3") @@ -185,12 +164,6 @@ endif(${BUILD_WITH_SQLITE}) # s3 if(${BUILD_WITH_S3}) - #cat("${TD_SUPPORT_DIR}/mxml_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) - #cat("${TD_SUPPORT_DIR}/apr_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) - #cat("${TD_SUPPORT_DIR}/apr-util_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) - #cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) - #INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.1/include) - cat("${TD_SUPPORT_DIR}/ssl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) cat("${TD_SUPPORT_DIR}/xml2_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) From f26ec41b8e11804ba87524e5accc8cc5cd459af1 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 18 Oct 2023 15:30:02 +0800 Subject: [PATCH 27/59] vnode/cos: fix get callback buffer --- source/common/CMakeLists.txt | 3 +++ source/common/src/tglobal.c | 2 +- source/dnode/vnode/CMakeLists.txt | 1 - source/dnode/vnode/src/vnd/vnodeCos.c | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/source/common/CMakeLists.txt b/source/common/CMakeLists.txt index b010467f20..6bdcdf7d1d 100644 --- a/source/common/CMakeLists.txt +++ b/source/common/CMakeLists.txt @@ -22,6 +22,9 @@ IF (TD_STORAGE) add_definitions(-DUSE_COS) ENDIF(${BUILD_WITH_COS}) + IF(${BUILD_WITH_S3}) + add_definitions(-DUSE_S3) + ENDIF(${BUILD_WITH_S3}) ENDIF(${TD_LINUX}) ENDIF () diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 8ea5528fbb..f3af6e0ab0 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -329,7 +329,7 @@ int32_t taosSetS3Cfg(SConfig *pCfg) { } } if (tsS3BucketName[0] != '<' && tsDiskCfgNum > 1) { -#ifdef USE_COS +#if defined(USE_COS) || defined(USE_S3) tsS3Enabled = true; #endif } diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index e554d94e4d..7e2693e8af 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -164,7 +164,6 @@ target_link_libraries( if(${TD_LINUX}) if(${BUILD_WITH_S3}) - MESSAGE("build with s3: ${BUILD_WITH_S3}") target_include_directories( vnode diff --git a/source/dnode/vnode/src/vnd/vnodeCos.c b/source/dnode/vnode/src/vnd/vnodeCos.c index b52d4809b2..c50f133987 100644 --- a/source/dnode/vnode/src/vnd/vnodeCos.c +++ b/source/dnode/vnode/src/vnd/vnodeCos.c @@ -721,7 +721,7 @@ static S3Status getObjectDataCallback(int bufferSize, const char *buffer, void * char *buf = taosMemoryCalloc(1, bufferSize); if (buf) { memcpy(buf, buffer, bufferSize); - + cbd->buf = buf; cbd->status = S3StatusOK; return S3StatusOK; } else { From e7d9de06cb0f73f9fb5148f30694f675dfcfee03 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Wed, 18 Oct 2023 15:59:04 +0800 Subject: [PATCH 28/59] test: support replica3 in splited/transform tmqvnode --- .../system-test/7-tmq/tmqVnodeTransform-db.py | 298 ++++++++++++++++++ .../7-tmq/tmqVnodeTransform-stb.py | 165 +--------- 2 files changed, 300 insertions(+), 163 deletions(-) create mode 100644 tests/system-test/7-tmq/tmqVnodeTransform-db.py diff --git a/tests/system-test/7-tmq/tmqVnodeTransform-db.py b/tests/system-test/7-tmq/tmqVnodeTransform-db.py new file mode 100644 index 0000000000..005bca70d6 --- /dev/null +++ b/tests/system-test/7-tmq/tmqVnodeTransform-db.py @@ -0,0 +1,298 @@ + +import taos +import sys +import time +import socket +import os +import threading +import math + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +from util.cluster import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def __init__(self): + self.vgroups = 1 + self.ctbNum = 10 + self.rowsPerTbl = 10000 + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def getDataPath(self): + selfPath = tdCom.getBuildPath() + + return selfPath + '/../sim/dnode%d/data/vnode/vnode%d/wal/*'; + + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 30, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tdCom.drop_all_db() + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], wal_retention_period=36000,vgroups=paraDict["vgroups"],replica=self.replicaVar) + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + # tdLog.info("create ctb") + # tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + # ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + # tdLog.info("insert data") + # tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + # ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + # startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + + # tdLog.info("restart taosd to ensure that the data falls into the disk") + # tdDnodes.stop(1) + # tdDnodes.start(1) + # tdSql.query("flush database %s"%(paraDict['dbName'])) + return + + def restartAndRemoveWal(self): + tdDnodes = cluster.dnodes + tdSql.query("select * from information_schema.ins_vnodes") + for result in tdSql.queryResult: + if result[2] == 'dbt': + tdLog.debug("dnode is %d"%(result[0])) + dnodeId = result[0] + vnodeId = result[1] + + tdDnodes[dnodeId - 1].stoptaosd() + time.sleep(1) + dataPath = self.getDataPath() + dataPath = dataPath%(dnodeId,vnodeId) + os.system('rm -rf ' + dataPath) + tdLog.debug("dataPath:%s"%dataPath) + tdDnodes[dnodeId - 1].starttaosd() + time.sleep(1) + break + tdLog.debug("restart dnode ok") + + def redistributeVgroups(self): + dnodesList = [] + tdSql.query("show dnodes") + for result in tdSql.queryResult: + dnodesList.append(result[0]) + print("dnodeList:",dnodesList) + tdSql.query("select * from information_schema.ins_vnodes") + vnodeId = 0 + for result in tdSql.queryResult: + if result[2] == 'dbt': + tdLog.debug("dnode is %d"%(result[0])) + dnodesList.remove(result[0]) + vnodeId = result[1] + print("its all data",dnodesList) + # if self.replicaVar == 1: + # redistributeSql = "redistribute vgroup %d dnode %d" %(vnodeId, dnodesList[0]) + # else: + redistributeSql = f"redistribute vgroup {vnodeId} " + for vgdnode in dnodesList: + redistributeSql += f"dnode {vgdnode} " + print(redistributeSql) + + tdLog.debug(f"redistributeSql:{redistributeSql}") + tdSql.query(redistributeSql) + tdLog.debug("redistributeSql ok") + + + def tmqCaseStableSelect(self): + tdLog.printNoPrefix("======== test case 3 subscrib column start : ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stbn', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 10, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + topicNameList = ['topic3'] + tmqCom.initConsumerTable() + + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + + tdLog.info("create ctb") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + tdLog.info("insert data") + tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("create topics from stb with filter") + queryString = "select * from %s.%s where c2 > 0 "%(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + + time.sleep(1) + #restart dnode & remove wal + # self.restartAndRemoveWal() + + # redistribute vgroup + self.redistributeVgroups() + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + expectRows = 2 + resultList = tmqCom.selectConsumeResult(expectRows) + + time.sleep(6) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 3 subscrib column end ...... ") + + def tmqCaseDbname(self): + tdLog.printNoPrefix("======== test case 4 subscrib Dbname start: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stbn', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 10, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + topicNameList = ['topic4'] + tmqCom.initConsumerTable() + + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + + tdLog.info("create ctb") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + tdLog.info("insert data") + tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("create topics from database ") + queryString = "database %s "%(paraDict['dbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + + time.sleep(1) + #restart dnode & remove wal + # self.restartAndRemoveWal() + + # redistribute vgroup + self.redistributeVgroups() + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + expectRows = 2 + resultList = tmqCom.selectConsumeResult(expectRows) + + time.sleep(6) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 4 subscrib Dbname end ...... ") + + def run(self): + self.prepareTestEnv() + self.tmqCaseStableSelect() + self.prepareTestEnv() + self.tmqCaseDbname() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/tmqVnodeTransform-stb.py b/tests/system-test/7-tmq/tmqVnodeTransform-stb.py index 006498d514..2dd756b788 100644 --- a/tests/system-test/7-tmq/tmqVnodeTransform-stb.py +++ b/tests/system-test/7-tmq/tmqVnodeTransform-stb.py @@ -49,7 +49,7 @@ class TDTestCase: 'rowsPerTbl': 10000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 60, + 'pollDelay': 30, 'showMsg': 1, 'showRow': 1, 'snapshot': 0} @@ -140,7 +140,7 @@ class TDTestCase: 'rowsPerTbl': 10000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 60, + 'pollDelay': 30, 'showMsg': 1, 'showRow': 1, 'snapshot': 0} @@ -255,173 +255,12 @@ class TDTestCase: tdLog.printNoPrefix("======== test case 2 end ...... ") - def tmqCaseStableSelect(self): - tdLog.printNoPrefix("======== test case 3 subscrib column start : ") - paraDict = {'dbName': 'dbt', - 'dropFlag': 1, - 'event': '', - 'vgroups': 1, - 'stbName': 'stbn', - 'colPrefix': 'c', - 'tagPrefix': 't', - 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], - 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], - 'ctbPrefix': 'ctb', - 'ctbStartIdx': 0, - 'ctbNum': 10, - 'rowsPerTbl': 10000, - 'batchNum': 10, - 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 2, - 'showMsg': 1, - 'showRow': 1, - 'snapshot': 0} - - paraDict['vgroups'] = self.vgroups - paraDict['ctbNum'] = self.ctbNum - paraDict['rowsPerTbl'] = self.rowsPerTbl - - topicNameList = ['topic3'] - tmqCom.initConsumerTable() - - tdLog.info("create stb") - tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) - - tdLog.info("create ctb") - tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], - ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) - tdLog.info("insert data") - tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], - ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], - startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) - - tdLog.info("create topics from stb with filter") - queryString = "select * from %s.%s where c2 > 0 "%(paraDict['dbName'], paraDict['stbName']) - sqlString = "create topic %s as %s" %(topicNameList[0], queryString) - tdLog.info("create topic sql: %s"%sqlString) - tdSql.execute(sqlString) - - # init consume info, and start tmq_sim, then check consume result - tdLog.info("insert consume info to consume processor") - consumerId = 0 - expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] - topicList = topicNameList[0] - ifcheckdata = 1 - ifManualCommit = 1 - keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' - tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) - - tdLog.info("start consume processor") - tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) - tdLog.info("wait the consume result") - - time.sleep(1) - #restart dnode & remove wal - # self.restartAndRemoveWal() - - # redistribute vgroup - self.redistributeVgroups() - - tdLog.info("start consume processor") - tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) - tdLog.info("wait the consume result") - expectRows = 1 - resultList = tmqCom.selectConsumeResult(expectRows) - - time.sleep(5) - for i in range(len(topicNameList)): - tdSql.query("drop topic %s"%topicNameList[i]) - - tdLog.printNoPrefix("======== test case 3 subscrib column end ...... ") - - def tmqCaseDbname(self): - tdLog.printNoPrefix("======== test case 4 subscrib Dbname start: ") - paraDict = {'dbName': 'dbt', - 'dropFlag': 1, - 'event': '', - 'vgroups': 1, - 'stbName': 'stbn', - 'colPrefix': 'c', - 'tagPrefix': 't', - 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], - 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], - 'ctbPrefix': 'ctb', - 'ctbStartIdx': 0, - 'ctbNum': 10, - 'rowsPerTbl': 10000, - 'batchNum': 10, - 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 2, - 'showMsg': 1, - 'showRow': 1, - 'snapshot': 0} - - paraDict['vgroups'] = self.vgroups - paraDict['ctbNum'] = self.ctbNum - paraDict['rowsPerTbl'] = self.rowsPerTbl - - topicNameList = ['topic4'] - tmqCom.initConsumerTable() - - tdLog.info("create stb") - tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) - - tdLog.info("create ctb") - tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], - ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) - tdLog.info("insert data") - tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], - ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], - startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) - - tdLog.info("create topics from database ") - queryString = "database %s "%(paraDict['dbName']) - sqlString = "create topic %s as %s" %(topicNameList[0], queryString) - tdLog.info("create topic sql: %s"%sqlString) - tdSql.execute(sqlString) - - # init consume info, and start tmq_sim, then check consume result - tdLog.info("insert consume info to consume processor") - consumerId = 0 - expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] - topicList = topicNameList[0] - ifcheckdata = 1 - ifManualCommit = 1 - keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' - tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) - - tdLog.info("start consume processor") - tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) - tdLog.info("wait the consume result") - - time.sleep(1) - #restart dnode & remove wal - # self.restartAndRemoveWal() - - # redistribute vgroup - self.redistributeVgroups() - - tdLog.info("start consume processor") - tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) - tdLog.info("wait the consume result") - expectRows = 1 - resultList = tmqCom.selectConsumeResult(expectRows) - - time.sleep(5) - for i in range(len(topicNameList)): - tdSql.query("drop topic %s"%topicNameList[i]) - - tdLog.printNoPrefix("======== test case 4 subscrib Dbname end ...... ") def run(self): self.prepareTestEnv() self.tmqCaseStable() self.prepareTestEnv() self.tmqCaseNtable() - self.prepareTestEnv() - self.tmqCaseStableSelect() - self.prepareTestEnv() - self.tmqCaseDbname() def stop(self): tdSql.close() From 52c8bc91f98ebfea693b1b7c352eba2fdaf44a7c Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 18 Oct 2023 18:21:45 +0800 Subject: [PATCH 29/59] cos/delete: fix delete object cond and double free --- source/dnode/vnode/src/vnd/vnodeCos.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeCos.c b/source/dnode/vnode/src/vnd/vnodeCos.c index c50f133987..ccda20e049 100644 --- a/source/dnode/vnode/src/vnd/vnodeCos.c +++ b/source/dnode/vnode/src/vnd/vnodeCos.c @@ -644,7 +644,7 @@ static S3Status listBucketCallback(int isTruncated, const char *nextMarker, int } static void s3FreeObjectKey(void *pItem) { - char *key = (char *)pItem; + char *key = *(char **)pItem; taosMemoryFree(key); } @@ -682,7 +682,7 @@ void s3DeleteObjectsByPrefix(const char *prefix) { } while (data.isTruncated && (!maxkeys || (data.keyCount < maxkeys))); if (data.status == S3StatusOK) { - if (!data.keyCount) { + if (data.keyCount > 0) { // printListBucketHeader(allDetails); s3DeleteObjects(TARRAY_DATA(data.objectArray), TARRAY_SIZE(data.objectArray)); } From 788f21fd946afc1a9332d250305d8faac28941fa Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Wed, 18 Oct 2023 19:38:24 +0800 Subject: [PATCH 30/59] test: support replica3 in splited/transform tmqvnode --- tests/parallel_test/cases.task | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 426646426d..c749864eec 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -173,6 +173,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -n 3 -i True ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-stb.py -N 2 -n 1 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-stb.py -N 6 -n 3 +,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-db.py -N 6 -n 3 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select.py -N 2 -n 1 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py -N 3 -n 3 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select.py -N 3 -n 3 From 720982374cc222c205de4c185a6ad6980240be13 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Fri, 20 Oct 2023 10:36:52 +0800 Subject: [PATCH 31/59] fix: union all cast func resType check error --- source/libs/parser/src/parAstCreater.c | 4 ++++ source/libs/parser/src/parTranslater.c | 5 +++-- source/libs/parser/src/parUtil.c | 2 +- tests/system-test/2-query/varchar.py | 6 ++++++ 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 12062e0d4a..e8614147f4 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -1379,6 +1379,10 @@ SNode* createColumnDefNode(SAstCreateContext* pCxt, SToken* pColName, SDataType if (!checkColumnName(pCxt, pColName) || !checkComment(pCxt, pComment, false)) { return NULL; } + if (IS_VAR_DATA_TYPE(dataType.type) && dataType.bytes == 0) { + pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN); + return NULL; + } SColumnDefNode* pCol = (SColumnDefNode*)nodesMakeNode(QUERY_NODE_COLUMN_DEF); CHECK_OUT_OF_MEM(pCol); COPY_STRING_FORM_ID_TOKEN(pCol->colName, pColName); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 42ab2f7a2f..756a1f867b 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2185,11 +2185,12 @@ static int32_t createCastFunc(STranslateContext* pCxt, SNode* pExpr, SDataType d nodesDestroyNode((SNode*)pFunc); return TSDB_CODE_OUT_OF_MEMORY; } - if (TSDB_CODE_SUCCESS != getFuncInfo(pCxt, pFunc)) { + int32_t code = getFuncInfo(pCxt, pFunc); + if (TSDB_CODE_SUCCESS != code) { nodesClearList(pFunc->pParameterList); pFunc->pParameterList = NULL; nodesDestroyNode((SNode*)pFunc); - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)pExpr)->aliasName); + return code; } *pCast = (SNode*)pFunc; return TSDB_CODE_SUCCESS; diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 1b8bac4cbc..0ab8927bd0 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -126,7 +126,7 @@ static char* getSyntaxErrFormat(int32_t errCode) { case TSDB_CODE_PAR_INVALID_FIRST_COLUMN: return "First column must be timestamp"; case TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN: - return "Invalid binary/nchar column length"; + return "Invalid column length for var length type"; case TSDB_CODE_PAR_INVALID_TAGS_NUM: return "Invalid number of tag columns"; case TSDB_CODE_PAR_INVALID_INTERNAL_PK: diff --git a/tests/system-test/2-query/varchar.py b/tests/system-test/2-query/varchar.py index f0849010c6..debcd1f95e 100644 --- a/tests/system-test/2-query/varchar.py +++ b/tests/system-test/2-query/varchar.py @@ -76,6 +76,12 @@ class TDTestCase: for i in range(tdSql.queryRows): tdSql.checkData(i,0, data_ct1_c8[i]) + tdSql.error("create stable empty_col_stable(ts timestamp, c2 varchar(0)) tags(tg1 int)") + tdSql.error("create stable empty_col_stable(ts timestamp, c2 varchar(10)) tags(tg1 varchar(0))") + tdSql.error("create stable empty_col_stable(ts timestamp, c2 varchar(10)) tags(tg1 nchar(0))") + tdSql.error("create stable empty_col_stable(ts timestamp, c2 varchar(10)) tags(tg1 binary(0))") + tdSql.error("create stable empty_col_stable(ts timestamp, c2 varchar(10)) tags(tg1 varbinary(0))") + # tdSql.query("select c8 from ct4") # data_ct4 = [tdSql.getData(i,0) for i in range(tdSql.queryRows)] From 74d05c2771c4c04b59cb3bf927b2eb6811838e92 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 20 Oct 2023 13:47:25 +0800 Subject: [PATCH 32/59] vnode/cos: fix multipart uploading --- source/dnode/vnode/src/vnd/vnodeCos.c | 71 ++++++++++++++++----------- 1 file changed, 41 insertions(+), 30 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeCos.c b/source/dnode/vnode/src/vnd/vnodeCos.c index ccda20e049..9c69f2452c 100644 --- a/source/dnode/vnode/src/vnd/vnodeCos.c +++ b/source/dnode/vnode/src/vnd/vnodeCos.c @@ -211,33 +211,6 @@ typedef struct put_object_callback_data { int noStatus; } put_object_callback_data; -static int putObjectDataCallback(int bufferSize, char *buffer, void *callbackData) { - put_object_callback_data *data = (put_object_callback_data *)callbackData; - - int ret = 0; - - if (data->contentLength) { - int toRead = ((data->contentLength > (unsigned)bufferSize) ? (unsigned)bufferSize : data->contentLength); - if (data->gb) { - growbuffer_read(&(data->gb), toRead, &ret, buffer); - } else if (data->infileFD) { - // ret = fread(buffer, 1, toRead, data->infile); - ret = taosReadFile(data->infileFD, buffer, toRead); - } - } - - data->contentLength -= ret; - data->totalContentLength -= ret; - - if (data->contentLength && !data->noStatus) { - vTrace("%llu bytes remaining ", (unsigned long long)data->totalContentLength); - vTrace("(%d%% complete) ...\n", (int)(((data->totalOriginalContentLength - data->totalContentLength) * 100) / - data->totalOriginalContentLength)); - } - - return ret; -} - #define MULTIPART_CHUNK_SIZE (768 << 20) // multipart is 768M typedef struct UploadManager { @@ -280,6 +253,37 @@ typedef struct MultipartPartData { UploadManager *manager; } MultipartPartData; +static int putObjectDataCallback(int bufferSize, char *buffer, void *callbackData) { + put_object_callback_data *data = (put_object_callback_data *)callbackData; + if (data->infileFD == 0) { + MultipartPartData *mpd = (MultipartPartData *)callbackData; + data = &mpd->put_object_data; + } + + int ret = 0; + + if (data->contentLength) { + int toRead = ((data->contentLength > (unsigned)bufferSize) ? (unsigned)bufferSize : data->contentLength); + if (data->gb) { + growbuffer_read(&(data->gb), toRead, &ret, buffer); + } else if (data->infileFD) { + // ret = fread(buffer, 1, toRead, data->infile); + ret = taosReadFile(data->infileFD, buffer, toRead); + } + } + + data->contentLength -= ret; + data->totalContentLength -= ret; + /* log too many open files + if (data->contentLength && !data->noStatus) { + vTrace("%llu bytes remaining ", (unsigned long long)data->totalContentLength); + vTrace("(%d%% complete) ...\n", (int)(((data->totalOriginalContentLength - data->totalContentLength) * 100) / + data->totalOriginalContentLength)); + } + */ + return ret; +} + S3Status initial_multipart_callback(const char *upload_id, void *callbackData) { UploadManager *manager = (UploadManager *)callbackData; manager->upload_id = strdup(upload_id); @@ -308,7 +312,7 @@ static int multipartPutXmlCallback(int bufferSize, char *buffer, void *callbackD manager->remaining -= ret; return ret; } - +/* static S3Status listPartsCallback(int isTruncated, const char *nextPartNumberMarker, const char *initiatorId, const char *initiatorDisplayName, const char *ownerId, const char *ownerDisplayName, const char *storageClass, int partsCount, int handlePartsStart, @@ -418,7 +422,7 @@ static int try_get_parts_info(const char *bucketName, const char *key, UploadMan return 0; } - +*/ int32_t s3PutObjectFromFile2(const char *file, const char *object) { int32_t code = 0; const char *key = object; @@ -480,9 +484,11 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { if (data.status != S3StatusOK) { s3PrintError(__func__, data.status, data.err_msg); + code = TAOS_SYSTEM_ERROR(EIO); } else if (data.contentLength) { vError("ERROR: %s Failed to read remaining %llu bytes from input", __func__, (unsigned long long)data.contentLength); + code = TAOS_SYSTEM_ERROR(EIO); } } else { uint64_t totalContentLength = contentLength; @@ -493,7 +499,9 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { // div round up int seq; + // uint64_t chunk_size = MULTIPART_CHUNK_SIZE >> 8; int totalSeq = ((contentLength + MULTIPART_CHUNK_SIZE - 1) / MULTIPART_CHUNK_SIZE); + // int totalSeq = ((contentLength + chunk_size - 1) / chunk_size); MultipartPartData partData; memset(&partData, 0, sizeof(MultipartPartData)); @@ -530,6 +538,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { if (manager.upload_id == 0 || manager.status != S3StatusOK) { s3PrintError(__func__, manager.status, manager.err_msg); + code = TAOS_SYSTEM_ERROR(EIO); goto clean; } @@ -554,6 +563,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { } while (S3_status_is_retryable(partData.status) && should_retry()); if (partData.status != S3StatusOK) { s3PrintError(__func__, partData.status, partData.err_msg); + code = TAOS_SYSTEM_ERROR(EIO); goto clean; } contentLength -= MULTIPART_CHUNK_SIZE; @@ -581,6 +591,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { } while (S3_status_is_retryable(manager.status) && should_retry()); if (manager.status != S3StatusOK) { s3PrintError(__func__, manager.status, manager.err_msg); + code = TAOS_SYSTEM_ERROR(EIO); goto clean; } @@ -595,7 +606,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { taosMemoryFree(manager.etags); } - return 0; + return code; } typedef struct list_bucket_callback_data { From f016dab40af2446310ecfa58871405af880b11ea Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 20 Oct 2023 13:59:01 +0800 Subject: [PATCH 33/59] common/global: comment s3BlockSize altering out --- source/common/src/tglobal.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index f3af6e0ab0..9bc981b520 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -1683,14 +1683,14 @@ void taosCfgDynamicOptions(const char *option, const char *value) { } return; } - + /* cannot alter s3BlockSize if (strcasecmp(option, "s3BlockSize") == 0) { int32_t newS3BlockSize = atoi(value); uInfo("s3BlockSize set from %d to %d", tsS3BlockSize, newS3BlockSize); tsS3BlockSize = newS3BlockSize; return; } - + */ if (strcasecmp(option, "s3BlockCacheSize") == 0) { int32_t newS3BlockCacheSize = atoi(value); uInfo("s3BlockCacheSize set from %d to %d", tsS3BlockCacheSize, newS3BlockCacheSize); From 55647f9ce9188e79405345935838bcd009f6f898 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 20 Oct 2023 15:15:03 +0800 Subject: [PATCH 34/59] cos/put: make chunk small to not block other reqs --- source/dnode/vnode/src/vnd/vnodeCos.c | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeCos.c b/source/dnode/vnode/src/vnd/vnodeCos.c index 9c69f2452c..6e36739f5a 100644 --- a/source/dnode/vnode/src/vnd/vnodeCos.c +++ b/source/dnode/vnode/src/vnd/vnodeCos.c @@ -498,10 +498,9 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { manager.gb = 0; // div round up - int seq; - // uint64_t chunk_size = MULTIPART_CHUNK_SIZE >> 8; - int totalSeq = ((contentLength + MULTIPART_CHUNK_SIZE - 1) / MULTIPART_CHUNK_SIZE); - // int totalSeq = ((contentLength + chunk_size - 1) / chunk_size); + int seq; + uint64_t chunk_size = MULTIPART_CHUNK_SIZE >> 8; + int totalSeq = ((contentLength + chunk_size - 1) / chunk_size); MultipartPartData partData; memset(&partData, 0, sizeof(MultipartPartData)); @@ -543,14 +542,14 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { } upload: - todoContentLength -= MULTIPART_CHUNK_SIZE * manager.next_etags_pos; + todoContentLength -= chunk_size * manager.next_etags_pos; for (seq = manager.next_etags_pos + 1; seq <= totalSeq; seq++) { partData.manager = &manager; partData.seq = seq; if (partData.put_object_data.gb == NULL) { partData.put_object_data = data; } - partContentLength = ((contentLength > MULTIPART_CHUNK_SIZE) ? MULTIPART_CHUNK_SIZE : contentLength); + partContentLength = ((contentLength > chunk_size) ? chunk_size : contentLength); // printf("%s Part Seq %d, length=%d\n", srcSize ? "Copying" : "Sending", seq, partContentLength); partData.put_object_data.contentLength = partContentLength; partData.put_object_data.originalContentLength = partContentLength; @@ -566,8 +565,8 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { code = TAOS_SYSTEM_ERROR(EIO); goto clean; } - contentLength -= MULTIPART_CHUNK_SIZE; - todoContentLength -= MULTIPART_CHUNK_SIZE; + contentLength -= chunk_size; + todoContentLength -= chunk_size; } int i; From f630fc38f805e178ef1c3e0b27b68a16036f2d80 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 23 Oct 2023 15:46:32 +0800 Subject: [PATCH 35/59] tsdb/commit: skip last tier ts data committing --- source/dnode/vnode/src/tsdb/tsdbCommit2.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c index d4cb63fb7b..5bb486de4a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c @@ -128,8 +128,16 @@ static int32_t tsdbCommitTSData(SCommitter2 *committer) { } } + extern int8_t tsS3Enabled; + + int32_t nlevel = tfsGetLevel(committer->tsdb->pVnode->pTfs); + bool skipRow = false; + if (tsS3Enabled && nlevel > 1 && committer->ctx->did.level == nlevel - 1) { + skipRow = true; + } + int64_t ts = TSDBROW_TS(&row->row); - if (ts > committer->ctx->maxKey) { + if (ts > committer->ctx->maxKey || skipRow) { committer->ctx->nextKey = TMIN(committer->ctx->nextKey, ts); code = tsdbIterMergerSkipTableData(committer->dataIterMerger, committer->ctx->tbid); TSDB_CHECK_CODE(code, lino, _exit); From f8097979995e3620a44dd8106a7cbf540a4e4d08 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 23 Oct 2023 16:04:45 +0800 Subject: [PATCH 36/59] enh: taosd build info --- cmake/cmake.version | 14 +++++++++++++- source/util/src/version.c.in | 2 +- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/cmake/cmake.version b/cmake/cmake.version index fa6ec4df17..c0ddae569c 100644 --- a/cmake/cmake.version +++ b/cmake/cmake.version @@ -50,7 +50,19 @@ ENDIF () IF (DEFINED VERDATE) SET(TD_VER_DATE ${VERDATE}) ELSE () - STRING(TIMESTAMP TD_VER_DATE "%Y-%m-%d %H:%M:%S") + STRING(COMPARE GREATER_EQUAL "${CMAKE_VERSION}" "3.26" TD_CMAKE_SUPPORT_TZ) + + IF (TD_CMAKE_SUPPORT_TZ) + STRING(TIMESTAMP TD_VER_DATE "%Y-%m-%d %H:%M:%S %z") + ELSE () + IF (TD_WINDOWS) + STRING(TIMESTAMP TD_VER_DATE "%Y-%m-%d %H:%M:%S") + ELSE () + EXECUTE_PROCESS(COMMAND date +"%F %T %z" OUTPUT_VARIABLE TD_VER_DATE) + STRING(REPLACE "\"" "" TD_VER_DATE ${TD_VER_DATE}) + STRING(STRIP ${TD_VER_DATE} TD_VER_DATE) + ENDIF () + ENDIF () ENDIF () IF (DEFINED VERTYPE) diff --git a/source/util/src/version.c.in b/source/util/src/version.c.in index 71998e3321..ec6449a02f 100644 --- a/source/util/src/version.c.in +++ b/source/util/src/version.c.in @@ -2,6 +2,6 @@ char version[64] = "${TD_VER_NUMBER}"; char compatible_version[12] = "${TD_VER_COMPATIBLE}"; char gitinfo[48] = "${TD_VER_GIT}"; char gitinfoOfInternal[48] = "${TD_VER_GIT_INTERNAL}"; -char buildinfo[64] = "Built at ${TD_VER_DATE}"; +char buildinfo[64] = "Built ${TD_VER_OSTYPE}-${TD_VER_CPUTYPE} at ${TD_VER_DATE}"; void libtaos_${TD_LIB_VER_NUMBER}_${TD_VER_OSTYPE}_${TD_VER_CPUTYPE}_${TD_VER_VERTYPE}() {}; From 62cd6f3ed26d1aa8b8e064de24e651b1b2d1d0eb Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 23 Oct 2023 16:50:51 +0800 Subject: [PATCH 37/59] cmake/curl, libs3, ssl: clean source dir before building --- cmake/curl_CMakeLists.txt.in | 7 ++++--- cmake/libs3_CMakeLists.txt.in | 4 ++-- cmake/ssl_CMakeLists.txt.in | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/cmake/curl_CMakeLists.txt.in b/cmake/curl_CMakeLists.txt.in index 3f7b9ee1fe..2fe84b9721 100644 --- a/cmake/curl_CMakeLists.txt.in +++ b/cmake/curl_CMakeLists.txt.in @@ -12,9 +12,10 @@ ExternalProject_Add(curl BUILD_ALWAYS 1 #UPDATE_COMMAND "" #CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --without-ssl --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd - CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --with-ssl=$ENV{HOME}/.cos-local.1 --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd --without-libidn2 + CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --with-ssl=$ENV{HOME}/.cos-local.1 --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd --without-libidn2 #--enable-debug + #CONFIGURE_COMMAND ./configure --without-ssl - BUILD_COMMAND make - INSTALL_COMMAND make install + BUILD_COMMAND "" + INSTALL_COMMAND make clean && make install TEST_COMMAND "" ) diff --git a/cmake/libs3_CMakeLists.txt.in b/cmake/libs3_CMakeLists.txt.in index b2d2d19059..6c911775ee 100644 --- a/cmake/libs3_CMakeLists.txt.in +++ b/cmake/libs3_CMakeLists.txt.in @@ -23,7 +23,7 @@ ExternalProject_Add(libs3 CONFIGURE_COMMAND cp ${TD_SUPPORT_DIR}/libs3.GNUmakefile GNUmakefile && sed -i "s|CFLAGS += -Wall -Werror|CFLAGS += -I'$ENV{HOME}/.cos-local.1/include' -L'$ENV{HOME}/.cos-local.1/lib' |" ./GNUmakefile #BUILD_COMMAND make CFLAGS=${s3_flags} DESTDIR=$ENV{HOME}/.cos-local.1 build/lib/libs3.a #BUILD_COMMAND make DESTDIR="$ENV{HOME}/.cos-local.1" build/lib/libs3.a - BUILD_COMMAND make build/lib/libs3.a - INSTALL_COMMAND make install_static + BUILD_COMMAND "" + INSTALL_COMMAND make clean && make build/lib/libs3.a && make install_static TEST_COMMAND "" ) diff --git a/cmake/ssl_CMakeLists.txt.in b/cmake/ssl_CMakeLists.txt.in index 65a02d7fd6..8301dbe222 100644 --- a/cmake/ssl_CMakeLists.txt.in +++ b/cmake/ssl_CMakeLists.txt.in @@ -10,6 +10,6 @@ ExternalProject_Add(openssl #UPDATE_COMMAND "" CONFIGURE_COMMAND ./Configure --prefix=$ENV{HOME}/.cos-local.1 -static #--no-shared BUILD_COMMAND "" - INSTALL_COMMAND make install_sw -j + INSTALL_COMMAND make clean && make install_sw -j TEST_COMMAND "" ) From b864c87b7c91d8663c592f25ee681f85cc3e761f Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 23 Oct 2023 17:00:52 +0800 Subject: [PATCH 38/59] cmake/ssl: make depend --- cmake/ssl_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/ssl_CMakeLists.txt.in b/cmake/ssl_CMakeLists.txt.in index 8301dbe222..c17043ff22 100644 --- a/cmake/ssl_CMakeLists.txt.in +++ b/cmake/ssl_CMakeLists.txt.in @@ -10,6 +10,6 @@ ExternalProject_Add(openssl #UPDATE_COMMAND "" CONFIGURE_COMMAND ./Configure --prefix=$ENV{HOME}/.cos-local.1 -static #--no-shared BUILD_COMMAND "" - INSTALL_COMMAND make clean && make install_sw -j + INSTALL_COMMAND make clean && make depend && make install_sw -j TEST_COMMAND "" ) From 7cf72a567da657f1c2f62401722dada5e757283b Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 23 Oct 2023 19:19:34 +0800 Subject: [PATCH 39/59] cmake/ssl: use build command --- cmake/ssl_CMakeLists.txt.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/ssl_CMakeLists.txt.in b/cmake/ssl_CMakeLists.txt.in index c17043ff22..1f3fd0a30d 100644 --- a/cmake/ssl_CMakeLists.txt.in +++ b/cmake/ssl_CMakeLists.txt.in @@ -9,7 +9,7 @@ ExternalProject_Add(openssl #BUILD_ALWAYS 1 #UPDATE_COMMAND "" CONFIGURE_COMMAND ./Configure --prefix=$ENV{HOME}/.cos-local.1 -static #--no-shared - BUILD_COMMAND "" - INSTALL_COMMAND make clean && make depend && make install_sw -j + BUILD_COMMAND make -j + INSTALL_COMMAND make install_sw -j TEST_COMMAND "" ) From 7d17c4cc90385350805fb90a3501796486dd48e6 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 23 Oct 2023 19:35:22 +0800 Subject: [PATCH 40/59] cmake/curl, libs3: use build command --- cmake/curl_CMakeLists.txt.in | 4 ++-- cmake/libs3_CMakeLists.txt.in | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmake/curl_CMakeLists.txt.in b/cmake/curl_CMakeLists.txt.in index 2fe84b9721..b013ed1807 100644 --- a/cmake/curl_CMakeLists.txt.in +++ b/cmake/curl_CMakeLists.txt.in @@ -15,7 +15,7 @@ ExternalProject_Add(curl CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --with-ssl=$ENV{HOME}/.cos-local.1 --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd --without-libidn2 #--enable-debug #CONFIGURE_COMMAND ./configure --without-ssl - BUILD_COMMAND "" - INSTALL_COMMAND make clean && make install + BUILD_COMMAND make -j + INSTALL_COMMAND make install TEST_COMMAND "" ) diff --git a/cmake/libs3_CMakeLists.txt.in b/cmake/libs3_CMakeLists.txt.in index 6c911775ee..b2d2d19059 100644 --- a/cmake/libs3_CMakeLists.txt.in +++ b/cmake/libs3_CMakeLists.txt.in @@ -23,7 +23,7 @@ ExternalProject_Add(libs3 CONFIGURE_COMMAND cp ${TD_SUPPORT_DIR}/libs3.GNUmakefile GNUmakefile && sed -i "s|CFLAGS += -Wall -Werror|CFLAGS += -I'$ENV{HOME}/.cos-local.1/include' -L'$ENV{HOME}/.cos-local.1/lib' |" ./GNUmakefile #BUILD_COMMAND make CFLAGS=${s3_flags} DESTDIR=$ENV{HOME}/.cos-local.1 build/lib/libs3.a #BUILD_COMMAND make DESTDIR="$ENV{HOME}/.cos-local.1" build/lib/libs3.a - BUILD_COMMAND "" - INSTALL_COMMAND make clean && make build/lib/libs3.a && make install_static + BUILD_COMMAND make build/lib/libs3.a + INSTALL_COMMAND make install_static TEST_COMMAND "" ) From 7fdf6cc696fae234f993479f5737451a384cfda3 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 24 Oct 2023 09:04:24 +0800 Subject: [PATCH 41/59] cmake: use new dir for libs3 --- cmake/curl_CMakeLists.txt.in | 7 ++----- cmake/libs3.GNUmakefile | 2 +- cmake/libs3_CMakeLists.txt.in | 15 +-------------- cmake/ssl_CMakeLists.txt.in | 2 +- contrib/CMakeLists.txt | 4 ++-- source/dnode/vnode/CMakeLists.txt | 8 ++++---- tests/parallel_test/container_build.sh | 4 ++-- 7 files changed, 13 insertions(+), 29 deletions(-) diff --git a/cmake/curl_CMakeLists.txt.in b/cmake/curl_CMakeLists.txt.in index b013ed1807..140fc48103 100644 --- a/cmake/curl_CMakeLists.txt.in +++ b/cmake/curl_CMakeLists.txt.in @@ -10,11 +10,8 @@ ExternalProject_Add(curl DEPENDS openssl BUILD_IN_SOURCE TRUE BUILD_ALWAYS 1 - #UPDATE_COMMAND "" - #CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --without-ssl --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd - CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --with-ssl=$ENV{HOME}/.cos-local.1 --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd --without-libidn2 #--enable-debug - - #CONFIGURE_COMMAND ./configure --without-ssl + UPDATE_COMMAND "" + CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.2 --with-ssl=$ENV{HOME}/.cos-local.2 --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd --without-libidn2 #--enable-debug BUILD_COMMAND make -j INSTALL_COMMAND make install TEST_COMMAND "" diff --git a/cmake/libs3.GNUmakefile b/cmake/libs3.GNUmakefile index abf954f5c1..98f98d8224 100644 --- a/cmake/libs3.GNUmakefile +++ b/cmake/libs3.GNUmakefile @@ -91,7 +91,7 @@ endif # -------------------------------------------------------------------------- # DESTDIR directory ifndef DESTDIR - DESTDIR := ${HOME}/.cos-local.1 + DESTDIR := ${HOME}/.cos-local.2 endif # -------------------------------------------------------------------------- diff --git a/cmake/libs3_CMakeLists.txt.in b/cmake/libs3_CMakeLists.txt.in index b2d2d19059..b5e9c438e6 100644 --- a/cmake/libs3_CMakeLists.txt.in +++ b/cmake/libs3_CMakeLists.txt.in @@ -1,15 +1,5 @@ # libs3 -set(s3_flags "${CMAKE_C_FLAGS} -O3 -Iinc -I$ENV{HOME}/.cos-local.1/include -Iinc ") - -set(SED_CMD "s/CFLAGS += -Wall -Werror/CFLAGS += -I$ENV{HOME}/.cos-local.1/include /") - -function(update_cflags) - file(READ "GNUmakefile" S3_CONTENT) - string(REPLACE "CFLAGS += -Wall -Werror" "CFLAGS += -I$ENV{HOME}/.cos-local.1/include " S3_CONTENT "${S3_CONTENT}") - file(WRITE "GNUmakefile" "${S3_CONTENT}") -endfunction(update_cflags) - ExternalProject_Add(libs3 GIT_REPOSITORY https://github.com/bji/libs3 #GIT_TAG v5.0.16 @@ -19,10 +9,7 @@ ExternalProject_Add(libs3 BUILD_IN_SOURCE TRUE BUILD_ALWAYS 1 UPDATE_COMMAND "" - #sed_cmd "s/CFLAGS += -Wall -Werror/CFLAGS += -I$ENV{HOME}/.cos-local.1/include /" - CONFIGURE_COMMAND cp ${TD_SUPPORT_DIR}/libs3.GNUmakefile GNUmakefile && sed -i "s|CFLAGS += -Wall -Werror|CFLAGS += -I'$ENV{HOME}/.cos-local.1/include' -L'$ENV{HOME}/.cos-local.1/lib' |" ./GNUmakefile - #BUILD_COMMAND make CFLAGS=${s3_flags} DESTDIR=$ENV{HOME}/.cos-local.1 build/lib/libs3.a - #BUILD_COMMAND make DESTDIR="$ENV{HOME}/.cos-local.1" build/lib/libs3.a + CONFIGURE_COMMAND cp ${TD_SUPPORT_DIR}/libs3.GNUmakefile GNUmakefile && sed -i "s|CFLAGS += -Wall -Werror|CFLAGS += -I'$ENV{HOME}/.cos-local.2/include' -L'$ENV{HOME}/.cos-local.2/lib' |" ./GNUmakefile BUILD_COMMAND make build/lib/libs3.a INSTALL_COMMAND make install_static TEST_COMMAND "" diff --git a/cmake/ssl_CMakeLists.txt.in b/cmake/ssl_CMakeLists.txt.in index 1f3fd0a30d..c9f836bade 100644 --- a/cmake/ssl_CMakeLists.txt.in +++ b/cmake/ssl_CMakeLists.txt.in @@ -8,7 +8,7 @@ ExternalProject_Add(openssl BUILD_IN_SOURCE TRUE #BUILD_ALWAYS 1 #UPDATE_COMMAND "" - CONFIGURE_COMMAND ./Configure --prefix=$ENV{HOME}/.cos-local.1 -static #--no-shared + CONFIGURE_COMMAND ./Configure --prefix=$ENV{HOME}/.cos-local.2 -static #--no-shared BUILD_COMMAND make -j INSTALL_COMMAND make install_sw -j TEST_COMMAND "" diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 013269c583..e2a5378f0d 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -9,7 +9,7 @@ endfunction(cat IN_FILE OUT_FILE) if(${TD_LINUX}) if(${BUILD_WITH_S3}) - file(MAKE_DIRECTORY $ENV{HOME}/.cos-local.1/) + file(MAKE_DIRECTORY $ENV{HOME}/.cos-local.2/) else() @@ -414,7 +414,7 @@ if (${BUILD_WITH_ROCKSDB}) endif() if(${BUILD_WITH_S3}) - INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.1/include) + INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.2/include) MESSAGE("build with s3: ${BUILD_WITH_S3}") else() diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 7e2693e8af..bc93ab575e 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -167,15 +167,15 @@ if(${BUILD_WITH_S3}) target_include_directories( vnode - PUBLIC "$ENV{HOME}/.cos-local.1/include" + PUBLIC "$ENV{HOME}/.cos-local.2/include" ) set(CMAKE_FIND_LIBRARY_SUFFIXES ".a") - set(CMAKE_PREFIX_PATH $ENV{HOME}/.cos-local.1) + set(CMAKE_PREFIX_PATH $ENV{HOME}/.cos-local.2) find_library(S3_LIBRARY s3) find_library(CURL_LIBRARY curl) - find_library(SSL_LIBRARY ssl PATHS $ENV{HOME}/.cos-local.1/lib64) - find_library(CRYPTO_LIBRARY crypto PATHS $ENV{HOME}/.cos-local.1/lib64) + find_library(SSL_LIBRARY ssl PATHS $ENV{HOME}/.cos-local.2/lib64) + find_library(CRYPTO_LIBRARY crypto PATHS $ENV{HOME}/.cos-local.2/lib64) target_link_libraries( vnode diff --git a/tests/parallel_test/container_build.sh b/tests/parallel_test/container_build.sh index f5e426057e..94704b1c25 100755 --- a/tests/parallel_test/container_build.sh +++ b/tests/parallel_test/container_build.sh @@ -60,7 +60,7 @@ docker run \ -v /root/.cargo/git:/root/.cargo/git \ -v /root/go/pkg/mod:/root/go/pkg/mod \ -v /root/.cache/go-build:/root/.cache/go-build \ - -v /root/.cos-local.1:/root/.cos-local.1 \ + -v /root/.cos-local.1:/root/.cos-local.2 \ -v ${REP_REAL_PATH}/enterprise/src/plugins/taosx/target:${REP_DIR}/enterprise/src/plugins/taosx/target \ -v ${REP_REAL_PATH}/community/tools/taosws-rs/target:${REP_DIR}/community/tools/taosws-rs/target \ -v ${REP_REAL_PATH}/community/contrib/cJson/:${REP_DIR}/community/contrib/cJson \ @@ -89,7 +89,7 @@ docker run \ -v /root/.cargo/git:/root/.cargo/git \ -v /root/go/pkg/mod:/root/go/pkg/mod \ -v /root/.cache/go-build:/root/.cache/go-build \ - -v /root/.cos-local.1:/root/.cos-local.1 \ + -v /root/.cos-local.1:/root/.cos-local.2 \ -v ${REP_REAL_PATH}/enterprise/src/plugins/taosx/target:${REP_DIR}/enterprise/src/plugins/taosx/target \ -v ${REP_REAL_PATH}/community/tools/taosws-rs/target:${REP_DIR}/community/tools/taosws-rs/target \ -v ${REP_REAL_PATH}/community/contrib/cJson/:${REP_DIR}/community/contrib/cJson \ From fd87f43f8cdfbb3668bfe011b14d1ce2d3794567 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 24 Oct 2023 09:10:58 +0800 Subject: [PATCH 42/59] enh: set CPUTYPE in cmake --- cmake/cmake.platform | 6 ++++++ cmake/cmake.version | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/cmake/cmake.platform b/cmake/cmake.platform index 18fd17f018..ca2cbbec2e 100644 --- a/cmake/cmake.platform +++ b/cmake/cmake.platform @@ -96,33 +96,39 @@ IF ("${CPUTYPE}" STREQUAL "") IF (CMAKE_SYSTEM_PROCESSOR MATCHES "(amd64)|(AMD64)") MESSAGE(STATUS "Current platform is amd64") SET(PLATFORM_ARCH_STR "amd64") + SET(CPUTYPE "x64") SET(TD_INTEL_64 TRUE) ADD_DEFINITIONS("-D_TD_X86_") ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)") MESSAGE(STATUS "Current platform is x86") SET(PLATFORM_ARCH_STR "i386") + SET(CPUTYPE "x86") SET(TD_INTEL_32 TRUE) ADD_DEFINITIONS("-D_TD_X86_") ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "armv7l") MESSAGE(STATUS "Current platform is aarch32") SET(PLATFORM_ARCH_STR "arm") + SET(CPUTYPE "arm32") SET(TD_ARM_32 TRUE) ADD_DEFINITIONS("-D_TD_ARM_") ADD_DEFINITIONS("-D_TD_ARM_32") ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "(aarch64)|(arm64)") MESSAGE(STATUS "Current platform is aarch64") SET(PLATFORM_ARCH_STR "arm64") + SET(CPUTYPE "arm64") SET(TD_ARM_64 TRUE) ADD_DEFINITIONS("-D_TD_ARM_") ADD_DEFINITIONS("-D_TD_ARM_64") ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "loongarch64") MESSAGE(STATUS "The current platform is loongarch64") SET(PLATFORM_ARCH_STR "loongarch64") + SET(CPUTYPE "loongarch64") SET(TD_LOONGARCH_64 TRUE) ADD_DEFINITIONS("-D_TD_LOONGARCH_") ADD_DEFINITIONS("-D_TD_LOONGARCH_64") ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "mips64") SET(PLATFORM_ARCH_STR "mips") + SET(CPUTYPE "mips64") MESSAGE(STATUS "input cpuType: mips64") SET(TD_MIPS_64 TRUE) ADD_DEFINITIONS("-D_TD_MIPS_") diff --git a/cmake/cmake.version b/cmake/cmake.version index c0ddae569c..b8871dbce3 100644 --- a/cmake/cmake.version +++ b/cmake/cmake.version @@ -79,9 +79,9 @@ ELSE () ELSEIF (TD_LINUX_32) SET(TD_VER_CPUTYPE "x86") ELSEIF (TD_ARM_32) - SET(TD_VER_CPUTYPE "x86") + SET(TD_VER_CPUTYPE "arm32") ELSEIF (TD_MIPS_32) - SET(TD_VER_CPUTYPE "x86") + SET(TD_VER_CPUTYPE "mips32") ELSE () SET(TD_VER_CPUTYPE "x64") ENDIF () From 0e65ed0202bb772908a8bc6867e6d1d2078c4f91 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 24 Oct 2023 09:16:09 +0800 Subject: [PATCH 43/59] cmake/curl: use new dir curl2 for s3 version --- cmake/curl_CMakeLists.txt.in | 4 ++-- cmake/libs3_CMakeLists.txt.in | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/curl_CMakeLists.txt.in b/cmake/curl_CMakeLists.txt.in index 140fc48103..458a518092 100644 --- a/cmake/curl_CMakeLists.txt.in +++ b/cmake/curl_CMakeLists.txt.in @@ -1,12 +1,12 @@ # curl -ExternalProject_Add(curl +ExternalProject_Add(curl2 URL https://curl.se/download/curl-8.2.1.tar.gz URL_HASH MD5=b25588a43556068be05e1624e0e74d41 DOWNLOAD_NO_PROGRESS 1 DOWNLOAD_DIR "${TD_CONTRIB_DIR}/deps-download" #GIT_REPOSITORY https://github.com/curl/curl.git #GIT_TAG curl-7_88_1 - SOURCE_DIR "${TD_CONTRIB_DIR}/curl" + SOURCE_DIR "${TD_CONTRIB_DIR}/curl2" DEPENDS openssl BUILD_IN_SOURCE TRUE BUILD_ALWAYS 1 diff --git a/cmake/libs3_CMakeLists.txt.in b/cmake/libs3_CMakeLists.txt.in index b5e9c438e6..f2b6cac953 100644 --- a/cmake/libs3_CMakeLists.txt.in +++ b/cmake/libs3_CMakeLists.txt.in @@ -3,7 +3,7 @@ ExternalProject_Add(libs3 GIT_REPOSITORY https://github.com/bji/libs3 #GIT_TAG v5.0.16 - DEPENDS curl xml2 + DEPENDS curl2 xml2 SOURCE_DIR "${TD_CONTRIB_DIR}/libs3" #BINARY_DIR "" BUILD_IN_SOURCE TRUE From e826532592d520de945ca6fcf771123ae967e8fa Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 24 Oct 2023 10:10:43 +0800 Subject: [PATCH 44/59] cmake/vnode: specify no default path for ssl & crypto --- source/dnode/vnode/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index bc93ab575e..b83d64c569 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -174,8 +174,8 @@ if(${BUILD_WITH_S3}) set(CMAKE_PREFIX_PATH $ENV{HOME}/.cos-local.2) find_library(S3_LIBRARY s3) find_library(CURL_LIBRARY curl) - find_library(SSL_LIBRARY ssl PATHS $ENV{HOME}/.cos-local.2/lib64) - find_library(CRYPTO_LIBRARY crypto PATHS $ENV{HOME}/.cos-local.2/lib64) + find_library(SSL_LIBRARY ssl $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH) + find_library(CRYPTO_LIBRARY crypto $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH) target_link_libraries( vnode From be7791f9cd25b1362907b414773ce6b5daf4ff33 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Tue, 24 Oct 2023 13:58:30 +0800 Subject: [PATCH 45/59] fix: disable life extending of conn cache obj when show queries/conns --- include/util/tcache.h | 2 ++ source/dnode/mnode/impl/src/mndProfile.c | 13 +++++++------ source/util/src/tcache.c | 12 ++++++++++++ 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/include/util/tcache.h b/include/util/tcache.h index d8ab018570..474e5274de 100644 --- a/include/util/tcache.h +++ b/include/util/tcache.h @@ -151,6 +151,8 @@ void *taosCacheIterGetData(const SCacheIter *pIter, size_t *dataLen); void *taosCacheIterGetKey(const SCacheIter *pIter, size_t *keyLen); void taosCacheDestroyIter(SCacheIter *pIter); +void taosCacheTryExtendLifeSpan(SCacheObj *pCacheObj, void **data); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 1f8c3b161b..4366053237 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -59,7 +59,7 @@ static SConnObj *mndCreateConn(SMnode *pMnode, const char *user, int8_t connType int32_t pid, const char *app, int64_t startTime); static void mndFreeConn(SConnObj *pConn); static SConnObj *mndAcquireConn(SMnode *pMnode, uint32_t connId); -static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn); +static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn, bool extendLifespan); static void *mndGetNextConn(SMnode *pMnode, SCacheIter *pIter); static void mndCancelGetNextConn(SMnode *pMnode, void *pIter); static int32_t mndProcessHeartBeatReq(SRpcMsg *pReq); @@ -79,7 +79,7 @@ int32_t mndInitProfile(SMnode *pMnode) { // in ms int32_t checkTime = tsShellActivityTimer * 2 * 1000; - pMgmt->connCache = taosCacheInit(TSDB_DATA_TYPE_UINT, checkTime, true, (__cache_free_fn_t)mndFreeConn, "conn"); + pMgmt->connCache = taosCacheInit(TSDB_DATA_TYPE_UINT, checkTime, false, (__cache_free_fn_t)mndFreeConn, "conn"); if (pMgmt->connCache == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; mError("failed to alloc profile cache since %s", terrstr()); @@ -185,11 +185,12 @@ static SConnObj *mndAcquireConn(SMnode *pMnode, uint32_t connId) { return pConn; } -static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn) { +static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn, bool extendLifespan) { if (pConn == NULL) return; mTrace("conn:%u, released from cache, data:%p", pConn->id, pConn); SProfileMgmt *pMgmt = &pMnode->profileMgmt; + if (extendLifespan) taosCacheTryExtendLifeSpan(pMgmt->connCache, (void **)&pConn); taosCacheRelease(pMgmt->connCache, (void **)&pConn, false); } @@ -323,7 +324,7 @@ _OVER: mndReleaseUser(pMnode, pUser); mndReleaseDb(pMnode, pDb); - mndReleaseConn(pMnode, pConn); + mndReleaseConn(pMnode, pConn, true); return code; } @@ -485,7 +486,7 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb SQueryHbRspBasic *rspBasic = taosMemoryCalloc(1, sizeof(SQueryHbRspBasic)); if (rspBasic == NULL) { - mndReleaseConn(pMnode, pConn); + mndReleaseConn(pMnode, pConn, true); terrno = TSDB_CODE_OUT_OF_MEMORY; mError("user:%s, conn:%u failed to process hb while since %s", pConn->user, pBasic->connId, terrstr()); return -1; @@ -508,7 +509,7 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb mndCreateQnodeList(pMnode, &rspBasic->pQnodeList, -1); - mndReleaseConn(pMnode, pConn); + mndReleaseConn(pMnode, pConn, true); hbRsp.query = rspBasic; } else { diff --git a/source/util/src/tcache.c b/source/util/src/tcache.c index 392ac5d8b2..11f8df4c93 100644 --- a/source/util/src/tcache.c +++ b/source/util/src/tcache.c @@ -997,3 +997,15 @@ void taosCacheDestroyIter(SCacheIter *pIter) { taosMemoryFreeClear(pIter->pCurrent); taosMemoryFreeClear(pIter); } + +void taosCacheTryExtendLifeSpan(SCacheObj *pCacheObj, void **data) { + if (!pCacheObj || !(*data)) return; + + SCacheNode *pNode = (SCacheNode *)((char *)(*data) - sizeof(SCacheNode)); + if (pNode->signature != pNode) return; + + if (!pNode->inTrashcan) { + atomic_store_64(&pNode->expireTime, pNode->lifespan + taosGetTimestampMs()); + uDebug("cache:%s, data:%p extend expire time: %" PRId64, pCacheObj->name, pNode->data, pNode->expireTime); + } +} From 956aa1742cb640a60b4b88a7894ce3f61de08f37 Mon Sep 17 00:00:00 2001 From: dmchen Date: Tue, 24 Oct 2023 05:32:48 +0000 Subject: [PATCH 46/59] TD-26884 --- source/common/src/tglobal.c | 2 +- source/dnode/mnode/impl/src/mndProfile.c | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index d9b1b2e0ed..f8feb949c7 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -98,7 +98,7 @@ int32_t tsMonitorMaxLogs = 100; bool tsMonitorComp = false; // audit -bool tsEnableAudit = false; +bool tsEnableAudit = true; char tsAuditFqdn[TSDB_FQDN_LEN] = {0}; uint16_t tsAuditPort = 6043; diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 1f8c3b161b..159774efdc 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -314,8 +314,7 @@ _CONNECT: sprintf(obj, "%s:%d", ip, pConn->port); char detail[1000] = {0}; - sprintf(detail, "connType:%d, db:%s, pid:%d, startTime:%" PRId64 ", sVer:%s, app:%s", - connReq.connType, connReq.db, connReq.pid, connReq.startTime, connReq.sVer, connReq.app); + sprintf(detail, "app:%s", connReq.app); auditRecord(pReq, pMnode->clusterId, "login", connReq.user, obj, detail, strlen(detail)); From 162066235b29533b05806425d1b6c4870271f8b4 Mon Sep 17 00:00:00 2001 From: dmchen Date: Tue, 24 Oct 2023 06:17:13 +0000 Subject: [PATCH 47/59] create table --- include/common/tglobal.h | 1 + packaging/cfg/taos.cfg | 3 +++ source/common/src/tglobal.c | 3 +++ source/dnode/vnode/src/vnd/vnodeSvr.c | 4 ++-- 4 files changed, 9 insertions(+), 2 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index c4037ed2ea..3e29703070 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -107,6 +107,7 @@ extern bool tsMonitorComp; extern bool tsEnableAudit; extern char tsAuditFqdn[]; extern uint16_t tsAuditPort; +extern bool tsEnableAuditCreateTable; // telem extern bool tsEnableTelem; diff --git a/packaging/cfg/taos.cfg b/packaging/cfg/taos.cfg index 743edb6ca3..6d25db843d 100644 --- a/packaging/cfg/taos.cfg +++ b/packaging/cfg/taos.cfg @@ -101,6 +101,9 @@ # enable/disable audit log # audit 1 +# enable/disable audit create table +# auditCreateTable 1 + # The following parameter is used to limit the maximum number of lines in log files. # max number of lines per log filters # numOfLogLines 10000000 diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index f8feb949c7..0a155f4ea1 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -101,6 +101,7 @@ bool tsMonitorComp = false; bool tsEnableAudit = true; char tsAuditFqdn[TSDB_FQDN_LEN] = {0}; uint16_t tsAuditPort = 6043; +bool tsEnableAuditCreateTable = true; // telem bool tsEnableTelem = true; @@ -632,6 +633,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddBool(pCfg, "audit", tsEnableAudit, CFG_SCOPE_SERVER) != 0) return -1; if (cfgAddString(pCfg, "auditFqdn", tsAuditFqdn, CFG_SCOPE_SERVER) != 0) return -1; if (cfgAddInt32(pCfg, "auditPort", tsAuditPort, 1, 65056, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddBool(pCfg, "auditCreateTable", tsEnableAuditCreateTable, CFG_SCOPE_SERVER) != 0) return -1; if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, CFG_SCOPE_BOTH) != 0) return -1; if (cfgAddBool(pCfg, "telemetryReporting", tsEnableTelem, CFG_SCOPE_BOTH) != 0) return -1; @@ -1050,6 +1052,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsEnableAudit = cfgGetItem(pCfg, "audit")->bval; tstrncpy(tsAuditFqdn, cfgGetItem(pCfg, "auditFqdn")->str, TSDB_FQDN_LEN); tsAuditPort = (uint16_t)cfgGetItem(pCfg, "auditPort")->i32; + tsEnableAuditCreateTable = cfgGetItem(pCfg, "auditCreateTable")->bval; tsEnableTelem = cfgGetItem(pCfg, "telemetryReporting")->bval; tsEnableCrashReport = cfgGetItem(pCfg, "crashReporting")->bval; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index efa722d41a..e3aa459e1b 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -947,13 +947,13 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, taosArrayPush(rsp.pArray, &cRsp); - if(pCreateReq->sqlLen > 0){ //skip auto create table, not set sql when auto create table + if(tsEnableAuditCreateTable){ int32_t clusterId = pVnode->config.syncCfg.nodeInfo[0].clusterId; SName name = {0}; tNameFromString(&name, pVnode->config.dbname, T_NAME_ACCT | T_NAME_DB); - auditRecord(pReq, clusterId, "createTable", name.dbname, pCreateReq->name, pCreateReq->sql, pCreateReq->sqlLen); + auditRecord(pReq, clusterId, "createTable", name.dbname, "", pCreateReq->name, strlen(pCreateReq->name)); } } From f9ca8fe9620e57f0c64c3ebb62d4dca2647352d9 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 24 Oct 2023 16:10:46 +0800 Subject: [PATCH 48/59] cmake: new option: BUILD_S3 --- CMakeLists.txt | 4 +--- cmake/cmake.options | 28 +++++++++++++++++++++------- contrib/CMakeLists.txt | 23 +++++++---------------- source/common/CMakeLists.txt | 14 +++++--------- source/dnode/vnode/CMakeLists.txt | 6 +++--- 5 files changed, 37 insertions(+), 38 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5048287cf6..66a6fd328d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,8 +16,6 @@ set(TD_SUPPORT_DIR "${TD_SOURCE_DIR}/cmake") set(TD_CONTRIB_DIR "${TD_SOURCE_DIR}/contrib") - - include(${TD_SUPPORT_DIR}/cmake.platform) include(${TD_SUPPORT_DIR}/cmake.define) include(${TD_SUPPORT_DIR}/cmake.options) @@ -46,4 +44,4 @@ add_subdirectory(examples/c) include(${TD_SUPPORT_DIR}/cmake.install) # docs -add_subdirectory(docs/doxgen) \ No newline at end of file +add_subdirectory(docs/doxgen) diff --git a/cmake/cmake.options b/cmake/cmake.options index bacc5adfb7..d34c34dd89 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -128,12 +128,16 @@ option( IF(${TD_LINUX}) option( - BUILD_WITH_S3 + BUILD_S3 "If build with s3" ON ) -IF(${BUILD_WITH_S3}) +option( + BUILD_WITH_S3 + "If build with s3" + ON +) option( BUILD_WITH_COS @@ -141,16 +145,26 @@ option( OFF ) +ENDIF () + +IF(${BUILD_S3}) + +IF(${BUILD_WITH_S3}) + +option(BUILD_WITH_COS "If build with cos" OFF) + ELSE () -option( - BUILD_WITH_COS - "If build with cos" - ON -) +option(BUILD_WITH_COS "If build with cos" ON) ENDIF () +ELSE () + +option(BUILD_WITH_S3 "If build with s3" OFF) + +option(BUILD_WITH_COS "If build with cos" OFF) + ENDIF () option( diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index e2a5378f0d..d631ceff11 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -6,12 +6,10 @@ function(cat IN_FILE OUT_FILE) file(APPEND ${OUT_FILE} "${CONTENTS}") endfunction(cat IN_FILE OUT_FILE) -if(${TD_LINUX}) - if(${BUILD_WITH_S3}) file(MAKE_DIRECTORY $ENV{HOME}/.cos-local.2/) -else() +elseif(${BUILD_WITH_COS}) set(CONTRIB_TMP_FILE3 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in3") configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3}) @@ -42,9 +40,8 @@ execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" . execute_process(COMMAND "${CMAKE_COMMAND}" --build . WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download") -endif(${BUILD_WITH_S3}) +endif() -endif(${TD_LINUX}) set(CONTRIB_TMP_FILE "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in") configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) @@ -170,19 +167,16 @@ if(${BUILD_WITH_S3}) cat("${TD_SUPPORT_DIR}/libs3_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) add_definitions(-DUSE_S3) -else() - # cos -if(${BUILD_WITH_COS}) +elseif(${BUILD_WITH_COS}) #cat("${TD_SUPPORT_DIR}/mxml_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) #cat("${TD_SUPPORT_DIR}/apr_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) #cat("${TD_SUPPORT_DIR}/apr-util_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) #cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) cat("${TD_SUPPORT_DIR}/cos_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) add_definitions(-DUSE_COS) -endif(${BUILD_WITH_COS}) -endif(${BUILD_WITH_S3}) +endif() # lucene if(${BUILD_WITH_LUCENE}) @@ -269,7 +263,7 @@ unset(CMAKE_PROJECT_INCLUDE_BEFORE) # xml2 if(${BUILD_WITH_S3}) add_subdirectory(xml2 EXCLUDE_FROM_ALL) -endif(${BUILD_WITH_S3}) +endif() # lz4 add_subdirectory(lz4/build/cmake EXCLUDE_FROM_ALL) @@ -417,10 +411,8 @@ if(${BUILD_WITH_S3}) INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.2/include) MESSAGE("build with s3: ${BUILD_WITH_S3}") -else() - # cos -if(${BUILD_WITH_COS}) +elseif(${BUILD_WITH_COS}) if(${TD_LINUX}) set(CMAKE_PREFIX_PATH $ENV{HOME}/.cos-local.1) #ADD_DEFINITIONS(-DMINIXML_LIBRARY=${CMAKE_BINARY_DIR}/build/lib/libxml.a) @@ -442,9 +434,8 @@ if(${BUILD_WITH_COS}) else() endif(${TD_LINUX}) -endif(${BUILD_WITH_COS}) -endif(${BUILD_WITH_S3}) +endif() # lucene # To support build on ubuntu: sudo apt-get install libboost-all-dev diff --git a/source/common/CMakeLists.txt b/source/common/CMakeLists.txt index 6bdcdf7d1d..c35845f9df 100644 --- a/source/common/CMakeLists.txt +++ b/source/common/CMakeLists.txt @@ -17,15 +17,11 @@ IF (TD_STORAGE) ADD_DEFINITIONS(-D_STORAGE) TARGET_LINK_LIBRARIES(common PRIVATE storage) - IF(${TD_LINUX}) - IF(${BUILD_WITH_COS}) - add_definitions(-DUSE_COS) - ENDIF(${BUILD_WITH_COS}) - - IF(${BUILD_WITH_S3}) - add_definitions(-DUSE_S3) - ENDIF(${BUILD_WITH_S3}) - ENDIF(${TD_LINUX}) + IF(${BUILD_WITH_S3}) + add_definitions(-DUSE_S3) + ELSEIF(${BUILD_WITH_COS}) + add_definitions(-DUSE_COS) + ENDIF() ENDIF () diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index b83d64c569..a10b38a1bf 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -161,7 +161,7 @@ target_link_libraries( PUBLIC index ) -if(${TD_LINUX}) +if(${BUILD_S3}) if(${BUILD_WITH_S3}) target_include_directories( @@ -188,7 +188,7 @@ if(${BUILD_WITH_S3}) ) add_definitions(-DUSE_S3) -endif(${BUILD_WITH_S3}) +endif() if(${BUILD_WITH_COS}) @@ -227,7 +227,7 @@ target_include_directories( add_definitions(-DUSE_COS) endif(${BUILD_WITH_COS}) -endif(${TD_LINUX}) +endif() IF (TD_GRANT) TARGET_LINK_LIBRARIES(vnode PUBLIC grant) From aa7fd64b9b964781493da67985b591f713605be4 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 24 Oct 2023 18:04:50 +0800 Subject: [PATCH 49/59] enh: cmake system process support x86_64 --- cmake/cmake.platform | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/cmake.platform b/cmake/cmake.platform index ca2cbbec2e..9b96ebe7cb 100644 --- a/cmake/cmake.platform +++ b/cmake/cmake.platform @@ -93,7 +93,7 @@ ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "Windows") ENDIF() IF ("${CPUTYPE}" STREQUAL "") - IF (CMAKE_SYSTEM_PROCESSOR MATCHES "(amd64)|(AMD64)") + IF (CMAKE_SYSTEM_PROCESSOR MATCHES "(amd64)|(AMD64)|(x86_64)|(X86_64)") MESSAGE(STATUS "Current platform is amd64") SET(PLATFORM_ARCH_STR "amd64") SET(CPUTYPE "x64") From c67853888f80030a29878149e608e20324619903 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 24 Oct 2023 18:06:19 +0800 Subject: [PATCH 50/59] cmake/xml2: build xml2 before s3 --- cmake/xml2_CMakeLists.txt.in | 16 ++++++++++------ contrib/CMakeLists.txt | 6 +++--- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/cmake/xml2_CMakeLists.txt.in b/cmake/xml2_CMakeLists.txt.in index 4c234d7a37..25eb5a69ac 100644 --- a/cmake/xml2_CMakeLists.txt.in +++ b/cmake/xml2_CMakeLists.txt.in @@ -1,12 +1,16 @@ # xml2 ExternalProject_Add(xml2 - GIT_REPOSITORY https://github.com/GNOME/libxml2 - GIT_TAG v2.11.5 + URL https://github.com/GNOME/libxml2/archive/refs/tags/v2.11.5.tar.gz + #GIT_REPOSITORY https://github.com/GNOME/libxml2 + #GIT_TAG v2.11.5 + DOWNLOAD_NO_PROGRESS 1 + DOWNLOAD_DIR "${TD_CONTRIB_DIR}/deps-download" SOURCE_DIR "${TD_CONTRIB_DIR}/xml2" - BINARY_DIR "" - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND "" + #BINARY_DIR "" + BUILD_IN_SOURCE TRUE + CONFIGURE_COMMAND ./autogen.sh && ./configure --prefix=$ENV{HOME}/.cos-local.2 --enable-shared=no --enable-static=yes --without-python + BUILD_COMMAND make -j + INSTALL_COMMAND make install && ln -s $ENV{HOME}/.cos-local.2/include/libxml2/libxml $ENV{HOME}/.cos-local.2/include/libxml TEST_COMMAND "" ) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index d631ceff11..c5715bd53f 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -261,9 +261,9 @@ target_include_directories( unset(CMAKE_PROJECT_INCLUDE_BEFORE) # xml2 -if(${BUILD_WITH_S3}) - add_subdirectory(xml2 EXCLUDE_FROM_ALL) -endif() +#if(${BUILD_WITH_S3}) +# add_subdirectory(xml2 EXCLUDE_FROM_ALL) +#endif() # lz4 add_subdirectory(lz4/build/cmake EXCLUDE_FROM_ALL) From 14be9c81e268ee9665c138869c24fd2544a5feed Mon Sep 17 00:00:00 2001 From: dmchen Date: Wed, 25 Oct 2023 06:49:23 +0000 Subject: [PATCH 51/59] create table user and clusterid --- source/dnode/vnode/src/vnd/vnodeSvr.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index e3aa459e1b..c46ea15111 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -948,12 +948,12 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, taosArrayPush(rsp.pArray, &cRsp); if(tsEnableAuditCreateTable){ - int32_t clusterId = pVnode->config.syncCfg.nodeInfo[0].clusterId; + int64_t clusterId = pVnode->config.syncCfg.nodeInfo[0].clusterId; SName name = {0}; tNameFromString(&name, pVnode->config.dbname, T_NAME_ACCT | T_NAME_DB); - auditRecord(pReq, clusterId, "createTable", name.dbname, "", pCreateReq->name, strlen(pCreateReq->name)); + auditRecord(NULL, clusterId, "createTable", name.dbname, "", pCreateReq->name, strlen(pCreateReq->name)); } } From d77ecbbb1b150611e5ccb9a6314213499ae0769c Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 25 Oct 2023 17:41:55 +0800 Subject: [PATCH 52/59] cmake/xml2: use sha256 to verify distribution release --- cmake/xml2_CMakeLists.txt.in | 6 ++++-- source/dnode/vnode/CMakeLists.txt | 5 +++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/cmake/xml2_CMakeLists.txt.in b/cmake/xml2_CMakeLists.txt.in index 25eb5a69ac..ad0704cdb9 100644 --- a/cmake/xml2_CMakeLists.txt.in +++ b/cmake/xml2_CMakeLists.txt.in @@ -1,7 +1,9 @@ # xml2 ExternalProject_Add(xml2 - URL https://github.com/GNOME/libxml2/archive/refs/tags/v2.11.5.tar.gz + URL https://download.gnome.org/sources/libxml2/2.11/libxml2-2.11.5.tar.xz + URL_HASH SHA256=3727b078c360ec69fa869de14bd6f75d7ee8d36987b071e6928d4720a28df3a6 + #https://github.com/GNOME/libxml2/archive/refs/tags/v2.11.5.tar.gz #GIT_REPOSITORY https://github.com/GNOME/libxml2 #GIT_TAG v2.11.5 DOWNLOAD_NO_PROGRESS 1 @@ -9,7 +11,7 @@ ExternalProject_Add(xml2 SOURCE_DIR "${TD_CONTRIB_DIR}/xml2" #BINARY_DIR "" BUILD_IN_SOURCE TRUE - CONFIGURE_COMMAND ./autogen.sh && ./configure --prefix=$ENV{HOME}/.cos-local.2 --enable-shared=no --enable-static=yes --without-python + CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.2 --enable-shared=no --enable-static=yes --without-python --without-lzma BUILD_COMMAND make -j INSTALL_COMMAND make install && ln -s $ENV{HOME}/.cos-local.2/include/libxml2/libxml $ENV{HOME}/.cos-local.2/include/libxml TEST_COMMAND "" diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index a10b38a1bf..dcc9f9a115 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -173,7 +173,8 @@ if(${BUILD_WITH_S3}) set(CMAKE_FIND_LIBRARY_SUFFIXES ".a") set(CMAKE_PREFIX_PATH $ENV{HOME}/.cos-local.2) find_library(S3_LIBRARY s3) - find_library(CURL_LIBRARY curl) + find_library(CURL_LIBRARY curl $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH) + find_library(XML2_LIBRARY xml2) find_library(SSL_LIBRARY ssl $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH) find_library(CRYPTO_LIBRARY crypto $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH) target_link_libraries( @@ -184,7 +185,7 @@ if(${BUILD_WITH_S3}) PUBLIC ${CURL_LIBRARY} PUBLIC ${SSL_LIBRARY} PUBLIC ${CRYPTO_LIBRARY} - PUBLIC xml2 + PUBLIC ${XML2_LIBRARY} ) add_definitions(-DUSE_S3) From 6c9c06552d8679e7f65da4008b856ad3e9caa2ac Mon Sep 17 00:00:00 2001 From: dmchen Date: Thu, 26 Oct 2023 06:10:09 +0000 Subject: [PATCH 53/59] fix case --- tests/system-test/0-others/taosdMonitor.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/system-test/0-others/taosdMonitor.py b/tests/system-test/0-others/taosdMonitor.py index a07d7f411e..169c70e675 100644 --- a/tests/system-test/0-others/taosdMonitor.py +++ b/tests/system-test/0-others/taosdMonitor.py @@ -268,6 +268,8 @@ class TDTestCase: updatecfgDict["monitorMaxLogs"] = "10" updatecfgDict["monitorComp"] = "1" + updatecfgDict["audit"] = '0' + print ("===================: ", updatecfgDict) def init(self, conn, logSql, replicaVar=1): From b3147581086a8bd0adfb540b3ea8abe71039d547 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 26 Oct 2023 16:24:06 +0800 Subject: [PATCH 54/59] tsdb/commit: make ts big enough to skip row --- source/dnode/vnode/src/tsdb/tsdbCommit2.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c index a2de5a81ca..6dc492f420 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c @@ -137,7 +137,12 @@ static int32_t tsdbCommitTSData(SCommitter2 *committer) { } int64_t ts = TSDBROW_TS(&row->row); - if (ts > committer->ctx->maxKey || skipRow) { + + if (skipRow && ts <= committer->ctx->maxKey) { + ts = committer->ctx->maxKey + 1; + } + + if (ts > committer->ctx->maxKey) { committer->ctx->nextKey = TMIN(committer->ctx->nextKey, ts); code = tsdbIterMergerSkipTableData(committer->dataIterMerger, committer->ctx->tbid); TSDB_CHECK_CODE(code, lino, _exit); From 58bab7cf68ab9a6eb6d97d51fba9426e8dd6d93e Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 26 Oct 2023 19:09:37 +0800 Subject: [PATCH 55/59] test: support replica3 in splited/transform tmqvnode --- tests/system-test/7-tmq/tmqVnodeTransform-stb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/7-tmq/tmqVnodeTransform-stb.py b/tests/system-test/7-tmq/tmqVnodeTransform-stb.py index 2dd756b788..18c5f8880b 100644 --- a/tests/system-test/7-tmq/tmqVnodeTransform-stb.py +++ b/tests/system-test/7-tmq/tmqVnodeTransform-stb.py @@ -140,7 +140,7 @@ class TDTestCase: 'rowsPerTbl': 10000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 30, + 'pollDelay': 60, 'showMsg': 1, 'showRow': 1, 'snapshot': 0} From 3f6f928399df2a35d0d779a2b05386536af77c91 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 26 Oct 2023 20:25:01 +0800 Subject: [PATCH 56/59] test:delete zlib cache in ci --- tests/parallel_test/container_build.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/parallel_test/container_build.sh b/tests/parallel_test/container_build.sh index 94704b1c25..8e309a8182 100755 --- a/tests/parallel_test/container_build.sh +++ b/tests/parallel_test/container_build.sh @@ -68,9 +68,10 @@ docker run \ -v ${REP_REAL_PATH}/community/contrib/cpp-stub/:${REP_DIR}/community/contrib/cpp-stub \ -v ${REP_REAL_PATH}/community/contrib/libuv/:${REP_DIR}/community/contrib/libuv \ -v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \ - -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \ + --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_TAOSX=true -DJEMALLOC_ENABLED=0;make -j 10|| exit 1" # -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \ +# -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \ if [[ -d ${WORKDIR}/debugNoSan ]] ;then echo "delete ${WORKDIR}/debugNoSan" @@ -97,10 +98,11 @@ docker run \ -v ${REP_REAL_PATH}/community/contrib/cpp-stub/:${REP_DIR}/community/contrib/cpp-stub \ -v ${REP_REAL_PATH}/community/contrib/libuv/:${REP_DIR}/community/contrib/libuv \ -v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \ - -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \ + -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \ --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_SANITIZER=1 -DTOOLS_SANITIZE=true -DTOOLS_BUILD_TYPE=Debug -DBUILD_TAOSX=true -DJEMALLOC_ENABLED=0;make -j 10|| exit 1 " + # -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \ mv ${REP_REAL_PATH}/debug ${WORKDIR}/debugSan ret=$? From 310793cbf0ccaa502a3f85838cfb299febfa3c2e Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 26 Oct 2023 20:33:51 +0800 Subject: [PATCH 57/59] test:delete zlib cache in ci --- tests/parallel_test/container_build.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/parallel_test/container_build.sh b/tests/parallel_test/container_build.sh index 8e309a8182..8652161d17 100755 --- a/tests/parallel_test/container_build.sh +++ b/tests/parallel_test/container_build.sh @@ -68,7 +68,6 @@ docker run \ -v ${REP_REAL_PATH}/community/contrib/cpp-stub/:${REP_DIR}/community/contrib/cpp-stub \ -v ${REP_REAL_PATH}/community/contrib/libuv/:${REP_DIR}/community/contrib/libuv \ -v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \ - --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_TAOSX=true -DJEMALLOC_ENABLED=0;make -j 10|| exit 1" # -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \ # -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \ @@ -98,7 +97,6 @@ docker run \ -v ${REP_REAL_PATH}/community/contrib/cpp-stub/:${REP_DIR}/community/contrib/cpp-stub \ -v ${REP_REAL_PATH}/community/contrib/libuv/:${REP_DIR}/community/contrib/libuv \ -v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \ - -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \ --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_SANITIZER=1 -DTOOLS_SANITIZE=true -DTOOLS_BUILD_TYPE=Debug -DBUILD_TAOSX=true -DJEMALLOC_ENABLED=0;make -j 10|| exit 1 " From 2b0c144c8eeaa8389b7b442fd904c36c78ebd722 Mon Sep 17 00:00:00 2001 From: haoranchen Date: Fri, 27 Oct 2023 09:52:59 +0800 Subject: [PATCH 58/59] Revert "test:delete zlib cache in ci" --- tests/parallel_test/container_build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/parallel_test/container_build.sh b/tests/parallel_test/container_build.sh index 8652161d17..94704b1c25 100755 --- a/tests/parallel_test/container_build.sh +++ b/tests/parallel_test/container_build.sh @@ -68,9 +68,9 @@ docker run \ -v ${REP_REAL_PATH}/community/contrib/cpp-stub/:${REP_DIR}/community/contrib/cpp-stub \ -v ${REP_REAL_PATH}/community/contrib/libuv/:${REP_DIR}/community/contrib/libuv \ -v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \ + -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \ --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_TAOSX=true -DJEMALLOC_ENABLED=0;make -j 10|| exit 1" # -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \ -# -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \ if [[ -d ${WORKDIR}/debugNoSan ]] ;then echo "delete ${WORKDIR}/debugNoSan" @@ -97,10 +97,10 @@ docker run \ -v ${REP_REAL_PATH}/community/contrib/cpp-stub/:${REP_DIR}/community/contrib/cpp-stub \ -v ${REP_REAL_PATH}/community/contrib/libuv/:${REP_DIR}/community/contrib/libuv \ -v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \ + -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \ -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \ --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_SANITIZER=1 -DTOOLS_SANITIZE=true -DTOOLS_BUILD_TYPE=Debug -DBUILD_TAOSX=true -DJEMALLOC_ENABLED=0;make -j 10|| exit 1 " - # -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \ mv ${REP_REAL_PATH}/debug ${WORKDIR}/debugSan ret=$? From e6df389151d25994ff51c9db9d05760f09c384c6 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 27 Oct 2023 11:46:59 +0800 Subject: [PATCH 59/59] fix:add req_id for raw block interface --- include/client/taos.h | 3 +++ source/client/src/clientRawBlockWrite.c | 13 +++++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/include/client/taos.h b/include/client/taos.h index f75a84baa8..45dc85f6d9 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -338,8 +338,11 @@ typedef struct tmq_raw_data { DLL_EXPORT int32_t tmq_get_raw(TAOS_RES *res, tmq_raw_data *raw); DLL_EXPORT int32_t tmq_write_raw(TAOS *taos, tmq_raw_data raw); DLL_EXPORT int taos_write_raw_block(TAOS *taos, int numOfRows, char *pData, const char *tbname); +DLL_EXPORT int taos_write_raw_block_with_reqid(TAOS *taos, int numOfRows, char *pData, const char *tbname, int64_t reqid); DLL_EXPORT int taos_write_raw_block_with_fields(TAOS *taos, int rows, char *pData, const char *tbname, TAOS_FIELD *fields, int numFields); +DLL_EXPORT int taos_write_raw_block_with_fields_with_reqid(TAOS *taos, int rows, char *pData, const char *tbname, + TAOS_FIELD *fields, int numFields, int64_t reqid); DLL_EXPORT void tmq_free_raw(tmq_raw_data raw); // Returning null means error. Returned result need to be freed by tmq_free_json_meta diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index e7ba30d78c..a12dc8d839 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -1338,6 +1338,11 @@ end: int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const char* tbname, TAOS_FIELD* fields, int numFields) { + return taos_write_raw_block_with_fields_with_reqid(taos, rows, pData, tbname, fields, numFields, 0); +} + +int taos_write_raw_block_with_fields_with_reqid(TAOS *taos, int rows, char *pData, const char *tbname, + TAOS_FIELD *fields, int numFields, int64_t reqid){ if (!taos || !pData || !tbname) { return TSDB_CODE_INVALID_PARA; } @@ -1347,7 +1352,7 @@ int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const ch SHashObj* pVgHash = NULL; uDebug("taos_write_raw_block_with_fields called"); - SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0); + SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, reqid); if (!pRequest) { uError("WriteRaw:createRequest error request is null"); code = terrno; @@ -1427,6 +1432,10 @@ end: } int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) { + return taos_write_raw_block_with_reqid(taos, rows, pData, tbname, 0); +} + +int taos_write_raw_block_with_reqid(TAOS* taos, int rows, char* pData, const char* tbname, int64_t reqid) { if (!taos || !pData || !tbname) { return TSDB_CODE_INVALID_PARA; } @@ -1436,7 +1445,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) SHashObj* pVgHash = NULL; uDebug("taos_write_raw_block called"); - SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0); + SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, reqid); if (!pRequest) { uError("WriteRaw:createRequest error request is null"); code = terrno;