From 29458b55e50301b4e077e4b0969af2d2df9119f1 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Mon, 25 Sep 2023 17:16:19 +0800 Subject: [PATCH 01/51] test: increase numbers of cases in windows --- tests/system-test/simpletest.bat | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/system-test/simpletest.bat b/tests/system-test/simpletest.bat index 5ae2d3feb3..31b76cad4a 100644 --- a/tests/system-test/simpletest.bat +++ b/tests/system-test/simpletest.bat @@ -18,7 +18,7 @@ python3 .\test.py -f 1-insert\influxdb_line_taosc_insert.py @REM #python3 .\test.py -f 1-insert\test_stmt_muti_insert_query.py @REM python3 .\test.py -f 1-insert\alter_stable.py @REM python3 .\test.py -f 1-insert\alter_table.py -@REM python3 .\test.py -f 2-query\between.py +python3 .\test.py -f 2-query\between.py @REM python3 .\test.py -f 2-query\distinct.py @REM python3 .\test.py -f 2-query\varchar.py @REM python3 .\test.py -f 2-query\ltrim.py @@ -101,3 +101,4 @@ python3 .\test.py -f 7-tmq\subscribeStb.py @REM python3 .\test.py -f 7-tmq\subscribeStb3.py @REM python3 .\test.py -f 7-tmq\subscribeStb4.py @REM python3 .\test.py -f 7-tmq\db.py +python3 .\test.py -f 6-cluster\5dnode3mnodeSep1VnodeStopDnodeModifyMeta.py -N 6 -M 3 \ No newline at end of file From b1014f31bca665cf17bddc1881fb8c4e2f5b3e5a Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Tue, 26 Sep 2023 20:45:40 +0800 Subject: [PATCH 02/51] test: add testcase of TS-4074 --- tests/system-test/0-others/compatibility.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/system-test/0-others/compatibility.py b/tests/system-test/0-others/compatibility.py index cb804aad0c..83bfb2bed7 100644 --- a/tests/system-test/0-others/compatibility.py +++ b/tests/system-test/0-others/compatibility.py @@ -30,7 +30,7 @@ class TDTestCase: self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) - self.deletedDataSql= '''drop database if exists deldata;create database deldata duration 300;use deldata; + self.deletedDataSql= '''drop database if exists deldata;create database deldata duration 300 stt_trigger 4; ;use deldata; create table deldata.stb1 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) tags (t1 int); create table deldata.ct1 using deldata.stb1 tags ( 1 ); insert into deldata.ct1 values ( now()-0s, 0, 0, 0, 0, 0.0, 0.0, 0, 'binary0', 'nchar0', now()+0a ) ( now()-10s, 1, 11111, 111, 11, 1.11, 11.11, 1, 'binary1', 'nchar1', now()+1a ) ( now()-20s, 2, 22222, 222, 22, 2.22, 22.22, 0, 'binary2', 'nchar2', now()+2a ) ( now()-30s, 3, 33333, 333, 33, 3.33, 33.33, 1, 'binary3', 'nchar3', now()+3a ); @@ -38,7 +38,9 @@ class TDTestCase: delete from deldata.stb1; flush database deldata; insert into deldata.ct1 values ( now()-0s, 0, 0, 0, 0, 0.0, 0.0, 0, 'binary0', 'nchar0', now()+0a ) ( now()-10s, 1, 11111, 111, 11, 1.11, 11.11, 1, 'binary1', 'nchar1', now()+1a ) ( now()-20s, 2, 22222, 222, 22, 2.22, 22.22, 0, 'binary2', 'nchar2', now()+2a ) ( now()-30s, 3, 33333, 333, 33, 3.33, 33.33, 1, 'binary3', 'nchar3', now()+3a ); - delete from deldata.ct1;''' + delete from deldata.ct1; + insert into deldata.ct1 values ( now()-0s, 0, 0, 0, 0, 0.0, 0.0, 0, 'binary0', 'nchar0', now()+0a ); + flush database deldata;''' def checkProcessPid(self,processName): i=0 while i<60: @@ -262,7 +264,7 @@ class TDTestCase: if self.is_list_same_as_ordered_list(resultList,expectList): print("The unordered list is the same as the ordered list.") else: - tdlog.error("The unordered list is not the same as the ordered list.") + tdLog.exit("The unordered list is not the same as the ordered list.") tdsql.execute("insert into test.d80 values (now+1s, 11, 103, 0.21);") tdsql.execute("insert into test.d9 values (now+5s, 4.3, 104, 0.4);") From 65448e20269039b4d2ab165d09e7c8c6c9516f79 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Wed, 27 Sep 2023 10:56:54 +0800 Subject: [PATCH 03/51] test: support replica3 in splited/transform tmqvnode --- tests/parallel_test/cases.task | 2 + tests/system-test/7-tmq/tmqVnodeSplit.py | 1 - tests/system-test/7-tmq/tmqVnodeTransform.py | 44 ++++++++++++-------- 3 files changed, 28 insertions(+), 19 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 9995c2970f..cf161ab004 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -168,7 +168,9 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -i True ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -n 3 -i True ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform.py -N 2 -n 1 +,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform.py -N 6 -n 3 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit.py -N 2 -n 1 +,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit.py -N 3 -n 3 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeReplicate.py -M 3 -N 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-19201.py ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-21561.py diff --git a/tests/system-test/7-tmq/tmqVnodeSplit.py b/tests/system-test/7-tmq/tmqVnodeSplit.py index c6cdc2bf83..67deb84620 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit.py @@ -194,7 +194,6 @@ class TDTestCase: tdLog.printNoPrefix("======== test case 1 end ...... ") def run(self): - tdSql.prepare() self.prepareTestEnv() self.tmqCase1(True) self.prepareTestEnv() diff --git a/tests/system-test/7-tmq/tmqVnodeTransform.py b/tests/system-test/7-tmq/tmqVnodeTransform.py index fa50e46853..3ebf48b93a 100644 --- a/tests/system-test/7-tmq/tmqVnodeTransform.py +++ b/tests/system-test/7-tmq/tmqVnodeTransform.py @@ -58,7 +58,7 @@ class TDTestCase: paraDict['ctbNum'] = self.ctbNum paraDict['rowsPerTbl'] = self.rowsPerTbl - tdCom.drop_all_db(); + tdCom.drop_all_db() tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], wal_retention_period=36000,vgroups=paraDict["vgroups"],replica=self.replicaVar) tdLog.info("create stb") @@ -102,7 +102,7 @@ class TDTestCase: tdSql.query("show dnodes") for result in tdSql.queryResult: dnodesList.append(result[0]) - + print("dnodeList:",dnodesList) tdSql.query("select * from information_schema.ins_vnodes") vnodeId = 0 for result in tdSql.queryResult: @@ -110,9 +110,16 @@ class TDTestCase: tdLog.debug("dnode is %d"%(result[0])) dnodesList.remove(result[0]) vnodeId = result[1] - break - redistributeSql = "redistribute vgroup %d dnode %d" %(vnodeId, dnodesList[0]) - tdLog.debug("redistributeSql:%s"%(redistributeSql)) + print("its all data",dnodesList) + # if self.replicaVar == 1: + # redistributeSql = "redistribute vgroup %d dnode %d" %(vnodeId, dnodesList[0]) + # else: + redistributeSql = f"redistribute vgroup {vnodeId} " + for vgdnode in dnodesList: + redistributeSql += f"dnode {vgdnode} " + print(redistributeSql) + + tdLog.debug(f"redistributeSql:{redistributeSql}") tdSql.query(redistributeSql) tdLog.debug("redistributeSql ok") @@ -179,7 +186,7 @@ class TDTestCase: tmqCom.getStartCommitNotifyFromTmqsim() #restart dnode & remove wal - self.restartAndRemoveWal() + # self.restartAndRemoveWal() # redistribute vgroup self.redistributeVgroups(); @@ -228,7 +235,7 @@ class TDTestCase: tdSql.execute(sqlString) tdSql.query("flush database %s"%(paraDict['dbName'])) #restart dnode & remove wal - self.restartAndRemoveWal() + # self.restartAndRemoveWal() # redistribute vgroup self.redistributeVgroups(); @@ -236,7 +243,8 @@ class TDTestCase: sqlString = "alter table %s.%s modify column i nchar(16)" %(paraDict['dbName'], ntbName) tdLog.info("alter table sql: %s"%sqlString) tdSql.error(sqlString) - + expectRows = 0 + resultList = tmqCom.selectConsumeResult(expectRows) time.sleep(1) for i in range(len(topicNameList)): tdSql.query("drop topic %s"%topicNameList[i]) @@ -284,7 +292,7 @@ class TDTestCase: startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) tdLog.info("create topics from stb with filter") - queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName']) + queryString = "select * from %s.%s where c2 > 0 "%(paraDict['dbName'], paraDict['stbName']) sqlString = "create topic %s as %s" %(topicNameList[0], queryString) tdLog.info("create topic sql: %s"%sqlString) tdSql.execute(sqlString) @@ -305,27 +313,27 @@ class TDTestCase: time.sleep(5) #restart dnode & remove wal - self.restartAndRemoveWal() + # self.restartAndRemoveWal() # redistribute vgroup - self.redistributeVgroups(); + self.redistributeVgroups() tdLog.info("start consume processor") tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) tdLog.info("wait the consume result") - - time.sleep(10) + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + time.sleep(20) for i in range(len(topicNameList)): tdSql.query("drop topic %s"%topicNameList[i]) tdLog.printNoPrefix("======== test case 3 end ...... ") def run(self): - - tdSql.prepare() - self.prepareTestEnv() - self.tmqCase1() - self.tmqCase2() + # self.prepareTestEnv() + # self.tmqCase1() + # self.tmqCase2() self.prepareTestEnv() self.tmqCase3() From a30beeee6de93bb4bdf69940465af678ae8e3cf7 Mon Sep 17 00:00:00 2001 From: haoranchen Date: Sun, 8 Oct 2023 16:46:03 +0800 Subject: [PATCH 04/51] Update tmqVnodeTransform.py --- tests/system-test/7-tmq/tmqVnodeTransform.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/system-test/7-tmq/tmqVnodeTransform.py b/tests/system-test/7-tmq/tmqVnodeTransform.py index 3ebf48b93a..aab94bc7a2 100644 --- a/tests/system-test/7-tmq/tmqVnodeTransform.py +++ b/tests/system-test/7-tmq/tmqVnodeTransform.py @@ -331,9 +331,9 @@ class TDTestCase: tdLog.printNoPrefix("======== test case 3 end ...... ") def run(self): - # self.prepareTestEnv() - # self.tmqCase1() - # self.tmqCase2() + self.prepareTestEnv() + self.tmqCase1() + self.tmqCase2() self.prepareTestEnv() self.tmqCase3() From 8253874398cc78fa2d76df35963e0e290154c8a4 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 11 Oct 2023 14:47:26 +0800 Subject: [PATCH 05/51] enh(tsdb/cos): new sdk for s3 --- cmake/cmake.options | 18 ++ cmake/libs3.GNUmakefile | 430 ++++++++++++++++++++++++++++++ cmake/libs3_CMakeLists.txt.in | 28 ++ cmake/ssl_CMakeLists.txt.in | 15 ++ contrib/CMakeLists.txt | 54 ++++ source/dnode/vnode/CMakeLists.txt | 7 +- 6 files changed, 551 insertions(+), 1 deletion(-) create mode 100644 cmake/libs3.GNUmakefile create mode 100644 cmake/libs3_CMakeLists.txt.in create mode 100644 cmake/ssl_CMakeLists.txt.in diff --git a/cmake/cmake.options b/cmake/cmake.options index 1d4e9ba515..bacc5adfb7 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -127,6 +127,22 @@ option( IF(${TD_LINUX}) +option( + BUILD_WITH_S3 + "If build with s3" + ON +) + +IF(${BUILD_WITH_S3}) + +option( + BUILD_WITH_COS + "If build with cos" + OFF +) + +ELSE () + option( BUILD_WITH_COS "If build with cos" @@ -135,6 +151,8 @@ option( ENDIF () +ENDIF () + option( BUILD_WITH_SQLITE "If build with sqlite" diff --git a/cmake/libs3.GNUmakefile b/cmake/libs3.GNUmakefile new file mode 100644 index 0000000000..abf954f5c1 --- /dev/null +++ b/cmake/libs3.GNUmakefile @@ -0,0 +1,430 @@ +# GNUmakefile +# +# Copyright 2008 Bryan Ischo +# +# This file is part of libs3. +# +# libs3 is free software: you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation, version 3 or above of the License. You can also +# redistribute and/or modify it under the terms of the GNU General Public +# License, version 2 or above of the License. +# +# In addition, as a special exception, the copyright holders give +# permission to link the code of this library and its programs with the +# OpenSSL library, and distribute linked combinations including the two. +# +# libs3 is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# version 3 along with libs3, in a file named COPYING. If not, see +# . +# +# You should also have received a copy of the GNU General Public License +# version 2 along with libs3, in a file named COPYING-GPLv2. If not, see +# . + +# I tried to use the autoconf/automake/autolocal/etc (i.e. autohell) tools +# but I just couldn't stomach them. Since this is a Makefile for POSIX +# systems, I will simply do away with autohell completely and use a GNU +# Makefile. GNU make ought to be available pretty much everywhere, so I +# don't see this being a significant issue for portability. + +# All commands assume a GNU compiler. For systems which do not use a GNU +# compiler, write scripts with the same names as these commands, and taking +# the same arguments, and translate the arguments and commands into the +# appropriate non-POSIX ones as needed. libs3 assumes a GNU toolchain as +# the most portable way to build software possible. Non-POSIX, non-GNU +# systems can do the work of supporting this build infrastructure. + + +# -------------------------------------------------------------------------- +# Set libs3 version number, unless it is already set. + +LIBS3_VER_MAJOR ?= 4 +LIBS3_VER_MINOR ?= 1 +LIBS3_VER := $(LIBS3_VER_MAJOR).$(LIBS3_VER_MINOR) + + +# ----------------------------------------------------------------------------- +# Determine verbosity. VERBOSE_SHOW should be prepended to every command which +# should only be displayed if VERBOSE is set. QUIET_ECHO may be used to +# echo text only if VERBOSE is not set. Typically, a VERBOSE_SHOW command will +# be paired with a QUIET_ECHO command, to provide a command which is displayed +# in VERBOSE mode, along with text which is displayed in non-VERBOSE mode to +# describe the command. +# +# No matter what VERBOSE is defined to, it ends up as true if it's defined. +# This will be weird if you defined VERBOSE=false in the environment, and we +# switch it to true here; but the meaning of VERBOSE is, "if it's defined to +# any value, then verbosity is turned on". So don't define VERBOSE if you +# don't want verbosity in the build process. +# ----------------------------------------------------------------------------- + +ifdef VERBOSE + VERBOSE = true + VERBOSE_ECHO = @ echo + VERBOSE_SHOW = + QUIET_ECHO = @ echo > /dev/null +else + VERBOSE = false + VERBOSE_ECHO = @ echo > /dev/null + VERBOSE_SHOW = @ + QUIET_ECHO = @ echo +endif + + +# -------------------------------------------------------------------------- +# BUILD directory +ifndef BUILD + ifdef DEBUG + BUILD := build-debug + else + BUILD := build + endif +endif + + +# -------------------------------------------------------------------------- +# DESTDIR directory +ifndef DESTDIR + DESTDIR := ${HOME}/.cos-local.1 +endif + +# -------------------------------------------------------------------------- +# LIBDIR directory +ifndef LIBDIR + LIBDIR := ${DESTDIR}/lib +endif + +# -------------------------------------------------------------------------- +# Compiler CC handling +ifndef CC + CC := gcc +endif + +# -------------------------------------------------------------------------- +# Acquire configuration information for libraries that libs3 depends upon + +ifndef CURL_LIBS + CURL_LIBS := $(shell curl-config --libs) +endif + +ifndef CURL_CFLAGS + CURL_CFLAGS := $(shell curl-config --cflags) +endif + +ifndef LIBXML2_LIBS + LIBXML2_LIBS := $(shell xml2-config --libs) +endif + +ifndef LIBXML2_CFLAGS + LIBXML2_CFLAGS := $(shell xml2-config --cflags) +endif + +ifndef OPENSSL_LIBS + OPENSSL_LIBS := -lssl -lcrypto +endif + +# -------------------------------------------------------------------------- +# These CFLAGS assume a GNU compiler. For other compilers, write a script +# which converts these arguments into their equivalent for that particular +# compiler. + +ifndef CFLAGS + ifdef DEBUG + CFLAGS := -g + else + CFLAGS := -O3 + endif +endif + +CFLAGS += -Wall -Werror -Wshadow -Wextra -Iinc \ + $(CURL_CFLAGS) $(LIBXML2_CFLAGS) \ + -DLIBS3_VER_MAJOR=\"$(LIBS3_VER_MAJOR)\" \ + -DLIBS3_VER_MINOR=\"$(LIBS3_VER_MINOR)\" \ + -DLIBS3_VER=\"$(LIBS3_VER)\" \ + -D__STRICT_ANSI__ \ + -D_ISOC99_SOURCE \ + -D_POSIX_C_SOURCE=200112L + +LDFLAGS = $(CURL_LIBS) $(LIBXML2_LIBS) $(OPENSSL_LIBS) -lpthread + +STRIP ?= strip +INSTALL := install --strip-program=$(STRIP) + + +# -------------------------------------------------------------------------- +# Default targets are everything + +.PHONY: all +all: exported test + + +# -------------------------------------------------------------------------- +# Exported targets are the library and driver program + +.PHONY: exported +exported: libs3 s3 headers +exported_static: $(LIBS3_STATIC) + +# -------------------------------------------------------------------------- +# Install target + +.PHONY: install install_static +install_static: exported_static + $(QUIET_ECHO) $(LIBDIR)/libs3.a: Installing static library + $(VERBOSE_SHOW) $(INSTALL) -Dp -m u+rw,go+r $(BUILD)/lib/libs3.a \ + $(LIBDIR)/libs3.a + $(QUIET_ECHO) $(DESTDIR)/include/libs3.h: Installing header + $(VERBOSE_SHOW) $(INSTALL) -Dp -m u+rw,go+r inc/libs3.h \ + $(DESTDIR)/include/libs3.h + +install: exported + $(QUIET_ECHO) $(DESTDIR)/bin/s3: Installing executable + $(VERBOSE_SHOW) $(INSTALL) -Dps -m u+rwx,go+rx $(BUILD)/bin/s3 \ + $(DESTDIR)/bin/s3 + $(QUIET_ECHO) \ + $(LIBDIR)/libs3.so.$(LIBS3_VER): Installing shared library + $(VERBOSE_SHOW) $(INSTALL) -Dps -m u+rw,go+r \ + $(BUILD)/lib/libs3.so.$(LIBS3_VER_MAJOR) \ + $(LIBDIR)/libs3.so.$(LIBS3_VER) + $(QUIET_ECHO) \ + $(LIBDIR)/libs3.so.$(LIBS3_VER_MAJOR): Linking shared library + $(VERBOSE_SHOW) ln -sf libs3.so.$(LIBS3_VER) \ + $(LIBDIR)/libs3.so.$(LIBS3_VER_MAJOR) + $(QUIET_ECHO) $(LIBDIR)/libs3.so: Linking shared library + $(VERBOSE_SHOW) ln -sf libs3.so.$(LIBS3_VER_MAJOR) $(LIBDIR)/libs3.so + $(QUIET_ECHO) $(LIBDIR)/libs3.a: Installing static library + $(VERBOSE_SHOW) $(INSTALL) -Dp -m u+rw,go+r $(BUILD)/lib/libs3.a \ + $(LIBDIR)/libs3.a + $(QUIET_ECHO) $(DESTDIR)/include/libs3.h: Installing header + $(VERBOSE_SHOW) $(INSTALL) -Dp -m u+rw,go+r $(BUILD)/include/libs3.h \ + $(DESTDIR)/include/libs3.h + + +# -------------------------------------------------------------------------- +# Uninstall target + +.PHONY: uninstall +uninstall: + $(QUIET_ECHO) Installed files: Uninstalling + $(VERBOSE_SHOW) \ + rm -f $(DESTDIR)/bin/s3 \ + $(DESTDIR)/include/libs3.h \ + $(DESTDIR)/lib/libs3.a \ + $(DESTDIR)/lib/libs3.so \ + $(DESTDIR)/lib/libs3.so.$(LIBS3_VER_MAJOR) \ + $(DESTDIR)/lib/libs3.so.$(LIBS3_VER) + + +# -------------------------------------------------------------------------- +# Compile target patterns + +$(BUILD)/obj/%.o: src/%.c + $(QUIET_ECHO) $@: Compiling object + @ mkdir -p $(dir $(BUILD)/dep/$<) + @ $(CC) $(CFLAGS) -M -MG -MQ $@ -DCOMPILINGDEPENDENCIES \ + -o $(BUILD)/dep/$(<:%.c=%.d) -c $< + @ mkdir -p $(dir $@) + $(VERBOSE_SHOW) $(CC) $(CFLAGS) -o $@ -c $< + +$(BUILD)/obj/%.do: src/%.c + $(QUIET_ECHO) $@: Compiling dynamic object + $(QUIET_ECHO) cflags:${CFLAGS} + @ mkdir -p $(dir $(BUILD)/dep/$<) + @ $(CC) $(CFLAGS) -M -MG -MQ $@ -DCOMPILINGDEPENDENCIES \ + -o $(BUILD)/dep/$(<:%.c=%.dd) -c $< + @ mkdir -p $(dir $@) + $(VERBOSE_SHOW) $(CC) $(CFLAGS) -fpic -fPIC -o $@ -c $< + + +# -------------------------------------------------------------------------- +# libs3 library targets + +LIBS3_SHARED = $(BUILD)/lib/libs3.so.$(LIBS3_VER_MAJOR) +LIBS3_STATIC = $(BUILD)/lib/libs3.a + +.PHONY: libs3 +libs3: $(LIBS3_SHARED) $(LIBS3_STATIC) + +LIBS3_SOURCES := bucket.c bucket_metadata.c error_parser.c general.c \ + object.c request.c request_context.c \ + response_headers_handler.c service_access_logging.c \ + service.c simplexml.c util.c multipart.c + +$(LIBS3_SHARED): $(LIBS3_SOURCES:%.c=$(BUILD)/obj/%.do) + $(QUIET_ECHO) $@: Building shared library + @ mkdir -p $(dir $@) + $(VERBOSE_SHOW) $(CC) -shared -Wl,-soname,libs3.so.$(LIBS3_VER_MAJOR) \ + -o $@ $^ $(LDFLAGS) + +$(LIBS3_STATIC): $(LIBS3_SOURCES:%.c=$(BUILD)/obj/%.o) + $(QUIET_ECHO) $@: Building static library + @ mkdir -p $(dir $@) + $(VERBOSE_SHOW) $(AR) cr $@ $^ + + +# -------------------------------------------------------------------------- +# Driver program targets + +.PHONY: s3 +s3: $(BUILD)/bin/s3 + +$(BUILD)/bin/s3: $(BUILD)/obj/s3.o $(LIBS3_SHARED) + $(QUIET_ECHO) $@: Building executable + @ mkdir -p $(dir $@) + $(VERBOSE_SHOW) $(CC) -o $@ $^ $(LDFLAGS) + + +# -------------------------------------------------------------------------- +# libs3 header targets + +.PHONY: headers +headers: $(BUILD)/include/libs3.h + +$(BUILD)/include/libs3.h: inc/libs3.h + $(QUIET_ECHO) $@: Linking header + @ mkdir -p $(dir $@) + $(VERBOSE_SHOW) ln -sf $(abspath $<) $@ + + +# -------------------------------------------------------------------------- +# Test targets + +.PHONY: test +test: $(BUILD)/bin/testsimplexml + +$(BUILD)/bin/testsimplexml: $(BUILD)/obj/testsimplexml.o $(LIBS3_STATIC) + $(QUIET_ECHO) $@: Building executable + @ mkdir -p $(dir $@) + $(VERBOSE_SHOW) $(CC) -o $@ $^ $(LIBXML2_LIBS) + + +# -------------------------------------------------------------------------- +# Clean target + +.PHONY: clean +clean: + $(QUIET_ECHO) $(BUILD): Cleaning + $(VERBOSE_SHOW) rm -rf $(BUILD) + +.PHONY: distclean +distclean: + $(QUIET_ECHO) $(BUILD): Cleaning + $(VERBOSE_SHOW) rm -rf $(BUILD) + + +# -------------------------------------------------------------------------- +# Clean dependencies target + +.PHONY: cleandeps +cleandeps: + $(QUIET_ECHO) $(BUILD)/dep: Cleaning dependencies + $(VERBOSE_SHOW) rm -rf $(BUILD)/dep + + +# -------------------------------------------------------------------------- +# Dependencies + +ALL_SOURCES := $(LIBS3_SOURCES) s3.c testsimplexml.c + +$(foreach i, $(ALL_SOURCES), $(eval -include $(BUILD)/dep/src/$(i:%.c=%.d))) +$(foreach i, $(ALL_SOURCES), $(eval -include $(BUILD)/dep/src/$(i:%.c=%.dd))) + + +# -------------------------------------------------------------------------- +# Debian package target + +DEBPKG = $(BUILD)/pkg/libs3_$(LIBS3_VER).deb +DEBDEVPKG = $(BUILD)/pkg/libs3-dev_$(LIBS3_VER).deb + +.PHONY: deb +deb: $(DEBPKG) $(DEBDEVPKG) + +$(DEBPKG): DEBARCH = $(shell dpkg-architecture | grep ^DEB_BUILD_ARCH= | \ + cut -d '=' -f 2) +$(DEBPKG): exported $(BUILD)/deb/DEBIAN/control $(BUILD)/deb/DEBIAN/shlibs \ + $(BUILD)/deb/DEBIAN/postinst \ + $(BUILD)/deb/usr/share/doc/libs3/changelog.gz \ + $(BUILD)/deb/usr/share/doc/libs3/changelog.Debian.gz \ + $(BUILD)/deb/usr/share/doc/libs3/copyright + DESTDIR=$(BUILD)/deb/usr $(MAKE) install + rm -rf $(BUILD)/deb/usr/include + rm -f $(BUILD)/deb/usr/lib/libs3.a + @mkdir -p $(dir $@) + fakeroot dpkg-deb -b $(BUILD)/deb $@ + mv $@ $(BUILD)/pkg/libs3_$(LIBS3_VER)_$(DEBARCH).deb + +$(DEBDEVPKG): DEBARCH = $(shell dpkg-architecture | grep ^DEB_BUILD_ARCH= | \ + cut -d '=' -f 2) +$(DEBDEVPKG): exported $(BUILD)/deb-dev/DEBIAN/control \ + $(BUILD)/deb-dev/usr/share/doc/libs3-dev/changelog.gz \ + $(BUILD)/deb-dev/usr/share/doc/libs3-dev/changelog.Debian.gz \ + $(BUILD)/deb-dev/usr/share/doc/libs3-dev/copyright + DESTDIR=$(BUILD)/deb-dev/usr $(MAKE) install + rm -rf $(BUILD)/deb-dev/usr/bin + rm -f $(BUILD)/deb-dev/usr/lib/libs3.so* + @mkdir -p $(dir $@) + fakeroot dpkg-deb -b $(BUILD)/deb-dev $@ + mv $@ $(BUILD)/pkg/libs3-dev_$(LIBS3_VER)_$(DEBARCH).deb + +$(BUILD)/deb/DEBIAN/control: debian/control + @mkdir -p $(dir $@) + echo -n "Depends: " > $@ + dpkg-shlibdeps -Sbuild -O $(BUILD)/lib/libs3.so.$(LIBS3_VER_MAJOR) | \ + cut -d '=' -f 2- >> $@ + sed -e 's/LIBS3_VERSION/$(LIBS3_VER)/' \ + < $< | sed -e 's/DEBIAN_ARCHITECTURE/$(DEBARCH)/' | \ + grep -v ^Source: >> $@ + +$(BUILD)/deb-dev/DEBIAN/control: debian/control.dev + @mkdir -p $(dir $@) + sed -e 's/LIBS3_VERSION/$(LIBS3_VER)/' \ + < $< | sed -e 's/DEBIAN_ARCHITECTURE/$(DEBARCH)/' > $@ + +$(BUILD)/deb/DEBIAN/shlibs: + echo -n "libs3 $(LIBS3_VER_MAJOR) libs3 " > $@ + echo "(>= $(LIBS3_VER))" >> $@ + +$(BUILD)/deb/DEBIAN/postinst: debian/postinst + @mkdir -p $(dir $@) + cp $< $@ + +$(BUILD)/deb/usr/share/doc/libs3/copyright: LICENSE + @mkdir -p $(dir $@) + cp $< $@ + @echo >> $@ + @echo -n "An alternate location for the GNU General Public " >> $@ + @echo "License version 3 on Debian" >> $@ + @echo "systems is /usr/share/common-licenses/GPL-3." >> $@ + +$(BUILD)/deb-dev/usr/share/doc/libs3-dev/copyright: LICENSE + @mkdir -p $(dir $@) + cp $< $@ + @echo >> $@ + @echo -n "An alternate location for the GNU General Public " >> $@ + @echo "License version 3 on Debian" >> $@ + @echo "systems is /usr/share/common-licenses/GPL-3." >> $@ + +$(BUILD)/deb/usr/share/doc/libs3/changelog.gz: debian/changelog + @mkdir -p $(dir $@) + gzip --best -c $< > $@ + +$(BUILD)/deb-dev/usr/share/doc/libs3-dev/changelog.gz: debian/changelog + @mkdir -p $(dir $@) + gzip --best -c $< > $@ + +$(BUILD)/deb/usr/share/doc/libs3/changelog.Debian.gz: debian/changelog.Debian + @mkdir -p $(dir $@) + gzip --best -c $< > $@ + +$(BUILD)/deb-dev/usr/share/doc/libs3-dev/changelog.Debian.gz: \ + debian/changelog.Debian + @mkdir -p $(dir $@) + gzip --best -c $< > $@ + + diff --git a/cmake/libs3_CMakeLists.txt.in b/cmake/libs3_CMakeLists.txt.in new file mode 100644 index 0000000000..b77299baf3 --- /dev/null +++ b/cmake/libs3_CMakeLists.txt.in @@ -0,0 +1,28 @@ +# libs3 + +set(s3_flags "${CMAKE_C_FLAGS} -O3 -Iinc -I$ENV{HOME}/.cos-local.1/include -Iinc ") + +set(SED_CMD "s/CFLAGS += -Wall -Werror/CFLAGS += -I$ENV{HOME}/.cos-local.1/include /") + +function(update_cflags) + file(READ "GNUmakefile" S3_CONTENT) + string(REPLACE "CFLAGS += -Wall -Werror" "CFLAGS += -I$ENV{HOME}/.cos-local.1/include " S3_CONTENT "${S3_CONTENT}") + file(WRITE "GNUmakefile" "${S3_CONTENT}") +endfunction(update_cflags) + +ExternalProject_Add(libs3 + GIT_REPOSITORY https://github.com/bji/libs3 + #GIT_TAG v5.0.16 + SOURCE_DIR "${TD_CONTRIB_DIR}/libs3" + #BINARY_DIR "" + BUILD_IN_SOURCE TRUE + BUILD_ALWAYS 1 + UPDATE_COMMAND "" + #sed_cmd "s/CFLAGS += -Wall -Werror/CFLAGS += -I$ENV{HOME}/.cos-local.1/include /" + CONFIGURE_COMMAND cp ${TD_SUPPORT_DIR}/libs3.GNUmakefile GNUmakefile && sed -i "s|CFLAGS += -Wall -Werror|CFLAGS += -I'$ENV{HOME}/.cos-local.1/include' -L'$ENV{HOME}/.cos-local.1/lib' |" ./GNUmakefile + #BUILD_COMMAND make CFLAGS=${s3_flags} DESTDIR=$ENV{HOME}/.cos-local.1 build/lib/libs3.a + #BUILD_COMMAND make DESTDIR="$ENV{HOME}/.cos-local.1" build/lib/libs3.a + BUILD_COMMAND make build/lib/libs3.a + INSTALL_COMMAND make install_static + TEST_COMMAND "" +) diff --git a/cmake/ssl_CMakeLists.txt.in b/cmake/ssl_CMakeLists.txt.in new file mode 100644 index 0000000000..7821848618 --- /dev/null +++ b/cmake/ssl_CMakeLists.txt.in @@ -0,0 +1,15 @@ +# openssl +ExternalProject_Add(openssl + URL https://www.openssl.org/source/openssl-3.1.3.tar.gz + URL_HASH SHA256=f0316a2ebd89e7f2352976445458689f80302093788c466692fb2a188b2eacf6 + DOWNLOAD_NO_PROGRESS 1 + DOWNLOAD_DIR "${TD_CONTRIB_DIR}/deps-download" + SOURCE_DIR "${TD_CONTRIB_DIR}/openssl" + BUILD_IN_SOURCE TRUE + #BUILD_ALWAYS 1 + #UPDATE_COMMAND "" + CONFIGURE_COMMAND ./Configure --prefix=$ENV{HOME}/.cos-local.1 -static #--no-shared + BUILD_COMMAND make -j + INSTALL_COMMAND make install + TEST_COMMAND "" +) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index e3e48ac3a1..39bf85df8f 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -8,6 +8,37 @@ endfunction(cat IN_FILE OUT_FILE) if(${TD_LINUX}) + if(${BUILD_WITH_S3}) + +set(CONTRIB_TMP_FILE3 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in3") +configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3}) + +if(${BUILD_WITH_COS}) + file(MAKE_DIRECTORY $ENV{HOME}/.cos-local.1/) + cat("${TD_SUPPORT_DIR}/ssl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3}) +endif(${BUILD_WITH_COS}) + +configure_file(${CONTRIB_TMP_FILE3} "${TD_CONTRIB_DIR}/deps-download/CMakeLists.txt") +execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" . + WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download") +execute_process(COMMAND "${CMAKE_COMMAND}" --build . + WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download") + +set(CONTRIB_TMP_FILE2 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in2") +configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE2}) + +if(${BUILD_WITH_COS}) + cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE2}) +endif(${BUILD_WITH_COS}) + +configure_file(${CONTRIB_TMP_FILE2} "${TD_CONTRIB_DIR}/deps-download/CMakeLists.txt") +execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" . + WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download") +execute_process(COMMAND "${CMAKE_COMMAND}" --build . + WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download") + +else() + set(CONTRIB_TMP_FILE3 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in3") configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3}) @@ -37,6 +68,8 @@ execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" . execute_process(COMMAND "${CMAKE_COMMAND}" --build . WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download") +endif(${BUILD_WITH_S3}) + endif(${TD_LINUX}) set(CONTRIB_TMP_FILE "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in") @@ -155,6 +188,18 @@ if(${BUILD_WITH_SQLITE}) cat("${TD_SUPPORT_DIR}/sqlite_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) endif(${BUILD_WITH_SQLITE}) +# s3 +if(${BUILD_WITH_S3}) + #cat("${TD_SUPPORT_DIR}/mxml_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) + #cat("${TD_SUPPORT_DIR}/apr_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) + #cat("${TD_SUPPORT_DIR}/apr-util_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) + #cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) + #INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.1/include) + cat("${TD_SUPPORT_DIR}/libs3_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) + add_definitions(-DUSE_S3) + +else() + # cos if(${BUILD_WITH_COS}) #cat("${TD_SUPPORT_DIR}/mxml_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) @@ -165,6 +210,8 @@ if(${BUILD_WITH_COS}) add_definitions(-DUSE_COS) endif(${BUILD_WITH_COS}) +endif(${BUILD_WITH_S3}) + # lucene if(${BUILD_WITH_LUCENE}) cat("${TD_SUPPORT_DIR}/lucene_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) @@ -390,6 +437,11 @@ if (${BUILD_WITH_ROCKSDB}) endif() endif() +if(${BUILD_WITH_S3}) + INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.1/include) + +else() + # cos if(${BUILD_WITH_COS}) if(${TD_LINUX}) @@ -415,6 +467,8 @@ if(${BUILD_WITH_COS}) endif(${TD_LINUX}) endif(${BUILD_WITH_COS}) +endif(${BUILD_WITH_S3}) + # lucene # To support build on ubuntu: sudo apt-get install libboost-all-dev if(${BUILD_WITH_LUCENE}) diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 84d54f3350..516c8a8b69 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -162,6 +162,12 @@ target_link_libraries( ) if(${TD_LINUX}) + if(${BUILD_WITH_S3}) + + endif(${BUILD_WITH_S3}) + +if(${BUILD_WITH_COS}) + set(CMAKE_FIND_LIBRARY_SUFFIXES ".a") find_library(APR_LIBRARY apr-1 PATHS /usr/local/apr/lib/) find_library(APR_UTIL_LIBRARY aprutil-1 PATHS /usr/local/apr/lib/) @@ -194,7 +200,6 @@ target_include_directories( PUBLIC "$ENV{HOME}/.cos-local.1/include" ) -if(${BUILD_WITH_COS}) add_definitions(-DUSE_COS) endif(${BUILD_WITH_COS}) From 0e31231212417690974c3d641b5b29ffa25a34d8 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Fri, 13 Oct 2023 16:22:32 +0800 Subject: [PATCH 06/51] test: support replica3 in splited/transform tmqvnode --- .../5dnode3mnodeSep1VnodeStopDnodeCreateDb.py | 4 +- .../system-test/7-tmq/tmqVnodeSplit-column.py | 213 +++++++++ tests/system-test/7-tmq/tmqVnodeSplit-db.py | 214 +++++++++ ...tmqVnodeSplit-stb-select-duplicatedata.py} | 23 +- .../7-tmq/tmqVnodeSplit-stb-select.py | 213 +++++++++ tests/system-test/7-tmq/tmqVnodeSplit-stb.py | 213 +++++++++ .../7-tmq/tmqVnodeTransform-stb.py | 433 ++++++++++++++++++ 7 files changed, 1301 insertions(+), 12 deletions(-) create mode 100644 tests/system-test/7-tmq/tmqVnodeSplit-column.py create mode 100644 tests/system-test/7-tmq/tmqVnodeSplit-db.py rename tests/system-test/7-tmq/{tmqVnodeSplit.py => tmqVnodeSplit-stb-select-duplicatedata.py} (93%) create mode 100644 tests/system-test/7-tmq/tmqVnodeSplit-stb-select.py create mode 100644 tests/system-test/7-tmq/tmqVnodeSplit-stb.py create mode 100644 tests/system-test/7-tmq/tmqVnodeTransform-stb.py diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py index 3a972ff4e9..fb62110b14 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py @@ -146,9 +146,9 @@ class TDTestCase: # a11111=paraDict["dbNumbers"] # print(f"==================={dbNameIndex},{a11111}") threads.append(threading.Thread(target=clusterComCreate.createDeltedatabases, args=(newTdSql, dbNameIndex,repeatNumber,paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica']))) - + newTdSql2=tdCom.newTdSql() redbNameIndex = '%s%d'%(paraDict["dbName"],i+100) - threads.append(threading.Thread(target=clusterComCreate.createDeltedatabases, args=(newTdSql, redbNameIndex,1,paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica']))) + threads.append(threading.Thread(target=clusterComCreate.createDeltedatabases, args=(newTdSql2, redbNameIndex,1,paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica']))) for tr in threads: tr.start() diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-column.py b/tests/system-test/7-tmq/tmqVnodeSplit-column.py new file mode 100644 index 0000000000..95c35363a8 --- /dev/null +++ b/tests/system-test/7-tmq/tmqVnodeSplit-column.py @@ -0,0 +1,213 @@ + +import taos +import sys +import time +import socket +import os +import threading +import math + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +from util.cluster import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def __init__(self): + self.vgroups = 1 + self.ctbNum = 10 + self.rowsPerTbl = 10000 + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def getDataPath(self): + selfPath = tdCom.getBuildPath() + + return selfPath + '/../sim/dnode%d/data/vnode/vnode%d/wal/*'; + + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 60, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tdCom.drop_all_db() + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], wal_retention_period=36000,vgroups=paraDict["vgroups"],replica=self.replicaVar) + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + return + + def restartAndRemoveWal(self, deleteWal): + tdDnodes = cluster.dnodes + tdSql.query("select * from information_schema.ins_vnodes") + for result in tdSql.queryResult: + if result[2] == 'dbt': + tdLog.debug("dnode is %d"%(result[0])) + dnodeId = result[0] + vnodeId = result[1] + + tdDnodes[dnodeId - 1].stoptaosd() + time.sleep(1) + dataPath = self.getDataPath() + dataPath = dataPath%(dnodeId,vnodeId) + tdLog.debug("dataPath:%s"%dataPath) + if deleteWal: + if os.system('rm -rf ' + dataPath) != 0: + tdLog.exit("rm error") + + tdDnodes[dnodeId - 1].starttaosd() + time.sleep(1) + break + tdLog.debug("restart dnode ok") + + def splitVgroups(self): + tdSql.query("select * from information_schema.ins_vnodes") + vnodeId = 0 + for result in tdSql.queryResult: + if result[2] == 'dbt': + vnodeId = result[1] + tdLog.debug("vnode is %d"%(vnodeId)) + break + splitSql = "split vgroup %d" %(vnodeId) + tdLog.debug("splitSql:%s"%(splitSql)) + tdSql.query(splitSql) + tdLog.debug("splitSql ok") + + def tmqCase1(self, deleteWal=False): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb1', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 60, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + topicNameList = ['topic1'] + # expectRowsList = [] + tmqCom.initConsumerTable() + + tdLog.info("create topics from stb with filter") + queryString = "select * from %s.%s where c2 >= 0 "%(paraDict['dbName'], paraDict['stbName']) + # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + # tdSql.query(queryString) + # expectRowsList.append(tdSql.getRows()) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2 + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + + tdLog.info("create ctb1") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("create ctb2") + paraDict2 = paraDict.copy() + paraDict2['ctbPrefix'] = "ctb2" + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict2['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("insert ctb1 data") + pInsertThread = tmqCom.asyncInsertDataByInterlace(paraDict) + + tmqCom.getStartConsumeNotifyFromTmqsim() + tmqCom.getStartCommitNotifyFromTmqsim() + + #restart dnode & remove wal + self.restartAndRemoveWal(deleteWal) + + # split vgroup + self.splitVgroups() + + + tdLog.info("insert ctb2 data") + pInsertThread1 = tmqCom.asyncInsertDataByInterlace(paraDict2) + pInsertThread.join() + pInsertThread1.join() + + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + if expectrowcnt / 2 >= resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0])) + tdLog.exit("%d tmq consume rows error!"%consumerId) + + # tmqCom.checkFileContent(consumerId, queryString) + + time.sleep(2) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def run(self): + self.prepareTestEnv() + self.tmqCase1(True) + self.prepareTestEnv() + self.tmqCase1(False) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-db.py b/tests/system-test/7-tmq/tmqVnodeSplit-db.py new file mode 100644 index 0000000000..1c2e867758 --- /dev/null +++ b/tests/system-test/7-tmq/tmqVnodeSplit-db.py @@ -0,0 +1,214 @@ + +import taos +import sys +import time +import socket +import os +import threading +import math + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +from util.cluster import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def __init__(self): + self.vgroups = 1 + self.ctbNum = 10 + self.rowsPerTbl = 10000 + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def getDataPath(self): + selfPath = tdCom.getBuildPath() + + return selfPath + '/../sim/dnode%d/data/vnode/vnode%d/wal/*'; + + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 60, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tdCom.drop_all_db() + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], wal_retention_period=36000,vgroups=paraDict["vgroups"],replica=self.replicaVar) + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + return + + def restartAndRemoveWal(self, deleteWal): + tdDnodes = cluster.dnodes + tdSql.query("select * from information_schema.ins_vnodes") + for result in tdSql.queryResult: + if result[2] == 'dbt': + tdLog.debug("dnode is %d"%(result[0])) + dnodeId = result[0] + vnodeId = result[1] + + tdDnodes[dnodeId - 1].stoptaosd() + time.sleep(1) + dataPath = self.getDataPath() + dataPath = dataPath%(dnodeId,vnodeId) + tdLog.debug("dataPath:%s"%dataPath) + if deleteWal: + if os.system('rm -rf ' + dataPath) != 0: + tdLog.exit("rm error") + + tdDnodes[dnodeId - 1].starttaosd() + time.sleep(1) + break + tdLog.debug("restart dnode ok") + + def splitVgroups(self): + tdSql.query("select * from information_schema.ins_vnodes") + vnodeId = 0 + for result in tdSql.queryResult: + if result[2] == 'dbt': + vnodeId = result[1] + tdLog.debug("vnode is %d"%(vnodeId)) + break + splitSql = "split vgroup %d" %(vnodeId) + tdLog.debug("splitSql:%s"%(splitSql)) + tdSql.query(splitSql) + tdLog.debug("splitSql ok") + + def tmqCase1(self, deleteWal=False): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb1', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 60, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + topicNameList = ['topic1'] + # expectRowsList = [] + tmqCom.initConsumerTable() + + tdLog.info("create topics from db") + queryString = "database %s"%(paraDict['dbName']) + # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + # tdSql.query(queryString) + # expectRowsList.append(tdSql.getRows()) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2 + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + + tdLog.info("create ctb1") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("create ctb2") + paraDict2 = paraDict.copy() + + paraDict2['ctbPrefix'] = "ctb2" + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict2['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("insert ctb1 data") + pInsertThread = tmqCom.asyncInsertDataByInterlace(paraDict) + + tmqCom.getStartConsumeNotifyFromTmqsim() + tmqCom.getStartCommitNotifyFromTmqsim() + + #restart dnode & remove wal + self.restartAndRemoveWal(deleteWal) + + # split vgroup + self.splitVgroups() + + + tdLog.info("insert ctb2 data") + pInsertThread1 = tmqCom.asyncInsertDataByInterlace(paraDict2) + pInsertThread.join() + pInsertThread1.join() + + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + if expectrowcnt / 2 >= resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0])) + tdLog.exit("%d tmq consume rows error!"%consumerId) + + # tmqCom.checkFileContent(consumerId, queryString) + + time.sleep(2) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def run(self): + self.prepareTestEnv() + self.tmqCase1(True) + self.prepareTestEnv() + self.tmqCase1(False) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/tmqVnodeSplit.py b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py similarity index 93% rename from tests/system-test/7-tmq/tmqVnodeSplit.py rename to tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py index 67deb84620..086fe6e4c5 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py @@ -58,7 +58,7 @@ class TDTestCase: paraDict['ctbNum'] = self.ctbNum paraDict['rowsPerTbl'] = self.rowsPerTbl - tdCom.drop_all_db(); + tdCom.drop_all_db() tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], wal_retention_period=36000,vgroups=paraDict["vgroups"],replica=self.replicaVar) tdLog.info("create stb") @@ -112,13 +112,13 @@ class TDTestCase: 'tagPrefix': 't', 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], - 'ctbPrefix': 'ctb', + 'ctbPrefix': 'ctb1', 'ctbStartIdx': 0, 'ctbNum': 10, 'rowsPerTbl': 10000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 60, + 'pollDelay': 120, 'showMsg': 1, 'showRow': 1, 'snapshot': 0} @@ -157,7 +157,13 @@ class TDTestCase: tdLog.info("create ctb1") tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) - tdLog.info("insert data") + + tdLog.info("create ctb2") + paraDict['ctbPrefix'] = "ctb2" + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("insert ctb1 data") pInsertThread = tmqCom.asyncInsertDataByInterlace(paraDict) tmqCom.getStartConsumeNotifyFromTmqsim() @@ -169,11 +175,8 @@ class TDTestCase: # split vgroup self.splitVgroups() - tdLog.info("create ctb2") - paraDict['ctbPrefix'] = "ctbn" - tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], - ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) - tdLog.info("insert data") + + tdLog.info("insert ctb2 data") pInsertThread1 = tmqCom.asyncInsertDataByInterlace(paraDict) pInsertThread.join() pInsertThread1.join() @@ -181,7 +184,7 @@ class TDTestCase: expectRows = 1 resultList = tmqCom.selectConsumeResult(expectRows) - if expectrowcnt / 2 >= resultList[0]: + if expectrowcnt / 2 >= resultList[0] or expectrowcnt <= resultList[0]: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0])) tdLog.exit("%d tmq consume rows error!"%consumerId) diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select.py b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select.py new file mode 100644 index 0000000000..3214c2f5c4 --- /dev/null +++ b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select.py @@ -0,0 +1,213 @@ + +import taos +import sys +import time +import socket +import os +import threading +import math + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +from util.cluster import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def __init__(self): + self.vgroups = 1 + self.ctbNum = 10 + self.rowsPerTbl = 10000 + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def getDataPath(self): + selfPath = tdCom.getBuildPath() + + return selfPath + '/../sim/dnode%d/data/vnode/vnode%d/wal/*'; + + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 60, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tdCom.drop_all_db() + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], wal_retention_period=36000,vgroups=paraDict["vgroups"],replica=self.replicaVar) + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + return + + def restartAndRemoveWal(self, deleteWal): + tdDnodes = cluster.dnodes + tdSql.query("select * from information_schema.ins_vnodes") + for result in tdSql.queryResult: + if result[2] == 'dbt': + tdLog.debug("dnode is %d"%(result[0])) + dnodeId = result[0] + vnodeId = result[1] + + tdDnodes[dnodeId - 1].stoptaosd() + time.sleep(1) + dataPath = self.getDataPath() + dataPath = dataPath%(dnodeId,vnodeId) + tdLog.debug("dataPath:%s"%dataPath) + if deleteWal: + if os.system('rm -rf ' + dataPath) != 0: + tdLog.exit("rm error") + + tdDnodes[dnodeId - 1].starttaosd() + time.sleep(1) + break + tdLog.debug("restart dnode ok") + + def splitVgroups(self): + tdSql.query("select * from information_schema.ins_vnodes") + vnodeId = 0 + for result in tdSql.queryResult: + if result[2] == 'dbt': + vnodeId = result[1] + tdLog.debug("vnode is %d"%(vnodeId)) + break + splitSql = "split vgroup %d" %(vnodeId) + tdLog.debug("splitSql:%s"%(splitSql)) + tdSql.query(splitSql) + tdLog.debug("splitSql ok") + + def tmqCase1(self, deleteWal=False): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb1', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 120, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + topicNameList = ['topic1'] + # expectRowsList = [] + tmqCom.initConsumerTable() + + tdLog.info("create topics from stb with filter") + queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName']) + # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + # tdSql.query(queryString) + # expectRowsList.append(tdSql.getRows()) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2 + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + + tdLog.info("create ctb1") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("create ctb2") + paraDict2 = paraDict.copy() + paraDict2['ctbPrefix'] = "ctb2" + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict2['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("insert ctb1 data") + pInsertThread = tmqCom.asyncInsertDataByInterlace(paraDict) + + tmqCom.getStartConsumeNotifyFromTmqsim() + tmqCom.getStartCommitNotifyFromTmqsim() + + #restart dnode & remove wal + self.restartAndRemoveWal(deleteWal) + + # split vgroup + self.splitVgroups() + + + tdLog.info("insert ctb2 data") + pInsertThread1 = tmqCom.asyncInsertDataByInterlace(paraDict2) + pInsertThread.join() + pInsertThread1.join() + + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + if expectrowcnt / 2 >= resultList[0] or expectrowcnt < resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0])) + tdLog.exit("%d tmq consume rows error!"%consumerId) + + # tmqCom.checkFileContent(consumerId, queryString) + + time.sleep(2) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def run(self): + self.prepareTestEnv() + self.tmqCase1(True) + self.prepareTestEnv() + self.tmqCase1(False) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-stb.py b/tests/system-test/7-tmq/tmqVnodeSplit-stb.py new file mode 100644 index 0000000000..27d296ed0e --- /dev/null +++ b/tests/system-test/7-tmq/tmqVnodeSplit-stb.py @@ -0,0 +1,213 @@ + +import taos +import sys +import time +import socket +import os +import threading +import math + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +from util.cluster import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def __init__(self): + self.vgroups = 1 + self.ctbNum = 10 + self.rowsPerTbl = 10000 + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def getDataPath(self): + selfPath = tdCom.getBuildPath() + + return selfPath + '/../sim/dnode%d/data/vnode/vnode%d/wal/*'; + + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 60, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tdCom.drop_all_db() + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], wal_retention_period=36000,vgroups=paraDict["vgroups"],replica=self.replicaVar) + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + return + + def restartAndRemoveWal(self, deleteWal): + tdDnodes = cluster.dnodes + tdSql.query("select * from information_schema.ins_vnodes") + for result in tdSql.queryResult: + if result[2] == 'dbt': + tdLog.debug("dnode is %d"%(result[0])) + dnodeId = result[0] + vnodeId = result[1] + + tdDnodes[dnodeId - 1].stoptaosd() + time.sleep(1) + dataPath = self.getDataPath() + dataPath = dataPath%(dnodeId,vnodeId) + tdLog.debug("dataPath:%s"%dataPath) + if deleteWal: + if os.system('rm -rf ' + dataPath) != 0: + tdLog.exit("rm error") + + tdDnodes[dnodeId - 1].starttaosd() + time.sleep(1) + break + tdLog.debug("restart dnode ok") + + def splitVgroups(self): + tdSql.query("select * from information_schema.ins_vnodes") + vnodeId = 0 + for result in tdSql.queryResult: + if result[2] == 'dbt': + vnodeId = result[1] + tdLog.debug("vnode is %d"%(vnodeId)) + break + splitSql = "split vgroup %d" %(vnodeId) + tdLog.debug("splitSql:%s"%(splitSql)) + tdSql.query(splitSql) + tdLog.debug("splitSql ok") + + def tmqCase1(self, deleteWal=False): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb1', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 60, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + topicNameList = ['topic1'] + # expectRowsList = [] + tmqCom.initConsumerTable() + + tdLog.info("create topics from stb ") + queryString = "stable %s.%s"%(paraDict['dbName'], paraDict['stbName']) + # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + # tdSql.query(queryString) + # expectRowsList.append(tdSql.getRows()) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2 + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + + tdLog.info("create ctb1") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("create ctb2") + paraDict2 = paraDict.copy() + paraDict2['ctbPrefix'] = "ctb2" + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict2['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("insert ctb1 data") + pInsertThread = tmqCom.asyncInsertDataByInterlace(paraDict) + + tmqCom.getStartConsumeNotifyFromTmqsim() + tmqCom.getStartCommitNotifyFromTmqsim() + + #restart dnode & remove wal + self.restartAndRemoveWal(deleteWal) + + # split vgroup + self.splitVgroups() + + + tdLog.info("insert ctb2 data") + pInsertThread1 = tmqCom.asyncInsertDataByInterlace(paraDict2) + pInsertThread.join() + pInsertThread1.join() + + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + if expectrowcnt / 2 >= resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0])) + tdLog.exit("%d tmq consume rows error!"%consumerId) + + # tmqCom.checkFileContent(consumerId, queryString) + + time.sleep(2) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def run(self): + self.prepareTestEnv() + self.tmqCase1(True) + self.prepareTestEnv() + self.tmqCase1(False) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/tmqVnodeTransform-stb.py b/tests/system-test/7-tmq/tmqVnodeTransform-stb.py new file mode 100644 index 0000000000..18f9fafe9a --- /dev/null +++ b/tests/system-test/7-tmq/tmqVnodeTransform-stb.py @@ -0,0 +1,433 @@ + +import taos +import sys +import time +import socket +import os +import threading +import math + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +from util.cluster import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def __init__(self): + self.vgroups = 1 + self.ctbNum = 10 + self.rowsPerTbl = 10000 + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def getDataPath(self): + selfPath = tdCom.getBuildPath() + + return selfPath + '/../sim/dnode%d/data/vnode/vnode%d/wal/*'; + + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 60, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tdCom.drop_all_db() + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], wal_retention_period=36000,vgroups=paraDict["vgroups"],replica=self.replicaVar) + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + # tdLog.info("create ctb") + # tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + # ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + # tdLog.info("insert data") + # tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + # ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + # startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + + # tdLog.info("restart taosd to ensure that the data falls into the disk") + # tdDnodes.stop(1) + # tdDnodes.start(1) + # tdSql.query("flush database %s"%(paraDict['dbName'])) + return + + def restartAndRemoveWal(self): + tdDnodes = cluster.dnodes + tdSql.query("select * from information_schema.ins_vnodes") + for result in tdSql.queryResult: + if result[2] == 'dbt': + tdLog.debug("dnode is %d"%(result[0])) + dnodeId = result[0] + vnodeId = result[1] + + tdDnodes[dnodeId - 1].stoptaosd() + time.sleep(1) + dataPath = self.getDataPath() + dataPath = dataPath%(dnodeId,vnodeId) + os.system('rm -rf ' + dataPath) + tdLog.debug("dataPath:%s"%dataPath) + tdDnodes[dnodeId - 1].starttaosd() + time.sleep(1) + break + tdLog.debug("restart dnode ok") + + def redistributeVgroups(self): + dnodesList = [] + tdSql.query("show dnodes") + for result in tdSql.queryResult: + dnodesList.append(result[0]) + print("dnodeList:",dnodesList) + tdSql.query("select * from information_schema.ins_vnodes") + vnodeId = 0 + for result in tdSql.queryResult: + if result[2] == 'dbt': + tdLog.debug("dnode is %d"%(result[0])) + dnodesList.remove(result[0]) + vnodeId = result[1] + print("its all data",dnodesList) + # if self.replicaVar == 1: + # redistributeSql = "redistribute vgroup %d dnode %d" %(vnodeId, dnodesList[0]) + # else: + redistributeSql = f"redistribute vgroup {vnodeId} " + for vgdnode in dnodesList: + redistributeSql += f"dnode {vgdnode} " + print(redistributeSql) + + tdLog.debug(f"redistributeSql:{redistributeSql}") + tdSql.query(redistributeSql) + tdLog.debug("redistributeSql ok") + + def tmqCaseStable(self): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb1', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 60, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + topicNameList = ['topic1'] + # expectRowsList = [] + tmqCom.initConsumerTable() + + tdLog.info("create topics from stb with filter") + queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName']) + # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + # tdSql.query(queryString) + # expectRowsList.append(tdSql.getRows()) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2 + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + + tdLog.info("create ctb1") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("create ctb2") + paraDict2 = paraDict.copy() + paraDict2['ctbPrefix'] = "ctb2" + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict2['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("insert data") + pInsertThread = tmqCom.asyncInsertDataByInterlace(paraDict) + + tmqCom.getStartConsumeNotifyFromTmqsim() + tmqCom.getStartCommitNotifyFromTmqsim() + + #restart dnode & remove wal + # self.restartAndRemoveWal() + + # redistribute vgroup + self.redistributeVgroups() + + + tdLog.info("insert data") + pInsertThread1 = tmqCom.asyncInsertDataByInterlace(paraDict2) + pInsertThread.join() + pInsertThread1.join() + + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + if expectrowcnt / 2 >= resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0])) + tdLog.exit("%d tmq consume rows error!"%consumerId) + + # tmqCom.checkFileContent(consumerId, queryString) + + time.sleep(5) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def tmqCaseNtable(self): + tdLog.printNoPrefix("======== test case 2: ") + paraDict = {'dbName':'dbt'} + + ntbName = "ntb" + + topicNameList = ['topic2'] + tmqCom.initConsumerTable() + + sqlString = "create table %s.%s(ts timestamp, i nchar(8))" %(paraDict['dbName'], ntbName) + tdLog.info("create nomal table sql: %s"%sqlString) + tdSql.execute(sqlString) + + tdLog.info("create topics from nomal table") + queryString = "select * from %s.%s"%(paraDict['dbName'], ntbName) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + tdSql.query("flush database %s"%(paraDict['dbName'])) + #restart dnode & remove wal + # self.restartAndRemoveWal() + + # redistribute vgroup + self.redistributeVgroups() + + sqlString = "alter table %s.%s modify column i nchar(16)" %(paraDict['dbName'], ntbName) + tdLog.info("alter table sql: %s"%sqlString) + tdSql.error(sqlString) + expectRows = 0 + resultList = tmqCom.selectConsumeResult(expectRows) + time.sleep(5) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 2 end ...... ") + + def tmqCaseStableSelect(self): + tdLog.printNoPrefix("======== test case subscrib column start : ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stbn', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 2, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + topicNameList = ['topic3'] + tmqCom.initConsumerTable() + + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + + tdLog.info("create ctb") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + tdLog.info("insert data") + tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("create topics from stb with filter") + queryString = "select * from %s.%s where c2 > 0 "%(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + + time.sleep(5) + #restart dnode & remove wal + # self.restartAndRemoveWal() + + # redistribute vgroup + self.redistributeVgroups() + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + time.sleep(5) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case subscrib column end ...... ") + + def tmqCaseDbname(self): + tdLog.printNoPrefix("======== test case subscrib Dbname start: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stbn', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 2, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + topicNameList = ['topic4'] + tmqCom.initConsumerTable() + + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + + tdLog.info("create ctb") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + tdLog.info("insert data") + tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("create topics from database ") + queryString = "database %s "%(paraDict['dbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + + time.sleep(5) + #restart dnode & remove wal + # self.restartAndRemoveWal() + + # redistribute vgroup + self.redistributeVgroups() + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + time.sleep(5) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case subscrib Dbname end ...... ") + + def run(self): + self.prepareTestEnv() + self.tmqCaseStable() + self.prepareTestEnv() + self.tmqCaseNtable() + self.prepareTestEnv() + self.tmqCaseStableSelect() + self.prepareTestEnv() + self.tmqCaseDbname() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From 0a1ff3283e50db9bde8cfe62a8a99166bd04504d Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 13 Oct 2023 16:59:05 +0800 Subject: [PATCH 07/51] vnode/cos: first round size delte and get --- cmake/curl_CMakeLists.txt.in | 3 +- cmake/libs3_CMakeLists.txt.in | 1 + contrib/CMakeLists.txt | 14 +- source/common/src/tglobal.c | 16 +- source/dnode/vnode/CMakeLists.txt | 29 +++- source/dnode/vnode/src/vnd/vnodeCos.c | 205 +++++++++++++++++++++++++- 6 files changed, 251 insertions(+), 17 deletions(-) diff --git a/cmake/curl_CMakeLists.txt.in b/cmake/curl_CMakeLists.txt.in index 1f2291c519..24a1210422 100644 --- a/cmake/curl_CMakeLists.txt.in +++ b/cmake/curl_CMakeLists.txt.in @@ -10,7 +10,8 @@ ExternalProject_Add(curl BUILD_IN_SOURCE TRUE BUILD_ALWAYS 1 #UPDATE_COMMAND "" - CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --without-ssl --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd + #CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --without-ssl --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd + CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --with-ssl=$ENV{HOME}/.cos-local.1 --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd #CONFIGURE_COMMAND ./configure --without-ssl BUILD_COMMAND make INSTALL_COMMAND make install diff --git a/cmake/libs3_CMakeLists.txt.in b/cmake/libs3_CMakeLists.txt.in index b77299baf3..b54a084f66 100644 --- a/cmake/libs3_CMakeLists.txt.in +++ b/cmake/libs3_CMakeLists.txt.in @@ -13,6 +13,7 @@ endfunction(update_cflags) ExternalProject_Add(libs3 GIT_REPOSITORY https://github.com/bji/libs3 #GIT_TAG v5.0.16 + DEPENDS curl SOURCE_DIR "${TD_CONTRIB_DIR}/libs3" #BINARY_DIR "" BUILD_IN_SOURCE TRUE diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 39bf85df8f..1588eb4025 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -13,10 +13,8 @@ if(${TD_LINUX}) set(CONTRIB_TMP_FILE3 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in3") configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3}) -if(${BUILD_WITH_COS}) file(MAKE_DIRECTORY $ENV{HOME}/.cos-local.1/) cat("${TD_SUPPORT_DIR}/ssl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3}) -endif(${BUILD_WITH_COS}) configure_file(${CONTRIB_TMP_FILE3} "${TD_CONTRIB_DIR}/deps-download/CMakeLists.txt") execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" . @@ -27,9 +25,7 @@ execute_process(COMMAND "${CMAKE_COMMAND}" --build . set(CONTRIB_TMP_FILE2 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in2") configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE2}) -if(${BUILD_WITH_COS}) cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE2}) -endif(${BUILD_WITH_COS}) configure_file(${CONTRIB_TMP_FILE2} "${TD_CONTRIB_DIR}/deps-download/CMakeLists.txt") execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" . @@ -129,6 +125,9 @@ if(${BUILD_TEST}) cat("${TD_SUPPORT_DIR}/stub_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) endif(${BUILD_TEST}) +# xml2 +cat("${TD_SUPPORT_DIR}/xml2_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) + # lz4 cat("${TD_SUPPORT_DIR}/lz4_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) @@ -195,6 +194,7 @@ if(${BUILD_WITH_S3}) #cat("${TD_SUPPORT_DIR}/apr-util_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) #cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) #INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.1/include) + cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE2}) cat("${TD_SUPPORT_DIR}/libs3_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) add_definitions(-DUSE_S3) @@ -295,6 +295,9 @@ target_include_directories( ) unset(CMAKE_PROJECT_INCLUDE_BEFORE) +# xml2 +add_subdirectory(xml2 EXCLUDE_FROM_ALL) + # lz4 add_subdirectory(lz4/build/cmake EXCLUDE_FROM_ALL) target_include_directories( @@ -439,6 +442,7 @@ endif() if(${BUILD_WITH_S3}) INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.1/include) + MESSAGE("build with s3: ${BUILD_WITH_S3}") else() @@ -449,7 +453,7 @@ if(${BUILD_WITH_COS}) #ADD_DEFINITIONS(-DMINIXML_LIBRARY=${CMAKE_BINARY_DIR}/build/lib/libxml.a) option(ENABLE_TEST "Enable the tests" OFF) INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.1/include) - MESSAGE("$ENV{HOME}/.cos-local.1/include") + #MESSAGE("$ENV{HOME}/.cos-local.1/include") set(CMAKE_BUILD_TYPE debug) set(ORIG_CMAKE_PROJECT_NAME ${CMAKE_PROJECT_NAME}) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 92b5ff2828..8ea5528fbb 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -15,12 +15,12 @@ #define _DEFAULT_SOURCE #include "tglobal.h" +#include "defines.h" #include "os.h" #include "tconfig.h" #include "tgrant.h" #include "tlog.h" #include "tmisce.h" -#include "defines.h" #if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL) #include "cus_name.h" @@ -221,7 +221,7 @@ float tsFPrecision = 1E-8; // float column precision double tsDPrecision = 1E-16; // double column precision uint32_t tsMaxRange = 500; // max quantization intervals uint32_t tsCurRange = 100; // current quantization intervals -bool tsIfAdtFse = false; // ADT-FSE algorithom or original huffman algorithom +bool tsIfAdtFse = false; // ADT-FSE algorithom or original huffman algorithom char tsCompressor[32] = "ZSTD_COMPRESSOR"; // ZSTD_COMPRESSOR or GZIP_COMPRESSOR // udf @@ -266,6 +266,9 @@ char tsS3BucketName[TSDB_FQDN_LEN] = ""; char tsS3AppId[TSDB_FQDN_LEN] = ""; int8_t tsS3Enabled = false; +int8_t tsS3Https = true; +char tsS3Hostname[TSDB_FQDN_LEN] = ""; + int32_t tsS3BlockSize = 4096; // number of tsdb pages int32_t tsS3BlockCacheSize = 16; // number of blocks @@ -307,6 +310,14 @@ int32_t taosSetS3Cfg(SConfig *pCfg) { tstrncpy(tsS3AccessKeySecret, colon + 1, TSDB_FQDN_LEN); tstrncpy(tsS3Endpoint, cfgGetItem(pCfg, "s3Endpoint")->str, TSDB_FQDN_LEN); tstrncpy(tsS3BucketName, cfgGetItem(pCfg, "s3BucketName")->str, TSDB_FQDN_LEN); + char *proto = strstr(tsS3Endpoint, "https://"); + if (!proto) { + tsS3Https = false; + tstrncpy(tsS3Hostname, tsS3Endpoint + 7, TSDB_FQDN_LEN); + } else { + tstrncpy(tsS3Hostname, tsS3Endpoint + 8, TSDB_FQDN_LEN); + } + char *cos = strstr(tsS3Endpoint, "cos."); if (cos) { char *appid = strrchr(tsS3BucketName, '-'); @@ -1086,7 +1097,6 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsIfAdtFse = cfgGetItem(pCfg, "IfAdtFse")->bval; tstrncpy(tsCompressor, cfgGetItem(pCfg, "Compressor")->str, sizeof(tsCompressor)); - tsDisableStream = cfgGetItem(pCfg, "disableStream")->bval; tsStreamBufferSize = cfgGetItem(pCfg, "streamBufferSize")->i64; diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 516c8a8b69..e554d94e4d 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -162,9 +162,34 @@ target_link_libraries( ) if(${TD_LINUX}) - if(${BUILD_WITH_S3}) - endif(${BUILD_WITH_S3}) +if(${BUILD_WITH_S3}) + MESSAGE("build with s3: ${BUILD_WITH_S3}") + target_include_directories( + vnode + + PUBLIC "$ENV{HOME}/.cos-local.1/include" + ) + + set(CMAKE_FIND_LIBRARY_SUFFIXES ".a") + set(CMAKE_PREFIX_PATH $ENV{HOME}/.cos-local.1) + find_library(S3_LIBRARY s3) + find_library(CURL_LIBRARY curl) + find_library(SSL_LIBRARY ssl PATHS $ENV{HOME}/.cos-local.1/lib64) + find_library(CRYPTO_LIBRARY crypto PATHS $ENV{HOME}/.cos-local.1/lib64) + target_link_libraries( + vnode + + # s3 + PUBLIC ${S3_LIBRARY} + PUBLIC ${CURL_LIBRARY} + PUBLIC ${SSL_LIBRARY} + PUBLIC ${CRYPTO_LIBRARY} + PUBLIC xml2 + ) + + add_definitions(-DUSE_S3) +endif(${BUILD_WITH_S3}) if(${BUILD_WITH_COS}) diff --git a/source/dnode/vnode/src/vnd/vnodeCos.c b/source/dnode/vnode/src/vnd/vnodeCos.c index 5f650be97d..8984d33ad8 100644 --- a/source/dnode/vnode/src/vnd/vnodeCos.c +++ b/source/dnode/vnode/src/vnd/vnodeCos.c @@ -2,13 +2,206 @@ #include "vndCos.h" -extern char tsS3Endpoint[]; -extern char tsS3AccessKeyId[]; -extern char tsS3AccessKeySecret[]; -extern char tsS3BucketName[]; -extern char tsS3AppId[]; +extern char tsS3Endpoint[]; +extern char tsS3AccessKeyId[]; +extern char tsS3AccessKeySecret[]; +extern char tsS3BucketName[]; +extern char tsS3AppId[]; +extern char tsS3Hostname[]; +extern int8_t tsS3Https; + +#if defined(USE_S3) + +#include "libs3.h" + +static int verifyPeerG = 0; +static const char *awsRegionG = NULL; +static int forceG = 0; +static int showResponsePropertiesG = 0; +static S3Protocol protocolG = S3ProtocolHTTPS; +// static S3Protocol protocolG = S3ProtocolHTTP; +static S3UriStyle uriStyleG = S3UriStylePath; +static int retriesG = 5; +static int timeoutMsG = 0; + +static int32_t s3Begin() { + S3Status status; + const char *hostname = tsS3Hostname; + const char *env_hn = getenv("S3_HOSTNAME"); + + if (env_hn) { + hostname = env_hn; + } + + if ((status = S3_initialize("s3", verifyPeerG | S3_INIT_ALL, hostname)) != S3StatusOK) { + vError("Failed to initialize libs3: %s\n", S3_get_status_name(status)); + return -1; + } + + protocolG = !tsS3Https; + + return 0; +} + +static void s3End() { S3_deinitialize(); } +int32_t s3Init() { return s3Begin(); } + +void s3CleanUp() { s3End(); } + +static int should_retry() { + /* + if (retriesG--) { + // Sleep before next retry; start out with a 1 second sleep + static int retrySleepInterval = 1 * SLEEP_UNITS_PER_SECOND; + sleep(retrySleepInterval); + // Next sleep 1 second longer + retrySleepInterval++; + return 1; + } + */ + + return 0; +} + +typedef struct { + uint64_t content_length; + int status; + char *buf; + char err_msg[4096]; +} TS3SizeCBD; + +static S3Status responsePropertiesCallback(const S3ResponseProperties *properties, void *callbackData) { + //(void)callbackData; + TS3SizeCBD *cbd = callbackData; + if (properties->contentLength > 0) { + cbd->content_length = properties->contentLength; + } else { + cbd->content_length = 0; + } + + return S3StatusOK; +} + +static void responseCompleteCallback(S3Status status, const S3ErrorDetails *error, void *callbackData) { + TS3SizeCBD *cbd = callbackData; + cbd->status = status; + + int len = 0; + const int elen = sizeof(cbd->err_msg); + if (error) { + if (error->message) { + len += snprintf(&(cbd->err_msg[len]), elen - len, " Message: %s\n", error->message); + } + if (error->resource) { + len += snprintf(&(cbd->err_msg[len]), elen - len, " Resource: %s\n", error->resource); + } + if (error->furtherDetails) { + len += snprintf(&(cbd->err_msg[len]), elen - len, " Further Details: %s\n", error->furtherDetails); + } + if (error->extraDetailsCount) { + len += snprintf(&(cbd->err_msg[len]), elen - len, "%s", " Extra Details:\n"); + for (int i = 0; i < error->extraDetailsCount; i++) { + len += snprintf(&(cbd->err_msg[len]), elen - len, " %s: %s\n", error->extraDetails[i].name, + error->extraDetails[i].value); + } + } + } +} + +int32_t s3PutObjectFromFile2(const char *file, const char *object) { return 0; } +void s3DeleteObjectsByPrefix(const char *prefix) {} + +void s3DeleteObjects(const char *object_name[], int nobject) { + int status = 0; + S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, + 0, awsRegionG}; + S3ResponseHandler responseHandler = {0, &responseCompleteCallback}; + + for (int i = 0; i < nobject; ++i) { + TS3SizeCBD cbd = {0}; + do { + S3_delete_object(&bucketContext, object_name[i], 0, timeoutMsG, &responseHandler, &cbd); + } while (S3_status_is_retryable(cbd.status) && should_retry()); + + if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) { + vError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg); + } + } +} + +static S3Status getObjectDataCallback(int bufferSize, const char *buffer, void *callbackData) { + TS3SizeCBD *cbd = callbackData; + if (cbd->content_length != bufferSize) { + cbd->status = S3StatusAbortedByCallback; + return S3StatusAbortedByCallback; + } + + char *buf = taosMemoryCalloc(1, bufferSize); + if (buf) { + memcpy(buf, buffer, bufferSize); + + cbd->status = S3StatusOK; + return S3StatusOK; + } else { + cbd->status = S3StatusAbortedByCallback; + return S3StatusAbortedByCallback; + } +} + +int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size, uint8_t **ppBlock) { + int status = 0; + int64_t ifModifiedSince = -1, ifNotModifiedSince = -1; + const char *ifMatch = 0, *ifNotMatch = 0; + + S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, + 0, awsRegionG}; + S3GetConditions getConditions = {ifModifiedSince, ifNotModifiedSince, ifMatch, ifNotMatch}; + S3GetObjectHandler getObjectHandler = {{&responsePropertiesCallback, &responseCompleteCallback}, + &getObjectDataCallback}; + + TS3SizeCBD cbd = {0}; + cbd.content_length = size; + do { + S3_get_object(&bucketContext, object_name, &getConditions, offset, size, 0, 0, &getObjectHandler, &cbd); + } while (S3_status_is_retryable(cbd.status) && should_retry()); + + if (cbd.status != S3StatusOK) { + vError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg); + return TAOS_SYSTEM_ERROR(EIO); + } + + *ppBlock = cbd.buf; + + return 0; +} + +long s3Size(const char *object_name) { + long size = 0; + int status = 0; + + S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, + 0, awsRegionG}; + + S3ResponseHandler responseHandler = {&responsePropertiesCallback, &responseCompleteCallback}; + + TS3SizeCBD cbd = {0}; + do { + S3_head_object(&bucketContext, object_name, 0, 0, &responseHandler, &cbd); + } while (S3_status_is_retryable(cbd.status) && should_retry()); + + if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) { + vError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg); + } + + size = cbd.content_length; + + return size; +} + +void s3EvictCache(const char *path, long object_size) {} + +#elif defined(USE_COS) -#ifdef USE_COS #include "cos_api.h" #include "cos_http_io.h" #include "cos_log.h" From 11f7cdcf9d3b6faea3db55cbc4227ecbee29b4cc Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 13 Oct 2023 17:16:19 +0800 Subject: [PATCH 08/51] cos/xml2: cmake file for xml2 --- cmake/xml2_CMakeLists.txt.in | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 cmake/xml2_CMakeLists.txt.in diff --git a/cmake/xml2_CMakeLists.txt.in b/cmake/xml2_CMakeLists.txt.in new file mode 100644 index 0000000000..4c234d7a37 --- /dev/null +++ b/cmake/xml2_CMakeLists.txt.in @@ -0,0 +1,12 @@ + +# xml2 +ExternalProject_Add(xml2 + GIT_REPOSITORY https://github.com/GNOME/libxml2 + GIT_TAG v2.11.5 + SOURCE_DIR "${TD_CONTRIB_DIR}/xml2" + BINARY_DIR "" + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + TEST_COMMAND "" + ) From c538478d0f91c26c34f998a31b0461016a769e9a Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 13 Oct 2023 17:30:09 +0800 Subject: [PATCH 09/51] contrib/cmake: fix curl file --- contrib/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 1588eb4025..1291506136 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -194,7 +194,7 @@ if(${BUILD_WITH_S3}) #cat("${TD_SUPPORT_DIR}/apr-util_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) #cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) #INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.1/include) - cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE2}) + cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) cat("${TD_SUPPORT_DIR}/libs3_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) add_definitions(-DUSE_S3) From f66e57bf1f650139e5d40ba939387da72edd7d2c Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 13 Oct 2023 17:47:42 +0800 Subject: [PATCH 10/51] contrib/cos: make libs3 depends xml2 --- cmake/libs3_CMakeLists.txt.in | 2 +- contrib/CMakeLists.txt | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/cmake/libs3_CMakeLists.txt.in b/cmake/libs3_CMakeLists.txt.in index b54a084f66..93d0eff41a 100644 --- a/cmake/libs3_CMakeLists.txt.in +++ b/cmake/libs3_CMakeLists.txt.in @@ -13,7 +13,7 @@ endfunction(update_cflags) ExternalProject_Add(libs3 GIT_REPOSITORY https://github.com/bji/libs3 #GIT_TAG v5.0.16 - DEPENDS curl + DEPENDS curl xml2 SOURCE_DIR "${TD_CONTRIB_DIR}/libs3" #BINARY_DIR "" BUILD_IN_SOURCE TRUE diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 1291506136..24ad63db13 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -125,9 +125,6 @@ if(${BUILD_TEST}) cat("${TD_SUPPORT_DIR}/stub_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) endif(${BUILD_TEST}) -# xml2 -cat("${TD_SUPPORT_DIR}/xml2_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) - # lz4 cat("${TD_SUPPORT_DIR}/lz4_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) @@ -194,6 +191,8 @@ if(${BUILD_WITH_S3}) #cat("${TD_SUPPORT_DIR}/apr-util_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) #cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) #INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.1/include) + + cat("${TD_SUPPORT_DIR}/xml2_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) cat("${TD_SUPPORT_DIR}/libs3_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) add_definitions(-DUSE_S3) From 4a77a49a3e3c38d6519390eb6d00df15496a9d9b Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 16 Oct 2023 15:25:48 +0800 Subject: [PATCH 11/51] prefix/list: list bucket with prefix --- cmake/libs3_CMakeLists.txt.in | 2 +- contrib/CMakeLists.txt | 2 +- source/dnode/vnode/CMakeLists.txt | 10 +-- source/dnode/vnode/src/vnd/vnodeCos.c | 93 ++++++++++++++++++++++++++- 4 files changed, 97 insertions(+), 10 deletions(-) diff --git a/cmake/libs3_CMakeLists.txt.in b/cmake/libs3_CMakeLists.txt.in index 93d0eff41a..790a6fe312 100644 --- a/cmake/libs3_CMakeLists.txt.in +++ b/cmake/libs3_CMakeLists.txt.in @@ -13,7 +13,7 @@ endfunction(update_cflags) ExternalProject_Add(libs3 GIT_REPOSITORY https://github.com/bji/libs3 #GIT_TAG v5.0.16 - DEPENDS curl xml2 + DEPENDS curl xml2 openssl SOURCE_DIR "${TD_CONTRIB_DIR}/libs3" #BINARY_DIR "" BUILD_IN_SOURCE TRUE diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 24ad63db13..63a60fa59a 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -14,7 +14,6 @@ set(CONTRIB_TMP_FILE3 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in3") configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3}) file(MAKE_DIRECTORY $ENV{HOME}/.cos-local.1/) - cat("${TD_SUPPORT_DIR}/ssl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3}) configure_file(${CONTRIB_TMP_FILE3} "${TD_CONTRIB_DIR}/deps-download/CMakeLists.txt") execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" . @@ -192,6 +191,7 @@ if(${BUILD_WITH_S3}) #cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) #INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.1/include) + cat("${TD_SUPPORT_DIR}/ssl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3}) cat("${TD_SUPPORT_DIR}/xml2_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) cat("${TD_SUPPORT_DIR}/libs3_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index e554d94e4d..d4ea90f48a 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -181,11 +181,11 @@ if(${BUILD_WITH_S3}) vnode # s3 - PUBLIC ${S3_LIBRARY} - PUBLIC ${CURL_LIBRARY} - PUBLIC ${SSL_LIBRARY} - PUBLIC ${CRYPTO_LIBRARY} - PUBLIC xml2 + PRIVATE ${S3_LIBRARY} + PRIVATE ${CURL_LIBRARY} + PRIVATE ${SSL_LIBRARY} + PRIVATE ${CRYPTO_LIBRARY} + PRIVATE xml2 ) add_definitions(-DUSE_S3) diff --git a/source/dnode/vnode/src/vnd/vnodeCos.c b/source/dnode/vnode/src/vnd/vnodeCos.c index 8984d33ad8..cf19a08813 100644 --- a/source/dnode/vnode/src/vnd/vnodeCos.c +++ b/source/dnode/vnode/src/vnd/vnodeCos.c @@ -63,9 +63,17 @@ static int should_retry() { return 0; } +static void s3PrintError(const char *func, S3Status status, char error_details[]) { + if (status < S3StatusErrorAccessDenied) { + vError("%s: %s", __func__, S3_get_status_name(status)); + } else { + vError("%s: %s, %s", __func__, S3_get_status_name(status), error_details); + } +} + typedef struct { uint64_t content_length; - int status; + S3Status status; char *buf; char err_msg[4096]; } TS3SizeCBD; @@ -109,7 +117,86 @@ static void responseCompleteCallback(S3Status status, const S3ErrorDetails *erro } int32_t s3PutObjectFromFile2(const char *file, const char *object) { return 0; } -void s3DeleteObjectsByPrefix(const char *prefix) {} + +typedef struct list_bucket_callback_data { + int isTruncated; + char nextMarker[1024]; + int keyCount; + int allDetails; + S3Status status; + char err_msg[4096]; +} list_bucket_callback_data; + +static S3Status listBucketCallback(int isTruncated, const char *nextMarker, int contentsCount, + const S3ListBucketContent *contents, int commonPrefixesCount, + const char **commonPrefixes, void *callbackData) { + list_bucket_callback_data *data = (list_bucket_callback_data *)callbackData; + + data->isTruncated = isTruncated; + if ((!nextMarker || !nextMarker[0]) && contentsCount) { + nextMarker = contents[contentsCount - 1].key; + } + if (nextMarker) { + snprintf(data->nextMarker, sizeof(data->nextMarker), "%s", nextMarker); + } else { + data->nextMarker[0] = 0; + } + + if (contentsCount && !data->keyCount) { + // printListBucketHeader(data->allDetails); + } + + int i; + for (i = 0; i < contentsCount; ++i) { + const S3ListBucketContent *content = &(contents[i]); + printf("%-50s", content->key); + } + data->keyCount += contentsCount; + + for (i = 0; i < commonPrefixesCount; i++) { + // printf("\nCommon Prefix: %s\n", commonPrefixes[i]); + } + + return S3StatusOK; +} + +void s3DeleteObjectsByPrefix(const char *prefix) { + S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, + 0, awsRegionG}; + S3ListBucketHandler listBucketHandler = {{&responsePropertiesCallback, &responseCompleteCallback}, + &listBucketCallback}; + + const char *marker = 0, *delimiter = 0; + int maxkeys = 0, allDetails = 0; + list_bucket_callback_data data; + + if (marker) { + snprintf(data.nextMarker, sizeof(data.nextMarker), "%s", marker); + } else { + data.nextMarker[0] = 0; + } + data.keyCount = 0; + data.allDetails = allDetails; + + do { + data.isTruncated = 0; + do { + S3_list_bucket(&bucketContext, prefix, data.nextMarker, delimiter, maxkeys, 0, timeoutMsG, &listBucketHandler, + &data); + } while (S3_status_is_retryable(data.status) && should_retry()); + if (data.status != S3StatusOK) { + break; + } + } while (data.isTruncated && (!maxkeys || (data.keyCount < maxkeys))); + + if (data.status == S3StatusOK) { + if (!data.keyCount) { + // printListBucketHeader(allDetails); + } + } else { + s3PrintError(__func__, data.status, data.err_msg); + } +} void s3DeleteObjects(const char *object_name[], int nobject) { int status = 0; @@ -124,7 +211,7 @@ void s3DeleteObjects(const char *object_name[], int nobject) { } while (S3_status_is_retryable(cbd.status) && should_retry()); if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) { - vError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg); + s3PrintError(__func__, cbd.status, cbd.err_msg); } } } From 2479bb343e56f98f37ad749e6607d2449b8892a7 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 16 Oct 2023 15:54:51 +0800 Subject: [PATCH 12/51] prefix/delete: delete with prefix --- contrib/CMakeLists.txt | 2 +- source/dnode/vnode/src/vnd/vnodeCos.c | 19 +++++++++++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 63a60fa59a..b40a7101c3 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -191,7 +191,7 @@ if(${BUILD_WITH_S3}) #cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) #INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.1/include) - cat("${TD_SUPPORT_DIR}/ssl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3}) + cat("${TD_SUPPORT_DIR}/ssl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) cat("${TD_SUPPORT_DIR}/xml2_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) cat("${TD_SUPPORT_DIR}/libs3_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) diff --git a/source/dnode/vnode/src/vnd/vnodeCos.c b/source/dnode/vnode/src/vnd/vnodeCos.c index cf19a08813..726d78e167 100644 --- a/source/dnode/vnode/src/vnd/vnodeCos.c +++ b/source/dnode/vnode/src/vnd/vnodeCos.c @@ -123,6 +123,7 @@ typedef struct list_bucket_callback_data { char nextMarker[1024]; int keyCount; int allDetails; + SArray *objectArray; S3Status status; char err_msg[4096]; } list_bucket_callback_data; @@ -149,7 +150,9 @@ static S3Status listBucketCallback(int isTruncated, const char *nextMarker, int int i; for (i = 0; i < contentsCount; ++i) { const S3ListBucketContent *content = &(contents[i]); - printf("%-50s", content->key); + // printf("%-50s", content->key); + char *object_key = strdup(content->key); + taosArrayPush(data->objectArray, &object_key); } data->keyCount += contentsCount; @@ -160,6 +163,11 @@ static S3Status listBucketCallback(int isTruncated, const char *nextMarker, int return S3StatusOK; } +static void s3FreeObjectKey(void *pItem) { + char *key = (char *)pItem; + taosMemoryFree(key); +} + void s3DeleteObjectsByPrefix(const char *prefix) { S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, 0, awsRegionG}; @@ -169,7 +177,11 @@ void s3DeleteObjectsByPrefix(const char *prefix) { const char *marker = 0, *delimiter = 0; int maxkeys = 0, allDetails = 0; list_bucket_callback_data data; - + data.objectArray = taosArrayInit(32, POINTER_BYTES); + if (!data.objectArray) { + vError("%s: %s", __func__, "out of memoty"); + return; + } if (marker) { snprintf(data.nextMarker, sizeof(data.nextMarker), "%s", marker); } else { @@ -192,10 +204,13 @@ void s3DeleteObjectsByPrefix(const char *prefix) { if (data.status == S3StatusOK) { if (!data.keyCount) { // printListBucketHeader(allDetails); + s3DeleteObjects(TARRAY_DATA(data.objectArray), TARRAY_SIZE(data.objectArray)); } } else { s3PrintError(__func__, data.status, data.err_msg); } + + taosArrayDestroyEx(data.objectArray, s3FreeObjectKey); } void s3DeleteObjects(const char *object_name[], int nobject) { From 93b99e4573e315f1a5bd5d0133738c44d03a10f9 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Mon, 16 Oct 2023 18:21:19 +0800 Subject: [PATCH 13/51] test: support replica3 in splited/transform tmqvnode --- tests/parallel_test/cases.task | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 9a888c1225..b221d99666 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -172,9 +172,14 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -i True ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -n 3 -i True ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform.py -N 2 -n 1 -,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform.py -N 6 -n 3 -,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit.py -N 2 -n 1 -,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit.py -N 3 -n 3 +,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-stb.py -N 6 -n 3 +,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select.py -N 2 -n 1 +,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py -N 3 -n 3 +,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select.py -N 3 -n 3 +,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-stb.py -N 3 -n 3 +,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-column.py.py -N 3 -n 3 +,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-db.py -N 3 -n 3 +e ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeReplicate.py -M 3 -N 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-19201.py ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-21561.py From 38c1f7d48d0b692e29a32c657f948b9901f8ead3 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 16 Oct 2023 20:20:04 +0800 Subject: [PATCH 14/51] fix: add lock for trans --- source/dnode/mnode/impl/inc/mndDef.h | 2 +- source/dnode/mnode/impl/src/mndTrans.c | 12 +++--------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index f4236964ca..e5f46c03cd 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -181,7 +181,7 @@ typedef struct { SArray* pRpcArray; SRWLatch lockRpcArray; int64_t mTraceId; - TdThreadMutex mutex; + int8_t lock; } STrans; typedef struct { diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 1d8dd5e345..f8d6cba171 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -470,7 +470,6 @@ void mndTransDropData(STrans *pTrans) { pTrans->param = NULL; pTrans->paramLen = 0; } - (void)taosThreadMutexDestroy(&pTrans->mutex); } static int32_t mndTransDelete(SSdb *pSdb, STrans *pTrans, bool callFunc) { @@ -543,10 +542,6 @@ STrans *mndAcquireTrans(SMnode *pMnode, int32_t transId) { STrans *pTrans = sdbAcquire(pMnode->pSdb, SDB_TRANS, &transId); if (pTrans == NULL) { terrno = TSDB_CODE_MND_TRANS_NOT_EXIST; - } else { - #ifdef WINDOWS - taosThreadMutexInit(&pTrans->mutex, NULL); - #endif } return pTrans; } @@ -582,7 +577,6 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnConflct conflict, pTrans->pRpcArray = taosArrayInit(1, sizeof(SRpcHandleInfo)); pTrans->mTraceId = pReq ? TRACE_GET_ROOTID(&pReq->info.traceId) : tGenIdPI64(); taosInitRWLatch(&pTrans->lockRpcArray); - taosThreadMutexInit(&pTrans->mutex, NULL); if (pTrans->redoActions == NULL || pTrans->undoActions == NULL || pTrans->commitActions == NULL || pTrans->pRpcArray == NULL) { @@ -1264,10 +1258,10 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans) int32_t numOfActions = taosArrayGetSize(pTrans->redoActions); if (numOfActions == 0) return code; - taosThreadMutexLock(&pTrans->mutex); + if (atomic_val_compare_exchange_8(&pTrans->lock, 0, 1) != 0) return code; if (pTrans->redoActionPos >= numOfActions) { - taosThreadMutexUnlock(&pTrans->mutex); + atomic_store_8(&pTrans->lock, 0); return code; } @@ -1339,7 +1333,7 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans) } } - taosThreadMutexUnlock(&pTrans->mutex); + atomic_store_8(&pTrans->lock, 0); return code; } From a30eedafecc8a320afb0fac4dbb25a115c532ecd Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 17 Oct 2023 11:06:06 +0800 Subject: [PATCH 15/51] fix: trans mutex init --- source/dnode/mnode/impl/inc/mndDef.h | 2 +- source/dnode/mnode/impl/src/mndTrans.c | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index e5f46c03cd..f4236964ca 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -181,7 +181,7 @@ typedef struct { SArray* pRpcArray; SRWLatch lockRpcArray; int64_t mTraceId; - int8_t lock; + TdThreadMutex mutex; } STrans; typedef struct { diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index f8d6cba171..76d6e21c64 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -428,6 +428,8 @@ static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans) { mInfo("trans:%d, perform insert action, row:%p stage:%s, callfunc:1, startFunc:%d", pTrans->id, pTrans, mndTransStr(pTrans->stage), pTrans->startFunc); + taosThreadMutexInit(&pTrans->mutex, NULL); + if (pTrans->startFunc > 0) { TransCbFp fp = mndTransGetCbFp(pTrans->startFunc); if (fp) { @@ -1258,10 +1260,10 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans) int32_t numOfActions = taosArrayGetSize(pTrans->redoActions); if (numOfActions == 0) return code; - if (atomic_val_compare_exchange_8(&pTrans->lock, 0, 1) != 0) return code; + taosThreadMutexLock(&pTrans->mutex); if (pTrans->redoActionPos >= numOfActions) { - atomic_store_8(&pTrans->lock, 0); + taosThreadMutexUnlock(&pTrans->mutex); return code; } @@ -1333,7 +1335,7 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans) } } - atomic_store_8(&pTrans->lock, 0); + taosThreadMutexUnlock(&pTrans->mutex); return code; } From 29c6daaff5efd137cc9d16f5aaa5a5869a766e45 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 17 Oct 2023 11:12:52 +0800 Subject: [PATCH 16/51] fix: trans mutex init --- source/dnode/mnode/impl/src/mndTrans.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 76d6e21c64..29a8ae1f29 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -472,6 +472,7 @@ void mndTransDropData(STrans *pTrans) { pTrans->param = NULL; pTrans->paramLen = 0; } + (void)taosThreadMutexDestroy(&pTrans->mutex); } static int32_t mndTransDelete(SSdb *pSdb, STrans *pTrans, bool callFunc) { @@ -579,6 +580,7 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnConflct conflict, pTrans->pRpcArray = taosArrayInit(1, sizeof(SRpcHandleInfo)); pTrans->mTraceId = pReq ? TRACE_GET_ROOTID(&pReq->info.traceId) : tGenIdPI64(); taosInitRWLatch(&pTrans->lockRpcArray); + taosThreadMutexInit(&pTrans->mutex, NULL); if (pTrans->redoActions == NULL || pTrans->undoActions == NULL || pTrans->commitActions == NULL || pTrans->pRpcArray == NULL) { From 1459c82d75ef5d7cd43e525c7c55dcd0885e4513 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Tue, 17 Oct 2023 11:23:21 +0800 Subject: [PATCH 17/51] test: support replica3 in splited/transform tmqvnode --- tests/parallel_test/cases.task | 4 ++-- .../7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index b221d99666..66ffc7f6dd 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -171,13 +171,13 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqSubscribeStb-r3.py -N 5 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -i True ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -n 3 -i True -,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform.py -N 2 -n 1 +,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-stb.py -N 2 -n 1 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-stb.py -N 6 -n 3 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select.py -N 2 -n 1 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py -N 3 -n 3 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select.py -N 3 -n 3 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-stb.py -N 3 -n 3 -,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-column.py.py -N 3 -n 3 +,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-column.py -N 3 -n 3 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-db.py -N 3 -n 3 e ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeReplicate.py -M 3 -N 3 -n 3 diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py index 086fe6e4c5..c76188eee2 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py @@ -184,7 +184,7 @@ class TDTestCase: expectRows = 1 resultList = tmqCom.selectConsumeResult(expectRows) - if expectrowcnt / 2 >= resultList[0] or expectrowcnt <= resultList[0]: + if expectrowcnt / 2 >= resultList[0]: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0])) tdLog.exit("%d tmq consume rows error!"%consumerId) From 8fa8c40391fbea4dbb668202dabb9160a281d289 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 17 Oct 2023 14:59:03 +0800 Subject: [PATCH 18/51] cos/put: simple object putting(< 5 gigabytes) --- source/dnode/vnode/src/vnd/vnodeCos.c | 250 +++++++++++++++++++++++++- 1 file changed, 249 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/vnd/vnodeCos.c b/source/dnode/vnode/src/vnd/vnodeCos.c index 726d78e167..2a36a898e0 100644 --- a/source/dnode/vnode/src/vnd/vnodeCos.c +++ b/source/dnode/vnode/src/vnd/vnodeCos.c @@ -116,7 +116,255 @@ static void responseCompleteCallback(S3Status status, const S3ErrorDetails *erro } } -int32_t s3PutObjectFromFile2(const char *file, const char *object) { return 0; } +typedef struct growbuffer { + // The total number of bytes, and the start byte + int size; + // The start byte + int start; + // The blocks + char data[64 * 1024]; + struct growbuffer *prev, *next; +} growbuffer; + +// returns nonzero on success, zero on out of memory +static int growbuffer_append(growbuffer **gb, const char *data, int dataLen) { + int origDataLen = dataLen; + while (dataLen) { + growbuffer *buf = *gb ? (*gb)->prev : 0; + if (!buf || (buf->size == sizeof(buf->data))) { + buf = (growbuffer *)malloc(sizeof(growbuffer)); + if (!buf) { + return 0; + } + buf->size = 0; + buf->start = 0; + if (*gb && (*gb)->prev) { + buf->prev = (*gb)->prev; + buf->next = *gb; + (*gb)->prev->next = buf; + (*gb)->prev = buf; + } else { + buf->prev = buf->next = buf; + *gb = buf; + } + } + + int toCopy = (sizeof(buf->data) - buf->size); + if (toCopy > dataLen) { + toCopy = dataLen; + } + + memcpy(&(buf->data[buf->size]), data, toCopy); + + buf->size += toCopy, data += toCopy, dataLen -= toCopy; + } + + return origDataLen; +} + +static void growbuffer_read(growbuffer **gb, int amt, int *amtReturn, char *buffer) { + *amtReturn = 0; + + growbuffer *buf = *gb; + + if (!buf) { + return; + } + + *amtReturn = (buf->size > amt) ? amt : buf->size; + + memcpy(buffer, &(buf->data[buf->start]), *amtReturn); + + buf->start += *amtReturn, buf->size -= *amtReturn; + + if (buf->size == 0) { + if (buf->next == buf) { + *gb = 0; + } else { + *gb = buf->next; + buf->prev->next = buf->next; + buf->next->prev = buf->prev; + } + free(buf); + buf = NULL; + } +} + +static void growbuffer_destroy(growbuffer *gb) { + growbuffer *start = gb; + + while (gb) { + growbuffer *next = gb->next; + free(gb); + gb = (next == start) ? 0 : next; + } +} + +typedef struct put_object_callback_data { + // FILE *infile; + TdFilePtr infileFD; + growbuffer *gb; + uint64_t contentLength, originalContentLength; + uint64_t totalContentLength, totalOriginalContentLength; + int noStatus; + S3Status status; + char err_msg[4096]; +} put_object_callback_data; + +static int putObjectDataCallback(int bufferSize, char *buffer, void *callbackData) { + put_object_callback_data *data = (put_object_callback_data *)callbackData; + + int ret = 0; + + if (data->contentLength) { + int toRead = ((data->contentLength > (unsigned)bufferSize) ? (unsigned)bufferSize : data->contentLength); + if (data->gb) { + growbuffer_read(&(data->gb), toRead, &ret, buffer); + } else if (data->infileFD) { + // ret = fread(buffer, 1, toRead, data->infile); + ret = taosReadFile(data->infileFD, buffer, toRead); + } + } + + data->contentLength -= ret; + data->totalContentLength -= ret; + + if (data->contentLength && !data->noStatus) { + vTrace("%llu bytes remaining ", (unsigned long long)data->totalContentLength); + vTrace("(%d%% complete) ...\n", (int)(((data->totalOriginalContentLength - data->totalContentLength) * 100) / + data->totalOriginalContentLength)); + } + + return ret; +} + +#define MULTIPART_CHUNK_SIZE (768 << 20) // multipart is 768M + +typedef struct UploadManager { + // used for initial multipart + char *upload_id; + + // used for upload part object + char **etags; + int next_etags_pos; + + // used for commit Upload + growbuffer *gb; + int remaining; +} UploadManager; + +typedef struct MultipartPartData { + put_object_callback_data put_object_data; + int seq; + UploadManager *manager; +} MultipartPartData; + +S3Status initial_multipart_callback(const char *upload_id, void *callbackData) { + UploadManager *manager = (UploadManager *)callbackData; + manager->upload_id = strdup(upload_id); + return S3StatusOK; +} + +S3Status MultipartResponseProperiesCallback(const S3ResponseProperties *properties, void *callbackData) { + responsePropertiesCallback(properties, callbackData); + + MultipartPartData *data = (MultipartPartData *)callbackData; + int seq = data->seq; + const char *etag = properties->eTag; + data->manager->etags[seq - 1] = strdup(etag); + data->manager->next_etags_pos = seq; + return S3StatusOK; +} + +static int multipartPutXmlCallback(int bufferSize, char *buffer, void *callbackData) { + UploadManager *manager = (UploadManager *)callbackData; + int ret = 0; + + if (manager->remaining) { + int toRead = ((manager->remaining > bufferSize) ? bufferSize : manager->remaining); + growbuffer_read(&(manager->gb), toRead, &ret, buffer); + } + manager->remaining -= ret; + return ret; +} + +static int try_get_parts_info(const char *bucketName, const char *key, UploadManager *manager) { + // + + return 0; +} + +int32_t s3PutObjectFromFile2(const char *file, const char *object) { + int32_t code = 0; + const char *key = object; + const char *uploadId = 0; + const char *filename = 0; + uint64_t contentLength = 0; + const char *cacheControl = 0, *contentType = 0, *md5 = 0; + const char *contentDispositionFilename = 0, *contentEncoding = 0; + int64_t expires = -1; + S3CannedAcl cannedAcl = S3CannedAclPrivate; + int metaPropertiesCount = 0; + S3NameValue metaProperties[S3_MAX_METADATA_COUNT]; + char useServerSideEncryption = 0; + int noStatus = 0; + put_object_callback_data data; + + // data.infile = 0; + data.infileFD = NULL; + data.gb = 0; + data.noStatus = noStatus; + + if (taosStatFile(file, &contentLength, NULL, NULL) < 0) { + vError("ERROR: %s Failed to stat file %s: ", __func__, file); + code = TAOS_SYSTEM_ERROR(errno); + return code; + } + + if (!(data.infileFD = taosOpenFile(file, TD_FILE_READ))) { + vError("ERROR: %s Failed to open file %s: ", __func__, file); + code = TAOS_SYSTEM_ERROR(errno); + return code; + } + + data.totalContentLength = data.totalOriginalContentLength = data.contentLength = data.originalContentLength = + contentLength; + + S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, + 0, awsRegionG}; + + S3PutProperties putProperties = {contentType, md5, + cacheControl, contentDispositionFilename, + contentEncoding, expires, + cannedAcl, metaPropertiesCount, + metaProperties, useServerSideEncryption}; + + if (contentLength <= MULTIPART_CHUNK_SIZE) { + S3PutObjectHandler putObjectHandler = {{&responsePropertiesCallback, &responseCompleteCallback}, + &putObjectDataCallback}; + + do { + S3_put_object(&bucketContext, key, contentLength, &putProperties, 0, 0, &putObjectHandler, &data); + } while (S3_status_is_retryable(data.status) && should_retry()); + + if (data.infileFD) { + taosCloseFile(&data.infileFD); + } else if (data.gb) { + growbuffer_destroy(data.gb); + } + + if (data.status != S3StatusOK) { + s3PrintError(__func__, data.status, data.err_msg); + } else if (data.contentLength) { + vError("ERROR: %s Failed to read remaining %llu bytes from input", __func__, + (unsigned long long)data.contentLength); + } + } else { + uint64_t totalContentLength = contentLength; + uint64_t todoContentLength = contentLength; + } + return 0; +} typedef struct list_bucket_callback_data { int isTruncated; From d8b0d8085be4cc51903d489c915a2a8a5331b9d4 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 17 Oct 2023 15:16:20 +0800 Subject: [PATCH 19/51] cmake/ssl: make curl depends openssl --- cmake/curl_CMakeLists.txt.in | 1 + cmake/libs3_CMakeLists.txt.in | 4 ++-- contrib/CMakeLists.txt | 1 - 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/curl_CMakeLists.txt.in b/cmake/curl_CMakeLists.txt.in index 24a1210422..fed98c5f0e 100644 --- a/cmake/curl_CMakeLists.txt.in +++ b/cmake/curl_CMakeLists.txt.in @@ -7,6 +7,7 @@ ExternalProject_Add(curl #GIT_REPOSITORY https://github.com/curl/curl.git #GIT_TAG curl-7_88_1 SOURCE_DIR "${TD_CONTRIB_DIR}/curl" + DEPENDS openssl BUILD_IN_SOURCE TRUE BUILD_ALWAYS 1 #UPDATE_COMMAND "" diff --git a/cmake/libs3_CMakeLists.txt.in b/cmake/libs3_CMakeLists.txt.in index 790a6fe312..b2d2d19059 100644 --- a/cmake/libs3_CMakeLists.txt.in +++ b/cmake/libs3_CMakeLists.txt.in @@ -12,8 +12,8 @@ endfunction(update_cflags) ExternalProject_Add(libs3 GIT_REPOSITORY https://github.com/bji/libs3 - #GIT_TAG v5.0.16 - DEPENDS curl xml2 openssl + #GIT_TAG v5.0.16 + DEPENDS curl xml2 SOURCE_DIR "${TD_CONTRIB_DIR}/libs3" #BINARY_DIR "" BUILD_IN_SOURCE TRUE diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index b40a7101c3..07201f8713 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -277,7 +277,6 @@ if(${BUILD_TEST}) ) endif(${TD_DARWIN}) - endif(${BUILD_TEST}) # cJson From ddfdbb3207415134c8b5c684864debe6da716848 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 17 Oct 2023 15:28:36 +0800 Subject: [PATCH 20/51] xml2: add sub dir if building libs3 --- contrib/CMakeLists.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 07201f8713..de3d9cfd37 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -24,7 +24,7 @@ execute_process(COMMAND "${CMAKE_COMMAND}" --build . set(CONTRIB_TMP_FILE2 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in2") configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE2}) - cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE2}) + #cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE2}) configure_file(${CONTRIB_TMP_FILE2} "${TD_CONTRIB_DIR}/deps-download/CMakeLists.txt") execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" . @@ -294,7 +294,9 @@ target_include_directories( unset(CMAKE_PROJECT_INCLUDE_BEFORE) # xml2 -add_subdirectory(xml2 EXCLUDE_FROM_ALL) +if(${BUILD_WITH_S3}) + add_subdirectory(xml2 EXCLUDE_FROM_ALL) +endif(${BUILD_WITH_S3}) # lz4 add_subdirectory(lz4/build/cmake EXCLUDE_FROM_ALL) From dd21d8e3ade60225d74d2cf37ea49fe4027bf9ba Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 17 Oct 2023 17:24:50 +0800 Subject: [PATCH 21/51] cos/multipart: first round multipart uploading --- source/dnode/vnode/src/vnd/vnodeCos.c | 256 ++++++++++++++++++++++++-- 1 file changed, 244 insertions(+), 12 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeCos.c b/source/dnode/vnode/src/vnd/vnodeCos.c index 2a36a898e0..b52d4809b2 100644 --- a/source/dnode/vnode/src/vnd/vnodeCos.c +++ b/source/dnode/vnode/src/vnd/vnodeCos.c @@ -72,10 +72,10 @@ static void s3PrintError(const char *func, S3Status status, char error_details[] } typedef struct { - uint64_t content_length; + char err_msg[128]; S3Status status; + uint64_t content_length; char *buf; - char err_msg[4096]; } TS3SizeCBD; static S3Status responsePropertiesCallback(const S3ResponseProperties *properties, void *callbackData) { @@ -201,14 +201,14 @@ static void growbuffer_destroy(growbuffer *gb) { } typedef struct put_object_callback_data { + char err_msg[128]; + S3Status status; // FILE *infile; TdFilePtr infileFD; growbuffer *gb; uint64_t contentLength, originalContentLength; uint64_t totalContentLength, totalOriginalContentLength; int noStatus; - S3Status status; - char err_msg[4096]; } put_object_callback_data; static int putObjectDataCallback(int bufferSize, char *buffer, void *callbackData) { @@ -241,6 +241,8 @@ static int putObjectDataCallback(int bufferSize, char *buffer, void *callbackDat #define MULTIPART_CHUNK_SIZE (768 << 20) // multipart is 768M typedef struct UploadManager { + char err_msg[128]; + S3Status status; // used for initial multipart char *upload_id; @@ -253,7 +255,26 @@ typedef struct UploadManager { int remaining; } UploadManager; +typedef struct list_parts_callback_data { + char err_msg[128]; + S3Status status; + int isTruncated; + char nextPartNumberMarker[24]; + char initiatorId[256]; + char initiatorDisplayName[256]; + char ownerId[256]; + char ownerDisplayName[256]; + char storageClass[256]; + int partsCount; + int handlePartsStart; + int allDetails; + int noPrint; + UploadManager *manager; +} list_parts_callback_data; + typedef struct MultipartPartData { + char err_msg[128]; + S3Status status; put_object_callback_data put_object_data; int seq; UploadManager *manager; @@ -288,16 +309,120 @@ static int multipartPutXmlCallback(int bufferSize, char *buffer, void *callbackD return ret; } +static S3Status listPartsCallback(int isTruncated, const char *nextPartNumberMarker, const char *initiatorId, + const char *initiatorDisplayName, const char *ownerId, const char *ownerDisplayName, + const char *storageClass, int partsCount, int handlePartsStart, + const S3ListPart *parts, void *callbackData) { + list_parts_callback_data *data = (list_parts_callback_data *)callbackData; + + data->isTruncated = isTruncated; + data->handlePartsStart = handlePartsStart; + UploadManager *manager = data->manager; + + if (nextPartNumberMarker) { + snprintf(data->nextPartNumberMarker, sizeof(data->nextPartNumberMarker), "%s", nextPartNumberMarker); + } else { + data->nextPartNumberMarker[0] = 0; + } + + if (initiatorId) { + snprintf(data->initiatorId, sizeof(data->initiatorId), "%s", initiatorId); + } else { + data->initiatorId[0] = 0; + } + + if (initiatorDisplayName) { + snprintf(data->initiatorDisplayName, sizeof(data->initiatorDisplayName), "%s", initiatorDisplayName); + } else { + data->initiatorDisplayName[0] = 0; + } + + if (ownerId) { + snprintf(data->ownerId, sizeof(data->ownerId), "%s", ownerId); + } else { + data->ownerId[0] = 0; + } + + if (ownerDisplayName) { + snprintf(data->ownerDisplayName, sizeof(data->ownerDisplayName), "%s", ownerDisplayName); + } else { + data->ownerDisplayName[0] = 0; + } + + if (storageClass) { + snprintf(data->storageClass, sizeof(data->storageClass), "%s", storageClass); + } else { + data->storageClass[0] = 0; + } + + if (partsCount && !data->partsCount && !data->noPrint) { + // printListPartsHeader(); + } + + int i; + for (i = 0; i < partsCount; i++) { + const S3ListPart *part = &(parts[i]); + char timebuf[256]; + if (data->noPrint) { + manager->etags[handlePartsStart + i] = strdup(part->eTag); + manager->next_etags_pos++; + manager->remaining = manager->remaining - part->size; + } else { + time_t t = (time_t)part->lastModified; + strftime(timebuf, sizeof(timebuf), "%Y-%m-%dT%H:%M:%SZ", gmtime(&t)); + printf("%-30s", timebuf); + printf("%-15llu", (unsigned long long)part->partNumber); + printf("%-45s", part->eTag); + printf("%-15llu\n", (unsigned long long)part->size); + } + } + + data->partsCount += partsCount; + + return S3StatusOK; +} + static int try_get_parts_info(const char *bucketName, const char *key, UploadManager *manager) { - // + S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, + 0, awsRegionG}; + + S3ListPartsHandler listPartsHandler = {{&responsePropertiesCallback, &responseCompleteCallback}, &listPartsCallback}; + + list_parts_callback_data data; + + memset(&data, 0, sizeof(list_parts_callback_data)); + + data.partsCount = 0; + data.allDetails = 0; + data.manager = manager; + data.noPrint = 1; + do { + data.isTruncated = 0; + do { + S3_list_parts(&bucketContext, key, data.nextPartNumberMarker, manager->upload_id, 0, 0, 0, timeoutMsG, + &listPartsHandler, &data); + } while (S3_status_is_retryable(data.status) && should_retry()); + if (data.status != S3StatusOK) { + break; + } + } while (data.isTruncated); + + if (data.status == S3StatusOK) { + if (!data.partsCount) { + // printListMultipartHeader(data.allDetails); + } + } else { + s3PrintError(__func__, data.status, data.err_msg); + return -1; + } return 0; } int32_t s3PutObjectFromFile2(const char *file, const char *object) { - int32_t code = 0; - const char *key = object; - const char *uploadId = 0; + int32_t code = 0; + const char *key = object; + // const char *uploadId = 0; const char *filename = 0; uint64_t contentLength = 0; const char *cacheControl = 0, *contentType = 0, *md5 = 0; @@ -360,20 +485,127 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { (unsigned long long)data.contentLength); } } else { - uint64_t totalContentLength = contentLength; - uint64_t todoContentLength = contentLength; + uint64_t totalContentLength = contentLength; + uint64_t todoContentLength = contentLength; + UploadManager manager; + manager.upload_id = 0; + manager.gb = 0; + + // div round up + int seq; + int totalSeq = ((contentLength + MULTIPART_CHUNK_SIZE - 1) / MULTIPART_CHUNK_SIZE); + + MultipartPartData partData; + memset(&partData, 0, sizeof(MultipartPartData)); + int partContentLength = 0; + + S3MultipartInitialHandler handler = {{&responsePropertiesCallback, &responseCompleteCallback}, + &initial_multipart_callback}; + + S3PutObjectHandler putObjectHandler = {{&MultipartResponseProperiesCallback, &responseCompleteCallback}, + &putObjectDataCallback}; + + S3MultipartCommitHandler commit_handler = { + {&responsePropertiesCallback, &responseCompleteCallback}, &multipartPutXmlCallback, 0}; + + manager.etags = (char **)taosMemoryMalloc(sizeof(char *) * totalSeq); + manager.next_etags_pos = 0; + /* + if (uploadId) { + manager.upload_id = strdup(uploadId); + manager.remaining = contentLength; + if (!try_get_parts_info(tsS3BucketName, key, &manager)) { + fseek(data.infile, -(manager.remaining), 2); + taosLSeekFile(data.infileFD, -(manager.remaining), SEEK_END); + contentLength = manager.remaining; + goto upload; + } else { + goto clean; + } + } + */ + do { + S3_initiate_multipart(&bucketContext, key, 0, &handler, 0, timeoutMsG, &manager); + } while (S3_status_is_retryable(manager.status) && should_retry()); + + if (manager.upload_id == 0 || manager.status != S3StatusOK) { + s3PrintError(__func__, manager.status, manager.err_msg); + goto clean; + } + + upload: + todoContentLength -= MULTIPART_CHUNK_SIZE * manager.next_etags_pos; + for (seq = manager.next_etags_pos + 1; seq <= totalSeq; seq++) { + partData.manager = &manager; + partData.seq = seq; + if (partData.put_object_data.gb == NULL) { + partData.put_object_data = data; + } + partContentLength = ((contentLength > MULTIPART_CHUNK_SIZE) ? MULTIPART_CHUNK_SIZE : contentLength); + // printf("%s Part Seq %d, length=%d\n", srcSize ? "Copying" : "Sending", seq, partContentLength); + partData.put_object_data.contentLength = partContentLength; + partData.put_object_data.originalContentLength = partContentLength; + partData.put_object_data.totalContentLength = todoContentLength; + partData.put_object_data.totalOriginalContentLength = totalContentLength; + putProperties.md5 = 0; + do { + S3_upload_part(&bucketContext, key, &putProperties, &putObjectHandler, seq, manager.upload_id, + partContentLength, 0, timeoutMsG, &partData); + } while (S3_status_is_retryable(partData.status) && should_retry()); + if (partData.status != S3StatusOK) { + s3PrintError(__func__, partData.status, partData.err_msg); + goto clean; + } + contentLength -= MULTIPART_CHUNK_SIZE; + todoContentLength -= MULTIPART_CHUNK_SIZE; + } + + int i; + int size = 0; + size += growbuffer_append(&(manager.gb), "", strlen("")); + char buf[256]; + int n; + for (i = 0; i < totalSeq; i++) { + n = snprintf(buf, sizeof(buf), + "%d" + "%s", + i + 1, manager.etags[i]); + size += growbuffer_append(&(manager.gb), buf, n); + } + size += growbuffer_append(&(manager.gb), "", strlen("")); + manager.remaining = size; + + do { + S3_complete_multipart_upload(&bucketContext, key, &commit_handler, manager.upload_id, manager.remaining, 0, + timeoutMsG, &manager); + } while (S3_status_is_retryable(manager.status) && should_retry()); + if (manager.status != S3StatusOK) { + s3PrintError(__func__, manager.status, manager.err_msg); + goto clean; + } + + clean: + if (manager.upload_id) { + taosMemoryFree(manager.upload_id); + } + for (i = 0; i < manager.next_etags_pos; i++) { + taosMemoryFree(manager.etags[i]); + } + growbuffer_destroy(manager.gb); + taosMemoryFree(manager.etags); } + return 0; } typedef struct list_bucket_callback_data { + char err_msg[128]; + S3Status status; int isTruncated; char nextMarker[1024]; int keyCount; int allDetails; SArray *objectArray; - S3Status status; - char err_msg[4096]; } list_bucket_callback_data; static S3Status listBucketCallback(int isTruncated, const char *nextMarker, int contentsCount, From 4bc3c22e0c92e8831d1cf6ee55bdb4be4ef6385b Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 17 Oct 2023 17:46:11 +0800 Subject: [PATCH 22/51] cmake/ssl: install install_sw only --- cmake/ssl_CMakeLists.txt.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/ssl_CMakeLists.txt.in b/cmake/ssl_CMakeLists.txt.in index 7821848618..65a02d7fd6 100644 --- a/cmake/ssl_CMakeLists.txt.in +++ b/cmake/ssl_CMakeLists.txt.in @@ -9,7 +9,7 @@ ExternalProject_Add(openssl #BUILD_ALWAYS 1 #UPDATE_COMMAND "" CONFIGURE_COMMAND ./Configure --prefix=$ENV{HOME}/.cos-local.1 -static #--no-shared - BUILD_COMMAND make -j - INSTALL_COMMAND make install + BUILD_COMMAND "" + INSTALL_COMMAND make install_sw -j TEST_COMMAND "" ) From 11b65e277925323b65efe49095a8c2cff3e72278 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 17 Oct 2023 18:11:13 +0800 Subject: [PATCH 23/51] vnode/cos: make dependent lib public --- cmake/curl_CMakeLists.txt.in | 2 +- source/dnode/vnode/CMakeLists.txt | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cmake/curl_CMakeLists.txt.in b/cmake/curl_CMakeLists.txt.in index fed98c5f0e..3f7b9ee1fe 100644 --- a/cmake/curl_CMakeLists.txt.in +++ b/cmake/curl_CMakeLists.txt.in @@ -12,7 +12,7 @@ ExternalProject_Add(curl BUILD_ALWAYS 1 #UPDATE_COMMAND "" #CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --without-ssl --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd - CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --with-ssl=$ENV{HOME}/.cos-local.1 --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd + CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --with-ssl=$ENV{HOME}/.cos-local.1 --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd --without-libidn2 #CONFIGURE_COMMAND ./configure --without-ssl BUILD_COMMAND make INSTALL_COMMAND make install diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index d4ea90f48a..e554d94e4d 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -181,11 +181,11 @@ if(${BUILD_WITH_S3}) vnode # s3 - PRIVATE ${S3_LIBRARY} - PRIVATE ${CURL_LIBRARY} - PRIVATE ${SSL_LIBRARY} - PRIVATE ${CRYPTO_LIBRARY} - PRIVATE xml2 + PUBLIC ${S3_LIBRARY} + PUBLIC ${CURL_LIBRARY} + PUBLIC ${SSL_LIBRARY} + PUBLIC ${CRYPTO_LIBRARY} + PUBLIC xml2 ) add_definitions(-DUSE_S3) From 6d9732e87a850b6aee3be6e6e77a16240861cd68 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Tue, 17 Oct 2023 18:26:04 +0800 Subject: [PATCH 24/51] test: set taosd log leavel 135 --- tests/pytest/util/dnodes.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index c4fc1ce654..d0ed1d874d 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -36,12 +36,12 @@ class TDSimClient: "locale": "en_US.UTF-8", "charset": "UTF-8", "asyncLog": "0", - "rpcDebugFlag": "143", + "rpcDebugFlag": "135", "tmrDebugFlag": "131", - "cDebugFlag": "143", - "uDebugFlag": "143", - "jniDebugFlag": "143", - "qDebugFlag": "143", + "cDebugFlag": "135", + "uDebugFlag": "135", + "jniDebugFlag": "135", + "qDebugFlag": "135", "supportVnodes": "1024", "enableQueryHb": "1", "telemetryReporting": "0", @@ -130,18 +130,18 @@ class TDDnode: "locale": "en_US.UTF-8", "charset": "UTF-8", "asyncLog": "0", - "mDebugFlag": "143", - "dDebugFlag": "143", - "vDebugFlag": "143", - "tqDebugFlag": "143", - "cDebugFlag": "143", - "jniDebugFlag": "143", - "qDebugFlag": "143", - "rpcDebugFlag": "143", + "mDebugFlag": "135", + "dDebugFlag": "135", + "vDebugFlag": "135", + "tqDebugFlag": "135", + "cDebugFlag": "135", + "jniDebugFlag": "135", + "qDebugFlag": "135", + "rpcDebugFlag": "135", "tmrDebugFlag": "131", - "uDebugFlag": "143", - "sDebugFlag": "143", - "wDebugFlag": "143", + "uDebugFlag": "135", + "sDebugFlag": "135", + "wDebugFlag": "135", "numOfLogLines": "100000000", "statusInterval": "1", "enableQueryHb": "1", From d6579d0498bbfd4728c29196dd42c303e897f48c Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Tue, 17 Oct 2023 18:30:47 +0800 Subject: [PATCH 25/51] test: support replica3 in splited/transform tmqvnode --- tests/system-test/7-tmq/tmqVnodeTransform-stb.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/system-test/7-tmq/tmqVnodeTransform-stb.py b/tests/system-test/7-tmq/tmqVnodeTransform-stb.py index 18f9fafe9a..006498d514 100644 --- a/tests/system-test/7-tmq/tmqVnodeTransform-stb.py +++ b/tests/system-test/7-tmq/tmqVnodeTransform-stb.py @@ -256,7 +256,7 @@ class TDTestCase: tdLog.printNoPrefix("======== test case 2 end ...... ") def tmqCaseStableSelect(self): - tdLog.printNoPrefix("======== test case subscrib column start : ") + tdLog.printNoPrefix("======== test case 3 subscrib column start : ") paraDict = {'dbName': 'dbt', 'dropFlag': 1, 'event': '', @@ -315,7 +315,7 @@ class TDTestCase: tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) tdLog.info("wait the consume result") - time.sleep(5) + time.sleep(1) #restart dnode & remove wal # self.restartAndRemoveWal() @@ -332,10 +332,10 @@ class TDTestCase: for i in range(len(topicNameList)): tdSql.query("drop topic %s"%topicNameList[i]) - tdLog.printNoPrefix("======== test case subscrib column end ...... ") + tdLog.printNoPrefix("======== test case 3 subscrib column end ...... ") def tmqCaseDbname(self): - tdLog.printNoPrefix("======== test case subscrib Dbname start: ") + tdLog.printNoPrefix("======== test case 4 subscrib Dbname start: ") paraDict = {'dbName': 'dbt', 'dropFlag': 1, 'event': '', @@ -394,7 +394,7 @@ class TDTestCase: tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) tdLog.info("wait the consume result") - time.sleep(5) + time.sleep(1) #restart dnode & remove wal # self.restartAndRemoveWal() @@ -411,7 +411,7 @@ class TDTestCase: for i in range(len(topicNameList)): tdSql.query("drop topic %s"%topicNameList[i]) - tdLog.printNoPrefix("======== test case subscrib Dbname end ...... ") + tdLog.printNoPrefix("======== test case 4 subscrib Dbname end ...... ") def run(self): self.prepareTestEnv() From a9dfb400691ee59d8e4d35fe3c97e649a5480250 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 18 Oct 2023 10:51:47 +0800 Subject: [PATCH 26/51] contrib/cmake: remove unused steps --- contrib/CMakeLists.txt | 29 +---------------------------- 1 file changed, 1 insertion(+), 28 deletions(-) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index de3d9cfd37..30853660ed 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -8,30 +8,9 @@ endfunction(cat IN_FILE OUT_FILE) if(${TD_LINUX}) - if(${BUILD_WITH_S3}) - -set(CONTRIB_TMP_FILE3 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in3") -configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3}) - +if(${BUILD_WITH_S3}) file(MAKE_DIRECTORY $ENV{HOME}/.cos-local.1/) -configure_file(${CONTRIB_TMP_FILE3} "${TD_CONTRIB_DIR}/deps-download/CMakeLists.txt") -execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" . - WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download") -execute_process(COMMAND "${CMAKE_COMMAND}" --build . - WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download") - -set(CONTRIB_TMP_FILE2 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in2") -configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE2}) - - #cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE2}) - -configure_file(${CONTRIB_TMP_FILE2} "${TD_CONTRIB_DIR}/deps-download/CMakeLists.txt") -execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" . - WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download") -execute_process(COMMAND "${CMAKE_COMMAND}" --build . - WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download") - else() set(CONTRIB_TMP_FILE3 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in3") @@ -185,12 +164,6 @@ endif(${BUILD_WITH_SQLITE}) # s3 if(${BUILD_WITH_S3}) - #cat("${TD_SUPPORT_DIR}/mxml_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) - #cat("${TD_SUPPORT_DIR}/apr_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) - #cat("${TD_SUPPORT_DIR}/apr-util_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) - #cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) - #INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.1/include) - cat("${TD_SUPPORT_DIR}/ssl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) cat("${TD_SUPPORT_DIR}/xml2_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) From f26ec41b8e11804ba87524e5accc8cc5cd459af1 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 18 Oct 2023 15:30:02 +0800 Subject: [PATCH 27/51] vnode/cos: fix get callback buffer --- source/common/CMakeLists.txt | 3 +++ source/common/src/tglobal.c | 2 +- source/dnode/vnode/CMakeLists.txt | 1 - source/dnode/vnode/src/vnd/vnodeCos.c | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/source/common/CMakeLists.txt b/source/common/CMakeLists.txt index b010467f20..6bdcdf7d1d 100644 --- a/source/common/CMakeLists.txt +++ b/source/common/CMakeLists.txt @@ -22,6 +22,9 @@ IF (TD_STORAGE) add_definitions(-DUSE_COS) ENDIF(${BUILD_WITH_COS}) + IF(${BUILD_WITH_S3}) + add_definitions(-DUSE_S3) + ENDIF(${BUILD_WITH_S3}) ENDIF(${TD_LINUX}) ENDIF () diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 8ea5528fbb..f3af6e0ab0 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -329,7 +329,7 @@ int32_t taosSetS3Cfg(SConfig *pCfg) { } } if (tsS3BucketName[0] != '<' && tsDiskCfgNum > 1) { -#ifdef USE_COS +#if defined(USE_COS) || defined(USE_S3) tsS3Enabled = true; #endif } diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index e554d94e4d..7e2693e8af 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -164,7 +164,6 @@ target_link_libraries( if(${TD_LINUX}) if(${BUILD_WITH_S3}) - MESSAGE("build with s3: ${BUILD_WITH_S3}") target_include_directories( vnode diff --git a/source/dnode/vnode/src/vnd/vnodeCos.c b/source/dnode/vnode/src/vnd/vnodeCos.c index b52d4809b2..c50f133987 100644 --- a/source/dnode/vnode/src/vnd/vnodeCos.c +++ b/source/dnode/vnode/src/vnd/vnodeCos.c @@ -721,7 +721,7 @@ static S3Status getObjectDataCallback(int bufferSize, const char *buffer, void * char *buf = taosMemoryCalloc(1, bufferSize); if (buf) { memcpy(buf, buffer, bufferSize); - + cbd->buf = buf; cbd->status = S3StatusOK; return S3StatusOK; } else { From e7d9de06cb0f73f9fb5148f30694f675dfcfee03 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Wed, 18 Oct 2023 15:59:04 +0800 Subject: [PATCH 28/51] test: support replica3 in splited/transform tmqvnode --- .../system-test/7-tmq/tmqVnodeTransform-db.py | 298 ++++++++++++++++++ .../7-tmq/tmqVnodeTransform-stb.py | 165 +--------- 2 files changed, 300 insertions(+), 163 deletions(-) create mode 100644 tests/system-test/7-tmq/tmqVnodeTransform-db.py diff --git a/tests/system-test/7-tmq/tmqVnodeTransform-db.py b/tests/system-test/7-tmq/tmqVnodeTransform-db.py new file mode 100644 index 0000000000..005bca70d6 --- /dev/null +++ b/tests/system-test/7-tmq/tmqVnodeTransform-db.py @@ -0,0 +1,298 @@ + +import taos +import sys +import time +import socket +import os +import threading +import math + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +from util.cluster import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def __init__(self): + self.vgroups = 1 + self.ctbNum = 10 + self.rowsPerTbl = 10000 + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def getDataPath(self): + selfPath = tdCom.getBuildPath() + + return selfPath + '/../sim/dnode%d/data/vnode/vnode%d/wal/*'; + + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 30, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tdCom.drop_all_db() + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], wal_retention_period=36000,vgroups=paraDict["vgroups"],replica=self.replicaVar) + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + # tdLog.info("create ctb") + # tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + # ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + # tdLog.info("insert data") + # tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + # ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + # startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + + # tdLog.info("restart taosd to ensure that the data falls into the disk") + # tdDnodes.stop(1) + # tdDnodes.start(1) + # tdSql.query("flush database %s"%(paraDict['dbName'])) + return + + def restartAndRemoveWal(self): + tdDnodes = cluster.dnodes + tdSql.query("select * from information_schema.ins_vnodes") + for result in tdSql.queryResult: + if result[2] == 'dbt': + tdLog.debug("dnode is %d"%(result[0])) + dnodeId = result[0] + vnodeId = result[1] + + tdDnodes[dnodeId - 1].stoptaosd() + time.sleep(1) + dataPath = self.getDataPath() + dataPath = dataPath%(dnodeId,vnodeId) + os.system('rm -rf ' + dataPath) + tdLog.debug("dataPath:%s"%dataPath) + tdDnodes[dnodeId - 1].starttaosd() + time.sleep(1) + break + tdLog.debug("restart dnode ok") + + def redistributeVgroups(self): + dnodesList = [] + tdSql.query("show dnodes") + for result in tdSql.queryResult: + dnodesList.append(result[0]) + print("dnodeList:",dnodesList) + tdSql.query("select * from information_schema.ins_vnodes") + vnodeId = 0 + for result in tdSql.queryResult: + if result[2] == 'dbt': + tdLog.debug("dnode is %d"%(result[0])) + dnodesList.remove(result[0]) + vnodeId = result[1] + print("its all data",dnodesList) + # if self.replicaVar == 1: + # redistributeSql = "redistribute vgroup %d dnode %d" %(vnodeId, dnodesList[0]) + # else: + redistributeSql = f"redistribute vgroup {vnodeId} " + for vgdnode in dnodesList: + redistributeSql += f"dnode {vgdnode} " + print(redistributeSql) + + tdLog.debug(f"redistributeSql:{redistributeSql}") + tdSql.query(redistributeSql) + tdLog.debug("redistributeSql ok") + + + def tmqCaseStableSelect(self): + tdLog.printNoPrefix("======== test case 3 subscrib column start : ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stbn', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 10, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + topicNameList = ['topic3'] + tmqCom.initConsumerTable() + + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + + tdLog.info("create ctb") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + tdLog.info("insert data") + tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("create topics from stb with filter") + queryString = "select * from %s.%s where c2 > 0 "%(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + + time.sleep(1) + #restart dnode & remove wal + # self.restartAndRemoveWal() + + # redistribute vgroup + self.redistributeVgroups() + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + expectRows = 2 + resultList = tmqCom.selectConsumeResult(expectRows) + + time.sleep(6) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 3 subscrib column end ...... ") + + def tmqCaseDbname(self): + tdLog.printNoPrefix("======== test case 4 subscrib Dbname start: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stbn', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 10, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + topicNameList = ['topic4'] + tmqCom.initConsumerTable() + + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + + tdLog.info("create ctb") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + tdLog.info("insert data") + tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("create topics from database ") + queryString = "database %s "%(paraDict['dbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + + time.sleep(1) + #restart dnode & remove wal + # self.restartAndRemoveWal() + + # redistribute vgroup + self.redistributeVgroups() + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + expectRows = 2 + resultList = tmqCom.selectConsumeResult(expectRows) + + time.sleep(6) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 4 subscrib Dbname end ...... ") + + def run(self): + self.prepareTestEnv() + self.tmqCaseStableSelect() + self.prepareTestEnv() + self.tmqCaseDbname() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/tmqVnodeTransform-stb.py b/tests/system-test/7-tmq/tmqVnodeTransform-stb.py index 006498d514..2dd756b788 100644 --- a/tests/system-test/7-tmq/tmqVnodeTransform-stb.py +++ b/tests/system-test/7-tmq/tmqVnodeTransform-stb.py @@ -49,7 +49,7 @@ class TDTestCase: 'rowsPerTbl': 10000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 60, + 'pollDelay': 30, 'showMsg': 1, 'showRow': 1, 'snapshot': 0} @@ -140,7 +140,7 @@ class TDTestCase: 'rowsPerTbl': 10000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 60, + 'pollDelay': 30, 'showMsg': 1, 'showRow': 1, 'snapshot': 0} @@ -255,173 +255,12 @@ class TDTestCase: tdLog.printNoPrefix("======== test case 2 end ...... ") - def tmqCaseStableSelect(self): - tdLog.printNoPrefix("======== test case 3 subscrib column start : ") - paraDict = {'dbName': 'dbt', - 'dropFlag': 1, - 'event': '', - 'vgroups': 1, - 'stbName': 'stbn', - 'colPrefix': 'c', - 'tagPrefix': 't', - 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], - 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], - 'ctbPrefix': 'ctb', - 'ctbStartIdx': 0, - 'ctbNum': 10, - 'rowsPerTbl': 10000, - 'batchNum': 10, - 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 2, - 'showMsg': 1, - 'showRow': 1, - 'snapshot': 0} - - paraDict['vgroups'] = self.vgroups - paraDict['ctbNum'] = self.ctbNum - paraDict['rowsPerTbl'] = self.rowsPerTbl - - topicNameList = ['topic3'] - tmqCom.initConsumerTable() - - tdLog.info("create stb") - tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) - - tdLog.info("create ctb") - tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], - ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) - tdLog.info("insert data") - tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], - ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], - startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) - - tdLog.info("create topics from stb with filter") - queryString = "select * from %s.%s where c2 > 0 "%(paraDict['dbName'], paraDict['stbName']) - sqlString = "create topic %s as %s" %(topicNameList[0], queryString) - tdLog.info("create topic sql: %s"%sqlString) - tdSql.execute(sqlString) - - # init consume info, and start tmq_sim, then check consume result - tdLog.info("insert consume info to consume processor") - consumerId = 0 - expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] - topicList = topicNameList[0] - ifcheckdata = 1 - ifManualCommit = 1 - keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' - tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) - - tdLog.info("start consume processor") - tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) - tdLog.info("wait the consume result") - - time.sleep(1) - #restart dnode & remove wal - # self.restartAndRemoveWal() - - # redistribute vgroup - self.redistributeVgroups() - - tdLog.info("start consume processor") - tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) - tdLog.info("wait the consume result") - expectRows = 1 - resultList = tmqCom.selectConsumeResult(expectRows) - - time.sleep(5) - for i in range(len(topicNameList)): - tdSql.query("drop topic %s"%topicNameList[i]) - - tdLog.printNoPrefix("======== test case 3 subscrib column end ...... ") - - def tmqCaseDbname(self): - tdLog.printNoPrefix("======== test case 4 subscrib Dbname start: ") - paraDict = {'dbName': 'dbt', - 'dropFlag': 1, - 'event': '', - 'vgroups': 1, - 'stbName': 'stbn', - 'colPrefix': 'c', - 'tagPrefix': 't', - 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], - 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], - 'ctbPrefix': 'ctb', - 'ctbStartIdx': 0, - 'ctbNum': 10, - 'rowsPerTbl': 10000, - 'batchNum': 10, - 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 2, - 'showMsg': 1, - 'showRow': 1, - 'snapshot': 0} - - paraDict['vgroups'] = self.vgroups - paraDict['ctbNum'] = self.ctbNum - paraDict['rowsPerTbl'] = self.rowsPerTbl - - topicNameList = ['topic4'] - tmqCom.initConsumerTable() - - tdLog.info("create stb") - tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) - - tdLog.info("create ctb") - tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], - ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) - tdLog.info("insert data") - tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], - ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], - startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) - - tdLog.info("create topics from database ") - queryString = "database %s "%(paraDict['dbName']) - sqlString = "create topic %s as %s" %(topicNameList[0], queryString) - tdLog.info("create topic sql: %s"%sqlString) - tdSql.execute(sqlString) - - # init consume info, and start tmq_sim, then check consume result - tdLog.info("insert consume info to consume processor") - consumerId = 0 - expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] - topicList = topicNameList[0] - ifcheckdata = 1 - ifManualCommit = 1 - keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' - tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) - - tdLog.info("start consume processor") - tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) - tdLog.info("wait the consume result") - - time.sleep(1) - #restart dnode & remove wal - # self.restartAndRemoveWal() - - # redistribute vgroup - self.redistributeVgroups() - - tdLog.info("start consume processor") - tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) - tdLog.info("wait the consume result") - expectRows = 1 - resultList = tmqCom.selectConsumeResult(expectRows) - - time.sleep(5) - for i in range(len(topicNameList)): - tdSql.query("drop topic %s"%topicNameList[i]) - - tdLog.printNoPrefix("======== test case 4 subscrib Dbname end ...... ") def run(self): self.prepareTestEnv() self.tmqCaseStable() self.prepareTestEnv() self.tmqCaseNtable() - self.prepareTestEnv() - self.tmqCaseStableSelect() - self.prepareTestEnv() - self.tmqCaseDbname() def stop(self): tdSql.close() From 52c8bc91f98ebfea693b1b7c352eba2fdaf44a7c Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 18 Oct 2023 18:21:45 +0800 Subject: [PATCH 29/51] cos/delete: fix delete object cond and double free --- source/dnode/vnode/src/vnd/vnodeCos.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeCos.c b/source/dnode/vnode/src/vnd/vnodeCos.c index c50f133987..ccda20e049 100644 --- a/source/dnode/vnode/src/vnd/vnodeCos.c +++ b/source/dnode/vnode/src/vnd/vnodeCos.c @@ -644,7 +644,7 @@ static S3Status listBucketCallback(int isTruncated, const char *nextMarker, int } static void s3FreeObjectKey(void *pItem) { - char *key = (char *)pItem; + char *key = *(char **)pItem; taosMemoryFree(key); } @@ -682,7 +682,7 @@ void s3DeleteObjectsByPrefix(const char *prefix) { } while (data.isTruncated && (!maxkeys || (data.keyCount < maxkeys))); if (data.status == S3StatusOK) { - if (!data.keyCount) { + if (data.keyCount > 0) { // printListBucketHeader(allDetails); s3DeleteObjects(TARRAY_DATA(data.objectArray), TARRAY_SIZE(data.objectArray)); } From 788f21fd946afc1a9332d250305d8faac28941fa Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Wed, 18 Oct 2023 19:38:24 +0800 Subject: [PATCH 30/51] test: support replica3 in splited/transform tmqvnode --- tests/parallel_test/cases.task | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 426646426d..c749864eec 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -173,6 +173,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -n 3 -i True ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-stb.py -N 2 -n 1 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-stb.py -N 6 -n 3 +,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-db.py -N 6 -n 3 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select.py -N 2 -n 1 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py -N 3 -n 3 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select.py -N 3 -n 3 From 28b69d279ff96c88b72a717a08137a0a88925a5f Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 19 Oct 2023 18:28:52 +0800 Subject: [PATCH 31/51] fix:add except characters for smlAutoChildTableNameDelimiter --- docs/en/14-reference/13-schemaless/13-schemaless.md | 2 +- docs/zh/14-reference/13-schemaless/13-schemaless.md | 2 +- tests/pytest/util/dnodes.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en/14-reference/13-schemaless/13-schemaless.md b/docs/en/14-reference/13-schemaless/13-schemaless.md index 11d3c0d0ab..eb336f4633 100644 --- a/docs/en/14-reference/13-schemaless/13-schemaless.md +++ b/docs/en/14-reference/13-schemaless/13-schemaless.md @@ -94,7 +94,7 @@ The string's MD5 hash value "md5_val" is calculated after the ranking is complet ::: If you do not want to use an automatically generated table name, there are two ways to specify sub table names, the first one has a higher priority. -You can configure smlAutoChildTableNameDelimiter in taos.cfg, for example, `smlAutoChildTableNameDelimiter=tname`. You can insert `st,t0=cpul,t1=4 c1=3 1626006833639000000` and the table name will be cpu1-4. +You can configure smlAutoChildTableNameDelimiter in taos.cfg(except for `@ # space \r \t \n`), for example, `smlAutoChildTableNameDelimiter=tname`. You can insert `st,t0=cpul,t1=4 c1=3 1626006833639000000` and the table name will be cpu1-4. You can configure smlChildTableName in taos.cfg to specify table names, for example, `smlChildTableName=tname`. You can insert `st,tname=cpul,t1=4 c1=3 1626006833639000000` and the cpu1 table will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored. 2. If the super table obtained by parsing the line protocol does not exist, this super table is created. diff --git a/docs/zh/14-reference/13-schemaless/13-schemaless.md b/docs/zh/14-reference/13-schemaless/13-schemaless.md index 084e31d810..723531676c 100644 --- a/docs/zh/14-reference/13-schemaless/13-schemaless.md +++ b/docs/zh/14-reference/13-schemaless/13-schemaless.md @@ -96,7 +96,7 @@ st,t1=3,t2=4,t3=t3 c1=3i64,c3="passit",c2=false,c4=4f64 1626006833639000000 排列完成以后计算该字符串的 MD5 散列值 "md5_val"。然后将计算的结果与字符串组合生成表名：“t_md5_val”。其中的 “t_” 是固定的前缀，每个通过该映射关系自动生成的表都具有该前缀。 :::tip 如果不想用自动生成的表名，有两种指定子表名的方式，第一种优先级更高： -通过在taos.cfg里配置 smlAutoChildTableNameDelimiter 参数来指定。 +通过在taos.cfg里配置 smlAutoChildTableNameDelimiter 参数来指定（`@ # 空格 回车 换行 制表符`除外)。 举例如下：配置 smlAutoChildTableNameDelimiter=- 插入数据为 st,t0=cpu1,t1=4 c1=3 1626006833639000000 则创建的表名为 cpu1-4。 通过在taos.cfg里配置 smlChildTableName 参数来指定。 举例如下：配置 smlChildTableName=tname 插入数据为 st,tname=cpu1,t1=4 c1=3 1626006833639000000 则创建的表名为 cpu1，注意如果多行数据 tname 相同，但是后面的 tag_set 不同，则使用第一行自动建表时指定的 tag_set，其他的行会忽略）。 diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index c4fc1ce654..5f1522bfd9 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -64,7 +64,7 @@ class TDSimClient: self.cfgDict.update({option: value}) def cfg(self, option, value): - cmd = "echo %s %s >> %s" % (option, value, self.cfgPath) + cmd = "echo %s '%s' >> %s" % (option, value, self.cfgPath) if os.system(cmd) != 0: tdLog.exit(cmd) From 720982374cc222c205de4c185a6ad6980240be13 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Fri, 20 Oct 2023 10:36:52 +0800 Subject: [PATCH 32/51] fix: union all cast func resType check error --- source/libs/parser/src/parAstCreater.c | 4 ++++ source/libs/parser/src/parTranslater.c | 5 +++-- source/libs/parser/src/parUtil.c | 2 +- tests/system-test/2-query/varchar.py | 6 ++++++ 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 12062e0d4a..e8614147f4 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -1379,6 +1379,10 @@ SNode* createColumnDefNode(SAstCreateContext* pCxt, SToken* pColName, SDataType if (!checkColumnName(pCxt, pColName) || !checkComment(pCxt, pComment, false)) { return NULL; } + if (IS_VAR_DATA_TYPE(dataType.type) && dataType.bytes == 0) { + pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN); + return NULL; + } SColumnDefNode* pCol = (SColumnDefNode*)nodesMakeNode(QUERY_NODE_COLUMN_DEF); CHECK_OUT_OF_MEM(pCol); COPY_STRING_FORM_ID_TOKEN(pCol->colName, pColName); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 42ab2f7a2f..756a1f867b 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2185,11 +2185,12 @@ static int32_t createCastFunc(STranslateContext* pCxt, SNode* pExpr, SDataType d nodesDestroyNode((SNode*)pFunc); return TSDB_CODE_OUT_OF_MEMORY; } - if (TSDB_CODE_SUCCESS != getFuncInfo(pCxt, pFunc)) { + int32_t code = getFuncInfo(pCxt, pFunc); + if (TSDB_CODE_SUCCESS != code) { nodesClearList(pFunc->pParameterList); pFunc->pParameterList = NULL; nodesDestroyNode((SNode*)pFunc); - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)pExpr)->aliasName); + return code; } *pCast = (SNode*)pFunc; return TSDB_CODE_SUCCESS; diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 1b8bac4cbc..0ab8927bd0 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -126,7 +126,7 @@ static char* getSyntaxErrFormat(int32_t errCode) { case TSDB_CODE_PAR_INVALID_FIRST_COLUMN: return "First column must be timestamp"; case TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN: - return "Invalid binary/nchar column length"; + return "Invalid column length for var length type"; case TSDB_CODE_PAR_INVALID_TAGS_NUM: return "Invalid number of tag columns"; case TSDB_CODE_PAR_INVALID_INTERNAL_PK: diff --git a/tests/system-test/2-query/varchar.py b/tests/system-test/2-query/varchar.py index f0849010c6..debcd1f95e 100644 --- a/tests/system-test/2-query/varchar.py +++ b/tests/system-test/2-query/varchar.py @@ -76,6 +76,12 @@ class TDTestCase: for i in range(tdSql.queryRows): tdSql.checkData(i,0, data_ct1_c8[i]) + tdSql.error("create stable empty_col_stable(ts timestamp, c2 varchar(0)) tags(tg1 int)") + tdSql.error("create stable empty_col_stable(ts timestamp, c2 varchar(10)) tags(tg1 varchar(0))") + tdSql.error("create stable empty_col_stable(ts timestamp, c2 varchar(10)) tags(tg1 nchar(0))") + tdSql.error("create stable empty_col_stable(ts timestamp, c2 varchar(10)) tags(tg1 binary(0))") + tdSql.error("create stable empty_col_stable(ts timestamp, c2 varchar(10)) tags(tg1 varbinary(0))") + # tdSql.query("select c8 from ct4") # data_ct4 = [tdSql.getData(i,0) for i in range(tdSql.queryRows)] From 74d05c2771c4c04b59cb3bf927b2eb6811838e92 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 20 Oct 2023 13:47:25 +0800 Subject: [PATCH 33/51] vnode/cos: fix multipart uploading --- source/dnode/vnode/src/vnd/vnodeCos.c | 71 ++++++++++++++++----------- 1 file changed, 41 insertions(+), 30 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeCos.c b/source/dnode/vnode/src/vnd/vnodeCos.c index ccda20e049..9c69f2452c 100644 --- a/source/dnode/vnode/src/vnd/vnodeCos.c +++ b/source/dnode/vnode/src/vnd/vnodeCos.c @@ -211,33 +211,6 @@ typedef struct put_object_callback_data { int noStatus; } put_object_callback_data; -static int putObjectDataCallback(int bufferSize, char *buffer, void *callbackData) { - put_object_callback_data *data = (put_object_callback_data *)callbackData; - - int ret = 0; - - if (data->contentLength) { - int toRead = ((data->contentLength > (unsigned)bufferSize) ? (unsigned)bufferSize : data->contentLength); - if (data->gb) { - growbuffer_read(&(data->gb), toRead, &ret, buffer); - } else if (data->infileFD) { - // ret = fread(buffer, 1, toRead, data->infile); - ret = taosReadFile(data->infileFD, buffer, toRead); - } - } - - data->contentLength -= ret; - data->totalContentLength -= ret; - - if (data->contentLength && !data->noStatus) { - vTrace("%llu bytes remaining ", (unsigned long long)data->totalContentLength); - vTrace("(%d%% complete) ...\n", (int)(((data->totalOriginalContentLength - data->totalContentLength) * 100) / - data->totalOriginalContentLength)); - } - - return ret; -} - #define MULTIPART_CHUNK_SIZE (768 << 20) // multipart is 768M typedef struct UploadManager { @@ -280,6 +253,37 @@ typedef struct MultipartPartData { UploadManager *manager; } MultipartPartData; +static int putObjectDataCallback(int bufferSize, char *buffer, void *callbackData) { + put_object_callback_data *data = (put_object_callback_data *)callbackData; + if (data->infileFD == 0) { + MultipartPartData *mpd = (MultipartPartData *)callbackData; + data = &mpd->put_object_data; + } + + int ret = 0; + + if (data->contentLength) { + int toRead = ((data->contentLength > (unsigned)bufferSize) ? (unsigned)bufferSize : data->contentLength); + if (data->gb) { + growbuffer_read(&(data->gb), toRead, &ret, buffer); + } else if (data->infileFD) { + // ret = fread(buffer, 1, toRead, data->infile); + ret = taosReadFile(data->infileFD, buffer, toRead); + } + } + + data->contentLength -= ret; + data->totalContentLength -= ret; + /* log too many open files + if (data->contentLength && !data->noStatus) { + vTrace("%llu bytes remaining ", (unsigned long long)data->totalContentLength); + vTrace("(%d%% complete) ...\n", (int)(((data->totalOriginalContentLength - data->totalContentLength) * 100) / + data->totalOriginalContentLength)); + } + */ + return ret; +} + S3Status initial_multipart_callback(const char *upload_id, void *callbackData) { UploadManager *manager = (UploadManager *)callbackData; manager->upload_id = strdup(upload_id); @@ -308,7 +312,7 @@ static int multipartPutXmlCallback(int bufferSize, char *buffer, void *callbackD manager->remaining -= ret; return ret; } - +/* static S3Status listPartsCallback(int isTruncated, const char *nextPartNumberMarker, const char *initiatorId, const char *initiatorDisplayName, const char *ownerId, const char *ownerDisplayName, const char *storageClass, int partsCount, int handlePartsStart, @@ -418,7 +422,7 @@ static int try_get_parts_info(const char *bucketName, const char *key, UploadMan return 0; } - +*/ int32_t s3PutObjectFromFile2(const char *file, const char *object) { int32_t code = 0; const char *key = object; @@ -480,9 +484,11 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { if (data.status != S3StatusOK) { s3PrintError(__func__, data.status, data.err_msg); + code = TAOS_SYSTEM_ERROR(EIO); } else if (data.contentLength) { vError("ERROR: %s Failed to read remaining %llu bytes from input", __func__, (unsigned long long)data.contentLength); + code = TAOS_SYSTEM_ERROR(EIO); } } else { uint64_t totalContentLength = contentLength; @@ -493,7 +499,9 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { // div round up int seq; + // uint64_t chunk_size = MULTIPART_CHUNK_SIZE >> 8; int totalSeq = ((contentLength + MULTIPART_CHUNK_SIZE - 1) / MULTIPART_CHUNK_SIZE); + // int totalSeq = ((contentLength + chunk_size - 1) / chunk_size); MultipartPartData partData; memset(&partData, 0, sizeof(MultipartPartData)); @@ -530,6 +538,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { if (manager.upload_id == 0 || manager.status != S3StatusOK) { s3PrintError(__func__, manager.status, manager.err_msg); + code = TAOS_SYSTEM_ERROR(EIO); goto clean; } @@ -554,6 +563,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { } while (S3_status_is_retryable(partData.status) && should_retry()); if (partData.status != S3StatusOK) { s3PrintError(__func__, partData.status, partData.err_msg); + code = TAOS_SYSTEM_ERROR(EIO); goto clean; } contentLength -= MULTIPART_CHUNK_SIZE; @@ -581,6 +591,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { } while (S3_status_is_retryable(manager.status) && should_retry()); if (manager.status != S3StatusOK) { s3PrintError(__func__, manager.status, manager.err_msg); + code = TAOS_SYSTEM_ERROR(EIO); goto clean; } @@ -595,7 +606,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { taosMemoryFree(manager.etags); } - return 0; + return code; } typedef struct list_bucket_callback_data { From f016dab40af2446310ecfa58871405af880b11ea Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 20 Oct 2023 13:59:01 +0800 Subject: [PATCH 34/51] common/global: comment s3BlockSize altering out --- source/common/src/tglobal.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index f3af6e0ab0..9bc981b520 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -1683,14 +1683,14 @@ void taosCfgDynamicOptions(const char *option, const char *value) { } return; } - + /* cannot alter s3BlockSize if (strcasecmp(option, "s3BlockSize") == 0) { int32_t newS3BlockSize = atoi(value); uInfo("s3BlockSize set from %d to %d", tsS3BlockSize, newS3BlockSize); tsS3BlockSize = newS3BlockSize; return; } - + */ if (strcasecmp(option, "s3BlockCacheSize") == 0) { int32_t newS3BlockCacheSize = atoi(value); uInfo("s3BlockCacheSize set from %d to %d", tsS3BlockCacheSize, newS3BlockCacheSize); From 55647f9ce9188e79405345935838bcd009f6f898 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 20 Oct 2023 15:15:03 +0800 Subject: [PATCH 35/51] cos/put: make chunk small to not block other reqs --- source/dnode/vnode/src/vnd/vnodeCos.c | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeCos.c b/source/dnode/vnode/src/vnd/vnodeCos.c index 9c69f2452c..6e36739f5a 100644 --- a/source/dnode/vnode/src/vnd/vnodeCos.c +++ b/source/dnode/vnode/src/vnd/vnodeCos.c @@ -498,10 +498,9 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { manager.gb = 0; // div round up - int seq; - // uint64_t chunk_size = MULTIPART_CHUNK_SIZE >> 8; - int totalSeq = ((contentLength + MULTIPART_CHUNK_SIZE - 1) / MULTIPART_CHUNK_SIZE); - // int totalSeq = ((contentLength + chunk_size - 1) / chunk_size); + int seq; + uint64_t chunk_size = MULTIPART_CHUNK_SIZE >> 8; + int totalSeq = ((contentLength + chunk_size - 1) / chunk_size); MultipartPartData partData; memset(&partData, 0, sizeof(MultipartPartData)); @@ -543,14 +542,14 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { } upload: - todoContentLength -= MULTIPART_CHUNK_SIZE * manager.next_etags_pos; + todoContentLength -= chunk_size * manager.next_etags_pos; for (seq = manager.next_etags_pos + 1; seq <= totalSeq; seq++) { partData.manager = &manager; partData.seq = seq; if (partData.put_object_data.gb == NULL) { partData.put_object_data = data; } - partContentLength = ((contentLength > MULTIPART_CHUNK_SIZE) ? MULTIPART_CHUNK_SIZE : contentLength); + partContentLength = ((contentLength > chunk_size) ? chunk_size : contentLength); // printf("%s Part Seq %d, length=%d\n", srcSize ? "Copying" : "Sending", seq, partContentLength); partData.put_object_data.contentLength = partContentLength; partData.put_object_data.originalContentLength = partContentLength; @@ -566,8 +565,8 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { code = TAOS_SYSTEM_ERROR(EIO); goto clean; } - contentLength -= MULTIPART_CHUNK_SIZE; - todoContentLength -= MULTIPART_CHUNK_SIZE; + contentLength -= chunk_size; + todoContentLength -= chunk_size; } int i; From f8097979995e3620a44dd8106a7cbf540a4e4d08 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 23 Oct 2023 16:04:45 +0800 Subject: [PATCH 36/51] enh: taosd build info --- cmake/cmake.version | 14 +++++++++++++- source/util/src/version.c.in | 2 +- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/cmake/cmake.version b/cmake/cmake.version index fa6ec4df17..c0ddae569c 100644 --- a/cmake/cmake.version +++ b/cmake/cmake.version @@ -50,7 +50,19 @@ ENDIF () IF (DEFINED VERDATE) SET(TD_VER_DATE ${VERDATE}) ELSE () - STRING(TIMESTAMP TD_VER_DATE "%Y-%m-%d %H:%M:%S") + STRING(COMPARE GREATER_EQUAL "${CMAKE_VERSION}" "3.26" TD_CMAKE_SUPPORT_TZ) + + IF (TD_CMAKE_SUPPORT_TZ) + STRING(TIMESTAMP TD_VER_DATE "%Y-%m-%d %H:%M:%S %z") + ELSE () + IF (TD_WINDOWS) + STRING(TIMESTAMP TD_VER_DATE "%Y-%m-%d %H:%M:%S") + ELSE () + EXECUTE_PROCESS(COMMAND date +"%F %T %z" OUTPUT_VARIABLE TD_VER_DATE) + STRING(REPLACE "\"" "" TD_VER_DATE ${TD_VER_DATE}) + STRING(STRIP ${TD_VER_DATE} TD_VER_DATE) + ENDIF () + ENDIF () ENDIF () IF (DEFINED VERTYPE) diff --git a/source/util/src/version.c.in b/source/util/src/version.c.in index 71998e3321..ec6449a02f 100644 --- a/source/util/src/version.c.in +++ b/source/util/src/version.c.in @@ -2,6 +2,6 @@ char version[64] = "${TD_VER_NUMBER}"; char compatible_version[12] = "${TD_VER_COMPATIBLE}"; char gitinfo[48] = "${TD_VER_GIT}"; char gitinfoOfInternal[48] = "${TD_VER_GIT_INTERNAL}"; -char buildinfo[64] = "Built at ${TD_VER_DATE}"; +char buildinfo[64] = "Built ${TD_VER_OSTYPE}-${TD_VER_CPUTYPE} at ${TD_VER_DATE}"; void libtaos_${TD_LIB_VER_NUMBER}_${TD_VER_OSTYPE}_${TD_VER_CPUTYPE}_${TD_VER_VERTYPE}() {}; From 62cd6f3ed26d1aa8b8e064de24e651b1b2d1d0eb Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 23 Oct 2023 16:50:51 +0800 Subject: [PATCH 37/51] cmake/curl, libs3, ssl: clean source dir before building --- cmake/curl_CMakeLists.txt.in | 7 ++++--- cmake/libs3_CMakeLists.txt.in | 4 ++-- cmake/ssl_CMakeLists.txt.in | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/cmake/curl_CMakeLists.txt.in b/cmake/curl_CMakeLists.txt.in index 3f7b9ee1fe..2fe84b9721 100644 --- a/cmake/curl_CMakeLists.txt.in +++ b/cmake/curl_CMakeLists.txt.in @@ -12,9 +12,10 @@ ExternalProject_Add(curl BUILD_ALWAYS 1 #UPDATE_COMMAND "" #CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --without-ssl --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd - CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --with-ssl=$ENV{HOME}/.cos-local.1 --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd --without-libidn2 + CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --with-ssl=$ENV{HOME}/.cos-local.1 --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd --without-libidn2 #--enable-debug + #CONFIGURE_COMMAND ./configure --without-ssl - BUILD_COMMAND make - INSTALL_COMMAND make install + BUILD_COMMAND "" + INSTALL_COMMAND make clean && make install TEST_COMMAND "" ) diff --git a/cmake/libs3_CMakeLists.txt.in b/cmake/libs3_CMakeLists.txt.in index b2d2d19059..6c911775ee 100644 --- a/cmake/libs3_CMakeLists.txt.in +++ b/cmake/libs3_CMakeLists.txt.in @@ -23,7 +23,7 @@ ExternalProject_Add(libs3 CONFIGURE_COMMAND cp ${TD_SUPPORT_DIR}/libs3.GNUmakefile GNUmakefile && sed -i "s|CFLAGS += -Wall -Werror|CFLAGS += -I'$ENV{HOME}/.cos-local.1/include' -L'$ENV{HOME}/.cos-local.1/lib' |" ./GNUmakefile #BUILD_COMMAND make CFLAGS=${s3_flags} DESTDIR=$ENV{HOME}/.cos-local.1 build/lib/libs3.a #BUILD_COMMAND make DESTDIR="$ENV{HOME}/.cos-local.1" build/lib/libs3.a - BUILD_COMMAND make build/lib/libs3.a - INSTALL_COMMAND make install_static + BUILD_COMMAND "" + INSTALL_COMMAND make clean && make build/lib/libs3.a && make install_static TEST_COMMAND "" ) diff --git a/cmake/ssl_CMakeLists.txt.in b/cmake/ssl_CMakeLists.txt.in index 65a02d7fd6..8301dbe222 100644 --- a/cmake/ssl_CMakeLists.txt.in +++ b/cmake/ssl_CMakeLists.txt.in @@ -10,6 +10,6 @@ ExternalProject_Add(openssl #UPDATE_COMMAND "" CONFIGURE_COMMAND ./Configure --prefix=$ENV{HOME}/.cos-local.1 -static #--no-shared BUILD_COMMAND "" - INSTALL_COMMAND make install_sw -j + INSTALL_COMMAND make clean && make install_sw -j TEST_COMMAND "" ) From b864c87b7c91d8663c592f25ee681f85cc3e761f Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 23 Oct 2023 17:00:52 +0800 Subject: [PATCH 38/51] cmake/ssl: make depend --- cmake/ssl_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/ssl_CMakeLists.txt.in b/cmake/ssl_CMakeLists.txt.in index 8301dbe222..c17043ff22 100644 --- a/cmake/ssl_CMakeLists.txt.in +++ b/cmake/ssl_CMakeLists.txt.in @@ -10,6 +10,6 @@ ExternalProject_Add(openssl #UPDATE_COMMAND "" CONFIGURE_COMMAND ./Configure --prefix=$ENV{HOME}/.cos-local.1 -static #--no-shared BUILD_COMMAND "" - INSTALL_COMMAND make clean && make install_sw -j + INSTALL_COMMAND make clean && make depend && make install_sw -j TEST_COMMAND "" ) From 7cf72a567da657f1c2f62401722dada5e757283b Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 23 Oct 2023 19:19:34 +0800 Subject: [PATCH 39/51] cmake/ssl: use build command --- cmake/ssl_CMakeLists.txt.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/ssl_CMakeLists.txt.in b/cmake/ssl_CMakeLists.txt.in index c17043ff22..1f3fd0a30d 100644 --- a/cmake/ssl_CMakeLists.txt.in +++ b/cmake/ssl_CMakeLists.txt.in @@ -9,7 +9,7 @@ ExternalProject_Add(openssl #BUILD_ALWAYS 1 #UPDATE_COMMAND "" CONFIGURE_COMMAND ./Configure --prefix=$ENV{HOME}/.cos-local.1 -static #--no-shared - BUILD_COMMAND "" - INSTALL_COMMAND make clean && make depend && make install_sw -j + BUILD_COMMAND make -j + INSTALL_COMMAND make install_sw -j TEST_COMMAND "" ) From 7d17c4cc90385350805fb90a3501796486dd48e6 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 23 Oct 2023 19:35:22 +0800 Subject: [PATCH 40/51] cmake/curl, libs3: use build command --- cmake/curl_CMakeLists.txt.in | 4 ++-- cmake/libs3_CMakeLists.txt.in | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmake/curl_CMakeLists.txt.in b/cmake/curl_CMakeLists.txt.in index 2fe84b9721..b013ed1807 100644 --- a/cmake/curl_CMakeLists.txt.in +++ b/cmake/curl_CMakeLists.txt.in @@ -15,7 +15,7 @@ ExternalProject_Add(curl CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --with-ssl=$ENV{HOME}/.cos-local.1 --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd --without-libidn2 #--enable-debug #CONFIGURE_COMMAND ./configure --without-ssl - BUILD_COMMAND "" - INSTALL_COMMAND make clean && make install + BUILD_COMMAND make -j + INSTALL_COMMAND make install TEST_COMMAND "" ) diff --git a/cmake/libs3_CMakeLists.txt.in b/cmake/libs3_CMakeLists.txt.in index 6c911775ee..b2d2d19059 100644 --- a/cmake/libs3_CMakeLists.txt.in +++ b/cmake/libs3_CMakeLists.txt.in @@ -23,7 +23,7 @@ ExternalProject_Add(libs3 CONFIGURE_COMMAND cp ${TD_SUPPORT_DIR}/libs3.GNUmakefile GNUmakefile && sed -i "s|CFLAGS += -Wall -Werror|CFLAGS += -I'$ENV{HOME}/.cos-local.1/include' -L'$ENV{HOME}/.cos-local.1/lib' |" ./GNUmakefile #BUILD_COMMAND make CFLAGS=${s3_flags} DESTDIR=$ENV{HOME}/.cos-local.1 build/lib/libs3.a #BUILD_COMMAND make DESTDIR="$ENV{HOME}/.cos-local.1" build/lib/libs3.a - BUILD_COMMAND "" - INSTALL_COMMAND make clean && make build/lib/libs3.a && make install_static + BUILD_COMMAND make build/lib/libs3.a + INSTALL_COMMAND make install_static TEST_COMMAND "" ) From 7fdf6cc696fae234f993479f5737451a384cfda3 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 24 Oct 2023 09:04:24 +0800 Subject: [PATCH 41/51] cmake: use new dir for libs3 --- cmake/curl_CMakeLists.txt.in | 7 ++----- cmake/libs3.GNUmakefile | 2 +- cmake/libs3_CMakeLists.txt.in | 15 +-------------- cmake/ssl_CMakeLists.txt.in | 2 +- contrib/CMakeLists.txt | 4 ++-- source/dnode/vnode/CMakeLists.txt | 8 ++++---- tests/parallel_test/container_build.sh | 4 ++-- 7 files changed, 13 insertions(+), 29 deletions(-) diff --git a/cmake/curl_CMakeLists.txt.in b/cmake/curl_CMakeLists.txt.in index b013ed1807..140fc48103 100644 --- a/cmake/curl_CMakeLists.txt.in +++ b/cmake/curl_CMakeLists.txt.in @@ -10,11 +10,8 @@ ExternalProject_Add(curl DEPENDS openssl BUILD_IN_SOURCE TRUE BUILD_ALWAYS 1 - #UPDATE_COMMAND "" - #CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --without-ssl --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd - CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --with-ssl=$ENV{HOME}/.cos-local.1 --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd --without-libidn2 #--enable-debug - - #CONFIGURE_COMMAND ./configure --without-ssl + UPDATE_COMMAND "" + CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.2 --with-ssl=$ENV{HOME}/.cos-local.2 --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd --without-libidn2 #--enable-debug BUILD_COMMAND make -j INSTALL_COMMAND make install TEST_COMMAND "" diff --git a/cmake/libs3.GNUmakefile b/cmake/libs3.GNUmakefile index abf954f5c1..98f98d8224 100644 --- a/cmake/libs3.GNUmakefile +++ b/cmake/libs3.GNUmakefile @@ -91,7 +91,7 @@ endif # -------------------------------------------------------------------------- # DESTDIR directory ifndef DESTDIR - DESTDIR := ${HOME}/.cos-local.1 + DESTDIR := ${HOME}/.cos-local.2 endif # -------------------------------------------------------------------------- diff --git a/cmake/libs3_CMakeLists.txt.in b/cmake/libs3_CMakeLists.txt.in index b2d2d19059..b5e9c438e6 100644 --- a/cmake/libs3_CMakeLists.txt.in +++ b/cmake/libs3_CMakeLists.txt.in @@ -1,15 +1,5 @@ # libs3 -set(s3_flags "${CMAKE_C_FLAGS} -O3 -Iinc -I$ENV{HOME}/.cos-local.1/include -Iinc ") - -set(SED_CMD "s/CFLAGS += -Wall -Werror/CFLAGS += -I$ENV{HOME}/.cos-local.1/include /") - -function(update_cflags) - file(READ "GNUmakefile" S3_CONTENT) - string(REPLACE "CFLAGS += -Wall -Werror" "CFLAGS += -I$ENV{HOME}/.cos-local.1/include " S3_CONTENT "${S3_CONTENT}") - file(WRITE "GNUmakefile" "${S3_CONTENT}") -endfunction(update_cflags) - ExternalProject_Add(libs3 GIT_REPOSITORY https://github.com/bji/libs3 #GIT_TAG v5.0.16 @@ -19,10 +9,7 @@ ExternalProject_Add(libs3 BUILD_IN_SOURCE TRUE BUILD_ALWAYS 1 UPDATE_COMMAND "" - #sed_cmd "s/CFLAGS += -Wall -Werror/CFLAGS += -I$ENV{HOME}/.cos-local.1/include /" - CONFIGURE_COMMAND cp ${TD_SUPPORT_DIR}/libs3.GNUmakefile GNUmakefile && sed -i "s|CFLAGS += -Wall -Werror|CFLAGS += -I'$ENV{HOME}/.cos-local.1/include' -L'$ENV{HOME}/.cos-local.1/lib' |" ./GNUmakefile - #BUILD_COMMAND make CFLAGS=${s3_flags} DESTDIR=$ENV{HOME}/.cos-local.1 build/lib/libs3.a - #BUILD_COMMAND make DESTDIR="$ENV{HOME}/.cos-local.1" build/lib/libs3.a + CONFIGURE_COMMAND cp ${TD_SUPPORT_DIR}/libs3.GNUmakefile GNUmakefile && sed -i "s|CFLAGS += -Wall -Werror|CFLAGS += -I'$ENV{HOME}/.cos-local.2/include' -L'$ENV{HOME}/.cos-local.2/lib' |" ./GNUmakefile BUILD_COMMAND make build/lib/libs3.a INSTALL_COMMAND make install_static TEST_COMMAND "" diff --git a/cmake/ssl_CMakeLists.txt.in b/cmake/ssl_CMakeLists.txt.in index 1f3fd0a30d..c9f836bade 100644 --- a/cmake/ssl_CMakeLists.txt.in +++ b/cmake/ssl_CMakeLists.txt.in @@ -8,7 +8,7 @@ ExternalProject_Add(openssl BUILD_IN_SOURCE TRUE #BUILD_ALWAYS 1 #UPDATE_COMMAND "" - CONFIGURE_COMMAND ./Configure --prefix=$ENV{HOME}/.cos-local.1 -static #--no-shared + CONFIGURE_COMMAND ./Configure --prefix=$ENV{HOME}/.cos-local.2 -static #--no-shared BUILD_COMMAND make -j INSTALL_COMMAND make install_sw -j TEST_COMMAND "" diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 013269c583..e2a5378f0d 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -9,7 +9,7 @@ endfunction(cat IN_FILE OUT_FILE) if(${TD_LINUX}) if(${BUILD_WITH_S3}) - file(MAKE_DIRECTORY $ENV{HOME}/.cos-local.1/) + file(MAKE_DIRECTORY $ENV{HOME}/.cos-local.2/) else() @@ -414,7 +414,7 @@ if (${BUILD_WITH_ROCKSDB}) endif() if(${BUILD_WITH_S3}) - INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.1/include) + INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.2/include) MESSAGE("build with s3: ${BUILD_WITH_S3}") else() diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 7e2693e8af..bc93ab575e 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -167,15 +167,15 @@ if(${BUILD_WITH_S3}) target_include_directories( vnode - PUBLIC "$ENV{HOME}/.cos-local.1/include" + PUBLIC "$ENV{HOME}/.cos-local.2/include" ) set(CMAKE_FIND_LIBRARY_SUFFIXES ".a") - set(CMAKE_PREFIX_PATH $ENV{HOME}/.cos-local.1) + set(CMAKE_PREFIX_PATH $ENV{HOME}/.cos-local.2) find_library(S3_LIBRARY s3) find_library(CURL_LIBRARY curl) - find_library(SSL_LIBRARY ssl PATHS $ENV{HOME}/.cos-local.1/lib64) - find_library(CRYPTO_LIBRARY crypto PATHS $ENV{HOME}/.cos-local.1/lib64) + find_library(SSL_LIBRARY ssl PATHS $ENV{HOME}/.cos-local.2/lib64) + find_library(CRYPTO_LIBRARY crypto PATHS $ENV{HOME}/.cos-local.2/lib64) target_link_libraries( vnode diff --git a/tests/parallel_test/container_build.sh b/tests/parallel_test/container_build.sh index f5e426057e..94704b1c25 100755 --- a/tests/parallel_test/container_build.sh +++ b/tests/parallel_test/container_build.sh @@ -60,7 +60,7 @@ docker run \ -v /root/.cargo/git:/root/.cargo/git \ -v /root/go/pkg/mod:/root/go/pkg/mod \ -v /root/.cache/go-build:/root/.cache/go-build \ - -v /root/.cos-local.1:/root/.cos-local.1 \ + -v /root/.cos-local.1:/root/.cos-local.2 \ -v ${REP_REAL_PATH}/enterprise/src/plugins/taosx/target:${REP_DIR}/enterprise/src/plugins/taosx/target \ -v ${REP_REAL_PATH}/community/tools/taosws-rs/target:${REP_DIR}/community/tools/taosws-rs/target \ -v ${REP_REAL_PATH}/community/contrib/cJson/:${REP_DIR}/community/contrib/cJson \ @@ -89,7 +89,7 @@ docker run \ -v /root/.cargo/git:/root/.cargo/git \ -v /root/go/pkg/mod:/root/go/pkg/mod \ -v /root/.cache/go-build:/root/.cache/go-build \ - -v /root/.cos-local.1:/root/.cos-local.1 \ + -v /root/.cos-local.1:/root/.cos-local.2 \ -v ${REP_REAL_PATH}/enterprise/src/plugins/taosx/target:${REP_DIR}/enterprise/src/plugins/taosx/target \ -v ${REP_REAL_PATH}/community/tools/taosws-rs/target:${REP_DIR}/community/tools/taosws-rs/target \ -v ${REP_REAL_PATH}/community/contrib/cJson/:${REP_DIR}/community/contrib/cJson \ From fd87f43f8cdfbb3668bfe011b14d1ce2d3794567 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 24 Oct 2023 09:10:58 +0800 Subject: [PATCH 42/51] enh: set CPUTYPE in cmake --- cmake/cmake.platform | 6 ++++++ cmake/cmake.version | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/cmake/cmake.platform b/cmake/cmake.platform index 18fd17f018..ca2cbbec2e 100644 --- a/cmake/cmake.platform +++ b/cmake/cmake.platform @@ -96,33 +96,39 @@ IF ("${CPUTYPE}" STREQUAL "") IF (CMAKE_SYSTEM_PROCESSOR MATCHES "(amd64)|(AMD64)") MESSAGE(STATUS "Current platform is amd64") SET(PLATFORM_ARCH_STR "amd64") + SET(CPUTYPE "x64") SET(TD_INTEL_64 TRUE) ADD_DEFINITIONS("-D_TD_X86_") ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)") MESSAGE(STATUS "Current platform is x86") SET(PLATFORM_ARCH_STR "i386") + SET(CPUTYPE "x86") SET(TD_INTEL_32 TRUE) ADD_DEFINITIONS("-D_TD_X86_") ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "armv7l") MESSAGE(STATUS "Current platform is aarch32") SET(PLATFORM_ARCH_STR "arm") + SET(CPUTYPE "arm32") SET(TD_ARM_32 TRUE) ADD_DEFINITIONS("-D_TD_ARM_") ADD_DEFINITIONS("-D_TD_ARM_32") ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "(aarch64)|(arm64)") MESSAGE(STATUS "Current platform is aarch64") SET(PLATFORM_ARCH_STR "arm64") + SET(CPUTYPE "arm64") SET(TD_ARM_64 TRUE) ADD_DEFINITIONS("-D_TD_ARM_") ADD_DEFINITIONS("-D_TD_ARM_64") ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "loongarch64") MESSAGE(STATUS "The current platform is loongarch64") SET(PLATFORM_ARCH_STR "loongarch64") + SET(CPUTYPE "loongarch64") SET(TD_LOONGARCH_64 TRUE) ADD_DEFINITIONS("-D_TD_LOONGARCH_") ADD_DEFINITIONS("-D_TD_LOONGARCH_64") ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "mips64") SET(PLATFORM_ARCH_STR "mips") + SET(CPUTYPE "mips64") MESSAGE(STATUS "input cpuType: mips64") SET(TD_MIPS_64 TRUE) ADD_DEFINITIONS("-D_TD_MIPS_") diff --git a/cmake/cmake.version b/cmake/cmake.version index c0ddae569c..b8871dbce3 100644 --- a/cmake/cmake.version +++ b/cmake/cmake.version @@ -79,9 +79,9 @@ ELSE () ELSEIF (TD_LINUX_32) SET(TD_VER_CPUTYPE "x86") ELSEIF (TD_ARM_32) - SET(TD_VER_CPUTYPE "x86") + SET(TD_VER_CPUTYPE "arm32") ELSEIF (TD_MIPS_32) - SET(TD_VER_CPUTYPE "x86") + SET(TD_VER_CPUTYPE "mips32") ELSE () SET(TD_VER_CPUTYPE "x64") ENDIF () From 0e65ed0202bb772908a8bc6867e6d1d2078c4f91 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 24 Oct 2023 09:16:09 +0800 Subject: [PATCH 43/51] cmake/curl: use new dir curl2 for s3 version --- cmake/curl_CMakeLists.txt.in | 4 ++-- cmake/libs3_CMakeLists.txt.in | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/curl_CMakeLists.txt.in b/cmake/curl_CMakeLists.txt.in index 140fc48103..458a518092 100644 --- a/cmake/curl_CMakeLists.txt.in +++ b/cmake/curl_CMakeLists.txt.in @@ -1,12 +1,12 @@ # curl -ExternalProject_Add(curl +ExternalProject_Add(curl2 URL https://curl.se/download/curl-8.2.1.tar.gz URL_HASH MD5=b25588a43556068be05e1624e0e74d41 DOWNLOAD_NO_PROGRESS 1 DOWNLOAD_DIR "${TD_CONTRIB_DIR}/deps-download" #GIT_REPOSITORY https://github.com/curl/curl.git #GIT_TAG curl-7_88_1 - SOURCE_DIR "${TD_CONTRIB_DIR}/curl" + SOURCE_DIR "${TD_CONTRIB_DIR}/curl2" DEPENDS openssl BUILD_IN_SOURCE TRUE BUILD_ALWAYS 1 diff --git a/cmake/libs3_CMakeLists.txt.in b/cmake/libs3_CMakeLists.txt.in index b5e9c438e6..f2b6cac953 100644 --- a/cmake/libs3_CMakeLists.txt.in +++ b/cmake/libs3_CMakeLists.txt.in @@ -3,7 +3,7 @@ ExternalProject_Add(libs3 GIT_REPOSITORY https://github.com/bji/libs3 #GIT_TAG v5.0.16 - DEPENDS curl xml2 + DEPENDS curl2 xml2 SOURCE_DIR "${TD_CONTRIB_DIR}/libs3" #BINARY_DIR "" BUILD_IN_SOURCE TRUE From 37131564a3c3c86df52bd47c4500b5631ff100e3 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 24 Oct 2023 09:54:52 +0800 Subject: [PATCH 44/51] fix:rollback processing with # --- tests/pytest/util/dnodes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 5f1522bfd9..c4fc1ce654 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -64,7 +64,7 @@ class TDSimClient: self.cfgDict.update({option: value}) def cfg(self, option, value): - cmd = "echo %s '%s' >> %s" % (option, value, self.cfgPath) + cmd = "echo %s %s >> %s" % (option, value, self.cfgPath) if os.system(cmd) != 0: tdLog.exit(cmd) From e826532592d520de945ca6fcf771123ae967e8fa Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 24 Oct 2023 10:10:43 +0800 Subject: [PATCH 45/51] cmake/vnode: specify no default path for ssl & crypto --- source/dnode/vnode/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index bc93ab575e..b83d64c569 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -174,8 +174,8 @@ if(${BUILD_WITH_S3}) set(CMAKE_PREFIX_PATH $ENV{HOME}/.cos-local.2) find_library(S3_LIBRARY s3) find_library(CURL_LIBRARY curl) - find_library(SSL_LIBRARY ssl PATHS $ENV{HOME}/.cos-local.2/lib64) - find_library(CRYPTO_LIBRARY crypto PATHS $ENV{HOME}/.cos-local.2/lib64) + find_library(SSL_LIBRARY ssl $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH) + find_library(CRYPTO_LIBRARY crypto $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH) target_link_libraries( vnode From a3fd13d43aa1fe856fc50c53aca2c89e972863a0 Mon Sep 17 00:00:00 2001 From: haoranchen Date: Tue, 24 Oct 2023 10:14:12 +0800 Subject: [PATCH 46/51] Update test_R.sh --- tests/docs-examples-test/test_R.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/docs-examples-test/test_R.sh b/tests/docs-examples-test/test_R.sh index 707ea02704..d56f8973f7 100755 --- a/tests/docs-examples-test/test_R.sh +++ b/tests/docs-examples-test/test_R.sh @@ -6,7 +6,7 @@ pgrep taosd || taosd >> /dev/null 2>&1 & pgrep taosadapter || taosadapter >> /dev/null 2>&1 & cd ../../docs/examples/R -wget -N https://repo1.maven.org/maven2/com/taosdata/jdbc/taos-jdbcdriver/3.2.4/taos-jdbcdriver-3.2.4-dist.jar +wget -N https://maven.aliyun.com/repository/central/com/taosdata/jdbc/taos-jdbcdriver/3.2.5/taos-jdbcdriver-3.2.5-dist.jar jar_path=`find . -name taos-jdbcdriver-*-dist.jar` echo jar_path=$jar_path From be7791f9cd25b1362907b414773ce6b5daf4ff33 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Tue, 24 Oct 2023 13:58:30 +0800 Subject: [PATCH 47/51] fix: disable life extending of conn cache obj when show queries/conns --- include/util/tcache.h | 2 ++ source/dnode/mnode/impl/src/mndProfile.c | 13 +++++++------ source/util/src/tcache.c | 12 ++++++++++++ 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/include/util/tcache.h b/include/util/tcache.h index d8ab018570..474e5274de 100644 --- a/include/util/tcache.h +++ b/include/util/tcache.h @@ -151,6 +151,8 @@ void *taosCacheIterGetData(const SCacheIter *pIter, size_t *dataLen); void *taosCacheIterGetKey(const SCacheIter *pIter, size_t *keyLen); void taosCacheDestroyIter(SCacheIter *pIter); +void taosCacheTryExtendLifeSpan(SCacheObj *pCacheObj, void **data); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 1f8c3b161b..4366053237 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -59,7 +59,7 @@ static SConnObj *mndCreateConn(SMnode *pMnode, const char *user, int8_t connType int32_t pid, const char *app, int64_t startTime); static void mndFreeConn(SConnObj *pConn); static SConnObj *mndAcquireConn(SMnode *pMnode, uint32_t connId); -static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn); +static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn, bool extendLifespan); static void *mndGetNextConn(SMnode *pMnode, SCacheIter *pIter); static void mndCancelGetNextConn(SMnode *pMnode, void *pIter); static int32_t mndProcessHeartBeatReq(SRpcMsg *pReq); @@ -79,7 +79,7 @@ int32_t mndInitProfile(SMnode *pMnode) { // in ms int32_t checkTime = tsShellActivityTimer * 2 * 1000; - pMgmt->connCache = taosCacheInit(TSDB_DATA_TYPE_UINT, checkTime, true, (__cache_free_fn_t)mndFreeConn, "conn"); + pMgmt->connCache = taosCacheInit(TSDB_DATA_TYPE_UINT, checkTime, false, (__cache_free_fn_t)mndFreeConn, "conn"); if (pMgmt->connCache == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; mError("failed to alloc profile cache since %s", terrstr()); @@ -185,11 +185,12 @@ static SConnObj *mndAcquireConn(SMnode *pMnode, uint32_t connId) { return pConn; } -static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn) { +static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn, bool extendLifespan) { if (pConn == NULL) return; mTrace("conn:%u, released from cache, data:%p", pConn->id, pConn); SProfileMgmt *pMgmt = &pMnode->profileMgmt; + if (extendLifespan) taosCacheTryExtendLifeSpan(pMgmt->connCache, (void **)&pConn); taosCacheRelease(pMgmt->connCache, (void **)&pConn, false); } @@ -323,7 +324,7 @@ _OVER: mndReleaseUser(pMnode, pUser); mndReleaseDb(pMnode, pDb); - mndReleaseConn(pMnode, pConn); + mndReleaseConn(pMnode, pConn, true); return code; } @@ -485,7 +486,7 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb SQueryHbRspBasic *rspBasic = taosMemoryCalloc(1, sizeof(SQueryHbRspBasic)); if (rspBasic == NULL) { - mndReleaseConn(pMnode, pConn); + mndReleaseConn(pMnode, pConn, true); terrno = TSDB_CODE_OUT_OF_MEMORY; mError("user:%s, conn:%u failed to process hb while since %s", pConn->user, pBasic->connId, terrstr()); return -1; @@ -508,7 +509,7 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb mndCreateQnodeList(pMnode, &rspBasic->pQnodeList, -1); - mndReleaseConn(pMnode, pConn); + mndReleaseConn(pMnode, pConn, true); hbRsp.query = rspBasic; } else { diff --git a/source/util/src/tcache.c b/source/util/src/tcache.c index 392ac5d8b2..11f8df4c93 100644 --- a/source/util/src/tcache.c +++ b/source/util/src/tcache.c @@ -997,3 +997,15 @@ void taosCacheDestroyIter(SCacheIter *pIter) { taosMemoryFreeClear(pIter->pCurrent); taosMemoryFreeClear(pIter); } + +void taosCacheTryExtendLifeSpan(SCacheObj *pCacheObj, void **data) { + if (!pCacheObj || !(*data)) return; + + SCacheNode *pNode = (SCacheNode *)((char *)(*data) - sizeof(SCacheNode)); + if (pNode->signature != pNode) return; + + if (!pNode->inTrashcan) { + atomic_store_64(&pNode->expireTime, pNode->lifespan + taosGetTimestampMs()); + uDebug("cache:%s, data:%p extend expire time: %" PRId64, pCacheObj->name, pNode->data, pNode->expireTime); + } +} From f9ca8fe9620e57f0c64c3ebb62d4dca2647352d9 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 24 Oct 2023 16:10:46 +0800 Subject: [PATCH 48/51] cmake: new option: BUILD_S3 --- CMakeLists.txt | 4 +--- cmake/cmake.options | 28 +++++++++++++++++++++------- contrib/CMakeLists.txt | 23 +++++++---------------- source/common/CMakeLists.txt | 14 +++++--------- source/dnode/vnode/CMakeLists.txt | 6 +++--- 5 files changed, 37 insertions(+), 38 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5048287cf6..66a6fd328d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,8 +16,6 @@ set(TD_SUPPORT_DIR "${TD_SOURCE_DIR}/cmake") set(TD_CONTRIB_DIR "${TD_SOURCE_DIR}/contrib") - - include(${TD_SUPPORT_DIR}/cmake.platform) include(${TD_SUPPORT_DIR}/cmake.define) include(${TD_SUPPORT_DIR}/cmake.options) @@ -46,4 +44,4 @@ add_subdirectory(examples/c) include(${TD_SUPPORT_DIR}/cmake.install) # docs -add_subdirectory(docs/doxgen) \ No newline at end of file +add_subdirectory(docs/doxgen) diff --git a/cmake/cmake.options b/cmake/cmake.options index bacc5adfb7..d34c34dd89 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -128,12 +128,16 @@ option( IF(${TD_LINUX}) option( - BUILD_WITH_S3 + BUILD_S3 "If build with s3" ON ) -IF(${BUILD_WITH_S3}) +option( + BUILD_WITH_S3 + "If build with s3" + ON +) option( BUILD_WITH_COS @@ -141,16 +145,26 @@ option( OFF ) +ENDIF () + +IF(${BUILD_S3}) + +IF(${BUILD_WITH_S3}) + +option(BUILD_WITH_COS "If build with cos" OFF) + ELSE () -option( - BUILD_WITH_COS - "If build with cos" - ON -) +option(BUILD_WITH_COS "If build with cos" ON) ENDIF () +ELSE () + +option(BUILD_WITH_S3 "If build with s3" OFF) + +option(BUILD_WITH_COS "If build with cos" OFF) + ENDIF () option( diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index e2a5378f0d..d631ceff11 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -6,12 +6,10 @@ function(cat IN_FILE OUT_FILE) file(APPEND ${OUT_FILE} "${CONTENTS}") endfunction(cat IN_FILE OUT_FILE) -if(${TD_LINUX}) - if(${BUILD_WITH_S3}) file(MAKE_DIRECTORY $ENV{HOME}/.cos-local.2/) -else() +elseif(${BUILD_WITH_COS}) set(CONTRIB_TMP_FILE3 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in3") configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3}) @@ -42,9 +40,8 @@ execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" . execute_process(COMMAND "${CMAKE_COMMAND}" --build . WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download") -endif(${BUILD_WITH_S3}) +endif() -endif(${TD_LINUX}) set(CONTRIB_TMP_FILE "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in") configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) @@ -170,19 +167,16 @@ if(${BUILD_WITH_S3}) cat("${TD_SUPPORT_DIR}/libs3_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) add_definitions(-DUSE_S3) -else() - # cos -if(${BUILD_WITH_COS}) +elseif(${BUILD_WITH_COS}) #cat("${TD_SUPPORT_DIR}/mxml_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) #cat("${TD_SUPPORT_DIR}/apr_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) #cat("${TD_SUPPORT_DIR}/apr-util_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) #cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) cat("${TD_SUPPORT_DIR}/cos_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) add_definitions(-DUSE_COS) -endif(${BUILD_WITH_COS}) -endif(${BUILD_WITH_S3}) +endif() # lucene if(${BUILD_WITH_LUCENE}) @@ -269,7 +263,7 @@ unset(CMAKE_PROJECT_INCLUDE_BEFORE) # xml2 if(${BUILD_WITH_S3}) add_subdirectory(xml2 EXCLUDE_FROM_ALL) -endif(${BUILD_WITH_S3}) +endif() # lz4 add_subdirectory(lz4/build/cmake EXCLUDE_FROM_ALL) @@ -417,10 +411,8 @@ if(${BUILD_WITH_S3}) INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.2/include) MESSAGE("build with s3: ${BUILD_WITH_S3}") -else() - # cos -if(${BUILD_WITH_COS}) +elseif(${BUILD_WITH_COS}) if(${TD_LINUX}) set(CMAKE_PREFIX_PATH $ENV{HOME}/.cos-local.1) #ADD_DEFINITIONS(-DMINIXML_LIBRARY=${CMAKE_BINARY_DIR}/build/lib/libxml.a) @@ -442,9 +434,8 @@ if(${BUILD_WITH_COS}) else() endif(${TD_LINUX}) -endif(${BUILD_WITH_COS}) -endif(${BUILD_WITH_S3}) +endif() # lucene # To support build on ubuntu: sudo apt-get install libboost-all-dev diff --git a/source/common/CMakeLists.txt b/source/common/CMakeLists.txt index 6bdcdf7d1d..c35845f9df 100644 --- a/source/common/CMakeLists.txt +++ b/source/common/CMakeLists.txt @@ -17,15 +17,11 @@ IF (TD_STORAGE) ADD_DEFINITIONS(-D_STORAGE) TARGET_LINK_LIBRARIES(common PRIVATE storage) - IF(${TD_LINUX}) - IF(${BUILD_WITH_COS}) - add_definitions(-DUSE_COS) - ENDIF(${BUILD_WITH_COS}) - - IF(${BUILD_WITH_S3}) - add_definitions(-DUSE_S3) - ENDIF(${BUILD_WITH_S3}) - ENDIF(${TD_LINUX}) + IF(${BUILD_WITH_S3}) + add_definitions(-DUSE_S3) + ELSEIF(${BUILD_WITH_COS}) + add_definitions(-DUSE_COS) + ENDIF() ENDIF () diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index b83d64c569..a10b38a1bf 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -161,7 +161,7 @@ target_link_libraries( PUBLIC index ) -if(${TD_LINUX}) +if(${BUILD_S3}) if(${BUILD_WITH_S3}) target_include_directories( @@ -188,7 +188,7 @@ if(${BUILD_WITH_S3}) ) add_definitions(-DUSE_S3) -endif(${BUILD_WITH_S3}) +endif() if(${BUILD_WITH_COS}) @@ -227,7 +227,7 @@ target_include_directories( add_definitions(-DUSE_COS) endif(${BUILD_WITH_COS}) -endif(${TD_LINUX}) +endif() IF (TD_GRANT) TARGET_LINK_LIBRARIES(vnode PUBLIC grant) From aa7fd64b9b964781493da67985b591f713605be4 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 24 Oct 2023 18:04:50 +0800 Subject: [PATCH 49/51] enh: cmake system process support x86_64 --- cmake/cmake.platform | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/cmake.platform b/cmake/cmake.platform index ca2cbbec2e..9b96ebe7cb 100644 --- a/cmake/cmake.platform +++ b/cmake/cmake.platform @@ -93,7 +93,7 @@ ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "Windows") ENDIF() IF ("${CPUTYPE}" STREQUAL "") - IF (CMAKE_SYSTEM_PROCESSOR MATCHES "(amd64)|(AMD64)") + IF (CMAKE_SYSTEM_PROCESSOR MATCHES "(amd64)|(AMD64)|(x86_64)|(X86_64)") MESSAGE(STATUS "Current platform is amd64") SET(PLATFORM_ARCH_STR "amd64") SET(CPUTYPE "x64") From c67853888f80030a29878149e608e20324619903 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 24 Oct 2023 18:06:19 +0800 Subject: [PATCH 50/51] cmake/xml2: build xml2 before s3 --- cmake/xml2_CMakeLists.txt.in | 16 ++++++++++------ contrib/CMakeLists.txt | 6 +++--- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/cmake/xml2_CMakeLists.txt.in b/cmake/xml2_CMakeLists.txt.in index 4c234d7a37..25eb5a69ac 100644 --- a/cmake/xml2_CMakeLists.txt.in +++ b/cmake/xml2_CMakeLists.txt.in @@ -1,12 +1,16 @@ # xml2 ExternalProject_Add(xml2 - GIT_REPOSITORY https://github.com/GNOME/libxml2 - GIT_TAG v2.11.5 + URL https://github.com/GNOME/libxml2/archive/refs/tags/v2.11.5.tar.gz + #GIT_REPOSITORY https://github.com/GNOME/libxml2 + #GIT_TAG v2.11.5 + DOWNLOAD_NO_PROGRESS 1 + DOWNLOAD_DIR "${TD_CONTRIB_DIR}/deps-download" SOURCE_DIR "${TD_CONTRIB_DIR}/xml2" - BINARY_DIR "" - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND "" + #BINARY_DIR "" + BUILD_IN_SOURCE TRUE + CONFIGURE_COMMAND ./autogen.sh && ./configure --prefix=$ENV{HOME}/.cos-local.2 --enable-shared=no --enable-static=yes --without-python + BUILD_COMMAND make -j + INSTALL_COMMAND make install && ln -s $ENV{HOME}/.cos-local.2/include/libxml2/libxml $ENV{HOME}/.cos-local.2/include/libxml TEST_COMMAND "" ) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index d631ceff11..c5715bd53f 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -261,9 +261,9 @@ target_include_directories( unset(CMAKE_PROJECT_INCLUDE_BEFORE) # xml2 -if(${BUILD_WITH_S3}) - add_subdirectory(xml2 EXCLUDE_FROM_ALL) -endif() +#if(${BUILD_WITH_S3}) +# add_subdirectory(xml2 EXCLUDE_FROM_ALL) +#endif() # lz4 add_subdirectory(lz4/build/cmake EXCLUDE_FROM_ALL) From d77ecbbb1b150611e5ccb9a6314213499ae0769c Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 25 Oct 2023 17:41:55 +0800 Subject: [PATCH 51/51] cmake/xml2: use sha256 to verify distribution release --- cmake/xml2_CMakeLists.txt.in | 6 ++++-- source/dnode/vnode/CMakeLists.txt | 5 +++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/cmake/xml2_CMakeLists.txt.in b/cmake/xml2_CMakeLists.txt.in index 25eb5a69ac..ad0704cdb9 100644 --- a/cmake/xml2_CMakeLists.txt.in +++ b/cmake/xml2_CMakeLists.txt.in @@ -1,7 +1,9 @@ # xml2 ExternalProject_Add(xml2 - URL https://github.com/GNOME/libxml2/archive/refs/tags/v2.11.5.tar.gz + URL https://download.gnome.org/sources/libxml2/2.11/libxml2-2.11.5.tar.xz + URL_HASH SHA256=3727b078c360ec69fa869de14bd6f75d7ee8d36987b071e6928d4720a28df3a6 + #https://github.com/GNOME/libxml2/archive/refs/tags/v2.11.5.tar.gz #GIT_REPOSITORY https://github.com/GNOME/libxml2 #GIT_TAG v2.11.5 DOWNLOAD_NO_PROGRESS 1 @@ -9,7 +11,7 @@ ExternalProject_Add(xml2 SOURCE_DIR "${TD_CONTRIB_DIR}/xml2" #BINARY_DIR "" BUILD_IN_SOURCE TRUE - CONFIGURE_COMMAND ./autogen.sh && ./configure --prefix=$ENV{HOME}/.cos-local.2 --enable-shared=no --enable-static=yes --without-python + CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.2 --enable-shared=no --enable-static=yes --without-python --without-lzma BUILD_COMMAND make -j INSTALL_COMMAND make install && ln -s $ENV{HOME}/.cos-local.2/include/libxml2/libxml $ENV{HOME}/.cos-local.2/include/libxml TEST_COMMAND "" diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index a10b38a1bf..dcc9f9a115 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -173,7 +173,8 @@ if(${BUILD_WITH_S3}) set(CMAKE_FIND_LIBRARY_SUFFIXES ".a") set(CMAKE_PREFIX_PATH $ENV{HOME}/.cos-local.2) find_library(S3_LIBRARY s3) - find_library(CURL_LIBRARY curl) + find_library(CURL_LIBRARY curl $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH) + find_library(XML2_LIBRARY xml2) find_library(SSL_LIBRARY ssl $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH) find_library(CRYPTO_LIBRARY crypto $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH) target_link_libraries( @@ -184,7 +185,7 @@ if(${BUILD_WITH_S3}) PUBLIC ${CURL_LIBRARY} PUBLIC ${SSL_LIBRARY} PUBLIC ${CRYPTO_LIBRARY} - PUBLIC xml2 + PUBLIC ${XML2_LIBRARY} ) add_definitions(-DUSE_S3)