From 8253874398cc78fa2d76df35963e0e290154c8a4 Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Wed, 11 Oct 2023 14:47:26 +0800
Subject: [PATCH 01/26] enh(tsdb/cos): new sdk for s3

---
 cmake/cmake.options               |  18 ++
 cmake/libs3.GNUmakefile           | 430 ++++++++++++++++++++++++++++++
 cmake/libs3_CMakeLists.txt.in     |  28 ++
 cmake/ssl_CMakeLists.txt.in       |  15 ++
 contrib/CMakeLists.txt            |  54 ++++
 source/dnode/vnode/CMakeLists.txt |   7 +-
 6 files changed, 551 insertions(+), 1 deletion(-)
 create mode 100644 cmake/libs3.GNUmakefile
 create mode 100644 cmake/libs3_CMakeLists.txt.in
 create mode 100644 cmake/ssl_CMakeLists.txt.in

diff --git a/cmake/cmake.options b/cmake/cmake.options
index 1d4e9ba515..bacc5adfb7 100644
--- a/cmake/cmake.options
+++ b/cmake/cmake.options
@@ -127,6 +127,22 @@ option(
 
 IF(${TD_LINUX})
 
+option(
+    BUILD_WITH_S3
+    "If build with s3"
+    ON
+)
+
+IF(${BUILD_WITH_S3})
+
+option(
+    BUILD_WITH_COS
+    "If build with cos"
+    OFF
+)
+
+ELSE ()
+
 option(
     BUILD_WITH_COS
     "If build with cos"
@@ -135,6 +151,8 @@ option(
 
 ENDIF ()
 
+ENDIF ()
+
 option(
     BUILD_WITH_SQLITE
     "If build with sqlite" 
diff --git a/cmake/libs3.GNUmakefile b/cmake/libs3.GNUmakefile
new file mode 100644
index 0000000000..abf954f5c1
--- /dev/null
+++ b/cmake/libs3.GNUmakefile
@@ -0,0 +1,430 @@
+# GNUmakefile
+#
+# Copyright 2008 Bryan Ischo <bryan@ischo.com>
+#
+# This file is part of libs3.
+#
+# libs3 is free software: you can redistribute it and/or modify it under the
+# terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, version 3 or above of the License.  You can also
+# redistribute and/or modify it under the terms of the GNU General Public
+# License, version 2 or above of the License.
+#
+# In addition, as a special exception, the copyright holders give
+# permission to link the code of this library and its programs with the
+# OpenSSL library, and distribute linked combinations including the two.
+#
+# libs3 is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# version 3 along with libs3, in a file named COPYING.  If not, see
+# <http://www.gnu.org/licenses/>.
+#
+# You should also have received a copy of the GNU General Public License
+# version 2 along with libs3, in a file named COPYING-GPLv2.  If not, see
+# <http://www.gnu.org/licenses/>.
+
+# I tried to use the autoconf/automake/autolocal/etc (i.e. autohell) tools
+# but I just couldn't stomach them.  Since this is a Makefile for POSIX
+# systems, I will simply do away with autohell completely and use a GNU
+# Makefile.  GNU make ought to be available pretty much everywhere, so I
+# don't see this being a significant issue for portability.
+
+# All commands assume a GNU compiler.  For systems which do not use a GNU
+# compiler, write scripts with the same names as these commands, and taking
+# the same arguments, and translate the arguments and commands into the
+# appropriate non-POSIX ones as needed.  libs3 assumes a GNU toolchain as
+# the most portable way to build software possible.  Non-POSIX, non-GNU
+# systems can do the work of supporting this build infrastructure.
+
+
+# --------------------------------------------------------------------------
+# Set libs3 version number, unless it is already set.
+
+LIBS3_VER_MAJOR ?= 4
+LIBS3_VER_MINOR ?= 1
+LIBS3_VER := $(LIBS3_VER_MAJOR).$(LIBS3_VER_MINOR)
+
+
+# -----------------------------------------------------------------------------
+# Determine verbosity.  VERBOSE_SHOW should be prepended to every command which
+# should only be displayed if VERBOSE is set.  QUIET_ECHO may be used to
+# echo text only if VERBOSE is not set.  Typically, a VERBOSE_SHOW command will
+# be paired with a QUIET_ECHO command, to provide a command which is displayed
+# in VERBOSE mode, along with text which is displayed in non-VERBOSE mode to
+# describe the command.
+#
+# No matter what VERBOSE is defined to, it ends up as true if it's defined.
+# This will be weird if you defined VERBOSE=false in the environment, and we
+# switch it to true here; but the meaning of VERBOSE is, "if it's defined to
+# any value, then verbosity is turned on".  So don't define VERBOSE if you
+# don't want verbosity in the build process.
+# -----------------------------------------------------------------------------
+
+ifdef VERBOSE
+        VERBOSE = true
+        VERBOSE_ECHO = @ echo
+        VERBOSE_SHOW =
+        QUIET_ECHO = @ echo > /dev/null
+else
+        VERBOSE = false
+        VERBOSE_ECHO = @ echo > /dev/null
+        VERBOSE_SHOW = @
+        QUIET_ECHO = @ echo
+endif
+
+
+# --------------------------------------------------------------------------
+# BUILD directory
+ifndef BUILD
+    ifdef DEBUG
+        BUILD := build-debug
+    else
+        BUILD := build
+    endif
+endif
+
+
+# --------------------------------------------------------------------------
+# DESTDIR directory
+ifndef DESTDIR
+    DESTDIR := ${HOME}/.cos-local.1
+endif
+
+# --------------------------------------------------------------------------
+# LIBDIR directory
+ifndef LIBDIR
+    LIBDIR := ${DESTDIR}/lib
+endif
+
+# --------------------------------------------------------------------------
+# Compiler CC handling
+ifndef CC
+    CC := gcc
+endif
+
+# --------------------------------------------------------------------------
+# Acquire configuration information for libraries that libs3 depends upon
+
+ifndef CURL_LIBS
+    CURL_LIBS := $(shell curl-config --libs)
+endif
+
+ifndef CURL_CFLAGS
+    CURL_CFLAGS := $(shell curl-config --cflags)
+endif
+
+ifndef LIBXML2_LIBS
+    LIBXML2_LIBS := $(shell xml2-config --libs)
+endif
+
+ifndef LIBXML2_CFLAGS
+    LIBXML2_CFLAGS := $(shell xml2-config --cflags)
+endif
+
+ifndef OPENSSL_LIBS
+    OPENSSL_LIBS := -lssl -lcrypto
+endif
+
+# --------------------------------------------------------------------------
+# These CFLAGS assume a GNU compiler.  For other compilers, write a script
+# which converts these arguments into their equivalent for that particular
+# compiler.
+
+ifndef CFLAGS
+    ifdef DEBUG
+        CFLAGS := -g
+    else
+        CFLAGS := -O3
+    endif
+endif
+
+CFLAGS += -Wall -Werror -Wshadow -Wextra -Iinc \
+          $(CURL_CFLAGS) $(LIBXML2_CFLAGS) \
+          -DLIBS3_VER_MAJOR=\"$(LIBS3_VER_MAJOR)\" \
+          -DLIBS3_VER_MINOR=\"$(LIBS3_VER_MINOR)\" \
+          -DLIBS3_VER=\"$(LIBS3_VER)\" \
+          -D__STRICT_ANSI__ \
+          -D_ISOC99_SOURCE \
+          -D_POSIX_C_SOURCE=200112L
+
+LDFLAGS = $(CURL_LIBS) $(LIBXML2_LIBS) $(OPENSSL_LIBS) -lpthread
+
+STRIP ?= strip
+INSTALL := install --strip-program=$(STRIP)
+
+
+# --------------------------------------------------------------------------
+# Default targets are everything
+
+.PHONY: all
+all: exported test
+
+
+# --------------------------------------------------------------------------
+# Exported targets are the library and driver program
+
+.PHONY: exported
+exported: libs3 s3 headers
+exported_static: $(LIBS3_STATIC)
+
+# --------------------------------------------------------------------------
+# Install target
+
+.PHONY: install install_static
+install_static: exported_static
+	$(QUIET_ECHO) $(LIBDIR)/libs3.a: Installing static library
+	$(VERBOSE_SHOW) $(INSTALL) -Dp -m u+rw,go+r $(BUILD)/lib/libs3.a \
+                    $(LIBDIR)/libs3.a
+	$(QUIET_ECHO) $(DESTDIR)/include/libs3.h: Installing header
+	$(VERBOSE_SHOW) $(INSTALL) -Dp -m u+rw,go+r inc/libs3.h \
+                    $(DESTDIR)/include/libs3.h
+
+install: exported
+	$(QUIET_ECHO) $(DESTDIR)/bin/s3: Installing executable
+	$(VERBOSE_SHOW) $(INSTALL) -Dps -m u+rwx,go+rx $(BUILD)/bin/s3 \
+                    $(DESTDIR)/bin/s3
+	$(QUIET_ECHO) \
+        $(LIBDIR)/libs3.so.$(LIBS3_VER): Installing shared library
+	$(VERBOSE_SHOW) $(INSTALL) -Dps -m u+rw,go+r \
+               $(BUILD)/lib/libs3.so.$(LIBS3_VER_MAJOR) \
+               $(LIBDIR)/libs3.so.$(LIBS3_VER)
+	$(QUIET_ECHO) \
+        $(LIBDIR)/libs3.so.$(LIBS3_VER_MAJOR): Linking shared library
+	$(VERBOSE_SHOW) ln -sf libs3.so.$(LIBS3_VER) \
+               $(LIBDIR)/libs3.so.$(LIBS3_VER_MAJOR)
+	$(QUIET_ECHO) $(LIBDIR)/libs3.so: Linking shared library
+	$(VERBOSE_SHOW) ln -sf libs3.so.$(LIBS3_VER_MAJOR) $(LIBDIR)/libs3.so
+	$(QUIET_ECHO) $(LIBDIR)/libs3.a: Installing static library
+	$(VERBOSE_SHOW) $(INSTALL) -Dp -m u+rw,go+r $(BUILD)/lib/libs3.a \
+                    $(LIBDIR)/libs3.a
+	$(QUIET_ECHO) $(DESTDIR)/include/libs3.h: Installing header
+	$(VERBOSE_SHOW) $(INSTALL) -Dp -m u+rw,go+r $(BUILD)/include/libs3.h \
+                    $(DESTDIR)/include/libs3.h
+
+
+# --------------------------------------------------------------------------
+# Uninstall target
+
+.PHONY: uninstall
+uninstall:
+	$(QUIET_ECHO) Installed files: Uninstalling
+	$(VERBOSE_SHOW) \
+	    rm -f $(DESTDIR)/bin/s3 \
+              $(DESTDIR)/include/libs3.h \
+              $(DESTDIR)/lib/libs3.a \
+              $(DESTDIR)/lib/libs3.so \
+              $(DESTDIR)/lib/libs3.so.$(LIBS3_VER_MAJOR) \
+              $(DESTDIR)/lib/libs3.so.$(LIBS3_VER)
+
+
+# --------------------------------------------------------------------------
+# Compile target patterns
+
+$(BUILD)/obj/%.o: src/%.c
+	$(QUIET_ECHO) $@: Compiling object
+	@ mkdir -p $(dir $(BUILD)/dep/$<)
+	@ $(CC) $(CFLAGS) -M -MG -MQ $@ -DCOMPILINGDEPENDENCIES \
+        -o $(BUILD)/dep/$(<:%.c=%.d) -c $<
+	@ mkdir -p $(dir $@)
+	$(VERBOSE_SHOW) $(CC) $(CFLAGS) -o $@ -c $<
+
+$(BUILD)/obj/%.do: src/%.c
+	$(QUIET_ECHO) $@: Compiling dynamic object
+	$(QUIET_ECHO) cflags:${CFLAGS}
+	@ mkdir -p $(dir $(BUILD)/dep/$<)
+	@ $(CC) $(CFLAGS) -M -MG -MQ $@ -DCOMPILINGDEPENDENCIES \
+        -o $(BUILD)/dep/$(<:%.c=%.dd) -c $<
+	@ mkdir -p $(dir $@)
+	$(VERBOSE_SHOW) $(CC) $(CFLAGS) -fpic -fPIC -o $@ -c $<
+
+
+# --------------------------------------------------------------------------
+# libs3 library targets
+
+LIBS3_SHARED = $(BUILD)/lib/libs3.so.$(LIBS3_VER_MAJOR)
+LIBS3_STATIC = $(BUILD)/lib/libs3.a
+
+.PHONY: libs3
+libs3: $(LIBS3_SHARED) $(LIBS3_STATIC)
+
+LIBS3_SOURCES := bucket.c bucket_metadata.c error_parser.c general.c \
+                 object.c request.c request_context.c \
+                 response_headers_handler.c service_access_logging.c \
+                 service.c simplexml.c util.c multipart.c
+
+$(LIBS3_SHARED): $(LIBS3_SOURCES:%.c=$(BUILD)/obj/%.do)
+	$(QUIET_ECHO) $@: Building shared library
+	@ mkdir -p $(dir $@)
+	$(VERBOSE_SHOW) $(CC) -shared -Wl,-soname,libs3.so.$(LIBS3_VER_MAJOR) \
+        -o $@ $^ $(LDFLAGS)
+
+$(LIBS3_STATIC): $(LIBS3_SOURCES:%.c=$(BUILD)/obj/%.o)
+	$(QUIET_ECHO) $@: Building static library
+	@ mkdir -p $(dir $@)
+	$(VERBOSE_SHOW) $(AR) cr $@ $^
+
+
+# --------------------------------------------------------------------------
+# Driver program targets
+
+.PHONY: s3
+s3: $(BUILD)/bin/s3
+
+$(BUILD)/bin/s3: $(BUILD)/obj/s3.o $(LIBS3_SHARED)
+	$(QUIET_ECHO) $@: Building executable
+	@ mkdir -p $(dir $@)
+	$(VERBOSE_SHOW) $(CC) -o $@ $^ $(LDFLAGS)
+
+
+# --------------------------------------------------------------------------
+# libs3 header targets
+
+.PHONY: headers
+headers: $(BUILD)/include/libs3.h
+
+$(BUILD)/include/libs3.h: inc/libs3.h
+	$(QUIET_ECHO) $@: Linking header
+	@ mkdir -p $(dir $@)
+	$(VERBOSE_SHOW) ln -sf $(abspath $<) $@
+
+
+# --------------------------------------------------------------------------
+# Test targets
+
+.PHONY: test
+test: $(BUILD)/bin/testsimplexml
+
+$(BUILD)/bin/testsimplexml: $(BUILD)/obj/testsimplexml.o $(LIBS3_STATIC)
+	$(QUIET_ECHO) $@: Building executable
+	@ mkdir -p $(dir $@)
+	$(VERBOSE_SHOW) $(CC) -o $@ $^ $(LIBXML2_LIBS)
+
+
+# --------------------------------------------------------------------------
+# Clean target
+
+.PHONY: clean
+clean:
+	$(QUIET_ECHO) $(BUILD): Cleaning
+	$(VERBOSE_SHOW) rm -rf $(BUILD)
+
+.PHONY: distclean
+distclean:
+	$(QUIET_ECHO) $(BUILD): Cleaning
+	$(VERBOSE_SHOW) rm -rf $(BUILD)
+
+
+# --------------------------------------------------------------------------
+# Clean dependencies target
+
+.PHONY: cleandeps
+cleandeps:
+	$(QUIET_ECHO) $(BUILD)/dep: Cleaning dependencies
+	$(VERBOSE_SHOW) rm -rf $(BUILD)/dep
+
+
+# --------------------------------------------------------------------------
+# Dependencies
+
+ALL_SOURCES := $(LIBS3_SOURCES) s3.c testsimplexml.c
+
+$(foreach i, $(ALL_SOURCES), $(eval -include $(BUILD)/dep/src/$(i:%.c=%.d)))
+$(foreach i, $(ALL_SOURCES), $(eval -include $(BUILD)/dep/src/$(i:%.c=%.dd)))
+
+
+# --------------------------------------------------------------------------
+# Debian package target
+
+DEBPKG = $(BUILD)/pkg/libs3_$(LIBS3_VER).deb
+DEBDEVPKG = $(BUILD)/pkg/libs3-dev_$(LIBS3_VER).deb
+
+.PHONY: deb
+deb: $(DEBPKG) $(DEBDEVPKG)
+
+$(DEBPKG): DEBARCH = $(shell dpkg-architecture | grep ^DEB_BUILD_ARCH= | \
+                       cut -d '=' -f 2)
+$(DEBPKG): exported $(BUILD)/deb/DEBIAN/control $(BUILD)/deb/DEBIAN/shlibs \
+           $(BUILD)/deb/DEBIAN/postinst \
+           $(BUILD)/deb/usr/share/doc/libs3/changelog.gz \
+           $(BUILD)/deb/usr/share/doc/libs3/changelog.Debian.gz \
+           $(BUILD)/deb/usr/share/doc/libs3/copyright
+	DESTDIR=$(BUILD)/deb/usr $(MAKE) install
+	rm -rf $(BUILD)/deb/usr/include
+	rm -f $(BUILD)/deb/usr/lib/libs3.a
+	@mkdir -p $(dir $@)
+	fakeroot dpkg-deb -b $(BUILD)/deb $@
+	mv $@ $(BUILD)/pkg/libs3_$(LIBS3_VER)_$(DEBARCH).deb
+
+$(DEBDEVPKG): DEBARCH = $(shell dpkg-architecture | grep ^DEB_BUILD_ARCH= | \
+                          cut -d '=' -f 2)
+$(DEBDEVPKG): exported $(BUILD)/deb-dev/DEBIAN/control \
+           $(BUILD)/deb-dev/usr/share/doc/libs3-dev/changelog.gz \
+           $(BUILD)/deb-dev/usr/share/doc/libs3-dev/changelog.Debian.gz \
+           $(BUILD)/deb-dev/usr/share/doc/libs3-dev/copyright
+	DESTDIR=$(BUILD)/deb-dev/usr $(MAKE) install
+	rm -rf $(BUILD)/deb-dev/usr/bin
+	rm -f $(BUILD)/deb-dev/usr/lib/libs3.so*
+	@mkdir -p $(dir $@)
+	fakeroot dpkg-deb -b $(BUILD)/deb-dev $@
+	mv $@ $(BUILD)/pkg/libs3-dev_$(LIBS3_VER)_$(DEBARCH).deb
+
+$(BUILD)/deb/DEBIAN/control: debian/control
+	@mkdir -p $(dir $@)
+	echo -n "Depends: " > $@
+	dpkg-shlibdeps -Sbuild -O $(BUILD)/lib/libs3.so.$(LIBS3_VER_MAJOR) | \
+            cut -d '=' -f 2- >> $@
+	sed -e 's/LIBS3_VERSION/$(LIBS3_VER)/' \
+            < $< | sed -e 's/DEBIAN_ARCHITECTURE/$(DEBARCH)/' | \
+            grep -v ^Source: >> $@
+
+$(BUILD)/deb-dev/DEBIAN/control: debian/control.dev
+	@mkdir -p $(dir $@)
+	sed -e 's/LIBS3_VERSION/$(LIBS3_VER)/' \
+            < $< | sed -e 's/DEBIAN_ARCHITECTURE/$(DEBARCH)/' > $@
+
+$(BUILD)/deb/DEBIAN/shlibs:
+	echo -n "libs3 $(LIBS3_VER_MAJOR) libs3 " > $@
+	echo "(>= $(LIBS3_VER))" >> $@
+
+$(BUILD)/deb/DEBIAN/postinst: debian/postinst
+	@mkdir -p $(dir $@)
+	cp $< $@
+
+$(BUILD)/deb/usr/share/doc/libs3/copyright: LICENSE
+	@mkdir -p $(dir $@)
+	cp $< $@
+	@echo >> $@
+	@echo -n "An alternate location for the GNU General Public " >> $@
+	@echo "License version 3 on Debian" >> $@
+	@echo "systems is /usr/share/common-licenses/GPL-3." >> $@
+
+$(BUILD)/deb-dev/usr/share/doc/libs3-dev/copyright: LICENSE
+	@mkdir -p $(dir $@)
+	cp $< $@
+	@echo >> $@
+	@echo -n "An alternate location for the GNU General Public " >> $@
+	@echo "License version 3 on Debian" >> $@
+	@echo "systems is /usr/share/common-licenses/GPL-3." >> $@
+
+$(BUILD)/deb/usr/share/doc/libs3/changelog.gz: debian/changelog
+	@mkdir -p $(dir $@)
+	gzip --best -c $< > $@
+
+$(BUILD)/deb-dev/usr/share/doc/libs3-dev/changelog.gz: debian/changelog
+	@mkdir -p $(dir $@)
+	gzip --best -c $< > $@
+
+$(BUILD)/deb/usr/share/doc/libs3/changelog.Debian.gz: debian/changelog.Debian
+	@mkdir -p $(dir $@)
+	gzip --best -c $< > $@
+
+$(BUILD)/deb-dev/usr/share/doc/libs3-dev/changelog.Debian.gz: \
+    debian/changelog.Debian
+	@mkdir -p $(dir $@)
+	gzip --best -c $< > $@
+
+
diff --git a/cmake/libs3_CMakeLists.txt.in b/cmake/libs3_CMakeLists.txt.in
new file mode 100644
index 0000000000..b77299baf3
--- /dev/null
+++ b/cmake/libs3_CMakeLists.txt.in
@@ -0,0 +1,28 @@
+# libs3
+
+set(s3_flags "${CMAKE_C_FLAGS} -O3 -Iinc -I$ENV{HOME}/.cos-local.1/include -Iinc ")
+
+set(SED_CMD "s/CFLAGS += -Wall -Werror/CFLAGS += -I$ENV{HOME}/.cos-local.1/include /")
+
+function(update_cflags)
+    file(READ "GNUmakefile" S3_CONTENT)
+    string(REPLACE "CFLAGS += -Wall -Werror" "CFLAGS += -I$ENV{HOME}/.cos-local.1/include " S3_CONTENT "${S3_CONTENT}")
+    file(WRITE "GNUmakefile" "${S3_CONTENT}")
+endfunction(update_cflags)
+
+ExternalProject_Add(libs3
+        GIT_REPOSITORY https://github.com/bji/libs3
+	#GIT_TAG v5.0.16
+        SOURCE_DIR "${TD_CONTRIB_DIR}/libs3"
+        #BINARY_DIR ""
+        BUILD_IN_SOURCE TRUE
+        BUILD_ALWAYS 1
+	UPDATE_COMMAND ""
+	#sed_cmd "s/CFLAGS += -Wall -Werror/CFLAGS += -I$ENV{HOME}/.cos-local.1/include /"
+	CONFIGURE_COMMAND cp ${TD_SUPPORT_DIR}/libs3.GNUmakefile GNUmakefile &&  sed -i "s|CFLAGS += -Wall -Werror|CFLAGS += -I'$ENV{HOME}/.cos-local.1/include' -L'$ENV{HOME}/.cos-local.1/lib' |" ./GNUmakefile
+        #BUILD_COMMAND make CFLAGS=${s3_flags} DESTDIR=$ENV{HOME}/.cos-local.1 build/lib/libs3.a
+        #BUILD_COMMAND make DESTDIR="$ENV{HOME}/.cos-local.1" build/lib/libs3.a
+        BUILD_COMMAND make build/lib/libs3.a
+        INSTALL_COMMAND make install_static
+        TEST_COMMAND ""
+)
diff --git a/cmake/ssl_CMakeLists.txt.in b/cmake/ssl_CMakeLists.txt.in
new file mode 100644
index 0000000000..7821848618
--- /dev/null
+++ b/cmake/ssl_CMakeLists.txt.in
@@ -0,0 +1,15 @@
+# openssl
+ExternalProject_Add(openssl
+        URL https://www.openssl.org/source/openssl-3.1.3.tar.gz
+        URL_HASH SHA256=f0316a2ebd89e7f2352976445458689f80302093788c466692fb2a188b2eacf6
+        DOWNLOAD_NO_PROGRESS 1
+        DOWNLOAD_DIR "${TD_CONTRIB_DIR}/deps-download"
+        SOURCE_DIR "${TD_CONTRIB_DIR}/openssl"
+        BUILD_IN_SOURCE TRUE
+        #BUILD_ALWAYS 1
+        #UPDATE_COMMAND ""
+        CONFIGURE_COMMAND ./Configure --prefix=$ENV{HOME}/.cos-local.1 -static #--no-shared
+        BUILD_COMMAND make -j
+        INSTALL_COMMAND make install
+        TEST_COMMAND ""
+)
diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt
index e3e48ac3a1..39bf85df8f 100644
--- a/contrib/CMakeLists.txt
+++ b/contrib/CMakeLists.txt
@@ -8,6 +8,37 @@ endfunction(cat IN_FILE OUT_FILE)
 
 if(${TD_LINUX})
 
+  if(${BUILD_WITH_S3})
+
+set(CONTRIB_TMP_FILE3 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in3")
+configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3})
+
+if(${BUILD_WITH_COS})
+    file(MAKE_DIRECTORY $ENV{HOME}/.cos-local.1/)
+    cat("${TD_SUPPORT_DIR}/ssl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3})
+endif(${BUILD_WITH_COS})
+
+configure_file(${CONTRIB_TMP_FILE3} "${TD_CONTRIB_DIR}/deps-download/CMakeLists.txt")
+execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" .
+        WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download")
+execute_process(COMMAND "${CMAKE_COMMAND}" --build .
+        WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download")
+
+set(CONTRIB_TMP_FILE2 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in2")
+configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE2})
+
+if(${BUILD_WITH_COS})
+    cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE2})
+endif(${BUILD_WITH_COS})
+
+configure_file(${CONTRIB_TMP_FILE2} "${TD_CONTRIB_DIR}/deps-download/CMakeLists.txt")
+execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" .
+        WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download")
+execute_process(COMMAND "${CMAKE_COMMAND}" --build .
+        WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download")
+
+else()
+
 set(CONTRIB_TMP_FILE3 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in3")
 configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3})
 
@@ -37,6 +68,8 @@ execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" .
 execute_process(COMMAND "${CMAKE_COMMAND}" --build .
         WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download")
 
+endif(${BUILD_WITH_S3})
+
 endif(${TD_LINUX})
 
 set(CONTRIB_TMP_FILE "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in")
@@ -155,6 +188,18 @@ if(${BUILD_WITH_SQLITE})
     cat("${TD_SUPPORT_DIR}/sqlite_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
 endif(${BUILD_WITH_SQLITE})
 
+# s3
+if(${BUILD_WITH_S3})
+    #cat("${TD_SUPPORT_DIR}/mxml_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+    #cat("${TD_SUPPORT_DIR}/apr_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+    #cat("${TD_SUPPORT_DIR}/apr-util_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+    #cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+    #INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.1/include)
+    cat("${TD_SUPPORT_DIR}/libs3_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+    add_definitions(-DUSE_S3)
+
+else()
+
 # cos
 if(${BUILD_WITH_COS})
     #cat("${TD_SUPPORT_DIR}/mxml_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
@@ -165,6 +210,8 @@ if(${BUILD_WITH_COS})
     add_definitions(-DUSE_COS)
 endif(${BUILD_WITH_COS})
 
+endif(${BUILD_WITH_S3})
+
 # lucene
 if(${BUILD_WITH_LUCENE})
     cat("${TD_SUPPORT_DIR}/lucene_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
@@ -390,6 +437,11 @@ if (${BUILD_WITH_ROCKSDB})
     endif()
 endif()
 
+if(${BUILD_WITH_S3})
+    INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.1/include)
+
+else()
+
 # cos
 if(${BUILD_WITH_COS})
   if(${TD_LINUX})
@@ -415,6 +467,8 @@ if(${BUILD_WITH_COS})
   endif(${TD_LINUX})
 endif(${BUILD_WITH_COS})
 
+endif(${BUILD_WITH_S3})
+
 # lucene
 # To support build on ubuntu: sudo apt-get install libboost-all-dev
 if(${BUILD_WITH_LUCENE})
diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt
index 84d54f3350..516c8a8b69 100644
--- a/source/dnode/vnode/CMakeLists.txt
+++ b/source/dnode/vnode/CMakeLists.txt
@@ -162,6 +162,12 @@ target_link_libraries(
 )
 
 if(${TD_LINUX})
+    if(${BUILD_WITH_S3})
+
+    endif(${BUILD_WITH_S3})
+
+if(${BUILD_WITH_COS})
+
 set(CMAKE_FIND_LIBRARY_SUFFIXES ".a")
 find_library(APR_LIBRARY apr-1 PATHS /usr/local/apr/lib/)
 find_library(APR_UTIL_LIBRARY aprutil-1 PATHS /usr/local/apr/lib/)
@@ -194,7 +200,6 @@ target_include_directories(
   PUBLIC "$ENV{HOME}/.cos-local.1/include"
   )
 
-if(${BUILD_WITH_COS})
   add_definitions(-DUSE_COS)
 endif(${BUILD_WITH_COS})
 

From 0a1ff3283e50db9bde8cfe62a8a99166bd04504d Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Fri, 13 Oct 2023 16:59:05 +0800
Subject: [PATCH 02/26] vnode/cos: first round size delte and get

---
 cmake/curl_CMakeLists.txt.in          |   3 +-
 cmake/libs3_CMakeLists.txt.in         |   1 +
 contrib/CMakeLists.txt                |  14 +-
 source/common/src/tglobal.c           |  16 +-
 source/dnode/vnode/CMakeLists.txt     |  29 +++-
 source/dnode/vnode/src/vnd/vnodeCos.c | 205 +++++++++++++++++++++++++-
 6 files changed, 251 insertions(+), 17 deletions(-)

diff --git a/cmake/curl_CMakeLists.txt.in b/cmake/curl_CMakeLists.txt.in
index 1f2291c519..24a1210422 100644
--- a/cmake/curl_CMakeLists.txt.in
+++ b/cmake/curl_CMakeLists.txt.in
@@ -10,7 +10,8 @@ ExternalProject_Add(curl
         BUILD_IN_SOURCE TRUE
         BUILD_ALWAYS 1
         #UPDATE_COMMAND ""
-        CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --without-ssl --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd
+        #CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --without-ssl --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd
+        CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --with-ssl=$ENV{HOME}/.cos-local.1  --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd
         #CONFIGURE_COMMAND ./configure --without-ssl
         BUILD_COMMAND make
         INSTALL_COMMAND make install
diff --git a/cmake/libs3_CMakeLists.txt.in b/cmake/libs3_CMakeLists.txt.in
index b77299baf3..b54a084f66 100644
--- a/cmake/libs3_CMakeLists.txt.in
+++ b/cmake/libs3_CMakeLists.txt.in
@@ -13,6 +13,7 @@ endfunction(update_cflags)
 ExternalProject_Add(libs3
         GIT_REPOSITORY https://github.com/bji/libs3
 	#GIT_TAG v5.0.16
+	DEPENDS curl
         SOURCE_DIR "${TD_CONTRIB_DIR}/libs3"
         #BINARY_DIR ""
         BUILD_IN_SOURCE TRUE
diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt
index 39bf85df8f..1588eb4025 100644
--- a/contrib/CMakeLists.txt
+++ b/contrib/CMakeLists.txt
@@ -13,10 +13,8 @@ if(${TD_LINUX})
 set(CONTRIB_TMP_FILE3 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in3")
 configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3})
 
-if(${BUILD_WITH_COS})
     file(MAKE_DIRECTORY $ENV{HOME}/.cos-local.1/)
     cat("${TD_SUPPORT_DIR}/ssl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3})
-endif(${BUILD_WITH_COS})
 
 configure_file(${CONTRIB_TMP_FILE3} "${TD_CONTRIB_DIR}/deps-download/CMakeLists.txt")
 execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" .
@@ -27,9 +25,7 @@ execute_process(COMMAND "${CMAKE_COMMAND}" --build .
 set(CONTRIB_TMP_FILE2 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in2")
 configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE2})
 
-if(${BUILD_WITH_COS})
     cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE2})
-endif(${BUILD_WITH_COS})
 
 configure_file(${CONTRIB_TMP_FILE2} "${TD_CONTRIB_DIR}/deps-download/CMakeLists.txt")
 execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" .
@@ -129,6 +125,9 @@ if(${BUILD_TEST})
     cat("${TD_SUPPORT_DIR}/stub_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
 endif(${BUILD_TEST})
 
+# xml2
+cat("${TD_SUPPORT_DIR}/xml2_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+
 # lz4
 cat("${TD_SUPPORT_DIR}/lz4_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
 
@@ -195,6 +194,7 @@ if(${BUILD_WITH_S3})
     #cat("${TD_SUPPORT_DIR}/apr-util_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
     #cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
     #INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.1/include)
+    cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE2})
     cat("${TD_SUPPORT_DIR}/libs3_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
     add_definitions(-DUSE_S3)
 
@@ -295,6 +295,9 @@ target_include_directories(
 )
 unset(CMAKE_PROJECT_INCLUDE_BEFORE)
 
+# xml2
+add_subdirectory(xml2 EXCLUDE_FROM_ALL)
+
 # lz4
 add_subdirectory(lz4/build/cmake EXCLUDE_FROM_ALL)
 target_include_directories(
@@ -439,6 +442,7 @@ endif()
 
 if(${BUILD_WITH_S3})
     INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.1/include)
+    MESSAGE("build with s3: ${BUILD_WITH_S3}")
 
 else()
 
@@ -449,7 +453,7 @@ if(${BUILD_WITH_COS})
     #ADD_DEFINITIONS(-DMINIXML_LIBRARY=${CMAKE_BINARY_DIR}/build/lib/libxml.a)
     option(ENABLE_TEST "Enable the tests" OFF)
     INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.1/include)
-    MESSAGE("$ENV{HOME}/.cos-local.1/include")
+    #MESSAGE("$ENV{HOME}/.cos-local.1/include")
 
     set(CMAKE_BUILD_TYPE debug)
     set(ORIG_CMAKE_PROJECT_NAME ${CMAKE_PROJECT_NAME})
diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c
index 92b5ff2828..8ea5528fbb 100644
--- a/source/common/src/tglobal.c
+++ b/source/common/src/tglobal.c
@@ -15,12 +15,12 @@
 
 #define _DEFAULT_SOURCE
 #include "tglobal.h"
+#include "defines.h"
 #include "os.h"
 #include "tconfig.h"
 #include "tgrant.h"
 #include "tlog.h"
 #include "tmisce.h"
-#include "defines.h"
 
 #if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL)
 #include "cus_name.h"
@@ -221,7 +221,7 @@ float    tsFPrecision = 1E-8;                   // float column precision
 double   tsDPrecision = 1E-16;                  // double column precision
 uint32_t tsMaxRange = 500;                      // max quantization intervals
 uint32_t tsCurRange = 100;                      // current quantization intervals
-bool     tsIfAdtFse = false;                     // ADT-FSE algorithom or original huffman algorithom
+bool     tsIfAdtFse = false;                    // ADT-FSE algorithom or original huffman algorithom
 char     tsCompressor[32] = "ZSTD_COMPRESSOR";  // ZSTD_COMPRESSOR or GZIP_COMPRESSOR
 
 // udf
@@ -266,6 +266,9 @@ char   tsS3BucketName[TSDB_FQDN_LEN] = "<bucketname>";
 char   tsS3AppId[TSDB_FQDN_LEN] = "<appid>";
 int8_t tsS3Enabled = false;
 
+int8_t tsS3Https = true;
+char   tsS3Hostname[TSDB_FQDN_LEN] = "<hostname>";
+
 int32_t tsS3BlockSize = 4096;     // number of tsdb pages
 int32_t tsS3BlockCacheSize = 16;  // number of blocks
 
@@ -307,6 +310,14 @@ int32_t taosSetS3Cfg(SConfig *pCfg) {
   tstrncpy(tsS3AccessKeySecret, colon + 1, TSDB_FQDN_LEN);
   tstrncpy(tsS3Endpoint, cfgGetItem(pCfg, "s3Endpoint")->str, TSDB_FQDN_LEN);
   tstrncpy(tsS3BucketName, cfgGetItem(pCfg, "s3BucketName")->str, TSDB_FQDN_LEN);
+  char *proto = strstr(tsS3Endpoint, "https://");
+  if (!proto) {
+    tsS3Https = false;
+    tstrncpy(tsS3Hostname, tsS3Endpoint + 7, TSDB_FQDN_LEN);
+  } else {
+    tstrncpy(tsS3Hostname, tsS3Endpoint + 8, TSDB_FQDN_LEN);
+  }
+
   char *cos = strstr(tsS3Endpoint, "cos.");
   if (cos) {
     char *appid = strrchr(tsS3BucketName, '-');
@@ -1086,7 +1097,6 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
   tsIfAdtFse = cfgGetItem(pCfg, "IfAdtFse")->bval;
   tstrncpy(tsCompressor, cfgGetItem(pCfg, "Compressor")->str, sizeof(tsCompressor));
 
-
   tsDisableStream = cfgGetItem(pCfg, "disableStream")->bval;
   tsStreamBufferSize = cfgGetItem(pCfg, "streamBufferSize")->i64;
 
diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt
index 516c8a8b69..e554d94e4d 100644
--- a/source/dnode/vnode/CMakeLists.txt
+++ b/source/dnode/vnode/CMakeLists.txt
@@ -162,9 +162,34 @@ target_link_libraries(
 )
 
 if(${TD_LINUX})
-    if(${BUILD_WITH_S3})
 
-    endif(${BUILD_WITH_S3})
+if(${BUILD_WITH_S3})
+  MESSAGE("build with s3: ${BUILD_WITH_S3}")
+  target_include_directories(
+    vnode
+
+    PUBLIC "$ENV{HOME}/.cos-local.1/include"
+    )
+
+  set(CMAKE_FIND_LIBRARY_SUFFIXES ".a")
+  set(CMAKE_PREFIX_PATH $ENV{HOME}/.cos-local.1)
+  find_library(S3_LIBRARY s3)
+  find_library(CURL_LIBRARY curl)
+  find_library(SSL_LIBRARY ssl PATHS $ENV{HOME}/.cos-local.1/lib64)
+  find_library(CRYPTO_LIBRARY crypto PATHS $ENV{HOME}/.cos-local.1/lib64)
+  target_link_libraries(
+    vnode
+
+    # s3
+    PUBLIC ${S3_LIBRARY}
+    PUBLIC ${CURL_LIBRARY}
+    PUBLIC ${SSL_LIBRARY}
+    PUBLIC ${CRYPTO_LIBRARY}
+    PUBLIC xml2
+  )
+
+  add_definitions(-DUSE_S3)
+endif(${BUILD_WITH_S3})
 
 if(${BUILD_WITH_COS})
 
diff --git a/source/dnode/vnode/src/vnd/vnodeCos.c b/source/dnode/vnode/src/vnd/vnodeCos.c
index 5f650be97d..8984d33ad8 100644
--- a/source/dnode/vnode/src/vnd/vnodeCos.c
+++ b/source/dnode/vnode/src/vnd/vnodeCos.c
@@ -2,13 +2,206 @@
 
 #include "vndCos.h"
 
-extern char tsS3Endpoint[];
-extern char tsS3AccessKeyId[];
-extern char tsS3AccessKeySecret[];
-extern char tsS3BucketName[];
-extern char tsS3AppId[];
+extern char   tsS3Endpoint[];
+extern char   tsS3AccessKeyId[];
+extern char   tsS3AccessKeySecret[];
+extern char   tsS3BucketName[];
+extern char   tsS3AppId[];
+extern char   tsS3Hostname[];
+extern int8_t tsS3Https;
+
+#if defined(USE_S3)
+
+#include "libs3.h"
+
+static int         verifyPeerG = 0;
+static const char *awsRegionG = NULL;
+static int         forceG = 0;
+static int         showResponsePropertiesG = 0;
+static S3Protocol  protocolG = S3ProtocolHTTPS;
+//  static S3Protocol protocolG = S3ProtocolHTTP;
+static S3UriStyle uriStyleG = S3UriStylePath;
+static int        retriesG = 5;
+static int        timeoutMsG = 0;
+
+static int32_t s3Begin() {
+  S3Status    status;
+  const char *hostname = tsS3Hostname;
+  const char *env_hn = getenv("S3_HOSTNAME");
+
+  if (env_hn) {
+    hostname = env_hn;
+  }
+
+  if ((status = S3_initialize("s3", verifyPeerG | S3_INIT_ALL, hostname)) != S3StatusOK) {
+    vError("Failed to initialize libs3: %s\n", S3_get_status_name(status));
+    return -1;
+  }
+
+  protocolG = !tsS3Https;
+
+  return 0;
+}
+
+static void s3End() { S3_deinitialize(); }
+int32_t     s3Init() { return s3Begin(); }
+
+void s3CleanUp() { s3End(); }
+
+static int should_retry() {
+  /*
+  if (retriesG--) {
+    // Sleep before next retry; start out with a 1 second sleep
+    static int retrySleepInterval = 1 * SLEEP_UNITS_PER_SECOND;
+    sleep(retrySleepInterval);
+    // Next sleep 1 second longer
+    retrySleepInterval++;
+    return 1;
+  }
+  */
+
+  return 0;
+}
+
+typedef struct {
+  uint64_t content_length;
+  int      status;
+  char    *buf;
+  char     err_msg[4096];
+} TS3SizeCBD;
+
+static S3Status responsePropertiesCallback(const S3ResponseProperties *properties, void *callbackData) {
+  //(void)callbackData;
+  TS3SizeCBD *cbd = callbackData;
+  if (properties->contentLength > 0) {
+    cbd->content_length = properties->contentLength;
+  } else {
+    cbd->content_length = 0;
+  }
+
+  return S3StatusOK;
+}
+
+static void responseCompleteCallback(S3Status status, const S3ErrorDetails *error, void *callbackData) {
+  TS3SizeCBD *cbd = callbackData;
+  cbd->status = status;
+
+  int       len = 0;
+  const int elen = sizeof(cbd->err_msg);
+  if (error) {
+    if (error->message) {
+      len += snprintf(&(cbd->err_msg[len]), elen - len, "  Message: %s\n", error->message);
+    }
+    if (error->resource) {
+      len += snprintf(&(cbd->err_msg[len]), elen - len, "  Resource: %s\n", error->resource);
+    }
+    if (error->furtherDetails) {
+      len += snprintf(&(cbd->err_msg[len]), elen - len, "  Further Details: %s\n", error->furtherDetails);
+    }
+    if (error->extraDetailsCount) {
+      len += snprintf(&(cbd->err_msg[len]), elen - len, "%s", "  Extra Details:\n");
+      for (int i = 0; i < error->extraDetailsCount; i++) {
+        len += snprintf(&(cbd->err_msg[len]), elen - len, "    %s: %s\n", error->extraDetails[i].name,
+                        error->extraDetails[i].value);
+      }
+    }
+  }
+}
+
+int32_t s3PutObjectFromFile2(const char *file, const char *object) { return 0; }
+void    s3DeleteObjectsByPrefix(const char *prefix) {}
+
+void s3DeleteObjects(const char *object_name[], int nobject) {
+  int               status = 0;
+  S3BucketContext   bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret,
+                                     0, awsRegionG};
+  S3ResponseHandler responseHandler = {0, &responseCompleteCallback};
+
+  for (int i = 0; i < nobject; ++i) {
+    TS3SizeCBD cbd = {0};
+    do {
+      S3_delete_object(&bucketContext, object_name[i], 0, timeoutMsG, &responseHandler, &cbd);
+    } while (S3_status_is_retryable(cbd.status) && should_retry());
+
+    if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) {
+      vError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg);
+    }
+  }
+}
+
+static S3Status getObjectDataCallback(int bufferSize, const char *buffer, void *callbackData) {
+  TS3SizeCBD *cbd = callbackData;
+  if (cbd->content_length != bufferSize) {
+    cbd->status = S3StatusAbortedByCallback;
+    return S3StatusAbortedByCallback;
+  }
+
+  char *buf = taosMemoryCalloc(1, bufferSize);
+  if (buf) {
+    memcpy(buf, buffer, bufferSize);
+
+    cbd->status = S3StatusOK;
+    return S3StatusOK;
+  } else {
+    cbd->status = S3StatusAbortedByCallback;
+    return S3StatusAbortedByCallback;
+  }
+}
+
+int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size, uint8_t **ppBlock) {
+  int         status = 0;
+  int64_t     ifModifiedSince = -1, ifNotModifiedSince = -1;
+  const char *ifMatch = 0, *ifNotMatch = 0;
+
+  S3BucketContext    bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret,
+                                      0, awsRegionG};
+  S3GetConditions    getConditions = {ifModifiedSince, ifNotModifiedSince, ifMatch, ifNotMatch};
+  S3GetObjectHandler getObjectHandler = {{&responsePropertiesCallback, &responseCompleteCallback},
+                                         &getObjectDataCallback};
+
+  TS3SizeCBD cbd = {0};
+  cbd.content_length = size;
+  do {
+    S3_get_object(&bucketContext, object_name, &getConditions, offset, size, 0, 0, &getObjectHandler, &cbd);
+  } while (S3_status_is_retryable(cbd.status) && should_retry());
+
+  if (cbd.status != S3StatusOK) {
+    vError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg);
+    return TAOS_SYSTEM_ERROR(EIO);
+  }
+
+  *ppBlock = cbd.buf;
+
+  return 0;
+}
+
+long s3Size(const char *object_name) {
+  long size = 0;
+  int  status = 0;
+
+  S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret,
+                                   0, awsRegionG};
+
+  S3ResponseHandler responseHandler = {&responsePropertiesCallback, &responseCompleteCallback};
+
+  TS3SizeCBD cbd = {0};
+  do {
+    S3_head_object(&bucketContext, object_name, 0, 0, &responseHandler, &cbd);
+  } while (S3_status_is_retryable(cbd.status) && should_retry());
+
+  if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) {
+    vError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg);
+  }
+
+  size = cbd.content_length;
+
+  return size;
+}
+
+void s3EvictCache(const char *path, long object_size) {}
+
+#elif defined(USE_COS)
 
-#ifdef USE_COS
 #include "cos_api.h"
 #include "cos_http_io.h"
 #include "cos_log.h"

From 11f7cdcf9d3b6faea3db55cbc4227ecbee29b4cc Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Fri, 13 Oct 2023 17:16:19 +0800
Subject: [PATCH 03/26] cos/xml2: cmake file for xml2

---
 cmake/xml2_CMakeLists.txt.in | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 cmake/xml2_CMakeLists.txt.in

diff --git a/cmake/xml2_CMakeLists.txt.in b/cmake/xml2_CMakeLists.txt.in
new file mode 100644
index 0000000000..4c234d7a37
--- /dev/null
+++ b/cmake/xml2_CMakeLists.txt.in
@@ -0,0 +1,12 @@
+
+# xml2
+ExternalProject_Add(xml2
+        GIT_REPOSITORY https://github.com/GNOME/libxml2
+        GIT_TAG v2.11.5
+        SOURCE_DIR "${TD_CONTRIB_DIR}/xml2"
+        BINARY_DIR ""
+        CONFIGURE_COMMAND ""
+        BUILD_COMMAND ""
+        INSTALL_COMMAND ""
+        TEST_COMMAND ""
+        )

From c538478d0f91c26c34f998a31b0461016a769e9a Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Fri, 13 Oct 2023 17:30:09 +0800
Subject: [PATCH 04/26] contrib/cmake: fix curl file

---
 contrib/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt
index 1588eb4025..1291506136 100644
--- a/contrib/CMakeLists.txt
+++ b/contrib/CMakeLists.txt
@@ -194,7 +194,7 @@ if(${BUILD_WITH_S3})
     #cat("${TD_SUPPORT_DIR}/apr-util_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
     #cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
     #INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.1/include)
-    cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE2})
+    cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
     cat("${TD_SUPPORT_DIR}/libs3_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
     add_definitions(-DUSE_S3)
 

From f66e57bf1f650139e5d40ba939387da72edd7d2c Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Fri, 13 Oct 2023 17:47:42 +0800
Subject: [PATCH 05/26] contrib/cos: make libs3 depends xml2

---
 cmake/libs3_CMakeLists.txt.in | 2 +-
 contrib/CMakeLists.txt        | 5 ++---
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/cmake/libs3_CMakeLists.txt.in b/cmake/libs3_CMakeLists.txt.in
index b54a084f66..93d0eff41a 100644
--- a/cmake/libs3_CMakeLists.txt.in
+++ b/cmake/libs3_CMakeLists.txt.in
@@ -13,7 +13,7 @@ endfunction(update_cflags)
 ExternalProject_Add(libs3
         GIT_REPOSITORY https://github.com/bji/libs3
 	#GIT_TAG v5.0.16
-	DEPENDS curl
+	DEPENDS curl xml2
         SOURCE_DIR "${TD_CONTRIB_DIR}/libs3"
         #BINARY_DIR ""
         BUILD_IN_SOURCE TRUE
diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt
index 1291506136..24ad63db13 100644
--- a/contrib/CMakeLists.txt
+++ b/contrib/CMakeLists.txt
@@ -125,9 +125,6 @@ if(${BUILD_TEST})
     cat("${TD_SUPPORT_DIR}/stub_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
 endif(${BUILD_TEST})
 
-# xml2
-cat("${TD_SUPPORT_DIR}/xml2_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
-
 # lz4
 cat("${TD_SUPPORT_DIR}/lz4_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
 
@@ -194,6 +191,8 @@ if(${BUILD_WITH_S3})
     #cat("${TD_SUPPORT_DIR}/apr-util_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
     #cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
     #INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.1/include)
+
+    cat("${TD_SUPPORT_DIR}/xml2_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
     cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
     cat("${TD_SUPPORT_DIR}/libs3_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
     add_definitions(-DUSE_S3)

From 4a77a49a3e3c38d6519390eb6d00df15496a9d9b Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Mon, 16 Oct 2023 15:25:48 +0800
Subject: [PATCH 06/26] prefix/list: list bucket with prefix

---
 cmake/libs3_CMakeLists.txt.in         |  2 +-
 contrib/CMakeLists.txt                |  2 +-
 source/dnode/vnode/CMakeLists.txt     | 10 +--
 source/dnode/vnode/src/vnd/vnodeCos.c | 93 ++++++++++++++++++++++++++-
 4 files changed, 97 insertions(+), 10 deletions(-)

diff --git a/cmake/libs3_CMakeLists.txt.in b/cmake/libs3_CMakeLists.txt.in
index 93d0eff41a..790a6fe312 100644
--- a/cmake/libs3_CMakeLists.txt.in
+++ b/cmake/libs3_CMakeLists.txt.in
@@ -13,7 +13,7 @@ endfunction(update_cflags)
 ExternalProject_Add(libs3
         GIT_REPOSITORY https://github.com/bji/libs3
 	#GIT_TAG v5.0.16
-	DEPENDS curl xml2
+	DEPENDS curl xml2 openssl
         SOURCE_DIR "${TD_CONTRIB_DIR}/libs3"
         #BINARY_DIR ""
         BUILD_IN_SOURCE TRUE
diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt
index 24ad63db13..63a60fa59a 100644
--- a/contrib/CMakeLists.txt
+++ b/contrib/CMakeLists.txt
@@ -14,7 +14,6 @@ set(CONTRIB_TMP_FILE3 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in3")
 configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3})
 
     file(MAKE_DIRECTORY $ENV{HOME}/.cos-local.1/)
-    cat("${TD_SUPPORT_DIR}/ssl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3})
 
 configure_file(${CONTRIB_TMP_FILE3} "${TD_CONTRIB_DIR}/deps-download/CMakeLists.txt")
 execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" .
@@ -192,6 +191,7 @@ if(${BUILD_WITH_S3})
     #cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
     #INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.1/include)
 
+    cat("${TD_SUPPORT_DIR}/ssl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3})
     cat("${TD_SUPPORT_DIR}/xml2_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
     cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
     cat("${TD_SUPPORT_DIR}/libs3_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt
index e554d94e4d..d4ea90f48a 100644
--- a/source/dnode/vnode/CMakeLists.txt
+++ b/source/dnode/vnode/CMakeLists.txt
@@ -181,11 +181,11 @@ if(${BUILD_WITH_S3})
     vnode
 
     # s3
-    PUBLIC ${S3_LIBRARY}
-    PUBLIC ${CURL_LIBRARY}
-    PUBLIC ${SSL_LIBRARY}
-    PUBLIC ${CRYPTO_LIBRARY}
-    PUBLIC xml2
+    PRIVATE ${S3_LIBRARY}
+    PRIVATE ${CURL_LIBRARY}
+    PRIVATE ${SSL_LIBRARY}
+    PRIVATE ${CRYPTO_LIBRARY}
+    PRIVATE xml2
   )
 
   add_definitions(-DUSE_S3)
diff --git a/source/dnode/vnode/src/vnd/vnodeCos.c b/source/dnode/vnode/src/vnd/vnodeCos.c
index 8984d33ad8..cf19a08813 100644
--- a/source/dnode/vnode/src/vnd/vnodeCos.c
+++ b/source/dnode/vnode/src/vnd/vnodeCos.c
@@ -63,9 +63,17 @@ static int should_retry() {
   return 0;
 }
 
+static void s3PrintError(const char *func, S3Status status, char error_details[]) {
+  if (status < S3StatusErrorAccessDenied) {
+    vError("%s: %s", __func__, S3_get_status_name(status));
+  } else {
+    vError("%s: %s, %s", __func__, S3_get_status_name(status), error_details);
+  }
+}
+
 typedef struct {
   uint64_t content_length;
-  int      status;
+  S3Status status;
   char    *buf;
   char     err_msg[4096];
 } TS3SizeCBD;
@@ -109,7 +117,86 @@ static void responseCompleteCallback(S3Status status, const S3ErrorDetails *erro
 }
 
 int32_t s3PutObjectFromFile2(const char *file, const char *object) { return 0; }
-void    s3DeleteObjectsByPrefix(const char *prefix) {}
+
+typedef struct list_bucket_callback_data {
+  int      isTruncated;
+  char     nextMarker[1024];
+  int      keyCount;
+  int      allDetails;
+  S3Status status;
+  char     err_msg[4096];
+} list_bucket_callback_data;
+
+static S3Status listBucketCallback(int isTruncated, const char *nextMarker, int contentsCount,
+                                   const S3ListBucketContent *contents, int commonPrefixesCount,
+                                   const char **commonPrefixes, void *callbackData) {
+  list_bucket_callback_data *data = (list_bucket_callback_data *)callbackData;
+
+  data->isTruncated = isTruncated;
+  if ((!nextMarker || !nextMarker[0]) && contentsCount) {
+    nextMarker = contents[contentsCount - 1].key;
+  }
+  if (nextMarker) {
+    snprintf(data->nextMarker, sizeof(data->nextMarker), "%s", nextMarker);
+  } else {
+    data->nextMarker[0] = 0;
+  }
+
+  if (contentsCount && !data->keyCount) {
+    // printListBucketHeader(data->allDetails);
+  }
+
+  int i;
+  for (i = 0; i < contentsCount; ++i) {
+    const S3ListBucketContent *content = &(contents[i]);
+    printf("%-50s", content->key);
+  }
+  data->keyCount += contentsCount;
+
+  for (i = 0; i < commonPrefixesCount; i++) {
+    // printf("\nCommon Prefix: %s\n", commonPrefixes[i]);
+  }
+
+  return S3StatusOK;
+}
+
+void s3DeleteObjectsByPrefix(const char *prefix) {
+  S3BucketContext     bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret,
+                                       0, awsRegionG};
+  S3ListBucketHandler listBucketHandler = {{&responsePropertiesCallback, &responseCompleteCallback},
+                                           &listBucketCallback};
+
+  const char               *marker = 0, *delimiter = 0;
+  int                       maxkeys = 0, allDetails = 0;
+  list_bucket_callback_data data;
+
+  if (marker) {
+    snprintf(data.nextMarker, sizeof(data.nextMarker), "%s", marker);
+  } else {
+    data.nextMarker[0] = 0;
+  }
+  data.keyCount = 0;
+  data.allDetails = allDetails;
+
+  do {
+    data.isTruncated = 0;
+    do {
+      S3_list_bucket(&bucketContext, prefix, data.nextMarker, delimiter, maxkeys, 0, timeoutMsG, &listBucketHandler,
+                     &data);
+    } while (S3_status_is_retryable(data.status) && should_retry());
+    if (data.status != S3StatusOK) {
+      break;
+    }
+  } while (data.isTruncated && (!maxkeys || (data.keyCount < maxkeys)));
+
+  if (data.status == S3StatusOK) {
+    if (!data.keyCount) {
+      // printListBucketHeader(allDetails);
+    }
+  } else {
+    s3PrintError(__func__, data.status, data.err_msg);
+  }
+}
 
 void s3DeleteObjects(const char *object_name[], int nobject) {
   int               status = 0;
@@ -124,7 +211,7 @@ void s3DeleteObjects(const char *object_name[], int nobject) {
     } while (S3_status_is_retryable(cbd.status) && should_retry());
 
     if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) {
-      vError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg);
+      s3PrintError(__func__, cbd.status, cbd.err_msg);
     }
   }
 }

From 2479bb343e56f98f37ad749e6607d2449b8892a7 Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Mon, 16 Oct 2023 15:54:51 +0800
Subject: [PATCH 07/26] prefix/delete: delete with prefix

---
 contrib/CMakeLists.txt                |  2 +-
 source/dnode/vnode/src/vnd/vnodeCos.c | 19 +++++++++++++++++--
 2 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt
index 63a60fa59a..b40a7101c3 100644
--- a/contrib/CMakeLists.txt
+++ b/contrib/CMakeLists.txt
@@ -191,7 +191,7 @@ if(${BUILD_WITH_S3})
     #cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
     #INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.1/include)
 
-    cat("${TD_SUPPORT_DIR}/ssl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3})
+    cat("${TD_SUPPORT_DIR}/ssl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
     cat("${TD_SUPPORT_DIR}/xml2_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
     cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
     cat("${TD_SUPPORT_DIR}/libs3_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
diff --git a/source/dnode/vnode/src/vnd/vnodeCos.c b/source/dnode/vnode/src/vnd/vnodeCos.c
index cf19a08813..726d78e167 100644
--- a/source/dnode/vnode/src/vnd/vnodeCos.c
+++ b/source/dnode/vnode/src/vnd/vnodeCos.c
@@ -123,6 +123,7 @@ typedef struct list_bucket_callback_data {
   char     nextMarker[1024];
   int      keyCount;
   int      allDetails;
+  SArray  *objectArray;
   S3Status status;
   char     err_msg[4096];
 } list_bucket_callback_data;
@@ -149,7 +150,9 @@ static S3Status listBucketCallback(int isTruncated, const char *nextMarker, int
   int i;
   for (i = 0; i < contentsCount; ++i) {
     const S3ListBucketContent *content = &(contents[i]);
-    printf("%-50s", content->key);
+    // printf("%-50s", content->key);
+    char *object_key = strdup(content->key);
+    taosArrayPush(data->objectArray, &object_key);
   }
   data->keyCount += contentsCount;
 
@@ -160,6 +163,11 @@ static S3Status listBucketCallback(int isTruncated, const char *nextMarker, int
   return S3StatusOK;
 }
 
+static void s3FreeObjectKey(void *pItem) {
+  char *key = (char *)pItem;
+  taosMemoryFree(key);
+}
+
 void s3DeleteObjectsByPrefix(const char *prefix) {
   S3BucketContext     bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret,
                                        0, awsRegionG};
@@ -169,7 +177,11 @@ void s3DeleteObjectsByPrefix(const char *prefix) {
   const char               *marker = 0, *delimiter = 0;
   int                       maxkeys = 0, allDetails = 0;
   list_bucket_callback_data data;
-
+  data.objectArray = taosArrayInit(32, POINTER_BYTES);
+  if (!data.objectArray) {
+    vError("%s: %s", __func__, "out of memoty");
+    return;
+  }
   if (marker) {
     snprintf(data.nextMarker, sizeof(data.nextMarker), "%s", marker);
   } else {
@@ -192,10 +204,13 @@ void s3DeleteObjectsByPrefix(const char *prefix) {
   if (data.status == S3StatusOK) {
     if (!data.keyCount) {
       // printListBucketHeader(allDetails);
+      s3DeleteObjects(TARRAY_DATA(data.objectArray), TARRAY_SIZE(data.objectArray));
     }
   } else {
     s3PrintError(__func__, data.status, data.err_msg);
   }
+
+  taosArrayDestroyEx(data.objectArray, s3FreeObjectKey);
 }
 
 void s3DeleteObjects(const char *object_name[], int nobject) {

From 8fa8c40391fbea4dbb668202dabb9160a281d289 Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Tue, 17 Oct 2023 14:59:03 +0800
Subject: [PATCH 08/26] cos/put: simple object putting(< 5 gigabytes)

---
 source/dnode/vnode/src/vnd/vnodeCos.c | 250 +++++++++++++++++++++++++-
 1 file changed, 249 insertions(+), 1 deletion(-)

diff --git a/source/dnode/vnode/src/vnd/vnodeCos.c b/source/dnode/vnode/src/vnd/vnodeCos.c
index 726d78e167..2a36a898e0 100644
--- a/source/dnode/vnode/src/vnd/vnodeCos.c
+++ b/source/dnode/vnode/src/vnd/vnodeCos.c
@@ -116,7 +116,255 @@ static void responseCompleteCallback(S3Status status, const S3ErrorDetails *erro
   }
 }
 
-int32_t s3PutObjectFromFile2(const char *file, const char *object) { return 0; }
+typedef struct growbuffer {
+  // The total number of bytes, and the start byte
+  int size;
+  // The start byte
+  int start;
+  // The blocks
+  char               data[64 * 1024];
+  struct growbuffer *prev, *next;
+} growbuffer;
+
+// returns nonzero on success, zero on out of memory
+static int growbuffer_append(growbuffer **gb, const char *data, int dataLen) {
+  int origDataLen = dataLen;
+  while (dataLen) {
+    growbuffer *buf = *gb ? (*gb)->prev : 0;
+    if (!buf || (buf->size == sizeof(buf->data))) {
+      buf = (growbuffer *)malloc(sizeof(growbuffer));
+      if (!buf) {
+        return 0;
+      }
+      buf->size = 0;
+      buf->start = 0;
+      if (*gb && (*gb)->prev) {
+        buf->prev = (*gb)->prev;
+        buf->next = *gb;
+        (*gb)->prev->next = buf;
+        (*gb)->prev = buf;
+      } else {
+        buf->prev = buf->next = buf;
+        *gb = buf;
+      }
+    }
+
+    int toCopy = (sizeof(buf->data) - buf->size);
+    if (toCopy > dataLen) {
+      toCopy = dataLen;
+    }
+
+    memcpy(&(buf->data[buf->size]), data, toCopy);
+
+    buf->size += toCopy, data += toCopy, dataLen -= toCopy;
+  }
+
+  return origDataLen;
+}
+
+static void growbuffer_read(growbuffer **gb, int amt, int *amtReturn, char *buffer) {
+  *amtReturn = 0;
+
+  growbuffer *buf = *gb;
+
+  if (!buf) {
+    return;
+  }
+
+  *amtReturn = (buf->size > amt) ? amt : buf->size;
+
+  memcpy(buffer, &(buf->data[buf->start]), *amtReturn);
+
+  buf->start += *amtReturn, buf->size -= *amtReturn;
+
+  if (buf->size == 0) {
+    if (buf->next == buf) {
+      *gb = 0;
+    } else {
+      *gb = buf->next;
+      buf->prev->next = buf->next;
+      buf->next->prev = buf->prev;
+    }
+    free(buf);
+    buf = NULL;
+  }
+}
+
+static void growbuffer_destroy(growbuffer *gb) {
+  growbuffer *start = gb;
+
+  while (gb) {
+    growbuffer *next = gb->next;
+    free(gb);
+    gb = (next == start) ? 0 : next;
+  }
+}
+
+typedef struct put_object_callback_data {
+  // FILE       *infile;
+  TdFilePtr   infileFD;
+  growbuffer *gb;
+  uint64_t    contentLength, originalContentLength;
+  uint64_t    totalContentLength, totalOriginalContentLength;
+  int         noStatus;
+  S3Status    status;
+  char        err_msg[4096];
+} put_object_callback_data;
+
+static int putObjectDataCallback(int bufferSize, char *buffer, void *callbackData) {
+  put_object_callback_data *data = (put_object_callback_data *)callbackData;
+
+  int ret = 0;
+
+  if (data->contentLength) {
+    int toRead = ((data->contentLength > (unsigned)bufferSize) ? (unsigned)bufferSize : data->contentLength);
+    if (data->gb) {
+      growbuffer_read(&(data->gb), toRead, &ret, buffer);
+    } else if (data->infileFD) {
+      // ret = fread(buffer, 1, toRead, data->infile);
+      ret = taosReadFile(data->infileFD, buffer, toRead);
+    }
+  }
+
+  data->contentLength -= ret;
+  data->totalContentLength -= ret;
+
+  if (data->contentLength && !data->noStatus) {
+    vTrace("%llu bytes remaining ", (unsigned long long)data->totalContentLength);
+    vTrace("(%d%% complete) ...\n", (int)(((data->totalOriginalContentLength - data->totalContentLength) * 100) /
+                                          data->totalOriginalContentLength));
+  }
+
+  return ret;
+}
+
+#define MULTIPART_CHUNK_SIZE (768 << 20)  // multipart is 768M
+
+typedef struct UploadManager {
+  // used for initial multipart
+  char *upload_id;
+
+  // used for upload part object
+  char **etags;
+  int    next_etags_pos;
+
+  // used for commit Upload
+  growbuffer *gb;
+  int         remaining;
+} UploadManager;
+
+typedef struct MultipartPartData {
+  put_object_callback_data put_object_data;
+  int                      seq;
+  UploadManager           *manager;
+} MultipartPartData;
+
+S3Status initial_multipart_callback(const char *upload_id, void *callbackData) {
+  UploadManager *manager = (UploadManager *)callbackData;
+  manager->upload_id = strdup(upload_id);
+  return S3StatusOK;
+}
+
+S3Status MultipartResponseProperiesCallback(const S3ResponseProperties *properties, void *callbackData) {
+  responsePropertiesCallback(properties, callbackData);
+
+  MultipartPartData *data = (MultipartPartData *)callbackData;
+  int                seq = data->seq;
+  const char        *etag = properties->eTag;
+  data->manager->etags[seq - 1] = strdup(etag);
+  data->manager->next_etags_pos = seq;
+  return S3StatusOK;
+}
+
+static int multipartPutXmlCallback(int bufferSize, char *buffer, void *callbackData) {
+  UploadManager *manager = (UploadManager *)callbackData;
+  int            ret = 0;
+
+  if (manager->remaining) {
+    int toRead = ((manager->remaining > bufferSize) ? bufferSize : manager->remaining);
+    growbuffer_read(&(manager->gb), toRead, &ret, buffer);
+  }
+  manager->remaining -= ret;
+  return ret;
+}
+
+static int try_get_parts_info(const char *bucketName, const char *key, UploadManager *manager) {
+  //
+
+  return 0;
+}
+
+int32_t s3PutObjectFromFile2(const char *file, const char *object) {
+  int32_t                  code = 0;
+  const char              *key = object;
+  const char              *uploadId = 0;
+  const char              *filename = 0;
+  uint64_t                 contentLength = 0;
+  const char              *cacheControl = 0, *contentType = 0, *md5 = 0;
+  const char              *contentDispositionFilename = 0, *contentEncoding = 0;
+  int64_t                  expires = -1;
+  S3CannedAcl              cannedAcl = S3CannedAclPrivate;
+  int                      metaPropertiesCount = 0;
+  S3NameValue              metaProperties[S3_MAX_METADATA_COUNT];
+  char                     useServerSideEncryption = 0;
+  int                      noStatus = 0;
+  put_object_callback_data data;
+
+  // data.infile = 0;
+  data.infileFD = NULL;
+  data.gb = 0;
+  data.noStatus = noStatus;
+
+  if (taosStatFile(file, &contentLength, NULL, NULL) < 0) {
+    vError("ERROR: %s Failed to stat file %s: ", __func__, file);
+    code = TAOS_SYSTEM_ERROR(errno);
+    return code;
+  }
+
+  if (!(data.infileFD = taosOpenFile(file, TD_FILE_READ))) {
+    vError("ERROR: %s Failed to open file %s: ", __func__, file);
+    code = TAOS_SYSTEM_ERROR(errno);
+    return code;
+  }
+
+  data.totalContentLength = data.totalOriginalContentLength = data.contentLength = data.originalContentLength =
+      contentLength;
+
+  S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret,
+                                   0, awsRegionG};
+
+  S3PutProperties putProperties = {contentType,     md5,
+                                   cacheControl,    contentDispositionFilename,
+                                   contentEncoding, expires,
+                                   cannedAcl,       metaPropertiesCount,
+                                   metaProperties,  useServerSideEncryption};
+
+  if (contentLength <= MULTIPART_CHUNK_SIZE) {
+    S3PutObjectHandler putObjectHandler = {{&responsePropertiesCallback, &responseCompleteCallback},
+                                           &putObjectDataCallback};
+
+    do {
+      S3_put_object(&bucketContext, key, contentLength, &putProperties, 0, 0, &putObjectHandler, &data);
+    } while (S3_status_is_retryable(data.status) && should_retry());
+
+    if (data.infileFD) {
+      taosCloseFile(&data.infileFD);
+    } else if (data.gb) {
+      growbuffer_destroy(data.gb);
+    }
+
+    if (data.status != S3StatusOK) {
+      s3PrintError(__func__, data.status, data.err_msg);
+    } else if (data.contentLength) {
+      vError("ERROR: %s Failed to read remaining %llu bytes from input", __func__,
+             (unsigned long long)data.contentLength);
+    }
+  } else {
+    uint64_t totalContentLength = contentLength;
+    uint64_t todoContentLength = contentLength;
+  }
+  return 0;
+}
 
 typedef struct list_bucket_callback_data {
   int      isTruncated;

From d8b0d8085be4cc51903d489c915a2a8a5331b9d4 Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Tue, 17 Oct 2023 15:16:20 +0800
Subject: [PATCH 09/26] cmake/ssl: make curl depends openssl

---
 cmake/curl_CMakeLists.txt.in  | 1 +
 cmake/libs3_CMakeLists.txt.in | 4 ++--
 contrib/CMakeLists.txt        | 1 -
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/cmake/curl_CMakeLists.txt.in b/cmake/curl_CMakeLists.txt.in
index 24a1210422..fed98c5f0e 100644
--- a/cmake/curl_CMakeLists.txt.in
+++ b/cmake/curl_CMakeLists.txt.in
@@ -7,6 +7,7 @@ ExternalProject_Add(curl
         #GIT_REPOSITORY https://github.com/curl/curl.git
         #GIT_TAG curl-7_88_1
         SOURCE_DIR "${TD_CONTRIB_DIR}/curl"
+        DEPENDS openssl
         BUILD_IN_SOURCE TRUE
         BUILD_ALWAYS 1
         #UPDATE_COMMAND ""
diff --git a/cmake/libs3_CMakeLists.txt.in b/cmake/libs3_CMakeLists.txt.in
index 790a6fe312..b2d2d19059 100644
--- a/cmake/libs3_CMakeLists.txt.in
+++ b/cmake/libs3_CMakeLists.txt.in
@@ -12,8 +12,8 @@ endfunction(update_cflags)
 
 ExternalProject_Add(libs3
         GIT_REPOSITORY https://github.com/bji/libs3
-	#GIT_TAG v5.0.16
-	DEPENDS curl xml2 openssl
+        #GIT_TAG v5.0.16
+        DEPENDS curl xml2
         SOURCE_DIR "${TD_CONTRIB_DIR}/libs3"
         #BINARY_DIR ""
         BUILD_IN_SOURCE TRUE
diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt
index b40a7101c3..07201f8713 100644
--- a/contrib/CMakeLists.txt
+++ b/contrib/CMakeLists.txt
@@ -277,7 +277,6 @@ if(${BUILD_TEST})
         )
     endif(${TD_DARWIN})
 
-
 endif(${BUILD_TEST})
 
 # cJson

From ddfdbb3207415134c8b5c684864debe6da716848 Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Tue, 17 Oct 2023 15:28:36 +0800
Subject: [PATCH 10/26] xml2: add sub dir if building libs3

---
 contrib/CMakeLists.txt | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt
index 07201f8713..de3d9cfd37 100644
--- a/contrib/CMakeLists.txt
+++ b/contrib/CMakeLists.txt
@@ -24,7 +24,7 @@ execute_process(COMMAND "${CMAKE_COMMAND}" --build .
 set(CONTRIB_TMP_FILE2 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in2")
 configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE2})
 
-    cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE2})
+    #cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE2})
 
 configure_file(${CONTRIB_TMP_FILE2} "${TD_CONTRIB_DIR}/deps-download/CMakeLists.txt")
 execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" .
@@ -294,7 +294,9 @@ target_include_directories(
 unset(CMAKE_PROJECT_INCLUDE_BEFORE)
 
 # xml2
-add_subdirectory(xml2 EXCLUDE_FROM_ALL)
+if(${BUILD_WITH_S3})
+  add_subdirectory(xml2 EXCLUDE_FROM_ALL)
+endif(${BUILD_WITH_S3})
 
 # lz4
 add_subdirectory(lz4/build/cmake EXCLUDE_FROM_ALL)

From dd21d8e3ade60225d74d2cf37ea49fe4027bf9ba Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Tue, 17 Oct 2023 17:24:50 +0800
Subject: [PATCH 11/26] cos/multipart: first round multipart uploading

---
 source/dnode/vnode/src/vnd/vnodeCos.c | 256 ++++++++++++++++++++++++--
 1 file changed, 244 insertions(+), 12 deletions(-)

diff --git a/source/dnode/vnode/src/vnd/vnodeCos.c b/source/dnode/vnode/src/vnd/vnodeCos.c
index 2a36a898e0..b52d4809b2 100644
--- a/source/dnode/vnode/src/vnd/vnodeCos.c
+++ b/source/dnode/vnode/src/vnd/vnodeCos.c
@@ -72,10 +72,10 @@ static void s3PrintError(const char *func, S3Status status, char error_details[]
 }
 
 typedef struct {
-  uint64_t content_length;
+  char     err_msg[128];
   S3Status status;
+  uint64_t content_length;
   char    *buf;
-  char     err_msg[4096];
 } TS3SizeCBD;
 
 static S3Status responsePropertiesCallback(const S3ResponseProperties *properties, void *callbackData) {
@@ -201,14 +201,14 @@ static void growbuffer_destroy(growbuffer *gb) {
 }
 
 typedef struct put_object_callback_data {
+  char     err_msg[128];
+  S3Status status;
   // FILE       *infile;
   TdFilePtr   infileFD;
   growbuffer *gb;
   uint64_t    contentLength, originalContentLength;
   uint64_t    totalContentLength, totalOriginalContentLength;
   int         noStatus;
-  S3Status    status;
-  char        err_msg[4096];
 } put_object_callback_data;
 
 static int putObjectDataCallback(int bufferSize, char *buffer, void *callbackData) {
@@ -241,6 +241,8 @@ static int putObjectDataCallback(int bufferSize, char *buffer, void *callbackDat
 #define MULTIPART_CHUNK_SIZE (768 << 20)  // multipart is 768M
 
 typedef struct UploadManager {
+  char     err_msg[128];
+  S3Status status;
   // used for initial multipart
   char *upload_id;
 
@@ -253,7 +255,26 @@ typedef struct UploadManager {
   int         remaining;
 } UploadManager;
 
+typedef struct list_parts_callback_data {
+  char           err_msg[128];
+  S3Status       status;
+  int            isTruncated;
+  char           nextPartNumberMarker[24];
+  char           initiatorId[256];
+  char           initiatorDisplayName[256];
+  char           ownerId[256];
+  char           ownerDisplayName[256];
+  char           storageClass[256];
+  int            partsCount;
+  int            handlePartsStart;
+  int            allDetails;
+  int            noPrint;
+  UploadManager *manager;
+} list_parts_callback_data;
+
 typedef struct MultipartPartData {
+  char                     err_msg[128];
+  S3Status                 status;
   put_object_callback_data put_object_data;
   int                      seq;
   UploadManager           *manager;
@@ -288,16 +309,120 @@ static int multipartPutXmlCallback(int bufferSize, char *buffer, void *callbackD
   return ret;
 }
 
+static S3Status listPartsCallback(int isTruncated, const char *nextPartNumberMarker, const char *initiatorId,
+                                  const char *initiatorDisplayName, const char *ownerId, const char *ownerDisplayName,
+                                  const char *storageClass, int partsCount, int handlePartsStart,
+                                  const S3ListPart *parts, void *callbackData) {
+  list_parts_callback_data *data = (list_parts_callback_data *)callbackData;
+
+  data->isTruncated = isTruncated;
+  data->handlePartsStart = handlePartsStart;
+  UploadManager *manager = data->manager;
+
+  if (nextPartNumberMarker) {
+    snprintf(data->nextPartNumberMarker, sizeof(data->nextPartNumberMarker), "%s", nextPartNumberMarker);
+  } else {
+    data->nextPartNumberMarker[0] = 0;
+  }
+
+  if (initiatorId) {
+    snprintf(data->initiatorId, sizeof(data->initiatorId), "%s", initiatorId);
+  } else {
+    data->initiatorId[0] = 0;
+  }
+
+  if (initiatorDisplayName) {
+    snprintf(data->initiatorDisplayName, sizeof(data->initiatorDisplayName), "%s", initiatorDisplayName);
+  } else {
+    data->initiatorDisplayName[0] = 0;
+  }
+
+  if (ownerId) {
+    snprintf(data->ownerId, sizeof(data->ownerId), "%s", ownerId);
+  } else {
+    data->ownerId[0] = 0;
+  }
+
+  if (ownerDisplayName) {
+    snprintf(data->ownerDisplayName, sizeof(data->ownerDisplayName), "%s", ownerDisplayName);
+  } else {
+    data->ownerDisplayName[0] = 0;
+  }
+
+  if (storageClass) {
+    snprintf(data->storageClass, sizeof(data->storageClass), "%s", storageClass);
+  } else {
+    data->storageClass[0] = 0;
+  }
+
+  if (partsCount && !data->partsCount && !data->noPrint) {
+    // printListPartsHeader();
+  }
+
+  int i;
+  for (i = 0; i < partsCount; i++) {
+    const S3ListPart *part = &(parts[i]);
+    char              timebuf[256];
+    if (data->noPrint) {
+      manager->etags[handlePartsStart + i] = strdup(part->eTag);
+      manager->next_etags_pos++;
+      manager->remaining = manager->remaining - part->size;
+    } else {
+      time_t t = (time_t)part->lastModified;
+      strftime(timebuf, sizeof(timebuf), "%Y-%m-%dT%H:%M:%SZ", gmtime(&t));
+      printf("%-30s", timebuf);
+      printf("%-15llu", (unsigned long long)part->partNumber);
+      printf("%-45s", part->eTag);
+      printf("%-15llu\n", (unsigned long long)part->size);
+    }
+  }
+
+  data->partsCount += partsCount;
+
+  return S3StatusOK;
+}
+
 static int try_get_parts_info(const char *bucketName, const char *key, UploadManager *manager) {
-  //
+  S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret,
+                                   0, awsRegionG};
+
+  S3ListPartsHandler listPartsHandler = {{&responsePropertiesCallback, &responseCompleteCallback}, &listPartsCallback};
+
+  list_parts_callback_data data;
+
+  memset(&data, 0, sizeof(list_parts_callback_data));
+
+  data.partsCount = 0;
+  data.allDetails = 0;
+  data.manager = manager;
+  data.noPrint = 1;
+  do {
+    data.isTruncated = 0;
+    do {
+      S3_list_parts(&bucketContext, key, data.nextPartNumberMarker, manager->upload_id, 0, 0, 0, timeoutMsG,
+                    &listPartsHandler, &data);
+    } while (S3_status_is_retryable(data.status) && should_retry());
+    if (data.status != S3StatusOK) {
+      break;
+    }
+  } while (data.isTruncated);
+
+  if (data.status == S3StatusOK) {
+    if (!data.partsCount) {
+      // printListMultipartHeader(data.allDetails);
+    }
+  } else {
+    s3PrintError(__func__, data.status, data.err_msg);
+    return -1;
+  }
 
   return 0;
 }
 
 int32_t s3PutObjectFromFile2(const char *file, const char *object) {
-  int32_t                  code = 0;
-  const char              *key = object;
-  const char              *uploadId = 0;
+  int32_t     code = 0;
+  const char *key = object;
+  // const char              *uploadId = 0;
   const char              *filename = 0;
   uint64_t                 contentLength = 0;
   const char              *cacheControl = 0, *contentType = 0, *md5 = 0;
@@ -360,20 +485,127 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) {
              (unsigned long long)data.contentLength);
     }
   } else {
-    uint64_t totalContentLength = contentLength;
-    uint64_t todoContentLength = contentLength;
+    uint64_t      totalContentLength = contentLength;
+    uint64_t      todoContentLength = contentLength;
+    UploadManager manager;
+    manager.upload_id = 0;
+    manager.gb = 0;
+
+    // div round up
+    int seq;
+    int totalSeq = ((contentLength + MULTIPART_CHUNK_SIZE - 1) / MULTIPART_CHUNK_SIZE);
+
+    MultipartPartData partData;
+    memset(&partData, 0, sizeof(MultipartPartData));
+    int partContentLength = 0;
+
+    S3MultipartInitialHandler handler = {{&responsePropertiesCallback, &responseCompleteCallback},
+                                         &initial_multipart_callback};
+
+    S3PutObjectHandler putObjectHandler = {{&MultipartResponseProperiesCallback, &responseCompleteCallback},
+                                           &putObjectDataCallback};
+
+    S3MultipartCommitHandler commit_handler = {
+        {&responsePropertiesCallback, &responseCompleteCallback}, &multipartPutXmlCallback, 0};
+
+    manager.etags = (char **)taosMemoryMalloc(sizeof(char *) * totalSeq);
+    manager.next_etags_pos = 0;
+    /*
+    if (uploadId) {
+      manager.upload_id = strdup(uploadId);
+      manager.remaining = contentLength;
+      if (!try_get_parts_info(tsS3BucketName, key, &manager)) {
+        fseek(data.infile, -(manager.remaining), 2);
+        taosLSeekFile(data.infileFD, -(manager.remaining), SEEK_END);
+        contentLength = manager.remaining;
+        goto upload;
+      } else {
+        goto clean;
+      }
+    }
+    */
+    do {
+      S3_initiate_multipart(&bucketContext, key, 0, &handler, 0, timeoutMsG, &manager);
+    } while (S3_status_is_retryable(manager.status) && should_retry());
+
+    if (manager.upload_id == 0 || manager.status != S3StatusOK) {
+      s3PrintError(__func__, manager.status, manager.err_msg);
+      goto clean;
+    }
+
+  upload:
+    todoContentLength -= MULTIPART_CHUNK_SIZE * manager.next_etags_pos;
+    for (seq = manager.next_etags_pos + 1; seq <= totalSeq; seq++) {
+      partData.manager = &manager;
+      partData.seq = seq;
+      if (partData.put_object_data.gb == NULL) {
+        partData.put_object_data = data;
+      }
+      partContentLength = ((contentLength > MULTIPART_CHUNK_SIZE) ? MULTIPART_CHUNK_SIZE : contentLength);
+      // printf("%s Part Seq %d, length=%d\n", srcSize ? "Copying" : "Sending", seq, partContentLength);
+      partData.put_object_data.contentLength = partContentLength;
+      partData.put_object_data.originalContentLength = partContentLength;
+      partData.put_object_data.totalContentLength = todoContentLength;
+      partData.put_object_data.totalOriginalContentLength = totalContentLength;
+      putProperties.md5 = 0;
+      do {
+        S3_upload_part(&bucketContext, key, &putProperties, &putObjectHandler, seq, manager.upload_id,
+                       partContentLength, 0, timeoutMsG, &partData);
+      } while (S3_status_is_retryable(partData.status) && should_retry());
+      if (partData.status != S3StatusOK) {
+        s3PrintError(__func__, partData.status, partData.err_msg);
+        goto clean;
+      }
+      contentLength -= MULTIPART_CHUNK_SIZE;
+      todoContentLength -= MULTIPART_CHUNK_SIZE;
+    }
+
+    int i;
+    int size = 0;
+    size += growbuffer_append(&(manager.gb), "<CompleteMultipartUpload>", strlen("<CompleteMultipartUpload>"));
+    char buf[256];
+    int  n;
+    for (i = 0; i < totalSeq; i++) {
+      n = snprintf(buf, sizeof(buf),
+                   "<Part><PartNumber>%d</PartNumber>"
+                   "<ETag>%s</ETag></Part>",
+                   i + 1, manager.etags[i]);
+      size += growbuffer_append(&(manager.gb), buf, n);
+    }
+    size += growbuffer_append(&(manager.gb), "</CompleteMultipartUpload>", strlen("</CompleteMultipartUpload>"));
+    manager.remaining = size;
+
+    do {
+      S3_complete_multipart_upload(&bucketContext, key, &commit_handler, manager.upload_id, manager.remaining, 0,
+                                   timeoutMsG, &manager);
+    } while (S3_status_is_retryable(manager.status) && should_retry());
+    if (manager.status != S3StatusOK) {
+      s3PrintError(__func__, manager.status, manager.err_msg);
+      goto clean;
+    }
+
+  clean:
+    if (manager.upload_id) {
+      taosMemoryFree(manager.upload_id);
+    }
+    for (i = 0; i < manager.next_etags_pos; i++) {
+      taosMemoryFree(manager.etags[i]);
+    }
+    growbuffer_destroy(manager.gb);
+    taosMemoryFree(manager.etags);
   }
+
   return 0;
 }
 
 typedef struct list_bucket_callback_data {
+  char     err_msg[128];
+  S3Status status;
   int      isTruncated;
   char     nextMarker[1024];
   int      keyCount;
   int      allDetails;
   SArray  *objectArray;
-  S3Status status;
-  char     err_msg[4096];
 } list_bucket_callback_data;
 
 static S3Status listBucketCallback(int isTruncated, const char *nextMarker, int contentsCount,

From 4bc3c22e0c92e8831d1cf6ee55bdb4be4ef6385b Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Tue, 17 Oct 2023 17:46:11 +0800
Subject: [PATCH 12/26] cmake/ssl: install install_sw only

---
 cmake/ssl_CMakeLists.txt.in | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cmake/ssl_CMakeLists.txt.in b/cmake/ssl_CMakeLists.txt.in
index 7821848618..65a02d7fd6 100644
--- a/cmake/ssl_CMakeLists.txt.in
+++ b/cmake/ssl_CMakeLists.txt.in
@@ -9,7 +9,7 @@ ExternalProject_Add(openssl
         #BUILD_ALWAYS 1
         #UPDATE_COMMAND ""
         CONFIGURE_COMMAND ./Configure --prefix=$ENV{HOME}/.cos-local.1 -static #--no-shared
-        BUILD_COMMAND make -j
-        INSTALL_COMMAND make install
+        BUILD_COMMAND ""
+        INSTALL_COMMAND make install_sw -j
         TEST_COMMAND ""
 )

From 11b65e277925323b65efe49095a8c2cff3e72278 Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Tue, 17 Oct 2023 18:11:13 +0800
Subject: [PATCH 13/26] vnode/cos: make dependent lib public

---
 cmake/curl_CMakeLists.txt.in      |  2 +-
 source/dnode/vnode/CMakeLists.txt | 10 +++++-----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/cmake/curl_CMakeLists.txt.in b/cmake/curl_CMakeLists.txt.in
index fed98c5f0e..3f7b9ee1fe 100644
--- a/cmake/curl_CMakeLists.txt.in
+++ b/cmake/curl_CMakeLists.txt.in
@@ -12,7 +12,7 @@ ExternalProject_Add(curl
         BUILD_ALWAYS 1
         #UPDATE_COMMAND ""
         #CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --without-ssl --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd
-        CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --with-ssl=$ENV{HOME}/.cos-local.1  --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd
+        CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --with-ssl=$ENV{HOME}/.cos-local.1  --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd --without-libidn2
         #CONFIGURE_COMMAND ./configure --without-ssl
         BUILD_COMMAND make
         INSTALL_COMMAND make install
diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt
index d4ea90f48a..e554d94e4d 100644
--- a/source/dnode/vnode/CMakeLists.txt
+++ b/source/dnode/vnode/CMakeLists.txt
@@ -181,11 +181,11 @@ if(${BUILD_WITH_S3})
     vnode
 
     # s3
-    PRIVATE ${S3_LIBRARY}
-    PRIVATE ${CURL_LIBRARY}
-    PRIVATE ${SSL_LIBRARY}
-    PRIVATE ${CRYPTO_LIBRARY}
-    PRIVATE xml2
+    PUBLIC ${S3_LIBRARY}
+    PUBLIC ${CURL_LIBRARY}
+    PUBLIC ${SSL_LIBRARY}
+    PUBLIC ${CRYPTO_LIBRARY}
+    PUBLIC xml2
   )
 
   add_definitions(-DUSE_S3)

From a9dfb400691ee59d8e4d35fe3c97e649a5480250 Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Wed, 18 Oct 2023 10:51:47 +0800
Subject: [PATCH 14/26] contrib/cmake: remove unused steps

---
 contrib/CMakeLists.txt | 29 +----------------------------
 1 file changed, 1 insertion(+), 28 deletions(-)

diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt
index de3d9cfd37..30853660ed 100644
--- a/contrib/CMakeLists.txt
+++ b/contrib/CMakeLists.txt
@@ -8,30 +8,9 @@ endfunction(cat IN_FILE OUT_FILE)
 
 if(${TD_LINUX})
 
-  if(${BUILD_WITH_S3})
-
-set(CONTRIB_TMP_FILE3 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in3")
-configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3})
-
+if(${BUILD_WITH_S3})
     file(MAKE_DIRECTORY $ENV{HOME}/.cos-local.1/)
 
-configure_file(${CONTRIB_TMP_FILE3} "${TD_CONTRIB_DIR}/deps-download/CMakeLists.txt")
-execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" .
-        WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download")
-execute_process(COMMAND "${CMAKE_COMMAND}" --build .
-        WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download")
-
-set(CONTRIB_TMP_FILE2 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in2")
-configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE2})
-
-    #cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE2})
-
-configure_file(${CONTRIB_TMP_FILE2} "${TD_CONTRIB_DIR}/deps-download/CMakeLists.txt")
-execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" .
-        WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download")
-execute_process(COMMAND "${CMAKE_COMMAND}" --build .
-        WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download")
-
 else()
 
 set(CONTRIB_TMP_FILE3 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in3")
@@ -185,12 +164,6 @@ endif(${BUILD_WITH_SQLITE})
 
 # s3
 if(${BUILD_WITH_S3})
-    #cat("${TD_SUPPORT_DIR}/mxml_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
-    #cat("${TD_SUPPORT_DIR}/apr_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
-    #cat("${TD_SUPPORT_DIR}/apr-util_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
-    #cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
-    #INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.1/include)
-
     cat("${TD_SUPPORT_DIR}/ssl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
     cat("${TD_SUPPORT_DIR}/xml2_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
     cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})

From f26ec41b8e11804ba87524e5accc8cc5cd459af1 Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Wed, 18 Oct 2023 15:30:02 +0800
Subject: [PATCH 15/26] vnode/cos: fix get callback buffer

---
 source/common/CMakeLists.txt          | 3 +++
 source/common/src/tglobal.c           | 2 +-
 source/dnode/vnode/CMakeLists.txt     | 1 -
 source/dnode/vnode/src/vnd/vnodeCos.c | 2 +-
 4 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/source/common/CMakeLists.txt b/source/common/CMakeLists.txt
index b010467f20..6bdcdf7d1d 100644
--- a/source/common/CMakeLists.txt
+++ b/source/common/CMakeLists.txt
@@ -22,6 +22,9 @@ IF (TD_STORAGE)
       add_definitions(-DUSE_COS)
     ENDIF(${BUILD_WITH_COS})
 
+    IF(${BUILD_WITH_S3})
+      add_definitions(-DUSE_S3)
+    ENDIF(${BUILD_WITH_S3})
   ENDIF(${TD_LINUX})
 
 ENDIF ()
diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c
index 8ea5528fbb..f3af6e0ab0 100644
--- a/source/common/src/tglobal.c
+++ b/source/common/src/tglobal.c
@@ -329,7 +329,7 @@ int32_t taosSetS3Cfg(SConfig *pCfg) {
     }
   }
   if (tsS3BucketName[0] != '<' && tsDiskCfgNum > 1) {
-#ifdef USE_COS
+#if defined(USE_COS) || defined(USE_S3)
     tsS3Enabled = true;
 #endif
   }
diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt
index e554d94e4d..7e2693e8af 100644
--- a/source/dnode/vnode/CMakeLists.txt
+++ b/source/dnode/vnode/CMakeLists.txt
@@ -164,7 +164,6 @@ target_link_libraries(
 if(${TD_LINUX})
 
 if(${BUILD_WITH_S3})
-  MESSAGE("build with s3: ${BUILD_WITH_S3}")
   target_include_directories(
     vnode
 
diff --git a/source/dnode/vnode/src/vnd/vnodeCos.c b/source/dnode/vnode/src/vnd/vnodeCos.c
index b52d4809b2..c50f133987 100644
--- a/source/dnode/vnode/src/vnd/vnodeCos.c
+++ b/source/dnode/vnode/src/vnd/vnodeCos.c
@@ -721,7 +721,7 @@ static S3Status getObjectDataCallback(int bufferSize, const char *buffer, void *
   char *buf = taosMemoryCalloc(1, bufferSize);
   if (buf) {
     memcpy(buf, buffer, bufferSize);
-
+    cbd->buf = buf;
     cbd->status = S3StatusOK;
     return S3StatusOK;
   } else {

From 52c8bc91f98ebfea693b1b7c352eba2fdaf44a7c Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Wed, 18 Oct 2023 18:21:45 +0800
Subject: [PATCH 16/26] cos/delete: fix delete object cond and double free

---
 source/dnode/vnode/src/vnd/vnodeCos.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/source/dnode/vnode/src/vnd/vnodeCos.c b/source/dnode/vnode/src/vnd/vnodeCos.c
index c50f133987..ccda20e049 100644
--- a/source/dnode/vnode/src/vnd/vnodeCos.c
+++ b/source/dnode/vnode/src/vnd/vnodeCos.c
@@ -644,7 +644,7 @@ static S3Status listBucketCallback(int isTruncated, const char *nextMarker, int
 }
 
 static void s3FreeObjectKey(void *pItem) {
-  char *key = (char *)pItem;
+  char *key = *(char **)pItem;
   taosMemoryFree(key);
 }
 
@@ -682,7 +682,7 @@ void s3DeleteObjectsByPrefix(const char *prefix) {
   } while (data.isTruncated && (!maxkeys || (data.keyCount < maxkeys)));
 
   if (data.status == S3StatusOK) {
-    if (!data.keyCount) {
+    if (data.keyCount > 0) {
       // printListBucketHeader(allDetails);
       s3DeleteObjects(TARRAY_DATA(data.objectArray), TARRAY_SIZE(data.objectArray));
     }

From 74d05c2771c4c04b59cb3bf927b2eb6811838e92 Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Fri, 20 Oct 2023 13:47:25 +0800
Subject: [PATCH 17/26] vnode/cos: fix multipart uploading

---
 source/dnode/vnode/src/vnd/vnodeCos.c | 71 ++++++++++++++++-----------
 1 file changed, 41 insertions(+), 30 deletions(-)

diff --git a/source/dnode/vnode/src/vnd/vnodeCos.c b/source/dnode/vnode/src/vnd/vnodeCos.c
index ccda20e049..9c69f2452c 100644
--- a/source/dnode/vnode/src/vnd/vnodeCos.c
+++ b/source/dnode/vnode/src/vnd/vnodeCos.c
@@ -211,33 +211,6 @@ typedef struct put_object_callback_data {
   int         noStatus;
 } put_object_callback_data;
 
-static int putObjectDataCallback(int bufferSize, char *buffer, void *callbackData) {
-  put_object_callback_data *data = (put_object_callback_data *)callbackData;
-
-  int ret = 0;
-
-  if (data->contentLength) {
-    int toRead = ((data->contentLength > (unsigned)bufferSize) ? (unsigned)bufferSize : data->contentLength);
-    if (data->gb) {
-      growbuffer_read(&(data->gb), toRead, &ret, buffer);
-    } else if (data->infileFD) {
-      // ret = fread(buffer, 1, toRead, data->infile);
-      ret = taosReadFile(data->infileFD, buffer, toRead);
-    }
-  }
-
-  data->contentLength -= ret;
-  data->totalContentLength -= ret;
-
-  if (data->contentLength && !data->noStatus) {
-    vTrace("%llu bytes remaining ", (unsigned long long)data->totalContentLength);
-    vTrace("(%d%% complete) ...\n", (int)(((data->totalOriginalContentLength - data->totalContentLength) * 100) /
-                                          data->totalOriginalContentLength));
-  }
-
-  return ret;
-}
-
 #define MULTIPART_CHUNK_SIZE (768 << 20)  // multipart is 768M
 
 typedef struct UploadManager {
@@ -280,6 +253,37 @@ typedef struct MultipartPartData {
   UploadManager           *manager;
 } MultipartPartData;
 
+static int putObjectDataCallback(int bufferSize, char *buffer, void *callbackData) {
+  put_object_callback_data *data = (put_object_callback_data *)callbackData;
+  if (data->infileFD == 0) {
+    MultipartPartData *mpd = (MultipartPartData *)callbackData;
+    data = &mpd->put_object_data;
+  }
+
+  int ret = 0;
+
+  if (data->contentLength) {
+    int toRead = ((data->contentLength > (unsigned)bufferSize) ? (unsigned)bufferSize : data->contentLength);
+    if (data->gb) {
+      growbuffer_read(&(data->gb), toRead, &ret, buffer);
+    } else if (data->infileFD) {
+      // ret = fread(buffer, 1, toRead, data->infile);
+      ret = taosReadFile(data->infileFD, buffer, toRead);
+    }
+  }
+
+  data->contentLength -= ret;
+  data->totalContentLength -= ret;
+  /* log too many open files
+  if (data->contentLength && !data->noStatus) {
+    vTrace("%llu bytes remaining ", (unsigned long long)data->totalContentLength);
+    vTrace("(%d%% complete) ...\n", (int)(((data->totalOriginalContentLength - data->totalContentLength) * 100) /
+                                          data->totalOriginalContentLength));
+  }
+  */
+  return ret;
+}
+
 S3Status initial_multipart_callback(const char *upload_id, void *callbackData) {
   UploadManager *manager = (UploadManager *)callbackData;
   manager->upload_id = strdup(upload_id);
@@ -308,7 +312,7 @@ static int multipartPutXmlCallback(int bufferSize, char *buffer, void *callbackD
   manager->remaining -= ret;
   return ret;
 }
-
+/*
 static S3Status listPartsCallback(int isTruncated, const char *nextPartNumberMarker, const char *initiatorId,
                                   const char *initiatorDisplayName, const char *ownerId, const char *ownerDisplayName,
                                   const char *storageClass, int partsCount, int handlePartsStart,
@@ -418,7 +422,7 @@ static int try_get_parts_info(const char *bucketName, const char *key, UploadMan
 
   return 0;
 }
-
+*/
 int32_t s3PutObjectFromFile2(const char *file, const char *object) {
   int32_t     code = 0;
   const char *key = object;
@@ -480,9 +484,11 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) {
 
     if (data.status != S3StatusOK) {
       s3PrintError(__func__, data.status, data.err_msg);
+      code = TAOS_SYSTEM_ERROR(EIO);
     } else if (data.contentLength) {
       vError("ERROR: %s Failed to read remaining %llu bytes from input", __func__,
              (unsigned long long)data.contentLength);
+      code = TAOS_SYSTEM_ERROR(EIO);
     }
   } else {
     uint64_t      totalContentLength = contentLength;
@@ -493,7 +499,9 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) {
 
     // div round up
     int seq;
+    // uint64_t chunk_size = MULTIPART_CHUNK_SIZE >> 8;
     int totalSeq = ((contentLength + MULTIPART_CHUNK_SIZE - 1) / MULTIPART_CHUNK_SIZE);
+    // int totalSeq = ((contentLength + chunk_size - 1) / chunk_size);
 
     MultipartPartData partData;
     memset(&partData, 0, sizeof(MultipartPartData));
@@ -530,6 +538,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) {
 
     if (manager.upload_id == 0 || manager.status != S3StatusOK) {
       s3PrintError(__func__, manager.status, manager.err_msg);
+      code = TAOS_SYSTEM_ERROR(EIO);
       goto clean;
     }
 
@@ -554,6 +563,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) {
       } while (S3_status_is_retryable(partData.status) && should_retry());
       if (partData.status != S3StatusOK) {
         s3PrintError(__func__, partData.status, partData.err_msg);
+        code = TAOS_SYSTEM_ERROR(EIO);
         goto clean;
       }
       contentLength -= MULTIPART_CHUNK_SIZE;
@@ -581,6 +591,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) {
     } while (S3_status_is_retryable(manager.status) && should_retry());
     if (manager.status != S3StatusOK) {
       s3PrintError(__func__, manager.status, manager.err_msg);
+      code = TAOS_SYSTEM_ERROR(EIO);
       goto clean;
     }
 
@@ -595,7 +606,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) {
     taosMemoryFree(manager.etags);
   }
 
-  return 0;
+  return code;
 }
 
 typedef struct list_bucket_callback_data {

From f016dab40af2446310ecfa58871405af880b11ea Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Fri, 20 Oct 2023 13:59:01 +0800
Subject: [PATCH 18/26] common/global: comment s3BlockSize altering out

---
 source/common/src/tglobal.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c
index f3af6e0ab0..9bc981b520 100644
--- a/source/common/src/tglobal.c
+++ b/source/common/src/tglobal.c
@@ -1683,14 +1683,14 @@ void taosCfgDynamicOptions(const char *option, const char *value) {
     }
     return;
   }
-
+  /* cannot alter s3BlockSize
   if (strcasecmp(option, "s3BlockSize") == 0) {
     int32_t newS3BlockSize = atoi(value);
     uInfo("s3BlockSize set from %d to %d", tsS3BlockSize, newS3BlockSize);
     tsS3BlockSize = newS3BlockSize;
     return;
   }
-
+  */
   if (strcasecmp(option, "s3BlockCacheSize") == 0) {
     int32_t newS3BlockCacheSize = atoi(value);
     uInfo("s3BlockCacheSize set from %d to %d", tsS3BlockCacheSize, newS3BlockCacheSize);

From 55647f9ce9188e79405345935838bcd009f6f898 Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Fri, 20 Oct 2023 15:15:03 +0800
Subject: [PATCH 19/26] cos/put: make chunk small to not block other reqs

---
 source/dnode/vnode/src/vnd/vnodeCos.c | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/source/dnode/vnode/src/vnd/vnodeCos.c b/source/dnode/vnode/src/vnd/vnodeCos.c
index 9c69f2452c..6e36739f5a 100644
--- a/source/dnode/vnode/src/vnd/vnodeCos.c
+++ b/source/dnode/vnode/src/vnd/vnodeCos.c
@@ -498,10 +498,9 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) {
     manager.gb = 0;
 
     // div round up
-    int seq;
-    // uint64_t chunk_size = MULTIPART_CHUNK_SIZE >> 8;
-    int totalSeq = ((contentLength + MULTIPART_CHUNK_SIZE - 1) / MULTIPART_CHUNK_SIZE);
-    // int totalSeq = ((contentLength + chunk_size - 1) / chunk_size);
+    int      seq;
+    uint64_t chunk_size = MULTIPART_CHUNK_SIZE >> 8;
+    int      totalSeq = ((contentLength + chunk_size - 1) / chunk_size);
 
     MultipartPartData partData;
     memset(&partData, 0, sizeof(MultipartPartData));
@@ -543,14 +542,14 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) {
     }
 
   upload:
-    todoContentLength -= MULTIPART_CHUNK_SIZE * manager.next_etags_pos;
+    todoContentLength -= chunk_size * manager.next_etags_pos;
     for (seq = manager.next_etags_pos + 1; seq <= totalSeq; seq++) {
       partData.manager = &manager;
       partData.seq = seq;
       if (partData.put_object_data.gb == NULL) {
         partData.put_object_data = data;
       }
-      partContentLength = ((contentLength > MULTIPART_CHUNK_SIZE) ? MULTIPART_CHUNK_SIZE : contentLength);
+      partContentLength = ((contentLength > chunk_size) ? chunk_size : contentLength);
       // printf("%s Part Seq %d, length=%d\n", srcSize ? "Copying" : "Sending", seq, partContentLength);
       partData.put_object_data.contentLength = partContentLength;
       partData.put_object_data.originalContentLength = partContentLength;
@@ -566,8 +565,8 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) {
         code = TAOS_SYSTEM_ERROR(EIO);
         goto clean;
       }
-      contentLength -= MULTIPART_CHUNK_SIZE;
-      todoContentLength -= MULTIPART_CHUNK_SIZE;
+      contentLength -= chunk_size;
+      todoContentLength -= chunk_size;
     }
 
     int i;

From 62cd6f3ed26d1aa8b8e064de24e651b1b2d1d0eb Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Mon, 23 Oct 2023 16:50:51 +0800
Subject: [PATCH 20/26] cmake/curl, libs3, ssl: clean source dir before
 building

---
 cmake/curl_CMakeLists.txt.in  | 7 ++++---
 cmake/libs3_CMakeLists.txt.in | 4 ++--
 cmake/ssl_CMakeLists.txt.in   | 2 +-
 3 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/cmake/curl_CMakeLists.txt.in b/cmake/curl_CMakeLists.txt.in
index 3f7b9ee1fe..2fe84b9721 100644
--- a/cmake/curl_CMakeLists.txt.in
+++ b/cmake/curl_CMakeLists.txt.in
@@ -12,9 +12,10 @@ ExternalProject_Add(curl
         BUILD_ALWAYS 1
         #UPDATE_COMMAND ""
         #CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --without-ssl --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd
-        CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --with-ssl=$ENV{HOME}/.cos-local.1  --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd --without-libidn2
+        CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --with-ssl=$ENV{HOME}/.cos-local.1  --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd --without-libidn2 #--enable-debug
+
         #CONFIGURE_COMMAND ./configure --without-ssl
-        BUILD_COMMAND make
-        INSTALL_COMMAND make install
+        BUILD_COMMAND ""
+        INSTALL_COMMAND make clean && make install
         TEST_COMMAND ""
 )
diff --git a/cmake/libs3_CMakeLists.txt.in b/cmake/libs3_CMakeLists.txt.in
index b2d2d19059..6c911775ee 100644
--- a/cmake/libs3_CMakeLists.txt.in
+++ b/cmake/libs3_CMakeLists.txt.in
@@ -23,7 +23,7 @@ ExternalProject_Add(libs3
 	CONFIGURE_COMMAND cp ${TD_SUPPORT_DIR}/libs3.GNUmakefile GNUmakefile &&  sed -i "s|CFLAGS += -Wall -Werror|CFLAGS += -I'$ENV{HOME}/.cos-local.1/include' -L'$ENV{HOME}/.cos-local.1/lib' |" ./GNUmakefile
         #BUILD_COMMAND make CFLAGS=${s3_flags} DESTDIR=$ENV{HOME}/.cos-local.1 build/lib/libs3.a
         #BUILD_COMMAND make DESTDIR="$ENV{HOME}/.cos-local.1" build/lib/libs3.a
-        BUILD_COMMAND make build/lib/libs3.a
-        INSTALL_COMMAND make install_static
+        BUILD_COMMAND ""
+        INSTALL_COMMAND make clean && make build/lib/libs3.a && make install_static
         TEST_COMMAND ""
 )
diff --git a/cmake/ssl_CMakeLists.txt.in b/cmake/ssl_CMakeLists.txt.in
index 65a02d7fd6..8301dbe222 100644
--- a/cmake/ssl_CMakeLists.txt.in
+++ b/cmake/ssl_CMakeLists.txt.in
@@ -10,6 +10,6 @@ ExternalProject_Add(openssl
         #UPDATE_COMMAND ""
         CONFIGURE_COMMAND ./Configure --prefix=$ENV{HOME}/.cos-local.1 -static #--no-shared
         BUILD_COMMAND ""
-        INSTALL_COMMAND make install_sw -j
+        INSTALL_COMMAND make clean && make install_sw -j
         TEST_COMMAND ""
 )

From b864c87b7c91d8663c592f25ee681f85cc3e761f Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Mon, 23 Oct 2023 17:00:52 +0800
Subject: [PATCH 21/26] cmake/ssl: make depend

---
 cmake/ssl_CMakeLists.txt.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/ssl_CMakeLists.txt.in b/cmake/ssl_CMakeLists.txt.in
index 8301dbe222..c17043ff22 100644
--- a/cmake/ssl_CMakeLists.txt.in
+++ b/cmake/ssl_CMakeLists.txt.in
@@ -10,6 +10,6 @@ ExternalProject_Add(openssl
         #UPDATE_COMMAND ""
         CONFIGURE_COMMAND ./Configure --prefix=$ENV{HOME}/.cos-local.1 -static #--no-shared
         BUILD_COMMAND ""
-        INSTALL_COMMAND make clean && make install_sw -j
+        INSTALL_COMMAND make clean && make depend && make install_sw -j
         TEST_COMMAND ""
 )

From 7cf72a567da657f1c2f62401722dada5e757283b Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Mon, 23 Oct 2023 19:19:34 +0800
Subject: [PATCH 22/26] cmake/ssl: use build command

---
 cmake/ssl_CMakeLists.txt.in | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cmake/ssl_CMakeLists.txt.in b/cmake/ssl_CMakeLists.txt.in
index c17043ff22..1f3fd0a30d 100644
--- a/cmake/ssl_CMakeLists.txt.in
+++ b/cmake/ssl_CMakeLists.txt.in
@@ -9,7 +9,7 @@ ExternalProject_Add(openssl
         #BUILD_ALWAYS 1
         #UPDATE_COMMAND ""
         CONFIGURE_COMMAND ./Configure --prefix=$ENV{HOME}/.cos-local.1 -static #--no-shared
-        BUILD_COMMAND ""
-        INSTALL_COMMAND make clean && make depend && make install_sw -j
+        BUILD_COMMAND make -j
+        INSTALL_COMMAND make install_sw -j
         TEST_COMMAND ""
 )

From 7d17c4cc90385350805fb90a3501796486dd48e6 Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Mon, 23 Oct 2023 19:35:22 +0800
Subject: [PATCH 23/26] cmake/curl, libs3: use build command

---
 cmake/curl_CMakeLists.txt.in  | 4 ++--
 cmake/libs3_CMakeLists.txt.in | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/cmake/curl_CMakeLists.txt.in b/cmake/curl_CMakeLists.txt.in
index 2fe84b9721..b013ed1807 100644
--- a/cmake/curl_CMakeLists.txt.in
+++ b/cmake/curl_CMakeLists.txt.in
@@ -15,7 +15,7 @@ ExternalProject_Add(curl
         CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --with-ssl=$ENV{HOME}/.cos-local.1  --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd --without-libidn2 #--enable-debug
 
         #CONFIGURE_COMMAND ./configure --without-ssl
-        BUILD_COMMAND ""
-        INSTALL_COMMAND make clean && make install
+        BUILD_COMMAND make -j
+        INSTALL_COMMAND make install
         TEST_COMMAND ""
 )
diff --git a/cmake/libs3_CMakeLists.txt.in b/cmake/libs3_CMakeLists.txt.in
index 6c911775ee..b2d2d19059 100644
--- a/cmake/libs3_CMakeLists.txt.in
+++ b/cmake/libs3_CMakeLists.txt.in
@@ -23,7 +23,7 @@ ExternalProject_Add(libs3
 	CONFIGURE_COMMAND cp ${TD_SUPPORT_DIR}/libs3.GNUmakefile GNUmakefile &&  sed -i "s|CFLAGS += -Wall -Werror|CFLAGS += -I'$ENV{HOME}/.cos-local.1/include' -L'$ENV{HOME}/.cos-local.1/lib' |" ./GNUmakefile
         #BUILD_COMMAND make CFLAGS=${s3_flags} DESTDIR=$ENV{HOME}/.cos-local.1 build/lib/libs3.a
         #BUILD_COMMAND make DESTDIR="$ENV{HOME}/.cos-local.1" build/lib/libs3.a
-        BUILD_COMMAND ""
-        INSTALL_COMMAND make clean && make build/lib/libs3.a && make install_static
+        BUILD_COMMAND make build/lib/libs3.a
+        INSTALL_COMMAND make install_static
         TEST_COMMAND ""
 )

From 7fdf6cc696fae234f993479f5737451a384cfda3 Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Tue, 24 Oct 2023 09:04:24 +0800
Subject: [PATCH 24/26] cmake: use new dir for libs3

---
 cmake/curl_CMakeLists.txt.in           |  7 ++-----
 cmake/libs3.GNUmakefile                |  2 +-
 cmake/libs3_CMakeLists.txt.in          | 15 +--------------
 cmake/ssl_CMakeLists.txt.in            |  2 +-
 contrib/CMakeLists.txt                 |  4 ++--
 source/dnode/vnode/CMakeLists.txt      |  8 ++++----
 tests/parallel_test/container_build.sh |  4 ++--
 7 files changed, 13 insertions(+), 29 deletions(-)

diff --git a/cmake/curl_CMakeLists.txt.in b/cmake/curl_CMakeLists.txt.in
index b013ed1807..140fc48103 100644
--- a/cmake/curl_CMakeLists.txt.in
+++ b/cmake/curl_CMakeLists.txt.in
@@ -10,11 +10,8 @@ ExternalProject_Add(curl
         DEPENDS openssl
         BUILD_IN_SOURCE TRUE
         BUILD_ALWAYS 1
-        #UPDATE_COMMAND ""
-        #CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --without-ssl --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd
-        CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --with-ssl=$ENV{HOME}/.cos-local.1  --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd --without-libidn2 #--enable-debug
-
-        #CONFIGURE_COMMAND ./configure --without-ssl
+        UPDATE_COMMAND ""
+        CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.2 --with-ssl=$ENV{HOME}/.cos-local.2  --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd --without-libidn2 #--enable-debug
         BUILD_COMMAND make -j
         INSTALL_COMMAND make install
         TEST_COMMAND ""
diff --git a/cmake/libs3.GNUmakefile b/cmake/libs3.GNUmakefile
index abf954f5c1..98f98d8224 100644
--- a/cmake/libs3.GNUmakefile
+++ b/cmake/libs3.GNUmakefile
@@ -91,7 +91,7 @@ endif
 # --------------------------------------------------------------------------
 # DESTDIR directory
 ifndef DESTDIR
-    DESTDIR := ${HOME}/.cos-local.1
+    DESTDIR := ${HOME}/.cos-local.2
 endif
 
 # --------------------------------------------------------------------------
diff --git a/cmake/libs3_CMakeLists.txt.in b/cmake/libs3_CMakeLists.txt.in
index b2d2d19059..b5e9c438e6 100644
--- a/cmake/libs3_CMakeLists.txt.in
+++ b/cmake/libs3_CMakeLists.txt.in
@@ -1,15 +1,5 @@
 # libs3
 
-set(s3_flags "${CMAKE_C_FLAGS} -O3 -Iinc -I$ENV{HOME}/.cos-local.1/include -Iinc ")
-
-set(SED_CMD "s/CFLAGS += -Wall -Werror/CFLAGS += -I$ENV{HOME}/.cos-local.1/include /")
-
-function(update_cflags)
-    file(READ "GNUmakefile" S3_CONTENT)
-    string(REPLACE "CFLAGS += -Wall -Werror" "CFLAGS += -I$ENV{HOME}/.cos-local.1/include " S3_CONTENT "${S3_CONTENT}")
-    file(WRITE "GNUmakefile" "${S3_CONTENT}")
-endfunction(update_cflags)
-
 ExternalProject_Add(libs3
         GIT_REPOSITORY https://github.com/bji/libs3
         #GIT_TAG v5.0.16
@@ -19,10 +9,7 @@ ExternalProject_Add(libs3
         BUILD_IN_SOURCE TRUE
         BUILD_ALWAYS 1
 	UPDATE_COMMAND ""
-	#sed_cmd "s/CFLAGS += -Wall -Werror/CFLAGS += -I$ENV{HOME}/.cos-local.1/include /"
-	CONFIGURE_COMMAND cp ${TD_SUPPORT_DIR}/libs3.GNUmakefile GNUmakefile &&  sed -i "s|CFLAGS += -Wall -Werror|CFLAGS += -I'$ENV{HOME}/.cos-local.1/include' -L'$ENV{HOME}/.cos-local.1/lib' |" ./GNUmakefile
-        #BUILD_COMMAND make CFLAGS=${s3_flags} DESTDIR=$ENV{HOME}/.cos-local.1 build/lib/libs3.a
-        #BUILD_COMMAND make DESTDIR="$ENV{HOME}/.cos-local.1" build/lib/libs3.a
+	CONFIGURE_COMMAND cp ${TD_SUPPORT_DIR}/libs3.GNUmakefile GNUmakefile &&  sed -i "s|CFLAGS += -Wall -Werror|CFLAGS += -I'$ENV{HOME}/.cos-local.2/include' -L'$ENV{HOME}/.cos-local.2/lib' |" ./GNUmakefile
         BUILD_COMMAND make build/lib/libs3.a
         INSTALL_COMMAND make install_static
         TEST_COMMAND ""
diff --git a/cmake/ssl_CMakeLists.txt.in b/cmake/ssl_CMakeLists.txt.in
index 1f3fd0a30d..c9f836bade 100644
--- a/cmake/ssl_CMakeLists.txt.in
+++ b/cmake/ssl_CMakeLists.txt.in
@@ -8,7 +8,7 @@ ExternalProject_Add(openssl
         BUILD_IN_SOURCE TRUE
         #BUILD_ALWAYS 1
         #UPDATE_COMMAND ""
-        CONFIGURE_COMMAND ./Configure --prefix=$ENV{HOME}/.cos-local.1 -static #--no-shared
+        CONFIGURE_COMMAND ./Configure --prefix=$ENV{HOME}/.cos-local.2 -static #--no-shared
         BUILD_COMMAND make -j
         INSTALL_COMMAND make install_sw -j
         TEST_COMMAND ""
diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt
index 013269c583..e2a5378f0d 100644
--- a/contrib/CMakeLists.txt
+++ b/contrib/CMakeLists.txt
@@ -9,7 +9,7 @@ endfunction(cat IN_FILE OUT_FILE)
 if(${TD_LINUX})
 
 if(${BUILD_WITH_S3})
-    file(MAKE_DIRECTORY $ENV{HOME}/.cos-local.1/)
+    file(MAKE_DIRECTORY $ENV{HOME}/.cos-local.2/)
 
 else()
 
@@ -414,7 +414,7 @@ if (${BUILD_WITH_ROCKSDB})
 endif()
 
 if(${BUILD_WITH_S3})
-    INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.1/include)
+    INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.2/include)
     MESSAGE("build with s3: ${BUILD_WITH_S3}")
 
 else()
diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt
index 7e2693e8af..bc93ab575e 100644
--- a/source/dnode/vnode/CMakeLists.txt
+++ b/source/dnode/vnode/CMakeLists.txt
@@ -167,15 +167,15 @@ if(${BUILD_WITH_S3})
   target_include_directories(
     vnode
 
-    PUBLIC "$ENV{HOME}/.cos-local.1/include"
+    PUBLIC "$ENV{HOME}/.cos-local.2/include"
     )
 
   set(CMAKE_FIND_LIBRARY_SUFFIXES ".a")
-  set(CMAKE_PREFIX_PATH $ENV{HOME}/.cos-local.1)
+  set(CMAKE_PREFIX_PATH $ENV{HOME}/.cos-local.2)
   find_library(S3_LIBRARY s3)
   find_library(CURL_LIBRARY curl)
-  find_library(SSL_LIBRARY ssl PATHS $ENV{HOME}/.cos-local.1/lib64)
-  find_library(CRYPTO_LIBRARY crypto PATHS $ENV{HOME}/.cos-local.1/lib64)
+  find_library(SSL_LIBRARY ssl PATHS $ENV{HOME}/.cos-local.2/lib64)
+  find_library(CRYPTO_LIBRARY crypto PATHS $ENV{HOME}/.cos-local.2/lib64)
   target_link_libraries(
     vnode
 
diff --git a/tests/parallel_test/container_build.sh b/tests/parallel_test/container_build.sh
index f5e426057e..94704b1c25 100755
--- a/tests/parallel_test/container_build.sh
+++ b/tests/parallel_test/container_build.sh
@@ -60,7 +60,7 @@ docker run \
     -v /root/.cargo/git:/root/.cargo/git \
     -v /root/go/pkg/mod:/root/go/pkg/mod \
     -v /root/.cache/go-build:/root/.cache/go-build \
-    -v /root/.cos-local.1:/root/.cos-local.1 \
+    -v /root/.cos-local.1:/root/.cos-local.2 \
     -v ${REP_REAL_PATH}/enterprise/src/plugins/taosx/target:${REP_DIR}/enterprise/src/plugins/taosx/target \
     -v ${REP_REAL_PATH}/community/tools/taosws-rs/target:${REP_DIR}/community/tools/taosws-rs/target \
     -v ${REP_REAL_PATH}/community/contrib/cJson/:${REP_DIR}/community/contrib/cJson \
@@ -89,7 +89,7 @@ docker run \
     -v /root/.cargo/git:/root/.cargo/git \
     -v /root/go/pkg/mod:/root/go/pkg/mod \
     -v /root/.cache/go-build:/root/.cache/go-build \
-    -v /root/.cos-local.1:/root/.cos-local.1 \
+    -v /root/.cos-local.1:/root/.cos-local.2 \
     -v ${REP_REAL_PATH}/enterprise/src/plugins/taosx/target:${REP_DIR}/enterprise/src/plugins/taosx/target \
     -v ${REP_REAL_PATH}/community/tools/taosws-rs/target:${REP_DIR}/community/tools/taosws-rs/target \
     -v ${REP_REAL_PATH}/community/contrib/cJson/:${REP_DIR}/community/contrib/cJson \

From 0e65ed0202bb772908a8bc6867e6d1d2078c4f91 Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Tue, 24 Oct 2023 09:16:09 +0800
Subject: [PATCH 25/26] cmake/curl: use new dir curl2 for s3 version

---
 cmake/curl_CMakeLists.txt.in  | 4 ++--
 cmake/libs3_CMakeLists.txt.in | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/cmake/curl_CMakeLists.txt.in b/cmake/curl_CMakeLists.txt.in
index 140fc48103..458a518092 100644
--- a/cmake/curl_CMakeLists.txt.in
+++ b/cmake/curl_CMakeLists.txt.in
@@ -1,12 +1,12 @@
 # curl
-ExternalProject_Add(curl
+ExternalProject_Add(curl2
         URL  https://curl.se/download/curl-8.2.1.tar.gz
         URL_HASH MD5=b25588a43556068be05e1624e0e74d41
         DOWNLOAD_NO_PROGRESS 1
         DOWNLOAD_DIR "${TD_CONTRIB_DIR}/deps-download"
         #GIT_REPOSITORY https://github.com/curl/curl.git
         #GIT_TAG curl-7_88_1
-        SOURCE_DIR "${TD_CONTRIB_DIR}/curl"
+        SOURCE_DIR "${TD_CONTRIB_DIR}/curl2"
         DEPENDS openssl
         BUILD_IN_SOURCE TRUE
         BUILD_ALWAYS 1
diff --git a/cmake/libs3_CMakeLists.txt.in b/cmake/libs3_CMakeLists.txt.in
index b5e9c438e6..f2b6cac953 100644
--- a/cmake/libs3_CMakeLists.txt.in
+++ b/cmake/libs3_CMakeLists.txt.in
@@ -3,7 +3,7 @@
 ExternalProject_Add(libs3
         GIT_REPOSITORY https://github.com/bji/libs3
         #GIT_TAG v5.0.16
-        DEPENDS curl xml2
+        DEPENDS curl2 xml2
         SOURCE_DIR "${TD_CONTRIB_DIR}/libs3"
         #BINARY_DIR ""
         BUILD_IN_SOURCE TRUE

From e826532592d520de945ca6fcf771123ae967e8fa Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Tue, 24 Oct 2023 10:10:43 +0800
Subject: [PATCH 26/26] cmake/vnode: specify no default path for ssl & crypto

---
 source/dnode/vnode/CMakeLists.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt
index bc93ab575e..b83d64c569 100644
--- a/source/dnode/vnode/CMakeLists.txt
+++ b/source/dnode/vnode/CMakeLists.txt
@@ -174,8 +174,8 @@ if(${BUILD_WITH_S3})
   set(CMAKE_PREFIX_PATH $ENV{HOME}/.cos-local.2)
   find_library(S3_LIBRARY s3)
   find_library(CURL_LIBRARY curl)
-  find_library(SSL_LIBRARY ssl PATHS $ENV{HOME}/.cos-local.2/lib64)
-  find_library(CRYPTO_LIBRARY crypto PATHS $ENV{HOME}/.cos-local.2/lib64)
+  find_library(SSL_LIBRARY ssl $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH)
+  find_library(CRYPTO_LIBRARY crypto $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH)
   target_link_libraries(
     vnode