diff --git a/cmake/cmake.options b/cmake/cmake.options index 1d4e9ba515..bacc5adfb7 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -127,6 +127,22 @@ option( IF(${TD_LINUX}) +option( + BUILD_WITH_S3 + "If build with s3" + ON +) + +IF(${BUILD_WITH_S3}) + +option( + BUILD_WITH_COS + "If build with cos" + OFF +) + +ELSE () + option( BUILD_WITH_COS "If build with cos" @@ -135,6 +151,8 @@ option( ENDIF () +ENDIF () + option( BUILD_WITH_SQLITE "If build with sqlite" diff --git a/cmake/curl_CMakeLists.txt.in b/cmake/curl_CMakeLists.txt.in index 1f2291c519..458a518092 100644 --- a/cmake/curl_CMakeLists.txt.in +++ b/cmake/curl_CMakeLists.txt.in @@ -1,18 +1,18 @@ # curl -ExternalProject_Add(curl +ExternalProject_Add(curl2 URL https://curl.se/download/curl-8.2.1.tar.gz URL_HASH MD5=b25588a43556068be05e1624e0e74d41 DOWNLOAD_NO_PROGRESS 1 DOWNLOAD_DIR "${TD_CONTRIB_DIR}/deps-download" #GIT_REPOSITORY https://github.com/curl/curl.git #GIT_TAG curl-7_88_1 - SOURCE_DIR "${TD_CONTRIB_DIR}/curl" + SOURCE_DIR "${TD_CONTRIB_DIR}/curl2" + DEPENDS openssl BUILD_IN_SOURCE TRUE BUILD_ALWAYS 1 - #UPDATE_COMMAND "" - CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --without-ssl --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd - #CONFIGURE_COMMAND ./configure --without-ssl - BUILD_COMMAND make + UPDATE_COMMAND "" + CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.2 --with-ssl=$ENV{HOME}/.cos-local.2 --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd --without-libidn2 #--enable-debug + BUILD_COMMAND make -j INSTALL_COMMAND make install TEST_COMMAND "" ) diff --git a/cmake/libs3.GNUmakefile b/cmake/libs3.GNUmakefile new file mode 100644 index 0000000000..98f98d8224 --- /dev/null +++ b/cmake/libs3.GNUmakefile @@ -0,0 +1,430 @@ +# GNUmakefile +# +# Copyright 2008 Bryan Ischo +# +# This file is part of libs3. +# +# libs3 is free software: you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation, version 3 or above of the License. You can also +# redistribute and/or modify it under the terms of the GNU General Public +# License, version 2 or above of the License. +# +# In addition, as a special exception, the copyright holders give +# permission to link the code of this library and its programs with the +# OpenSSL library, and distribute linked combinations including the two. +# +# libs3 is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# version 3 along with libs3, in a file named COPYING. If not, see +# . +# +# You should also have received a copy of the GNU General Public License +# version 2 along with libs3, in a file named COPYING-GPLv2. If not, see +# . + +# I tried to use the autoconf/automake/autolocal/etc (i.e. autohell) tools +# but I just couldn't stomach them. Since this is a Makefile for POSIX +# systems, I will simply do away with autohell completely and use a GNU +# Makefile. GNU make ought to be available pretty much everywhere, so I +# don't see this being a significant issue for portability. + +# All commands assume a GNU compiler. For systems which do not use a GNU +# compiler, write scripts with the same names as these commands, and taking +# the same arguments, and translate the arguments and commands into the +# appropriate non-POSIX ones as needed. libs3 assumes a GNU toolchain as +# the most portable way to build software possible. Non-POSIX, non-GNU +# systems can do the work of supporting this build infrastructure. + + +# -------------------------------------------------------------------------- +# Set libs3 version number, unless it is already set. + +LIBS3_VER_MAJOR ?= 4 +LIBS3_VER_MINOR ?= 1 +LIBS3_VER := $(LIBS3_VER_MAJOR).$(LIBS3_VER_MINOR) + + +# ----------------------------------------------------------------------------- +# Determine verbosity. VERBOSE_SHOW should be prepended to every command which +# should only be displayed if VERBOSE is set. QUIET_ECHO may be used to +# echo text only if VERBOSE is not set. Typically, a VERBOSE_SHOW command will +# be paired with a QUIET_ECHO command, to provide a command which is displayed +# in VERBOSE mode, along with text which is displayed in non-VERBOSE mode to +# describe the command. +# +# No matter what VERBOSE is defined to, it ends up as true if it's defined. +# This will be weird if you defined VERBOSE=false in the environment, and we +# switch it to true here; but the meaning of VERBOSE is, "if it's defined to +# any value, then verbosity is turned on". So don't define VERBOSE if you +# don't want verbosity in the build process. +# ----------------------------------------------------------------------------- + +ifdef VERBOSE + VERBOSE = true + VERBOSE_ECHO = @ echo + VERBOSE_SHOW = + QUIET_ECHO = @ echo > /dev/null +else + VERBOSE = false + VERBOSE_ECHO = @ echo > /dev/null + VERBOSE_SHOW = @ + QUIET_ECHO = @ echo +endif + + +# -------------------------------------------------------------------------- +# BUILD directory +ifndef BUILD + ifdef DEBUG + BUILD := build-debug + else + BUILD := build + endif +endif + + +# -------------------------------------------------------------------------- +# DESTDIR directory +ifndef DESTDIR + DESTDIR := ${HOME}/.cos-local.2 +endif + +# -------------------------------------------------------------------------- +# LIBDIR directory +ifndef LIBDIR + LIBDIR := ${DESTDIR}/lib +endif + +# -------------------------------------------------------------------------- +# Compiler CC handling +ifndef CC + CC := gcc +endif + +# -------------------------------------------------------------------------- +# Acquire configuration information for libraries that libs3 depends upon + +ifndef CURL_LIBS + CURL_LIBS := $(shell curl-config --libs) +endif + +ifndef CURL_CFLAGS + CURL_CFLAGS := $(shell curl-config --cflags) +endif + +ifndef LIBXML2_LIBS + LIBXML2_LIBS := $(shell xml2-config --libs) +endif + +ifndef LIBXML2_CFLAGS + LIBXML2_CFLAGS := $(shell xml2-config --cflags) +endif + +ifndef OPENSSL_LIBS + OPENSSL_LIBS := -lssl -lcrypto +endif + +# -------------------------------------------------------------------------- +# These CFLAGS assume a GNU compiler. For other compilers, write a script +# which converts these arguments into their equivalent for that particular +# compiler. + +ifndef CFLAGS + ifdef DEBUG + CFLAGS := -g + else + CFLAGS := -O3 + endif +endif + +CFLAGS += -Wall -Werror -Wshadow -Wextra -Iinc \ + $(CURL_CFLAGS) $(LIBXML2_CFLAGS) \ + -DLIBS3_VER_MAJOR=\"$(LIBS3_VER_MAJOR)\" \ + -DLIBS3_VER_MINOR=\"$(LIBS3_VER_MINOR)\" \ + -DLIBS3_VER=\"$(LIBS3_VER)\" \ + -D__STRICT_ANSI__ \ + -D_ISOC99_SOURCE \ + -D_POSIX_C_SOURCE=200112L + +LDFLAGS = $(CURL_LIBS) $(LIBXML2_LIBS) $(OPENSSL_LIBS) -lpthread + +STRIP ?= strip +INSTALL := install --strip-program=$(STRIP) + + +# -------------------------------------------------------------------------- +# Default targets are everything + +.PHONY: all +all: exported test + + +# -------------------------------------------------------------------------- +# Exported targets are the library and driver program + +.PHONY: exported +exported: libs3 s3 headers +exported_static: $(LIBS3_STATIC) + +# -------------------------------------------------------------------------- +# Install target + +.PHONY: install install_static +install_static: exported_static + $(QUIET_ECHO) $(LIBDIR)/libs3.a: Installing static library + $(VERBOSE_SHOW) $(INSTALL) -Dp -m u+rw,go+r $(BUILD)/lib/libs3.a \ + $(LIBDIR)/libs3.a + $(QUIET_ECHO) $(DESTDIR)/include/libs3.h: Installing header + $(VERBOSE_SHOW) $(INSTALL) -Dp -m u+rw,go+r inc/libs3.h \ + $(DESTDIR)/include/libs3.h + +install: exported + $(QUIET_ECHO) $(DESTDIR)/bin/s3: Installing executable + $(VERBOSE_SHOW) $(INSTALL) -Dps -m u+rwx,go+rx $(BUILD)/bin/s3 \ + $(DESTDIR)/bin/s3 + $(QUIET_ECHO) \ + $(LIBDIR)/libs3.so.$(LIBS3_VER): Installing shared library + $(VERBOSE_SHOW) $(INSTALL) -Dps -m u+rw,go+r \ + $(BUILD)/lib/libs3.so.$(LIBS3_VER_MAJOR) \ + $(LIBDIR)/libs3.so.$(LIBS3_VER) + $(QUIET_ECHO) \ + $(LIBDIR)/libs3.so.$(LIBS3_VER_MAJOR): Linking shared library + $(VERBOSE_SHOW) ln -sf libs3.so.$(LIBS3_VER) \ + $(LIBDIR)/libs3.so.$(LIBS3_VER_MAJOR) + $(QUIET_ECHO) $(LIBDIR)/libs3.so: Linking shared library + $(VERBOSE_SHOW) ln -sf libs3.so.$(LIBS3_VER_MAJOR) $(LIBDIR)/libs3.so + $(QUIET_ECHO) $(LIBDIR)/libs3.a: Installing static library + $(VERBOSE_SHOW) $(INSTALL) -Dp -m u+rw,go+r $(BUILD)/lib/libs3.a \ + $(LIBDIR)/libs3.a + $(QUIET_ECHO) $(DESTDIR)/include/libs3.h: Installing header + $(VERBOSE_SHOW) $(INSTALL) -Dp -m u+rw,go+r $(BUILD)/include/libs3.h \ + $(DESTDIR)/include/libs3.h + + +# -------------------------------------------------------------------------- +# Uninstall target + +.PHONY: uninstall +uninstall: + $(QUIET_ECHO) Installed files: Uninstalling + $(VERBOSE_SHOW) \ + rm -f $(DESTDIR)/bin/s3 \ + $(DESTDIR)/include/libs3.h \ + $(DESTDIR)/lib/libs3.a \ + $(DESTDIR)/lib/libs3.so \ + $(DESTDIR)/lib/libs3.so.$(LIBS3_VER_MAJOR) \ + $(DESTDIR)/lib/libs3.so.$(LIBS3_VER) + + +# -------------------------------------------------------------------------- +# Compile target patterns + +$(BUILD)/obj/%.o: src/%.c + $(QUIET_ECHO) $@: Compiling object + @ mkdir -p $(dir $(BUILD)/dep/$<) + @ $(CC) $(CFLAGS) -M -MG -MQ $@ -DCOMPILINGDEPENDENCIES \ + -o $(BUILD)/dep/$(<:%.c=%.d) -c $< + @ mkdir -p $(dir $@) + $(VERBOSE_SHOW) $(CC) $(CFLAGS) -o $@ -c $< + +$(BUILD)/obj/%.do: src/%.c + $(QUIET_ECHO) $@: Compiling dynamic object + $(QUIET_ECHO) cflags:${CFLAGS} + @ mkdir -p $(dir $(BUILD)/dep/$<) + @ $(CC) $(CFLAGS) -M -MG -MQ $@ -DCOMPILINGDEPENDENCIES \ + -o $(BUILD)/dep/$(<:%.c=%.dd) -c $< + @ mkdir -p $(dir $@) + $(VERBOSE_SHOW) $(CC) $(CFLAGS) -fpic -fPIC -o $@ -c $< + + +# -------------------------------------------------------------------------- +# libs3 library targets + +LIBS3_SHARED = $(BUILD)/lib/libs3.so.$(LIBS3_VER_MAJOR) +LIBS3_STATIC = $(BUILD)/lib/libs3.a + +.PHONY: libs3 +libs3: $(LIBS3_SHARED) $(LIBS3_STATIC) + +LIBS3_SOURCES := bucket.c bucket_metadata.c error_parser.c general.c \ + object.c request.c request_context.c \ + response_headers_handler.c service_access_logging.c \ + service.c simplexml.c util.c multipart.c + +$(LIBS3_SHARED): $(LIBS3_SOURCES:%.c=$(BUILD)/obj/%.do) + $(QUIET_ECHO) $@: Building shared library + @ mkdir -p $(dir $@) + $(VERBOSE_SHOW) $(CC) -shared -Wl,-soname,libs3.so.$(LIBS3_VER_MAJOR) \ + -o $@ $^ $(LDFLAGS) + +$(LIBS3_STATIC): $(LIBS3_SOURCES:%.c=$(BUILD)/obj/%.o) + $(QUIET_ECHO) $@: Building static library + @ mkdir -p $(dir $@) + $(VERBOSE_SHOW) $(AR) cr $@ $^ + + +# -------------------------------------------------------------------------- +# Driver program targets + +.PHONY: s3 +s3: $(BUILD)/bin/s3 + +$(BUILD)/bin/s3: $(BUILD)/obj/s3.o $(LIBS3_SHARED) + $(QUIET_ECHO) $@: Building executable + @ mkdir -p $(dir $@) + $(VERBOSE_SHOW) $(CC) -o $@ $^ $(LDFLAGS) + + +# -------------------------------------------------------------------------- +# libs3 header targets + +.PHONY: headers +headers: $(BUILD)/include/libs3.h + +$(BUILD)/include/libs3.h: inc/libs3.h + $(QUIET_ECHO) $@: Linking header + @ mkdir -p $(dir $@) + $(VERBOSE_SHOW) ln -sf $(abspath $<) $@ + + +# -------------------------------------------------------------------------- +# Test targets + +.PHONY: test +test: $(BUILD)/bin/testsimplexml + +$(BUILD)/bin/testsimplexml: $(BUILD)/obj/testsimplexml.o $(LIBS3_STATIC) + $(QUIET_ECHO) $@: Building executable + @ mkdir -p $(dir $@) + $(VERBOSE_SHOW) $(CC) -o $@ $^ $(LIBXML2_LIBS) + + +# -------------------------------------------------------------------------- +# Clean target + +.PHONY: clean +clean: + $(QUIET_ECHO) $(BUILD): Cleaning + $(VERBOSE_SHOW) rm -rf $(BUILD) + +.PHONY: distclean +distclean: + $(QUIET_ECHO) $(BUILD): Cleaning + $(VERBOSE_SHOW) rm -rf $(BUILD) + + +# -------------------------------------------------------------------------- +# Clean dependencies target + +.PHONY: cleandeps +cleandeps: + $(QUIET_ECHO) $(BUILD)/dep: Cleaning dependencies + $(VERBOSE_SHOW) rm -rf $(BUILD)/dep + + +# -------------------------------------------------------------------------- +# Dependencies + +ALL_SOURCES := $(LIBS3_SOURCES) s3.c testsimplexml.c + +$(foreach i, $(ALL_SOURCES), $(eval -include $(BUILD)/dep/src/$(i:%.c=%.d))) +$(foreach i, $(ALL_SOURCES), $(eval -include $(BUILD)/dep/src/$(i:%.c=%.dd))) + + +# -------------------------------------------------------------------------- +# Debian package target + +DEBPKG = $(BUILD)/pkg/libs3_$(LIBS3_VER).deb +DEBDEVPKG = $(BUILD)/pkg/libs3-dev_$(LIBS3_VER).deb + +.PHONY: deb +deb: $(DEBPKG) $(DEBDEVPKG) + +$(DEBPKG): DEBARCH = $(shell dpkg-architecture | grep ^DEB_BUILD_ARCH= | \ + cut -d '=' -f 2) +$(DEBPKG): exported $(BUILD)/deb/DEBIAN/control $(BUILD)/deb/DEBIAN/shlibs \ + $(BUILD)/deb/DEBIAN/postinst \ + $(BUILD)/deb/usr/share/doc/libs3/changelog.gz \ + $(BUILD)/deb/usr/share/doc/libs3/changelog.Debian.gz \ + $(BUILD)/deb/usr/share/doc/libs3/copyright + DESTDIR=$(BUILD)/deb/usr $(MAKE) install + rm -rf $(BUILD)/deb/usr/include + rm -f $(BUILD)/deb/usr/lib/libs3.a + @mkdir -p $(dir $@) + fakeroot dpkg-deb -b $(BUILD)/deb $@ + mv $@ $(BUILD)/pkg/libs3_$(LIBS3_VER)_$(DEBARCH).deb + +$(DEBDEVPKG): DEBARCH = $(shell dpkg-architecture | grep ^DEB_BUILD_ARCH= | \ + cut -d '=' -f 2) +$(DEBDEVPKG): exported $(BUILD)/deb-dev/DEBIAN/control \ + $(BUILD)/deb-dev/usr/share/doc/libs3-dev/changelog.gz \ + $(BUILD)/deb-dev/usr/share/doc/libs3-dev/changelog.Debian.gz \ + $(BUILD)/deb-dev/usr/share/doc/libs3-dev/copyright + DESTDIR=$(BUILD)/deb-dev/usr $(MAKE) install + rm -rf $(BUILD)/deb-dev/usr/bin + rm -f $(BUILD)/deb-dev/usr/lib/libs3.so* + @mkdir -p $(dir $@) + fakeroot dpkg-deb -b $(BUILD)/deb-dev $@ + mv $@ $(BUILD)/pkg/libs3-dev_$(LIBS3_VER)_$(DEBARCH).deb + +$(BUILD)/deb/DEBIAN/control: debian/control + @mkdir -p $(dir $@) + echo -n "Depends: " > $@ + dpkg-shlibdeps -Sbuild -O $(BUILD)/lib/libs3.so.$(LIBS3_VER_MAJOR) | \ + cut -d '=' -f 2- >> $@ + sed -e 's/LIBS3_VERSION/$(LIBS3_VER)/' \ + < $< | sed -e 's/DEBIAN_ARCHITECTURE/$(DEBARCH)/' | \ + grep -v ^Source: >> $@ + +$(BUILD)/deb-dev/DEBIAN/control: debian/control.dev + @mkdir -p $(dir $@) + sed -e 's/LIBS3_VERSION/$(LIBS3_VER)/' \ + < $< | sed -e 's/DEBIAN_ARCHITECTURE/$(DEBARCH)/' > $@ + +$(BUILD)/deb/DEBIAN/shlibs: + echo -n "libs3 $(LIBS3_VER_MAJOR) libs3 " > $@ + echo "(>= $(LIBS3_VER))" >> $@ + +$(BUILD)/deb/DEBIAN/postinst: debian/postinst + @mkdir -p $(dir $@) + cp $< $@ + +$(BUILD)/deb/usr/share/doc/libs3/copyright: LICENSE + @mkdir -p $(dir $@) + cp $< $@ + @echo >> $@ + @echo -n "An alternate location for the GNU General Public " >> $@ + @echo "License version 3 on Debian" >> $@ + @echo "systems is /usr/share/common-licenses/GPL-3." >> $@ + +$(BUILD)/deb-dev/usr/share/doc/libs3-dev/copyright: LICENSE + @mkdir -p $(dir $@) + cp $< $@ + @echo >> $@ + @echo -n "An alternate location for the GNU General Public " >> $@ + @echo "License version 3 on Debian" >> $@ + @echo "systems is /usr/share/common-licenses/GPL-3." >> $@ + +$(BUILD)/deb/usr/share/doc/libs3/changelog.gz: debian/changelog + @mkdir -p $(dir $@) + gzip --best -c $< > $@ + +$(BUILD)/deb-dev/usr/share/doc/libs3-dev/changelog.gz: debian/changelog + @mkdir -p $(dir $@) + gzip --best -c $< > $@ + +$(BUILD)/deb/usr/share/doc/libs3/changelog.Debian.gz: debian/changelog.Debian + @mkdir -p $(dir $@) + gzip --best -c $< > $@ + +$(BUILD)/deb-dev/usr/share/doc/libs3-dev/changelog.Debian.gz: \ + debian/changelog.Debian + @mkdir -p $(dir $@) + gzip --best -c $< > $@ + + diff --git a/cmake/libs3_CMakeLists.txt.in b/cmake/libs3_CMakeLists.txt.in new file mode 100644 index 0000000000..f2b6cac953 --- /dev/null +++ b/cmake/libs3_CMakeLists.txt.in @@ -0,0 +1,16 @@ +# libs3 + +ExternalProject_Add(libs3 + GIT_REPOSITORY https://github.com/bji/libs3 + #GIT_TAG v5.0.16 + DEPENDS curl2 xml2 + SOURCE_DIR "${TD_CONTRIB_DIR}/libs3" + #BINARY_DIR "" + BUILD_IN_SOURCE TRUE + BUILD_ALWAYS 1 + UPDATE_COMMAND "" + CONFIGURE_COMMAND cp ${TD_SUPPORT_DIR}/libs3.GNUmakefile GNUmakefile && sed -i "s|CFLAGS += -Wall -Werror|CFLAGS += -I'$ENV{HOME}/.cos-local.2/include' -L'$ENV{HOME}/.cos-local.2/lib' |" ./GNUmakefile + BUILD_COMMAND make build/lib/libs3.a + INSTALL_COMMAND make install_static + TEST_COMMAND "" +) diff --git a/cmake/ssl_CMakeLists.txt.in b/cmake/ssl_CMakeLists.txt.in new file mode 100644 index 0000000000..c9f836bade --- /dev/null +++ b/cmake/ssl_CMakeLists.txt.in @@ -0,0 +1,15 @@ +# openssl +ExternalProject_Add(openssl + URL https://www.openssl.org/source/openssl-3.1.3.tar.gz + URL_HASH SHA256=f0316a2ebd89e7f2352976445458689f80302093788c466692fb2a188b2eacf6 + DOWNLOAD_NO_PROGRESS 1 + DOWNLOAD_DIR "${TD_CONTRIB_DIR}/deps-download" + SOURCE_DIR "${TD_CONTRIB_DIR}/openssl" + BUILD_IN_SOURCE TRUE + #BUILD_ALWAYS 1 + #UPDATE_COMMAND "" + CONFIGURE_COMMAND ./Configure --prefix=$ENV{HOME}/.cos-local.2 -static #--no-shared + BUILD_COMMAND make -j + INSTALL_COMMAND make install_sw -j + TEST_COMMAND "" +) diff --git a/cmake/xml2_CMakeLists.txt.in b/cmake/xml2_CMakeLists.txt.in new file mode 100644 index 0000000000..4c234d7a37 --- /dev/null +++ b/cmake/xml2_CMakeLists.txt.in @@ -0,0 +1,12 @@ + +# xml2 +ExternalProject_Add(xml2 + GIT_REPOSITORY https://github.com/GNOME/libxml2 + GIT_TAG v2.11.5 + SOURCE_DIR "${TD_CONTRIB_DIR}/xml2" + BINARY_DIR "" + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + TEST_COMMAND "" + ) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index a963e4497f..e2a5378f0d 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -8,6 +8,11 @@ endfunction(cat IN_FILE OUT_FILE) if(${TD_LINUX}) +if(${BUILD_WITH_S3}) + file(MAKE_DIRECTORY $ENV{HOME}/.cos-local.2/) + +else() + set(CONTRIB_TMP_FILE3 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in3") configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3}) @@ -37,6 +42,8 @@ execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" . execute_process(COMMAND "${CMAKE_COMMAND}" --build . WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download") +endif(${BUILD_WITH_S3}) + endif(${TD_LINUX}) set(CONTRIB_TMP_FILE "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in") @@ -155,6 +162,16 @@ if(${BUILD_WITH_SQLITE}) cat("${TD_SUPPORT_DIR}/sqlite_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) endif(${BUILD_WITH_SQLITE}) +# s3 +if(${BUILD_WITH_S3}) + cat("${TD_SUPPORT_DIR}/ssl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) + cat("${TD_SUPPORT_DIR}/xml2_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) + cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) + cat("${TD_SUPPORT_DIR}/libs3_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) + add_definitions(-DUSE_S3) + +else() + # cos if(${BUILD_WITH_COS}) #cat("${TD_SUPPORT_DIR}/mxml_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) @@ -165,6 +182,8 @@ if(${BUILD_WITH_COS}) add_definitions(-DUSE_COS) endif(${BUILD_WITH_COS}) +endif(${BUILD_WITH_S3}) + # lucene if(${BUILD_WITH_LUCENE}) cat("${TD_SUPPORT_DIR}/lucene_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) @@ -231,7 +250,6 @@ if(${BUILD_TEST}) ) endif(${TD_DARWIN}) - endif(${BUILD_TEST}) # cJson @@ -248,6 +266,11 @@ target_include_directories( ) unset(CMAKE_PROJECT_INCLUDE_BEFORE) +# xml2 +if(${BUILD_WITH_S3}) + add_subdirectory(xml2 EXCLUDE_FROM_ALL) +endif(${BUILD_WITH_S3}) + # lz4 add_subdirectory(lz4/build/cmake EXCLUDE_FROM_ALL) target_include_directories( @@ -390,6 +413,12 @@ if (${BUILD_WITH_ROCKSDB}) endif() endif() +if(${BUILD_WITH_S3}) + INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.2/include) + MESSAGE("build with s3: ${BUILD_WITH_S3}") + +else() + # cos if(${BUILD_WITH_COS}) if(${TD_LINUX}) @@ -397,7 +426,7 @@ if(${BUILD_WITH_COS}) #ADD_DEFINITIONS(-DMINIXML_LIBRARY=${CMAKE_BINARY_DIR}/build/lib/libxml.a) option(ENABLE_TEST "Enable the tests" OFF) INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.1/include) - MESSAGE("$ENV{HOME}/.cos-local.1/include") + #MESSAGE("$ENV{HOME}/.cos-local.1/include") set(CMAKE_BUILD_TYPE Release) set(ORIG_CMAKE_PROJECT_NAME ${CMAKE_PROJECT_NAME}) @@ -415,6 +444,8 @@ if(${BUILD_WITH_COS}) endif(${TD_LINUX}) endif(${BUILD_WITH_COS}) +endif(${BUILD_WITH_S3}) + # lucene # To support build on ubuntu: sudo apt-get install libboost-all-dev if(${BUILD_WITH_LUCENE}) diff --git a/source/common/CMakeLists.txt b/source/common/CMakeLists.txt index b010467f20..6bdcdf7d1d 100644 --- a/source/common/CMakeLists.txt +++ b/source/common/CMakeLists.txt @@ -22,6 +22,9 @@ IF (TD_STORAGE) add_definitions(-DUSE_COS) ENDIF(${BUILD_WITH_COS}) + IF(${BUILD_WITH_S3}) + add_definitions(-DUSE_S3) + ENDIF(${BUILD_WITH_S3}) ENDIF(${TD_LINUX}) ENDIF () diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index ed1776dc45..d9b1b2e0ed 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -15,12 +15,12 @@ #define _DEFAULT_SOURCE #include "tglobal.h" +#include "defines.h" #include "os.h" #include "tconfig.h" #include "tgrant.h" #include "tlog.h" #include "tmisce.h" -#include "defines.h" #if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL) #include "cus_name.h" @@ -222,7 +222,7 @@ float tsFPrecision = 1E-8; // float column precision double tsDPrecision = 1E-16; // double column precision uint32_t tsMaxRange = 500; // max quantization intervals uint32_t tsCurRange = 100; // current quantization intervals -bool tsIfAdtFse = false; // ADT-FSE algorithom or original huffman algorithom +bool tsIfAdtFse = false; // ADT-FSE algorithom or original huffman algorithom char tsCompressor[32] = "ZSTD_COMPRESSOR"; // ZSTD_COMPRESSOR or GZIP_COMPRESSOR // udf @@ -267,6 +267,9 @@ char tsS3BucketName[TSDB_FQDN_LEN] = ""; char tsS3AppId[TSDB_FQDN_LEN] = ""; int8_t tsS3Enabled = false; +int8_t tsS3Https = true; +char tsS3Hostname[TSDB_FQDN_LEN] = ""; + int32_t tsS3BlockSize = 4096; // number of tsdb pages int32_t tsS3BlockCacheSize = 16; // number of blocks @@ -308,6 +311,14 @@ int32_t taosSetS3Cfg(SConfig *pCfg) { tstrncpy(tsS3AccessKeySecret, colon + 1, TSDB_FQDN_LEN); tstrncpy(tsS3Endpoint, cfgGetItem(pCfg, "s3Endpoint")->str, TSDB_FQDN_LEN); tstrncpy(tsS3BucketName, cfgGetItem(pCfg, "s3BucketName")->str, TSDB_FQDN_LEN); + char *proto = strstr(tsS3Endpoint, "https://"); + if (!proto) { + tsS3Https = false; + tstrncpy(tsS3Hostname, tsS3Endpoint + 7, TSDB_FQDN_LEN); + } else { + tstrncpy(tsS3Hostname, tsS3Endpoint + 8, TSDB_FQDN_LEN); + } + char *cos = strstr(tsS3Endpoint, "cos."); if (cos) { char *appid = strrchr(tsS3BucketName, '-'); @@ -319,7 +330,7 @@ int32_t taosSetS3Cfg(SConfig *pCfg) { } } if (tsS3BucketName[0] != '<' && tsDiskCfgNum > 1) { -#ifdef USE_COS +#if defined(USE_COS) || defined(USE_S3) tsS3Enabled = true; #endif } @@ -1091,7 +1102,6 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsIfAdtFse = cfgGetItem(pCfg, "IfAdtFse")->bval; tstrncpy(tsCompressor, cfgGetItem(pCfg, "Compressor")->str, sizeof(tsCompressor)); - tsDisableStream = cfgGetItem(pCfg, "disableStream")->bval; tsStreamBufferSize = cfgGetItem(pCfg, "streamBufferSize")->i64; @@ -1680,14 +1690,14 @@ void taosCfgDynamicOptions(const char *option, const char *value) { } return; } - + /* cannot alter s3BlockSize if (strcasecmp(option, "s3BlockSize") == 0) { int32_t newS3BlockSize = atoi(value); uInfo("s3BlockSize set from %d to %d", tsS3BlockSize, newS3BlockSize); tsS3BlockSize = newS3BlockSize; return; } - + */ if (strcasecmp(option, "s3BlockCacheSize") == 0) { int32_t newS3BlockCacheSize = atoi(value); uInfo("s3BlockCacheSize set from %d to %d", tsS3BlockCacheSize, newS3BlockCacheSize); diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 84d54f3350..b83d64c569 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -162,6 +162,36 @@ target_link_libraries( ) if(${TD_LINUX}) + +if(${BUILD_WITH_S3}) + target_include_directories( + vnode + + PUBLIC "$ENV{HOME}/.cos-local.2/include" + ) + + set(CMAKE_FIND_LIBRARY_SUFFIXES ".a") + set(CMAKE_PREFIX_PATH $ENV{HOME}/.cos-local.2) + find_library(S3_LIBRARY s3) + find_library(CURL_LIBRARY curl) + find_library(SSL_LIBRARY ssl $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH) + find_library(CRYPTO_LIBRARY crypto $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH) + target_link_libraries( + vnode + + # s3 + PUBLIC ${S3_LIBRARY} + PUBLIC ${CURL_LIBRARY} + PUBLIC ${SSL_LIBRARY} + PUBLIC ${CRYPTO_LIBRARY} + PUBLIC xml2 + ) + + add_definitions(-DUSE_S3) +endif(${BUILD_WITH_S3}) + +if(${BUILD_WITH_COS}) + set(CMAKE_FIND_LIBRARY_SUFFIXES ".a") find_library(APR_LIBRARY apr-1 PATHS /usr/local/apr/lib/) find_library(APR_UTIL_LIBRARY aprutil-1 PATHS /usr/local/apr/lib/) @@ -194,7 +224,6 @@ target_include_directories( PUBLIC "$ENV{HOME}/.cos-local.1/include" ) -if(${BUILD_WITH_COS}) add_definitions(-DUSE_COS) endif(${BUILD_WITH_COS}) diff --git a/source/dnode/vnode/src/vnd/vnodeCos.c b/source/dnode/vnode/src/vnd/vnodeCos.c index 5f650be97d..6e36739f5a 100644 --- a/source/dnode/vnode/src/vnd/vnodeCos.c +++ b/source/dnode/vnode/src/vnd/vnodeCos.c @@ -2,13 +2,798 @@ #include "vndCos.h" -extern char tsS3Endpoint[]; -extern char tsS3AccessKeyId[]; -extern char tsS3AccessKeySecret[]; -extern char tsS3BucketName[]; -extern char tsS3AppId[]; +extern char tsS3Endpoint[]; +extern char tsS3AccessKeyId[]; +extern char tsS3AccessKeySecret[]; +extern char tsS3BucketName[]; +extern char tsS3AppId[]; +extern char tsS3Hostname[]; +extern int8_t tsS3Https; + +#if defined(USE_S3) + +#include "libs3.h" + +static int verifyPeerG = 0; +static const char *awsRegionG = NULL; +static int forceG = 0; +static int showResponsePropertiesG = 0; +static S3Protocol protocolG = S3ProtocolHTTPS; +// static S3Protocol protocolG = S3ProtocolHTTP; +static S3UriStyle uriStyleG = S3UriStylePath; +static int retriesG = 5; +static int timeoutMsG = 0; + +static int32_t s3Begin() { + S3Status status; + const char *hostname = tsS3Hostname; + const char *env_hn = getenv("S3_HOSTNAME"); + + if (env_hn) { + hostname = env_hn; + } + + if ((status = S3_initialize("s3", verifyPeerG | S3_INIT_ALL, hostname)) != S3StatusOK) { + vError("Failed to initialize libs3: %s\n", S3_get_status_name(status)); + return -1; + } + + protocolG = !tsS3Https; + + return 0; +} + +static void s3End() { S3_deinitialize(); } +int32_t s3Init() { return s3Begin(); } + +void s3CleanUp() { s3End(); } + +static int should_retry() { + /* + if (retriesG--) { + // Sleep before next retry; start out with a 1 second sleep + static int retrySleepInterval = 1 * SLEEP_UNITS_PER_SECOND; + sleep(retrySleepInterval); + // Next sleep 1 second longer + retrySleepInterval++; + return 1; + } + */ + + return 0; +} + +static void s3PrintError(const char *func, S3Status status, char error_details[]) { + if (status < S3StatusErrorAccessDenied) { + vError("%s: %s", __func__, S3_get_status_name(status)); + } else { + vError("%s: %s, %s", __func__, S3_get_status_name(status), error_details); + } +} + +typedef struct { + char err_msg[128]; + S3Status status; + uint64_t content_length; + char *buf; +} TS3SizeCBD; + +static S3Status responsePropertiesCallback(const S3ResponseProperties *properties, void *callbackData) { + //(void)callbackData; + TS3SizeCBD *cbd = callbackData; + if (properties->contentLength > 0) { + cbd->content_length = properties->contentLength; + } else { + cbd->content_length = 0; + } + + return S3StatusOK; +} + +static void responseCompleteCallback(S3Status status, const S3ErrorDetails *error, void *callbackData) { + TS3SizeCBD *cbd = callbackData; + cbd->status = status; + + int len = 0; + const int elen = sizeof(cbd->err_msg); + if (error) { + if (error->message) { + len += snprintf(&(cbd->err_msg[len]), elen - len, " Message: %s\n", error->message); + } + if (error->resource) { + len += snprintf(&(cbd->err_msg[len]), elen - len, " Resource: %s\n", error->resource); + } + if (error->furtherDetails) { + len += snprintf(&(cbd->err_msg[len]), elen - len, " Further Details: %s\n", error->furtherDetails); + } + if (error->extraDetailsCount) { + len += snprintf(&(cbd->err_msg[len]), elen - len, "%s", " Extra Details:\n"); + for (int i = 0; i < error->extraDetailsCount; i++) { + len += snprintf(&(cbd->err_msg[len]), elen - len, " %s: %s\n", error->extraDetails[i].name, + error->extraDetails[i].value); + } + } + } +} + +typedef struct growbuffer { + // The total number of bytes, and the start byte + int size; + // The start byte + int start; + // The blocks + char data[64 * 1024]; + struct growbuffer *prev, *next; +} growbuffer; + +// returns nonzero on success, zero on out of memory +static int growbuffer_append(growbuffer **gb, const char *data, int dataLen) { + int origDataLen = dataLen; + while (dataLen) { + growbuffer *buf = *gb ? (*gb)->prev : 0; + if (!buf || (buf->size == sizeof(buf->data))) { + buf = (growbuffer *)malloc(sizeof(growbuffer)); + if (!buf) { + return 0; + } + buf->size = 0; + buf->start = 0; + if (*gb && (*gb)->prev) { + buf->prev = (*gb)->prev; + buf->next = *gb; + (*gb)->prev->next = buf; + (*gb)->prev = buf; + } else { + buf->prev = buf->next = buf; + *gb = buf; + } + } + + int toCopy = (sizeof(buf->data) - buf->size); + if (toCopy > dataLen) { + toCopy = dataLen; + } + + memcpy(&(buf->data[buf->size]), data, toCopy); + + buf->size += toCopy, data += toCopy, dataLen -= toCopy; + } + + return origDataLen; +} + +static void growbuffer_read(growbuffer **gb, int amt, int *amtReturn, char *buffer) { + *amtReturn = 0; + + growbuffer *buf = *gb; + + if (!buf) { + return; + } + + *amtReturn = (buf->size > amt) ? amt : buf->size; + + memcpy(buffer, &(buf->data[buf->start]), *amtReturn); + + buf->start += *amtReturn, buf->size -= *amtReturn; + + if (buf->size == 0) { + if (buf->next == buf) { + *gb = 0; + } else { + *gb = buf->next; + buf->prev->next = buf->next; + buf->next->prev = buf->prev; + } + free(buf); + buf = NULL; + } +} + +static void growbuffer_destroy(growbuffer *gb) { + growbuffer *start = gb; + + while (gb) { + growbuffer *next = gb->next; + free(gb); + gb = (next == start) ? 0 : next; + } +} + +typedef struct put_object_callback_data { + char err_msg[128]; + S3Status status; + // FILE *infile; + TdFilePtr infileFD; + growbuffer *gb; + uint64_t contentLength, originalContentLength; + uint64_t totalContentLength, totalOriginalContentLength; + int noStatus; +} put_object_callback_data; + +#define MULTIPART_CHUNK_SIZE (768 << 20) // multipart is 768M + +typedef struct UploadManager { + char err_msg[128]; + S3Status status; + // used for initial multipart + char *upload_id; + + // used for upload part object + char **etags; + int next_etags_pos; + + // used for commit Upload + growbuffer *gb; + int remaining; +} UploadManager; + +typedef struct list_parts_callback_data { + char err_msg[128]; + S3Status status; + int isTruncated; + char nextPartNumberMarker[24]; + char initiatorId[256]; + char initiatorDisplayName[256]; + char ownerId[256]; + char ownerDisplayName[256]; + char storageClass[256]; + int partsCount; + int handlePartsStart; + int allDetails; + int noPrint; + UploadManager *manager; +} list_parts_callback_data; + +typedef struct MultipartPartData { + char err_msg[128]; + S3Status status; + put_object_callback_data put_object_data; + int seq; + UploadManager *manager; +} MultipartPartData; + +static int putObjectDataCallback(int bufferSize, char *buffer, void *callbackData) { + put_object_callback_data *data = (put_object_callback_data *)callbackData; + if (data->infileFD == 0) { + MultipartPartData *mpd = (MultipartPartData *)callbackData; + data = &mpd->put_object_data; + } + + int ret = 0; + + if (data->contentLength) { + int toRead = ((data->contentLength > (unsigned)bufferSize) ? (unsigned)bufferSize : data->contentLength); + if (data->gb) { + growbuffer_read(&(data->gb), toRead, &ret, buffer); + } else if (data->infileFD) { + // ret = fread(buffer, 1, toRead, data->infile); + ret = taosReadFile(data->infileFD, buffer, toRead); + } + } + + data->contentLength -= ret; + data->totalContentLength -= ret; + /* log too many open files + if (data->contentLength && !data->noStatus) { + vTrace("%llu bytes remaining ", (unsigned long long)data->totalContentLength); + vTrace("(%d%% complete) ...\n", (int)(((data->totalOriginalContentLength - data->totalContentLength) * 100) / + data->totalOriginalContentLength)); + } + */ + return ret; +} + +S3Status initial_multipart_callback(const char *upload_id, void *callbackData) { + UploadManager *manager = (UploadManager *)callbackData; + manager->upload_id = strdup(upload_id); + return S3StatusOK; +} + +S3Status MultipartResponseProperiesCallback(const S3ResponseProperties *properties, void *callbackData) { + responsePropertiesCallback(properties, callbackData); + + MultipartPartData *data = (MultipartPartData *)callbackData; + int seq = data->seq; + const char *etag = properties->eTag; + data->manager->etags[seq - 1] = strdup(etag); + data->manager->next_etags_pos = seq; + return S3StatusOK; +} + +static int multipartPutXmlCallback(int bufferSize, char *buffer, void *callbackData) { + UploadManager *manager = (UploadManager *)callbackData; + int ret = 0; + + if (manager->remaining) { + int toRead = ((manager->remaining > bufferSize) ? bufferSize : manager->remaining); + growbuffer_read(&(manager->gb), toRead, &ret, buffer); + } + manager->remaining -= ret; + return ret; +} +/* +static S3Status listPartsCallback(int isTruncated, const char *nextPartNumberMarker, const char *initiatorId, + const char *initiatorDisplayName, const char *ownerId, const char *ownerDisplayName, + const char *storageClass, int partsCount, int handlePartsStart, + const S3ListPart *parts, void *callbackData) { + list_parts_callback_data *data = (list_parts_callback_data *)callbackData; + + data->isTruncated = isTruncated; + data->handlePartsStart = handlePartsStart; + UploadManager *manager = data->manager; + + if (nextPartNumberMarker) { + snprintf(data->nextPartNumberMarker, sizeof(data->nextPartNumberMarker), "%s", nextPartNumberMarker); + } else { + data->nextPartNumberMarker[0] = 0; + } + + if (initiatorId) { + snprintf(data->initiatorId, sizeof(data->initiatorId), "%s", initiatorId); + } else { + data->initiatorId[0] = 0; + } + + if (initiatorDisplayName) { + snprintf(data->initiatorDisplayName, sizeof(data->initiatorDisplayName), "%s", initiatorDisplayName); + } else { + data->initiatorDisplayName[0] = 0; + } + + if (ownerId) { + snprintf(data->ownerId, sizeof(data->ownerId), "%s", ownerId); + } else { + data->ownerId[0] = 0; + } + + if (ownerDisplayName) { + snprintf(data->ownerDisplayName, sizeof(data->ownerDisplayName), "%s", ownerDisplayName); + } else { + data->ownerDisplayName[0] = 0; + } + + if (storageClass) { + snprintf(data->storageClass, sizeof(data->storageClass), "%s", storageClass); + } else { + data->storageClass[0] = 0; + } + + if (partsCount && !data->partsCount && !data->noPrint) { + // printListPartsHeader(); + } + + int i; + for (i = 0; i < partsCount; i++) { + const S3ListPart *part = &(parts[i]); + char timebuf[256]; + if (data->noPrint) { + manager->etags[handlePartsStart + i] = strdup(part->eTag); + manager->next_etags_pos++; + manager->remaining = manager->remaining - part->size; + } else { + time_t t = (time_t)part->lastModified; + strftime(timebuf, sizeof(timebuf), "%Y-%m-%dT%H:%M:%SZ", gmtime(&t)); + printf("%-30s", timebuf); + printf("%-15llu", (unsigned long long)part->partNumber); + printf("%-45s", part->eTag); + printf("%-15llu\n", (unsigned long long)part->size); + } + } + + data->partsCount += partsCount; + + return S3StatusOK; +} + +static int try_get_parts_info(const char *bucketName, const char *key, UploadManager *manager) { + S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, + 0, awsRegionG}; + + S3ListPartsHandler listPartsHandler = {{&responsePropertiesCallback, &responseCompleteCallback}, &listPartsCallback}; + + list_parts_callback_data data; + + memset(&data, 0, sizeof(list_parts_callback_data)); + + data.partsCount = 0; + data.allDetails = 0; + data.manager = manager; + data.noPrint = 1; + do { + data.isTruncated = 0; + do { + S3_list_parts(&bucketContext, key, data.nextPartNumberMarker, manager->upload_id, 0, 0, 0, timeoutMsG, + &listPartsHandler, &data); + } while (S3_status_is_retryable(data.status) && should_retry()); + if (data.status != S3StatusOK) { + break; + } + } while (data.isTruncated); + + if (data.status == S3StatusOK) { + if (!data.partsCount) { + // printListMultipartHeader(data.allDetails); + } + } else { + s3PrintError(__func__, data.status, data.err_msg); + return -1; + } + + return 0; +} +*/ +int32_t s3PutObjectFromFile2(const char *file, const char *object) { + int32_t code = 0; + const char *key = object; + // const char *uploadId = 0; + const char *filename = 0; + uint64_t contentLength = 0; + const char *cacheControl = 0, *contentType = 0, *md5 = 0; + const char *contentDispositionFilename = 0, *contentEncoding = 0; + int64_t expires = -1; + S3CannedAcl cannedAcl = S3CannedAclPrivate; + int metaPropertiesCount = 0; + S3NameValue metaProperties[S3_MAX_METADATA_COUNT]; + char useServerSideEncryption = 0; + int noStatus = 0; + put_object_callback_data data; + + // data.infile = 0; + data.infileFD = NULL; + data.gb = 0; + data.noStatus = noStatus; + + if (taosStatFile(file, &contentLength, NULL, NULL) < 0) { + vError("ERROR: %s Failed to stat file %s: ", __func__, file); + code = TAOS_SYSTEM_ERROR(errno); + return code; + } + + if (!(data.infileFD = taosOpenFile(file, TD_FILE_READ))) { + vError("ERROR: %s Failed to open file %s: ", __func__, file); + code = TAOS_SYSTEM_ERROR(errno); + return code; + } + + data.totalContentLength = data.totalOriginalContentLength = data.contentLength = data.originalContentLength = + contentLength; + + S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, + 0, awsRegionG}; + + S3PutProperties putProperties = {contentType, md5, + cacheControl, contentDispositionFilename, + contentEncoding, expires, + cannedAcl, metaPropertiesCount, + metaProperties, useServerSideEncryption}; + + if (contentLength <= MULTIPART_CHUNK_SIZE) { + S3PutObjectHandler putObjectHandler = {{&responsePropertiesCallback, &responseCompleteCallback}, + &putObjectDataCallback}; + + do { + S3_put_object(&bucketContext, key, contentLength, &putProperties, 0, 0, &putObjectHandler, &data); + } while (S3_status_is_retryable(data.status) && should_retry()); + + if (data.infileFD) { + taosCloseFile(&data.infileFD); + } else if (data.gb) { + growbuffer_destroy(data.gb); + } + + if (data.status != S3StatusOK) { + s3PrintError(__func__, data.status, data.err_msg); + code = TAOS_SYSTEM_ERROR(EIO); + } else if (data.contentLength) { + vError("ERROR: %s Failed to read remaining %llu bytes from input", __func__, + (unsigned long long)data.contentLength); + code = TAOS_SYSTEM_ERROR(EIO); + } + } else { + uint64_t totalContentLength = contentLength; + uint64_t todoContentLength = contentLength; + UploadManager manager; + manager.upload_id = 0; + manager.gb = 0; + + // div round up + int seq; + uint64_t chunk_size = MULTIPART_CHUNK_SIZE >> 8; + int totalSeq = ((contentLength + chunk_size - 1) / chunk_size); + + MultipartPartData partData; + memset(&partData, 0, sizeof(MultipartPartData)); + int partContentLength = 0; + + S3MultipartInitialHandler handler = {{&responsePropertiesCallback, &responseCompleteCallback}, + &initial_multipart_callback}; + + S3PutObjectHandler putObjectHandler = {{&MultipartResponseProperiesCallback, &responseCompleteCallback}, + &putObjectDataCallback}; + + S3MultipartCommitHandler commit_handler = { + {&responsePropertiesCallback, &responseCompleteCallback}, &multipartPutXmlCallback, 0}; + + manager.etags = (char **)taosMemoryMalloc(sizeof(char *) * totalSeq); + manager.next_etags_pos = 0; + /* + if (uploadId) { + manager.upload_id = strdup(uploadId); + manager.remaining = contentLength; + if (!try_get_parts_info(tsS3BucketName, key, &manager)) { + fseek(data.infile, -(manager.remaining), 2); + taosLSeekFile(data.infileFD, -(manager.remaining), SEEK_END); + contentLength = manager.remaining; + goto upload; + } else { + goto clean; + } + } + */ + do { + S3_initiate_multipart(&bucketContext, key, 0, &handler, 0, timeoutMsG, &manager); + } while (S3_status_is_retryable(manager.status) && should_retry()); + + if (manager.upload_id == 0 || manager.status != S3StatusOK) { + s3PrintError(__func__, manager.status, manager.err_msg); + code = TAOS_SYSTEM_ERROR(EIO); + goto clean; + } + + upload: + todoContentLength -= chunk_size * manager.next_etags_pos; + for (seq = manager.next_etags_pos + 1; seq <= totalSeq; seq++) { + partData.manager = &manager; + partData.seq = seq; + if (partData.put_object_data.gb == NULL) { + partData.put_object_data = data; + } + partContentLength = ((contentLength > chunk_size) ? chunk_size : contentLength); + // printf("%s Part Seq %d, length=%d\n", srcSize ? "Copying" : "Sending", seq, partContentLength); + partData.put_object_data.contentLength = partContentLength; + partData.put_object_data.originalContentLength = partContentLength; + partData.put_object_data.totalContentLength = todoContentLength; + partData.put_object_data.totalOriginalContentLength = totalContentLength; + putProperties.md5 = 0; + do { + S3_upload_part(&bucketContext, key, &putProperties, &putObjectHandler, seq, manager.upload_id, + partContentLength, 0, timeoutMsG, &partData); + } while (S3_status_is_retryable(partData.status) && should_retry()); + if (partData.status != S3StatusOK) { + s3PrintError(__func__, partData.status, partData.err_msg); + code = TAOS_SYSTEM_ERROR(EIO); + goto clean; + } + contentLength -= chunk_size; + todoContentLength -= chunk_size; + } + + int i; + int size = 0; + size += growbuffer_append(&(manager.gb), "", strlen("")); + char buf[256]; + int n; + for (i = 0; i < totalSeq; i++) { + n = snprintf(buf, sizeof(buf), + "%d" + "%s", + i + 1, manager.etags[i]); + size += growbuffer_append(&(manager.gb), buf, n); + } + size += growbuffer_append(&(manager.gb), "", strlen("")); + manager.remaining = size; + + do { + S3_complete_multipart_upload(&bucketContext, key, &commit_handler, manager.upload_id, manager.remaining, 0, + timeoutMsG, &manager); + } while (S3_status_is_retryable(manager.status) && should_retry()); + if (manager.status != S3StatusOK) { + s3PrintError(__func__, manager.status, manager.err_msg); + code = TAOS_SYSTEM_ERROR(EIO); + goto clean; + } + + clean: + if (manager.upload_id) { + taosMemoryFree(manager.upload_id); + } + for (i = 0; i < manager.next_etags_pos; i++) { + taosMemoryFree(manager.etags[i]); + } + growbuffer_destroy(manager.gb); + taosMemoryFree(manager.etags); + } + + return code; +} + +typedef struct list_bucket_callback_data { + char err_msg[128]; + S3Status status; + int isTruncated; + char nextMarker[1024]; + int keyCount; + int allDetails; + SArray *objectArray; +} list_bucket_callback_data; + +static S3Status listBucketCallback(int isTruncated, const char *nextMarker, int contentsCount, + const S3ListBucketContent *contents, int commonPrefixesCount, + const char **commonPrefixes, void *callbackData) { + list_bucket_callback_data *data = (list_bucket_callback_data *)callbackData; + + data->isTruncated = isTruncated; + if ((!nextMarker || !nextMarker[0]) && contentsCount) { + nextMarker = contents[contentsCount - 1].key; + } + if (nextMarker) { + snprintf(data->nextMarker, sizeof(data->nextMarker), "%s", nextMarker); + } else { + data->nextMarker[0] = 0; + } + + if (contentsCount && !data->keyCount) { + // printListBucketHeader(data->allDetails); + } + + int i; + for (i = 0; i < contentsCount; ++i) { + const S3ListBucketContent *content = &(contents[i]); + // printf("%-50s", content->key); + char *object_key = strdup(content->key); + taosArrayPush(data->objectArray, &object_key); + } + data->keyCount += contentsCount; + + for (i = 0; i < commonPrefixesCount; i++) { + // printf("\nCommon Prefix: %s\n", commonPrefixes[i]); + } + + return S3StatusOK; +} + +static void s3FreeObjectKey(void *pItem) { + char *key = *(char **)pItem; + taosMemoryFree(key); +} + +void s3DeleteObjectsByPrefix(const char *prefix) { + S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, + 0, awsRegionG}; + S3ListBucketHandler listBucketHandler = {{&responsePropertiesCallback, &responseCompleteCallback}, + &listBucketCallback}; + + const char *marker = 0, *delimiter = 0; + int maxkeys = 0, allDetails = 0; + list_bucket_callback_data data; + data.objectArray = taosArrayInit(32, POINTER_BYTES); + if (!data.objectArray) { + vError("%s: %s", __func__, "out of memoty"); + return; + } + if (marker) { + snprintf(data.nextMarker, sizeof(data.nextMarker), "%s", marker); + } else { + data.nextMarker[0] = 0; + } + data.keyCount = 0; + data.allDetails = allDetails; + + do { + data.isTruncated = 0; + do { + S3_list_bucket(&bucketContext, prefix, data.nextMarker, delimiter, maxkeys, 0, timeoutMsG, &listBucketHandler, + &data); + } while (S3_status_is_retryable(data.status) && should_retry()); + if (data.status != S3StatusOK) { + break; + } + } while (data.isTruncated && (!maxkeys || (data.keyCount < maxkeys))); + + if (data.status == S3StatusOK) { + if (data.keyCount > 0) { + // printListBucketHeader(allDetails); + s3DeleteObjects(TARRAY_DATA(data.objectArray), TARRAY_SIZE(data.objectArray)); + } + } else { + s3PrintError(__func__, data.status, data.err_msg); + } + + taosArrayDestroyEx(data.objectArray, s3FreeObjectKey); +} + +void s3DeleteObjects(const char *object_name[], int nobject) { + int status = 0; + S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, + 0, awsRegionG}; + S3ResponseHandler responseHandler = {0, &responseCompleteCallback}; + + for (int i = 0; i < nobject; ++i) { + TS3SizeCBD cbd = {0}; + do { + S3_delete_object(&bucketContext, object_name[i], 0, timeoutMsG, &responseHandler, &cbd); + } while (S3_status_is_retryable(cbd.status) && should_retry()); + + if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) { + s3PrintError(__func__, cbd.status, cbd.err_msg); + } + } +} + +static S3Status getObjectDataCallback(int bufferSize, const char *buffer, void *callbackData) { + TS3SizeCBD *cbd = callbackData; + if (cbd->content_length != bufferSize) { + cbd->status = S3StatusAbortedByCallback; + return S3StatusAbortedByCallback; + } + + char *buf = taosMemoryCalloc(1, bufferSize); + if (buf) { + memcpy(buf, buffer, bufferSize); + cbd->buf = buf; + cbd->status = S3StatusOK; + return S3StatusOK; + } else { + cbd->status = S3StatusAbortedByCallback; + return S3StatusAbortedByCallback; + } +} + +int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size, uint8_t **ppBlock) { + int status = 0; + int64_t ifModifiedSince = -1, ifNotModifiedSince = -1; + const char *ifMatch = 0, *ifNotMatch = 0; + + S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, + 0, awsRegionG}; + S3GetConditions getConditions = {ifModifiedSince, ifNotModifiedSince, ifMatch, ifNotMatch}; + S3GetObjectHandler getObjectHandler = {{&responsePropertiesCallback, &responseCompleteCallback}, + &getObjectDataCallback}; + + TS3SizeCBD cbd = {0}; + cbd.content_length = size; + do { + S3_get_object(&bucketContext, object_name, &getConditions, offset, size, 0, 0, &getObjectHandler, &cbd); + } while (S3_status_is_retryable(cbd.status) && should_retry()); + + if (cbd.status != S3StatusOK) { + vError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg); + return TAOS_SYSTEM_ERROR(EIO); + } + + *ppBlock = cbd.buf; + + return 0; +} + +long s3Size(const char *object_name) { + long size = 0; + int status = 0; + + S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, + 0, awsRegionG}; + + S3ResponseHandler responseHandler = {&responsePropertiesCallback, &responseCompleteCallback}; + + TS3SizeCBD cbd = {0}; + do { + S3_head_object(&bucketContext, object_name, 0, 0, &responseHandler, &cbd); + } while (S3_status_is_retryable(cbd.status) && should_retry()); + + if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) { + vError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg); + } + + size = cbd.content_length; + + return size; +} + +void s3EvictCache(const char *path, long object_size) {} + +#elif defined(USE_COS) -#ifdef USE_COS #include "cos_api.h" #include "cos_http_io.h" #include "cos_log.h" diff --git a/tests/parallel_test/container_build.sh b/tests/parallel_test/container_build.sh index f5e426057e..94704b1c25 100755 --- a/tests/parallel_test/container_build.sh +++ b/tests/parallel_test/container_build.sh @@ -60,7 +60,7 @@ docker run \ -v /root/.cargo/git:/root/.cargo/git \ -v /root/go/pkg/mod:/root/go/pkg/mod \ -v /root/.cache/go-build:/root/.cache/go-build \ - -v /root/.cos-local.1:/root/.cos-local.1 \ + -v /root/.cos-local.1:/root/.cos-local.2 \ -v ${REP_REAL_PATH}/enterprise/src/plugins/taosx/target:${REP_DIR}/enterprise/src/plugins/taosx/target \ -v ${REP_REAL_PATH}/community/tools/taosws-rs/target:${REP_DIR}/community/tools/taosws-rs/target \ -v ${REP_REAL_PATH}/community/contrib/cJson/:${REP_DIR}/community/contrib/cJson \ @@ -89,7 +89,7 @@ docker run \ -v /root/.cargo/git:/root/.cargo/git \ -v /root/go/pkg/mod:/root/go/pkg/mod \ -v /root/.cache/go-build:/root/.cache/go-build \ - -v /root/.cos-local.1:/root/.cos-local.1 \ + -v /root/.cos-local.1:/root/.cos-local.2 \ -v ${REP_REAL_PATH}/enterprise/src/plugins/taosx/target:${REP_DIR}/enterprise/src/plugins/taosx/target \ -v ${REP_REAL_PATH}/community/tools/taosws-rs/target:${REP_DIR}/community/tools/taosws-rs/target \ -v ${REP_REAL_PATH}/community/contrib/cJson/:${REP_DIR}/community/contrib/cJson \