Merge pull request #23223 from taosdata/fix/TD-26189-libs3
enh(tsdb/cos): new sdk for s3
This commit is contained in:
Hongze Cheng
GitHub
commit
56bc18bbd4
|
|
@ -127,6 +127,22 @@ option(
|
||||||
|
|
||||||
IF(${TD_LINUX})
|
IF(${TD_LINUX})
|
||||||
|
|
||||||
|
option(
|
||||||
|
BUILD_WITH_S3
|
||||||
|
"If build with s3"
|
||||||
|
ON
|
||||||
|
)
|
||||||
|
|
||||||
|
IF(${BUILD_WITH_S3})
|
||||||
|
|
||||||
|
option(
|
||||||
|
BUILD_WITH_COS
|
||||||
|
"If build with cos"
|
||||||
|
OFF
|
||||||
|
)
|
||||||
|
|
||||||
|
ELSE ()
|
||||||
|
|
||||||
option(
|
option(
|
||||||
BUILD_WITH_COS
|
BUILD_WITH_COS
|
||||||
"If build with cos"
|
"If build with cos"
|
||||||
|
|
@ -135,6 +151,8 @@ option(
|
||||||
|
|
||||||
ENDIF ()
|
ENDIF ()
|
||||||
|
|
||||||
|
ENDIF ()
|
||||||
|
|
||||||
option(
|
option(
|
||||||
BUILD_WITH_SQLITE
|
BUILD_WITH_SQLITE
|
||||||
"If build with sqlite"
|
"If build with sqlite"
|
||||||
|
|
|
||||||
|
|
@ -1,18 +1,18 @@
|
||||||
# curl
|
# curl
|
||||||
ExternalProject_Add(curl
|
ExternalProject_Add(curl2
|
||||||
URL https://curl.se/download/curl-8.2.1.tar.gz
|
URL https://curl.se/download/curl-8.2.1.tar.gz
|
||||||
URL_HASH MD5=b25588a43556068be05e1624e0e74d41
|
URL_HASH MD5=b25588a43556068be05e1624e0e74d41
|
||||||
DOWNLOAD_NO_PROGRESS 1
|
DOWNLOAD_NO_PROGRESS 1
|
||||||
DOWNLOAD_DIR "${TD_CONTRIB_DIR}/deps-download"
|
DOWNLOAD_DIR "${TD_CONTRIB_DIR}/deps-download"
|
||||||
#GIT_REPOSITORY https://github.com/curl/curl.git
|
#GIT_REPOSITORY https://github.com/curl/curl.git
|
||||||
#GIT_TAG curl-7_88_1
|
#GIT_TAG curl-7_88_1
|
||||||
SOURCE_DIR "${TD_CONTRIB_DIR}/curl"
|
SOURCE_DIR "${TD_CONTRIB_DIR}/curl2"
|
||||||
|
DEPENDS openssl
|
||||||
BUILD_IN_SOURCE TRUE
|
BUILD_IN_SOURCE TRUE
|
||||||
BUILD_ALWAYS 1
|
BUILD_ALWAYS 1
|
||||||
#UPDATE_COMMAND ""
|
UPDATE_COMMAND ""
|
||||||
CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --without-ssl --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd
|
CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.2 --with-ssl=$ENV{HOME}/.cos-local.2 --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd --without-libidn2 #--enable-debug
|
||||||
#CONFIGURE_COMMAND ./configure --without-ssl
|
BUILD_COMMAND make -j
|
||||||
BUILD_COMMAND make
|
|
||||||
INSTALL_COMMAND make install
|
INSTALL_COMMAND make install
|
||||||
TEST_COMMAND ""
|
TEST_COMMAND ""
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,430 @@
|
||||||
|
# GNUmakefile
|
||||||
|
#
|
||||||
|
# Copyright 2008 Bryan Ischo <bryan@ischo.com>
|
||||||
|
#
|
||||||
|
# This file is part of libs3.
|
||||||
|
#
|
||||||
|
# libs3 is free software: you can redistribute it and/or modify it under the
|
||||||
|
# terms of the GNU Lesser General Public License as published by the Free
|
||||||
|
# Software Foundation, version 3 or above of the License. You can also
|
||||||
|
# redistribute and/or modify it under the terms of the GNU General Public
|
||||||
|
# License, version 2 or above of the License.
|
||||||
|
#
|
||||||
|
# In addition, as a special exception, the copyright holders give
|
||||||
|
# permission to link the code of this library and its programs with the
|
||||||
|
# OpenSSL library, and distribute linked combinations including the two.
|
||||||
|
#
|
||||||
|
# libs3 is distributed in the hope that it will be useful, but WITHOUT ANY
|
||||||
|
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
|
||||||
|
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
|
||||||
|
# details.
|
||||||
|
#
|
||||||
|
# You should have received a copy of the GNU Lesser General Public License
|
||||||
|
# version 3 along with libs3, in a file named COPYING. If not, see
|
||||||
|
# <http://www.gnu.org/licenses/>.
|
||||||
|
#
|
||||||
|
# You should also have received a copy of the GNU General Public License
|
||||||
|
# version 2 along with libs3, in a file named COPYING-GPLv2. If not, see
|
||||||
|
# <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
# I tried to use the autoconf/automake/autolocal/etc (i.e. autohell) tools
|
||||||
|
# but I just couldn't stomach them. Since this is a Makefile for POSIX
|
||||||
|
# systems, I will simply do away with autohell completely and use a GNU
|
||||||
|
# Makefile. GNU make ought to be available pretty much everywhere, so I
|
||||||
|
# don't see this being a significant issue for portability.
|
||||||
|
|
||||||
|
# All commands assume a GNU compiler. For systems which do not use a GNU
|
||||||
|
# compiler, write scripts with the same names as these commands, and taking
|
||||||
|
# the same arguments, and translate the arguments and commands into the
|
||||||
|
# appropriate non-POSIX ones as needed. libs3 assumes a GNU toolchain as
|
||||||
|
# the most portable way to build software possible. Non-POSIX, non-GNU
|
||||||
|
# systems can do the work of supporting this build infrastructure.
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------
|
||||||
|
# Set libs3 version number, unless it is already set.
|
||||||
|
|
||||||
|
LIBS3_VER_MAJOR ?= 4
|
||||||
|
LIBS3_VER_MINOR ?= 1
|
||||||
|
LIBS3_VER := $(LIBS3_VER_MAJOR).$(LIBS3_VER_MINOR)
|
||||||
|
|
||||||
|
|
||||||
|
# -----------------------------------------------------------------------------
|
||||||
|
# Determine verbosity. VERBOSE_SHOW should be prepended to every command which
|
||||||
|
# should only be displayed if VERBOSE is set. QUIET_ECHO may be used to
|
||||||
|
# echo text only if VERBOSE is not set. Typically, a VERBOSE_SHOW command will
|
||||||
|
# be paired with a QUIET_ECHO command, to provide a command which is displayed
|
||||||
|
# in VERBOSE mode, along with text which is displayed in non-VERBOSE mode to
|
||||||
|
# describe the command.
|
||||||
|
#
|
||||||
|
# No matter what VERBOSE is defined to, it ends up as true if it's defined.
|
||||||
|
# This will be weird if you defined VERBOSE=false in the environment, and we
|
||||||
|
# switch it to true here; but the meaning of VERBOSE is, "if it's defined to
|
||||||
|
# any value, then verbosity is turned on". So don't define VERBOSE if you
|
||||||
|
# don't want verbosity in the build process.
|
||||||
|
# -----------------------------------------------------------------------------
|
||||||
|
|
||||||
|
ifdef VERBOSE
|
||||||
|
VERBOSE = true
|
||||||
|
VERBOSE_ECHO = @ echo
|
||||||
|
VERBOSE_SHOW =
|
||||||
|
QUIET_ECHO = @ echo > /dev/null
|
||||||
|
else
|
||||||
|
VERBOSE = false
|
||||||
|
VERBOSE_ECHO = @ echo > /dev/null
|
||||||
|
VERBOSE_SHOW = @
|
||||||
|
QUIET_ECHO = @ echo
|
||||||
|
endif
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------
|
||||||
|
# BUILD directory
|
||||||
|
ifndef BUILD
|
||||||
|
ifdef DEBUG
|
||||||
|
BUILD := build-debug
|
||||||
|
else
|
||||||
|
BUILD := build
|
||||||
|
endif
|
||||||
|
endif
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------
|
||||||
|
# DESTDIR directory
|
||||||
|
ifndef DESTDIR
|
||||||
|
DESTDIR := ${HOME}/.cos-local.2
|
||||||
|
endif
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------
|
||||||
|
# LIBDIR directory
|
||||||
|
ifndef LIBDIR
|
||||||
|
LIBDIR := ${DESTDIR}/lib
|
||||||
|
endif
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------
|
||||||
|
# Compiler CC handling
|
||||||
|
ifndef CC
|
||||||
|
CC := gcc
|
||||||
|
endif
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------
|
||||||
|
# Acquire configuration information for libraries that libs3 depends upon
|
||||||
|
|
||||||
|
ifndef CURL_LIBS
|
||||||
|
CURL_LIBS := $(shell curl-config --libs)
|
||||||
|
endif
|
||||||
|
|
||||||
|
ifndef CURL_CFLAGS
|
||||||
|
CURL_CFLAGS := $(shell curl-config --cflags)
|
||||||
|
endif
|
||||||
|
|
||||||
|
ifndef LIBXML2_LIBS
|
||||||
|
LIBXML2_LIBS := $(shell xml2-config --libs)
|
||||||
|
endif
|
||||||
|
|
||||||
|
ifndef LIBXML2_CFLAGS
|
||||||
|
LIBXML2_CFLAGS := $(shell xml2-config --cflags)
|
||||||
|
endif
|
||||||
|
|
||||||
|
ifndef OPENSSL_LIBS
|
||||||
|
OPENSSL_LIBS := -lssl -lcrypto
|
||||||
|
endif
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------
|
||||||
|
# These CFLAGS assume a GNU compiler. For other compilers, write a script
|
||||||
|
# which converts these arguments into their equivalent for that particular
|
||||||
|
# compiler.
|
||||||
|
|
||||||
|
ifndef CFLAGS
|
||||||
|
ifdef DEBUG
|
||||||
|
CFLAGS := -g
|
||||||
|
else
|
||||||
|
CFLAGS := -O3
|
||||||
|
endif
|
||||||
|
endif
|
||||||
|
|
||||||
|
CFLAGS += -Wall -Werror -Wshadow -Wextra -Iinc \
|
||||||
|
$(CURL_CFLAGS) $(LIBXML2_CFLAGS) \
|
||||||
|
-DLIBS3_VER_MAJOR=\"$(LIBS3_VER_MAJOR)\" \
|
||||||
|
-DLIBS3_VER_MINOR=\"$(LIBS3_VER_MINOR)\" \
|
||||||
|
-DLIBS3_VER=\"$(LIBS3_VER)\" \
|
||||||
|
-D__STRICT_ANSI__ \
|
||||||
|
-D_ISOC99_SOURCE \
|
||||||
|
-D_POSIX_C_SOURCE=200112L
|
||||||
|
|
||||||
|
LDFLAGS = $(CURL_LIBS) $(LIBXML2_LIBS) $(OPENSSL_LIBS) -lpthread
|
||||||
|
|
||||||
|
STRIP ?= strip
|
||||||
|
INSTALL := install --strip-program=$(STRIP)
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------
|
||||||
|
# Default targets are everything
|
||||||
|
|
||||||
|
.PHONY: all
|
||||||
|
all: exported test
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------
|
||||||
|
# Exported targets are the library and driver program
|
||||||
|
|
||||||
|
.PHONY: exported
|
||||||
|
exported: libs3 s3 headers
|
||||||
|
exported_static: $(LIBS3_STATIC)
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------
|
||||||
|
# Install target
|
||||||
|
|
||||||
|
.PHONY: install install_static
|
||||||
|
install_static: exported_static
|
||||||
|
$(QUIET_ECHO) $(LIBDIR)/libs3.a: Installing static library
|
||||||
|
$(VERBOSE_SHOW) $(INSTALL) -Dp -m u+rw,go+r $(BUILD)/lib/libs3.a \
|
||||||
|
$(LIBDIR)/libs3.a
|
||||||
|
$(QUIET_ECHO) $(DESTDIR)/include/libs3.h: Installing header
|
||||||
|
$(VERBOSE_SHOW) $(INSTALL) -Dp -m u+rw,go+r inc/libs3.h \
|
||||||
|
$(DESTDIR)/include/libs3.h
|
||||||
|
|
||||||
|
install: exported
|
||||||
|
$(QUIET_ECHO) $(DESTDIR)/bin/s3: Installing executable
|
||||||
|
$(VERBOSE_SHOW) $(INSTALL) -Dps -m u+rwx,go+rx $(BUILD)/bin/s3 \
|
||||||
|
$(DESTDIR)/bin/s3
|
||||||
|
$(QUIET_ECHO) \
|
||||||
|
$(LIBDIR)/libs3.so.$(LIBS3_VER): Installing shared library
|
||||||
|
$(VERBOSE_SHOW) $(INSTALL) -Dps -m u+rw,go+r \
|
||||||
|
$(BUILD)/lib/libs3.so.$(LIBS3_VER_MAJOR) \
|
||||||
|
$(LIBDIR)/libs3.so.$(LIBS3_VER)
|
||||||
|
$(QUIET_ECHO) \
|
||||||
|
$(LIBDIR)/libs3.so.$(LIBS3_VER_MAJOR): Linking shared library
|
||||||
|
$(VERBOSE_SHOW) ln -sf libs3.so.$(LIBS3_VER) \
|
||||||
|
$(LIBDIR)/libs3.so.$(LIBS3_VER_MAJOR)
|
||||||
|
$(QUIET_ECHO) $(LIBDIR)/libs3.so: Linking shared library
|
||||||
|
$(VERBOSE_SHOW) ln -sf libs3.so.$(LIBS3_VER_MAJOR) $(LIBDIR)/libs3.so
|
||||||
|
$(QUIET_ECHO) $(LIBDIR)/libs3.a: Installing static library
|
||||||
|
$(VERBOSE_SHOW) $(INSTALL) -Dp -m u+rw,go+r $(BUILD)/lib/libs3.a \
|
||||||
|
$(LIBDIR)/libs3.a
|
||||||
|
$(QUIET_ECHO) $(DESTDIR)/include/libs3.h: Installing header
|
||||||
|
$(VERBOSE_SHOW) $(INSTALL) -Dp -m u+rw,go+r $(BUILD)/include/libs3.h \
|
||||||
|
$(DESTDIR)/include/libs3.h
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------
|
||||||
|
# Uninstall target
|
||||||
|
|
||||||
|
.PHONY: uninstall
|
||||||
|
uninstall:
|
||||||
|
$(QUIET_ECHO) Installed files: Uninstalling
|
||||||
|
$(VERBOSE_SHOW) \
|
||||||
|
rm -f $(DESTDIR)/bin/s3 \
|
||||||
|
$(DESTDIR)/include/libs3.h \
|
||||||
|
$(DESTDIR)/lib/libs3.a \
|
||||||
|
$(DESTDIR)/lib/libs3.so \
|
||||||
|
$(DESTDIR)/lib/libs3.so.$(LIBS3_VER_MAJOR) \
|
||||||
|
$(DESTDIR)/lib/libs3.so.$(LIBS3_VER)
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------
|
||||||
|
# Compile target patterns
|
||||||
|
|
||||||
|
$(BUILD)/obj/%.o: src/%.c
|
||||||
|
$(QUIET_ECHO) $@: Compiling object
|
||||||
|
@ mkdir -p $(dir $(BUILD)/dep/$<)
|
||||||
|
@ $(CC) $(CFLAGS) -M -MG -MQ $@ -DCOMPILINGDEPENDENCIES \
|
||||||
|
-o $(BUILD)/dep/$(<:%.c=%.d) -c $<
|
||||||
|
@ mkdir -p $(dir $@)
|
||||||
|
$(VERBOSE_SHOW) $(CC) $(CFLAGS) -o $@ -c $<
|
||||||
|
|
||||||
|
$(BUILD)/obj/%.do: src/%.c
|
||||||
|
$(QUIET_ECHO) $@: Compiling dynamic object
|
||||||
|
$(QUIET_ECHO) cflags:${CFLAGS}
|
||||||
|
@ mkdir -p $(dir $(BUILD)/dep/$<)
|
||||||
|
@ $(CC) $(CFLAGS) -M -MG -MQ $@ -DCOMPILINGDEPENDENCIES \
|
||||||
|
-o $(BUILD)/dep/$(<:%.c=%.dd) -c $<
|
||||||
|
@ mkdir -p $(dir $@)
|
||||||
|
$(VERBOSE_SHOW) $(CC) $(CFLAGS) -fpic -fPIC -o $@ -c $<
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------
|
||||||
|
# libs3 library targets
|
||||||
|
|
||||||
|
LIBS3_SHARED = $(BUILD)/lib/libs3.so.$(LIBS3_VER_MAJOR)
|
||||||
|
LIBS3_STATIC = $(BUILD)/lib/libs3.a
|
||||||
|
|
||||||
|
.PHONY: libs3
|
||||||
|
libs3: $(LIBS3_SHARED) $(LIBS3_STATIC)
|
||||||
|
|
||||||
|
LIBS3_SOURCES := bucket.c bucket_metadata.c error_parser.c general.c \
|
||||||
|
object.c request.c request_context.c \
|
||||||
|
response_headers_handler.c service_access_logging.c \
|
||||||
|
service.c simplexml.c util.c multipart.c
|
||||||
|
|
||||||
|
$(LIBS3_SHARED): $(LIBS3_SOURCES:%.c=$(BUILD)/obj/%.do)
|
||||||
|
$(QUIET_ECHO) $@: Building shared library
|
||||||
|
@ mkdir -p $(dir $@)
|
||||||
|
$(VERBOSE_SHOW) $(CC) -shared -Wl,-soname,libs3.so.$(LIBS3_VER_MAJOR) \
|
||||||
|
-o $@ $^ $(LDFLAGS)
|
||||||
|
|
||||||
|
$(LIBS3_STATIC): $(LIBS3_SOURCES:%.c=$(BUILD)/obj/%.o)
|
||||||
|
$(QUIET_ECHO) $@: Building static library
|
||||||
|
@ mkdir -p $(dir $@)
|
||||||
|
$(VERBOSE_SHOW) $(AR) cr $@ $^
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------
|
||||||
|
# Driver program targets
|
||||||
|
|
||||||
|
.PHONY: s3
|
||||||
|
s3: $(BUILD)/bin/s3
|
||||||
|
|
||||||
|
$(BUILD)/bin/s3: $(BUILD)/obj/s3.o $(LIBS3_SHARED)
|
||||||
|
$(QUIET_ECHO) $@: Building executable
|
||||||
|
@ mkdir -p $(dir $@)
|
||||||
|
$(VERBOSE_SHOW) $(CC) -o $@ $^ $(LDFLAGS)
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------
|
||||||
|
# libs3 header targets
|
||||||
|
|
||||||
|
.PHONY: headers
|
||||||
|
headers: $(BUILD)/include/libs3.h
|
||||||
|
|
||||||
|
$(BUILD)/include/libs3.h: inc/libs3.h
|
||||||
|
$(QUIET_ECHO) $@: Linking header
|
||||||
|
@ mkdir -p $(dir $@)
|
||||||
|
$(VERBOSE_SHOW) ln -sf $(abspath $<) $@
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------
|
||||||
|
# Test targets
|
||||||
|
|
||||||
|
.PHONY: test
|
||||||
|
test: $(BUILD)/bin/testsimplexml
|
||||||
|
|
||||||
|
$(BUILD)/bin/testsimplexml: $(BUILD)/obj/testsimplexml.o $(LIBS3_STATIC)
|
||||||
|
$(QUIET_ECHO) $@: Building executable
|
||||||
|
@ mkdir -p $(dir $@)
|
||||||
|
$(VERBOSE_SHOW) $(CC) -o $@ $^ $(LIBXML2_LIBS)
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------
|
||||||
|
# Clean target
|
||||||
|
|
||||||
|
.PHONY: clean
|
||||||
|
clean:
|
||||||
|
$(QUIET_ECHO) $(BUILD): Cleaning
|
||||||
|
$(VERBOSE_SHOW) rm -rf $(BUILD)
|
||||||
|
|
||||||
|
.PHONY: distclean
|
||||||
|
distclean:
|
||||||
|
$(QUIET_ECHO) $(BUILD): Cleaning
|
||||||
|
$(VERBOSE_SHOW) rm -rf $(BUILD)
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------
|
||||||
|
# Clean dependencies target
|
||||||
|
|
||||||
|
.PHONY: cleandeps
|
||||||
|
cleandeps:
|
||||||
|
$(QUIET_ECHO) $(BUILD)/dep: Cleaning dependencies
|
||||||
|
$(VERBOSE_SHOW) rm -rf $(BUILD)/dep
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------
|
||||||
|
# Dependencies
|
||||||
|
|
||||||
|
ALL_SOURCES := $(LIBS3_SOURCES) s3.c testsimplexml.c
|
||||||
|
|
||||||
|
$(foreach i, $(ALL_SOURCES), $(eval -include $(BUILD)/dep/src/$(i:%.c=%.d)))
|
||||||
|
$(foreach i, $(ALL_SOURCES), $(eval -include $(BUILD)/dep/src/$(i:%.c=%.dd)))
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------
|
||||||
|
# Debian package target
|
||||||
|
|
||||||
|
DEBPKG = $(BUILD)/pkg/libs3_$(LIBS3_VER).deb
|
||||||
|
DEBDEVPKG = $(BUILD)/pkg/libs3-dev_$(LIBS3_VER).deb
|
||||||
|
|
||||||
|
.PHONY: deb
|
||||||
|
deb: $(DEBPKG) $(DEBDEVPKG)
|
||||||
|
|
||||||
|
$(DEBPKG): DEBARCH = $(shell dpkg-architecture | grep ^DEB_BUILD_ARCH= | \
|
||||||
|
cut -d '=' -f 2)
|
||||||
|
$(DEBPKG): exported $(BUILD)/deb/DEBIAN/control $(BUILD)/deb/DEBIAN/shlibs \
|
||||||
|
$(BUILD)/deb/DEBIAN/postinst \
|
||||||
|
$(BUILD)/deb/usr/share/doc/libs3/changelog.gz \
|
||||||
|
$(BUILD)/deb/usr/share/doc/libs3/changelog.Debian.gz \
|
||||||
|
$(BUILD)/deb/usr/share/doc/libs3/copyright
|
||||||
|
DESTDIR=$(BUILD)/deb/usr $(MAKE) install
|
||||||
|
rm -rf $(BUILD)/deb/usr/include
|
||||||
|
rm -f $(BUILD)/deb/usr/lib/libs3.a
|
||||||
|
@mkdir -p $(dir $@)
|
||||||
|
fakeroot dpkg-deb -b $(BUILD)/deb $@
|
||||||
|
mv $@ $(BUILD)/pkg/libs3_$(LIBS3_VER)_$(DEBARCH).deb
|
||||||
|
|
||||||
|
$(DEBDEVPKG): DEBARCH = $(shell dpkg-architecture | grep ^DEB_BUILD_ARCH= | \
|
||||||
|
cut -d '=' -f 2)
|
||||||
|
$(DEBDEVPKG): exported $(BUILD)/deb-dev/DEBIAN/control \
|
||||||
|
$(BUILD)/deb-dev/usr/share/doc/libs3-dev/changelog.gz \
|
||||||
|
$(BUILD)/deb-dev/usr/share/doc/libs3-dev/changelog.Debian.gz \
|
||||||
|
$(BUILD)/deb-dev/usr/share/doc/libs3-dev/copyright
|
||||||
|
DESTDIR=$(BUILD)/deb-dev/usr $(MAKE) install
|
||||||
|
rm -rf $(BUILD)/deb-dev/usr/bin
|
||||||
|
rm -f $(BUILD)/deb-dev/usr/lib/libs3.so*
|
||||||
|
@mkdir -p $(dir $@)
|
||||||
|
fakeroot dpkg-deb -b $(BUILD)/deb-dev $@
|
||||||
|
mv $@ $(BUILD)/pkg/libs3-dev_$(LIBS3_VER)_$(DEBARCH).deb
|
||||||
|
|
||||||
|
$(BUILD)/deb/DEBIAN/control: debian/control
|
||||||
|
@mkdir -p $(dir $@)
|
||||||
|
echo -n "Depends: " > $@
|
||||||
|
dpkg-shlibdeps -Sbuild -O $(BUILD)/lib/libs3.so.$(LIBS3_VER_MAJOR) | \
|
||||||
|
cut -d '=' -f 2- >> $@
|
||||||
|
sed -e 's/LIBS3_VERSION/$(LIBS3_VER)/' \
|
||||||
|
< $< | sed -e 's/DEBIAN_ARCHITECTURE/$(DEBARCH)/' | \
|
||||||
|
grep -v ^Source: >> $@
|
||||||
|
|
||||||
|
$(BUILD)/deb-dev/DEBIAN/control: debian/control.dev
|
||||||
|
@mkdir -p $(dir $@)
|
||||||
|
sed -e 's/LIBS3_VERSION/$(LIBS3_VER)/' \
|
||||||
|
< $< | sed -e 's/DEBIAN_ARCHITECTURE/$(DEBARCH)/' > $@
|
||||||
|
|
||||||
|
$(BUILD)/deb/DEBIAN/shlibs:
|
||||||
|
echo -n "libs3 $(LIBS3_VER_MAJOR) libs3 " > $@
|
||||||
|
echo "(>= $(LIBS3_VER))" >> $@
|
||||||
|
|
||||||
|
$(BUILD)/deb/DEBIAN/postinst: debian/postinst
|
||||||
|
@mkdir -p $(dir $@)
|
||||||
|
cp $< $@
|
||||||
|
|
||||||
|
$(BUILD)/deb/usr/share/doc/libs3/copyright: LICENSE
|
||||||
|
@mkdir -p $(dir $@)
|
||||||
|
cp $< $@
|
||||||
|
@echo >> $@
|
||||||
|
@echo -n "An alternate location for the GNU General Public " >> $@
|
||||||
|
@echo "License version 3 on Debian" >> $@
|
||||||
|
@echo "systems is /usr/share/common-licenses/GPL-3." >> $@
|
||||||
|
|
||||||
|
$(BUILD)/deb-dev/usr/share/doc/libs3-dev/copyright: LICENSE
|
||||||
|
@mkdir -p $(dir $@)
|
||||||
|
cp $< $@
|
||||||
|
@echo >> $@
|
||||||
|
@echo -n "An alternate location for the GNU General Public " >> $@
|
||||||
|
@echo "License version 3 on Debian" >> $@
|
||||||
|
@echo "systems is /usr/share/common-licenses/GPL-3." >> $@
|
||||||
|
|
||||||
|
$(BUILD)/deb/usr/share/doc/libs3/changelog.gz: debian/changelog
|
||||||
|
@mkdir -p $(dir $@)
|
||||||
|
gzip --best -c $< > $@
|
||||||
|
|
||||||
|
$(BUILD)/deb-dev/usr/share/doc/libs3-dev/changelog.gz: debian/changelog
|
||||||
|
@mkdir -p $(dir $@)
|
||||||
|
gzip --best -c $< > $@
|
||||||
|
|
||||||
|
$(BUILD)/deb/usr/share/doc/libs3/changelog.Debian.gz: debian/changelog.Debian
|
||||||
|
@mkdir -p $(dir $@)
|
||||||
|
gzip --best -c $< > $@
|
||||||
|
|
||||||
|
$(BUILD)/deb-dev/usr/share/doc/libs3-dev/changelog.Debian.gz: \
|
||||||
|
debian/changelog.Debian
|
||||||
|
@mkdir -p $(dir $@)
|
||||||
|
gzip --best -c $< > $@
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -0,0 +1,16 @@
|
||||||
|
# libs3
|
||||||
|
|
||||||
|
ExternalProject_Add(libs3
|
||||||
|
GIT_REPOSITORY https://github.com/bji/libs3
|
||||||
|
#GIT_TAG v5.0.16
|
||||||
|
DEPENDS curl2 xml2
|
||||||
|
SOURCE_DIR "${TD_CONTRIB_DIR}/libs3"
|
||||||
|
#BINARY_DIR ""
|
||||||
|
BUILD_IN_SOURCE TRUE
|
||||||
|
BUILD_ALWAYS 1
|
||||||
|
UPDATE_COMMAND ""
|
||||||
|
CONFIGURE_COMMAND cp ${TD_SUPPORT_DIR}/libs3.GNUmakefile GNUmakefile && sed -i "s|CFLAGS += -Wall -Werror|CFLAGS += -I'$ENV{HOME}/.cos-local.2/include' -L'$ENV{HOME}/.cos-local.2/lib' |" ./GNUmakefile
|
||||||
|
BUILD_COMMAND make build/lib/libs3.a
|
||||||
|
INSTALL_COMMAND make install_static
|
||||||
|
TEST_COMMAND ""
|
||||||
|
)
|
||||||
|
|
@ -0,0 +1,15 @@
|
||||||
|
# openssl
|
||||||
|
ExternalProject_Add(openssl
|
||||||
|
URL https://www.openssl.org/source/openssl-3.1.3.tar.gz
|
||||||
|
URL_HASH SHA256=f0316a2ebd89e7f2352976445458689f80302093788c466692fb2a188b2eacf6
|
||||||
|
DOWNLOAD_NO_PROGRESS 1
|
||||||
|
DOWNLOAD_DIR "${TD_CONTRIB_DIR}/deps-download"
|
||||||
|
SOURCE_DIR "${TD_CONTRIB_DIR}/openssl"
|
||||||
|
BUILD_IN_SOURCE TRUE
|
||||||
|
#BUILD_ALWAYS 1
|
||||||
|
#UPDATE_COMMAND ""
|
||||||
|
CONFIGURE_COMMAND ./Configure --prefix=$ENV{HOME}/.cos-local.2 -static #--no-shared
|
||||||
|
BUILD_COMMAND make -j
|
||||||
|
INSTALL_COMMAND make install_sw -j
|
||||||
|
TEST_COMMAND ""
|
||||||
|
)
|
||||||
|
|
@ -0,0 +1,12 @@
|
||||||
|
|
||||||
|
# xml2
|
||||||
|
ExternalProject_Add(xml2
|
||||||
|
GIT_REPOSITORY https://github.com/GNOME/libxml2
|
||||||
|
GIT_TAG v2.11.5
|
||||||
|
SOURCE_DIR "${TD_CONTRIB_DIR}/xml2"
|
||||||
|
BINARY_DIR ""
|
||||||
|
CONFIGURE_COMMAND ""
|
||||||
|
BUILD_COMMAND ""
|
||||||
|
INSTALL_COMMAND ""
|
||||||
|
TEST_COMMAND ""
|
||||||
|
)
|
||||||
|
|
@ -8,6 +8,11 @@ endfunction(cat IN_FILE OUT_FILE)
|
||||||
|
|
||||||
if(${TD_LINUX})
|
if(${TD_LINUX})
|
||||||
|
|
||||||
|
if(${BUILD_WITH_S3})
|
||||||
|
file(MAKE_DIRECTORY $ENV{HOME}/.cos-local.2/)
|
||||||
|
|
||||||
|
else()
|
||||||
|
|
||||||
set(CONTRIB_TMP_FILE3 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in3")
|
set(CONTRIB_TMP_FILE3 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in3")
|
||||||
configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3})
|
configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3})
|
||||||
|
|
||||||
|
|
@ -37,6 +42,8 @@ execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" .
|
||||||
execute_process(COMMAND "${CMAKE_COMMAND}" --build .
|
execute_process(COMMAND "${CMAKE_COMMAND}" --build .
|
||||||
WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download")
|
WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download")
|
||||||
|
|
||||||
|
endif(${BUILD_WITH_S3})
|
||||||
|
|
||||||
endif(${TD_LINUX})
|
endif(${TD_LINUX})
|
||||||
|
|
||||||
set(CONTRIB_TMP_FILE "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in")
|
set(CONTRIB_TMP_FILE "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in")
|
||||||
|
|
@ -155,6 +162,16 @@ if(${BUILD_WITH_SQLITE})
|
||||||
cat("${TD_SUPPORT_DIR}/sqlite_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
|
cat("${TD_SUPPORT_DIR}/sqlite_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
|
||||||
endif(${BUILD_WITH_SQLITE})
|
endif(${BUILD_WITH_SQLITE})
|
||||||
|
|
||||||
|
# s3
|
||||||
|
if(${BUILD_WITH_S3})
|
||||||
|
cat("${TD_SUPPORT_DIR}/ssl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
|
||||||
|
cat("${TD_SUPPORT_DIR}/xml2_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
|
||||||
|
cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
|
||||||
|
cat("${TD_SUPPORT_DIR}/libs3_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
|
||||||
|
add_definitions(-DUSE_S3)
|
||||||
|
|
||||||
|
else()
|
||||||
|
|
||||||
# cos
|
# cos
|
||||||
if(${BUILD_WITH_COS})
|
if(${BUILD_WITH_COS})
|
||||||
#cat("${TD_SUPPORT_DIR}/mxml_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
|
#cat("${TD_SUPPORT_DIR}/mxml_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
|
||||||
|
|
@ -165,6 +182,8 @@ if(${BUILD_WITH_COS})
|
||||||
add_definitions(-DUSE_COS)
|
add_definitions(-DUSE_COS)
|
||||||
endif(${BUILD_WITH_COS})
|
endif(${BUILD_WITH_COS})
|
||||||
|
|
||||||
|
endif(${BUILD_WITH_S3})
|
||||||
|
|
||||||
# lucene
|
# lucene
|
||||||
if(${BUILD_WITH_LUCENE})
|
if(${BUILD_WITH_LUCENE})
|
||||||
cat("${TD_SUPPORT_DIR}/lucene_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
|
cat("${TD_SUPPORT_DIR}/lucene_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
|
||||||
|
|
@ -231,7 +250,6 @@ if(${BUILD_TEST})
|
||||||
)
|
)
|
||||||
endif(${TD_DARWIN})
|
endif(${TD_DARWIN})
|
||||||
|
|
||||||
|
|
||||||
endif(${BUILD_TEST})
|
endif(${BUILD_TEST})
|
||||||
|
|
||||||
# cJson
|
# cJson
|
||||||
|
|
@ -248,6 +266,11 @@ target_include_directories(
|
||||||
)
|
)
|
||||||
unset(CMAKE_PROJECT_INCLUDE_BEFORE)
|
unset(CMAKE_PROJECT_INCLUDE_BEFORE)
|
||||||
|
|
||||||
|
# xml2
|
||||||
|
if(${BUILD_WITH_S3})
|
||||||
|
add_subdirectory(xml2 EXCLUDE_FROM_ALL)
|
||||||
|
endif(${BUILD_WITH_S3})
|
||||||
|
|
||||||
# lz4
|
# lz4
|
||||||
add_subdirectory(lz4/build/cmake EXCLUDE_FROM_ALL)
|
add_subdirectory(lz4/build/cmake EXCLUDE_FROM_ALL)
|
||||||
target_include_directories(
|
target_include_directories(
|
||||||
|
|
@ -390,6 +413,12 @@ if (${BUILD_WITH_ROCKSDB})
|
||||||
endif()
|
endif()
|
||||||
endif()
|
endif()
|
||||||
|
|
||||||
|
if(${BUILD_WITH_S3})
|
||||||
|
INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.2/include)
|
||||||
|
MESSAGE("build with s3: ${BUILD_WITH_S3}")
|
||||||
|
|
||||||
|
else()
|
||||||
|
|
||||||
# cos
|
# cos
|
||||||
if(${BUILD_WITH_COS})
|
if(${BUILD_WITH_COS})
|
||||||
if(${TD_LINUX})
|
if(${TD_LINUX})
|
||||||
|
|
@ -397,7 +426,7 @@ if(${BUILD_WITH_COS})
|
||||||
#ADD_DEFINITIONS(-DMINIXML_LIBRARY=${CMAKE_BINARY_DIR}/build/lib/libxml.a)
|
#ADD_DEFINITIONS(-DMINIXML_LIBRARY=${CMAKE_BINARY_DIR}/build/lib/libxml.a)
|
||||||
option(ENABLE_TEST "Enable the tests" OFF)
|
option(ENABLE_TEST "Enable the tests" OFF)
|
||||||
INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.1/include)
|
INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.1/include)
|
||||||
MESSAGE("$ENV{HOME}/.cos-local.1/include")
|
#MESSAGE("$ENV{HOME}/.cos-local.1/include")
|
||||||
|
|
||||||
set(CMAKE_BUILD_TYPE Release)
|
set(CMAKE_BUILD_TYPE Release)
|
||||||
set(ORIG_CMAKE_PROJECT_NAME ${CMAKE_PROJECT_NAME})
|
set(ORIG_CMAKE_PROJECT_NAME ${CMAKE_PROJECT_NAME})
|
||||||
|
|
@ -415,6 +444,8 @@ if(${BUILD_WITH_COS})
|
||||||
endif(${TD_LINUX})
|
endif(${TD_LINUX})
|
||||||
endif(${BUILD_WITH_COS})
|
endif(${BUILD_WITH_COS})
|
||||||
|
|
||||||
|
endif(${BUILD_WITH_S3})
|
||||||
|
|
||||||
# lucene
|
# lucene
|
||||||
# To support build on ubuntu: sudo apt-get install libboost-all-dev
|
# To support build on ubuntu: sudo apt-get install libboost-all-dev
|
||||||
if(${BUILD_WITH_LUCENE})
|
if(${BUILD_WITH_LUCENE})
|
||||||
|
|
|
||||||
|
|
@ -22,6 +22,9 @@ IF (TD_STORAGE)
|
||||||
add_definitions(-DUSE_COS)
|
add_definitions(-DUSE_COS)
|
||||||
ENDIF(${BUILD_WITH_COS})
|
ENDIF(${BUILD_WITH_COS})
|
||||||
|
|
||||||
|
IF(${BUILD_WITH_S3})
|
||||||
|
add_definitions(-DUSE_S3)
|
||||||
|
ENDIF(${BUILD_WITH_S3})
|
||||||
ENDIF(${TD_LINUX})
|
ENDIF(${TD_LINUX})
|
||||||
|
|
||||||
ENDIF ()
|
ENDIF ()
|
||||||
|
|
|
||||||
|
|
@ -15,12 +15,12 @@
|
||||||
|
|
||||||
#define _DEFAULT_SOURCE
|
#define _DEFAULT_SOURCE
|
||||||
#include "tglobal.h"
|
#include "tglobal.h"
|
||||||
|
#include "defines.h"
|
||||||
#include "os.h"
|
#include "os.h"
|
||||||
#include "tconfig.h"
|
#include "tconfig.h"
|
||||||
#include "tgrant.h"
|
#include "tgrant.h"
|
||||||
#include "tlog.h"
|
#include "tlog.h"
|
||||||
#include "tmisce.h"
|
#include "tmisce.h"
|
||||||
#include "defines.h"
|
|
||||||
|
|
||||||
#if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL)
|
#if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL)
|
||||||
#include "cus_name.h"
|
#include "cus_name.h"
|
||||||
|
|
@ -222,7 +222,7 @@ float tsFPrecision = 1E-8; // float column precision
|
||||||
double tsDPrecision = 1E-16; // double column precision
|
double tsDPrecision = 1E-16; // double column precision
|
||||||
uint32_t tsMaxRange = 500; // max quantization intervals
|
uint32_t tsMaxRange = 500; // max quantization intervals
|
||||||
uint32_t tsCurRange = 100; // current quantization intervals
|
uint32_t tsCurRange = 100; // current quantization intervals
|
||||||
bool tsIfAdtFse = false; // ADT-FSE algorithom or original huffman algorithom
|
bool tsIfAdtFse = false; // ADT-FSE algorithom or original huffman algorithom
|
||||||
char tsCompressor[32] = "ZSTD_COMPRESSOR"; // ZSTD_COMPRESSOR or GZIP_COMPRESSOR
|
char tsCompressor[32] = "ZSTD_COMPRESSOR"; // ZSTD_COMPRESSOR or GZIP_COMPRESSOR
|
||||||
|
|
||||||
// udf
|
// udf
|
||||||
|
|
@ -267,6 +267,9 @@ char tsS3BucketName[TSDB_FQDN_LEN] = "<bucketname>";
|
||||||
char tsS3AppId[TSDB_FQDN_LEN] = "<appid>";
|
char tsS3AppId[TSDB_FQDN_LEN] = "<appid>";
|
||||||
int8_t tsS3Enabled = false;
|
int8_t tsS3Enabled = false;
|
||||||
|
|
||||||
|
int8_t tsS3Https = true;
|
||||||
|
char tsS3Hostname[TSDB_FQDN_LEN] = "<hostname>";
|
||||||
|
|
||||||
int32_t tsS3BlockSize = 4096; // number of tsdb pages
|
int32_t tsS3BlockSize = 4096; // number of tsdb pages
|
||||||
int32_t tsS3BlockCacheSize = 16; // number of blocks
|
int32_t tsS3BlockCacheSize = 16; // number of blocks
|
||||||
|
|
||||||
|
|
@ -308,6 +311,14 @@ int32_t taosSetS3Cfg(SConfig *pCfg) {
|
||||||
tstrncpy(tsS3AccessKeySecret, colon + 1, TSDB_FQDN_LEN);
|
tstrncpy(tsS3AccessKeySecret, colon + 1, TSDB_FQDN_LEN);
|
||||||
tstrncpy(tsS3Endpoint, cfgGetItem(pCfg, "s3Endpoint")->str, TSDB_FQDN_LEN);
|
tstrncpy(tsS3Endpoint, cfgGetItem(pCfg, "s3Endpoint")->str, TSDB_FQDN_LEN);
|
||||||
tstrncpy(tsS3BucketName, cfgGetItem(pCfg, "s3BucketName")->str, TSDB_FQDN_LEN);
|
tstrncpy(tsS3BucketName, cfgGetItem(pCfg, "s3BucketName")->str, TSDB_FQDN_LEN);
|
||||||
|
char *proto = strstr(tsS3Endpoint, "https://");
|
||||||
|
if (!proto) {
|
||||||
|
tsS3Https = false;
|
||||||
|
tstrncpy(tsS3Hostname, tsS3Endpoint + 7, TSDB_FQDN_LEN);
|
||||||
|
} else {
|
||||||
|
tstrncpy(tsS3Hostname, tsS3Endpoint + 8, TSDB_FQDN_LEN);
|
||||||
|
}
|
||||||
|
|
||||||
char *cos = strstr(tsS3Endpoint, "cos.");
|
char *cos = strstr(tsS3Endpoint, "cos.");
|
||||||
if (cos) {
|
if (cos) {
|
||||||
char *appid = strrchr(tsS3BucketName, '-');
|
char *appid = strrchr(tsS3BucketName, '-');
|
||||||
|
|
@ -319,7 +330,7 @@ int32_t taosSetS3Cfg(SConfig *pCfg) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (tsS3BucketName[0] != '<' && tsDiskCfgNum > 1) {
|
if (tsS3BucketName[0] != '<' && tsDiskCfgNum > 1) {
|
||||||
#ifdef USE_COS
|
#if defined(USE_COS) || defined(USE_S3)
|
||||||
tsS3Enabled = true;
|
tsS3Enabled = true;
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
@ -1091,7 +1102,6 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
|
||||||
tsIfAdtFse = cfgGetItem(pCfg, "IfAdtFse")->bval;
|
tsIfAdtFse = cfgGetItem(pCfg, "IfAdtFse")->bval;
|
||||||
tstrncpy(tsCompressor, cfgGetItem(pCfg, "Compressor")->str, sizeof(tsCompressor));
|
tstrncpy(tsCompressor, cfgGetItem(pCfg, "Compressor")->str, sizeof(tsCompressor));
|
||||||
|
|
||||||
|
|
||||||
tsDisableStream = cfgGetItem(pCfg, "disableStream")->bval;
|
tsDisableStream = cfgGetItem(pCfg, "disableStream")->bval;
|
||||||
tsStreamBufferSize = cfgGetItem(pCfg, "streamBufferSize")->i64;
|
tsStreamBufferSize = cfgGetItem(pCfg, "streamBufferSize")->i64;
|
||||||
|
|
||||||
|
|
@ -1680,14 +1690,14 @@ void taosCfgDynamicOptions(const char *option, const char *value) {
|
||||||
}
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
/* cannot alter s3BlockSize
|
||||||
if (strcasecmp(option, "s3BlockSize") == 0) {
|
if (strcasecmp(option, "s3BlockSize") == 0) {
|
||||||
int32_t newS3BlockSize = atoi(value);
|
int32_t newS3BlockSize = atoi(value);
|
||||||
uInfo("s3BlockSize set from %d to %d", tsS3BlockSize, newS3BlockSize);
|
uInfo("s3BlockSize set from %d to %d", tsS3BlockSize, newS3BlockSize);
|
||||||
tsS3BlockSize = newS3BlockSize;
|
tsS3BlockSize = newS3BlockSize;
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
*/
|
||||||
if (strcasecmp(option, "s3BlockCacheSize") == 0) {
|
if (strcasecmp(option, "s3BlockCacheSize") == 0) {
|
||||||
int32_t newS3BlockCacheSize = atoi(value);
|
int32_t newS3BlockCacheSize = atoi(value);
|
||||||
uInfo("s3BlockCacheSize set from %d to %d", tsS3BlockCacheSize, newS3BlockCacheSize);
|
uInfo("s3BlockCacheSize set from %d to %d", tsS3BlockCacheSize, newS3BlockCacheSize);
|
||||||
|
|
|
||||||
|
|
@ -162,6 +162,36 @@ target_link_libraries(
|
||||||
)
|
)
|
||||||
|
|
||||||
if(${TD_LINUX})
|
if(${TD_LINUX})
|
||||||
|
|
||||||
|
if(${BUILD_WITH_S3})
|
||||||
|
target_include_directories(
|
||||||
|
vnode
|
||||||
|
|
||||||
|
PUBLIC "$ENV{HOME}/.cos-local.2/include"
|
||||||
|
)
|
||||||
|
|
||||||
|
set(CMAKE_FIND_LIBRARY_SUFFIXES ".a")
|
||||||
|
set(CMAKE_PREFIX_PATH $ENV{HOME}/.cos-local.2)
|
||||||
|
find_library(S3_LIBRARY s3)
|
||||||
|
find_library(CURL_LIBRARY curl)
|
||||||
|
find_library(SSL_LIBRARY ssl $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH)
|
||||||
|
find_library(CRYPTO_LIBRARY crypto $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH)
|
||||||
|
target_link_libraries(
|
||||||
|
vnode
|
||||||
|
|
||||||
|
# s3
|
||||||
|
PUBLIC ${S3_LIBRARY}
|
||||||
|
PUBLIC ${CURL_LIBRARY}
|
||||||
|
PUBLIC ${SSL_LIBRARY}
|
||||||
|
PUBLIC ${CRYPTO_LIBRARY}
|
||||||
|
PUBLIC xml2
|
||||||
|
)
|
||||||
|
|
||||||
|
add_definitions(-DUSE_S3)
|
||||||
|
endif(${BUILD_WITH_S3})
|
||||||
|
|
||||||
|
if(${BUILD_WITH_COS})
|
||||||
|
|
||||||
set(CMAKE_FIND_LIBRARY_SUFFIXES ".a")
|
set(CMAKE_FIND_LIBRARY_SUFFIXES ".a")
|
||||||
find_library(APR_LIBRARY apr-1 PATHS /usr/local/apr/lib/)
|
find_library(APR_LIBRARY apr-1 PATHS /usr/local/apr/lib/)
|
||||||
find_library(APR_UTIL_LIBRARY aprutil-1 PATHS /usr/local/apr/lib/)
|
find_library(APR_UTIL_LIBRARY aprutil-1 PATHS /usr/local/apr/lib/)
|
||||||
|
|
@ -194,7 +224,6 @@ target_include_directories(
|
||||||
PUBLIC "$ENV{HOME}/.cos-local.1/include"
|
PUBLIC "$ENV{HOME}/.cos-local.1/include"
|
||||||
)
|
)
|
||||||
|
|
||||||
if(${BUILD_WITH_COS})
|
|
||||||
add_definitions(-DUSE_COS)
|
add_definitions(-DUSE_COS)
|
||||||
endif(${BUILD_WITH_COS})
|
endif(${BUILD_WITH_COS})
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -2,13 +2,798 @@
|
||||||
|
|
||||||
#include "vndCos.h"
|
#include "vndCos.h"
|
||||||
|
|
||||||
extern char tsS3Endpoint[];
|
extern char tsS3Endpoint[];
|
||||||
extern char tsS3AccessKeyId[];
|
extern char tsS3AccessKeyId[];
|
||||||
extern char tsS3AccessKeySecret[];
|
extern char tsS3AccessKeySecret[];
|
||||||
extern char tsS3BucketName[];
|
extern char tsS3BucketName[];
|
||||||
extern char tsS3AppId[];
|
extern char tsS3AppId[];
|
||||||
|
extern char tsS3Hostname[];
|
||||||
|
extern int8_t tsS3Https;
|
||||||
|
|
||||||
|
#if defined(USE_S3)
|
||||||
|
|
||||||
|
#include "libs3.h"
|
||||||
|
|
||||||
|
static int verifyPeerG = 0;
|
||||||
|
static const char *awsRegionG = NULL;
|
||||||
|
static int forceG = 0;
|
||||||
|
static int showResponsePropertiesG = 0;
|
||||||
|
static S3Protocol protocolG = S3ProtocolHTTPS;
|
||||||
|
// static S3Protocol protocolG = S3ProtocolHTTP;
|
||||||
|
static S3UriStyle uriStyleG = S3UriStylePath;
|
||||||
|
static int retriesG = 5;
|
||||||
|
static int timeoutMsG = 0;
|
||||||
|
|
||||||
|
static int32_t s3Begin() {
|
||||||
|
S3Status status;
|
||||||
|
const char *hostname = tsS3Hostname;
|
||||||
|
const char *env_hn = getenv("S3_HOSTNAME");
|
||||||
|
|
||||||
|
if (env_hn) {
|
||||||
|
hostname = env_hn;
|
||||||
|
}
|
||||||
|
|
||||||
|
if ((status = S3_initialize("s3", verifyPeerG | S3_INIT_ALL, hostname)) != S3StatusOK) {
|
||||||
|
vError("Failed to initialize libs3: %s\n", S3_get_status_name(status));
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
protocolG = !tsS3Https;
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void s3End() { S3_deinitialize(); }
|
||||||
|
int32_t s3Init() { return s3Begin(); }
|
||||||
|
|
||||||
|
void s3CleanUp() { s3End(); }
|
||||||
|
|
||||||
|
static int should_retry() {
|
||||||
|
/*
|
||||||
|
if (retriesG--) {
|
||||||
|
// Sleep before next retry; start out with a 1 second sleep
|
||||||
|
static int retrySleepInterval = 1 * SLEEP_UNITS_PER_SECOND;
|
||||||
|
sleep(retrySleepInterval);
|
||||||
|
// Next sleep 1 second longer
|
||||||
|
retrySleepInterval++;
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
*/
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void s3PrintError(const char *func, S3Status status, char error_details[]) {
|
||||||
|
if (status < S3StatusErrorAccessDenied) {
|
||||||
|
vError("%s: %s", __func__, S3_get_status_name(status));
|
||||||
|
} else {
|
||||||
|
vError("%s: %s, %s", __func__, S3_get_status_name(status), error_details);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
typedef struct {
|
||||||
|
char err_msg[128];
|
||||||
|
S3Status status;
|
||||||
|
uint64_t content_length;
|
||||||
|
char *buf;
|
||||||
|
} TS3SizeCBD;
|
||||||
|
|
||||||
|
static S3Status responsePropertiesCallback(const S3ResponseProperties *properties, void *callbackData) {
|
||||||
|
//(void)callbackData;
|
||||||
|
TS3SizeCBD *cbd = callbackData;
|
||||||
|
if (properties->contentLength > 0) {
|
||||||
|
cbd->content_length = properties->contentLength;
|
||||||
|
} else {
|
||||||
|
cbd->content_length = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
return S3StatusOK;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void responseCompleteCallback(S3Status status, const S3ErrorDetails *error, void *callbackData) {
|
||||||
|
TS3SizeCBD *cbd = callbackData;
|
||||||
|
cbd->status = status;
|
||||||
|
|
||||||
|
int len = 0;
|
||||||
|
const int elen = sizeof(cbd->err_msg);
|
||||||
|
if (error) {
|
||||||
|
if (error->message) {
|
||||||
|
len += snprintf(&(cbd->err_msg[len]), elen - len, " Message: %s\n", error->message);
|
||||||
|
}
|
||||||
|
if (error->resource) {
|
||||||
|
len += snprintf(&(cbd->err_msg[len]), elen - len, " Resource: %s\n", error->resource);
|
||||||
|
}
|
||||||
|
if (error->furtherDetails) {
|
||||||
|
len += snprintf(&(cbd->err_msg[len]), elen - len, " Further Details: %s\n", error->furtherDetails);
|
||||||
|
}
|
||||||
|
if (error->extraDetailsCount) {
|
||||||
|
len += snprintf(&(cbd->err_msg[len]), elen - len, "%s", " Extra Details:\n");
|
||||||
|
for (int i = 0; i < error->extraDetailsCount; i++) {
|
||||||
|
len += snprintf(&(cbd->err_msg[len]), elen - len, " %s: %s\n", error->extraDetails[i].name,
|
||||||
|
error->extraDetails[i].value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
typedef struct growbuffer {
|
||||||
|
// The total number of bytes, and the start byte
|
||||||
|
int size;
|
||||||
|
// The start byte
|
||||||
|
int start;
|
||||||
|
// The blocks
|
||||||
|
char data[64 * 1024];
|
||||||
|
struct growbuffer *prev, *next;
|
||||||
|
} growbuffer;
|
||||||
|
|
||||||
|
// returns nonzero on success, zero on out of memory
|
||||||
|
static int growbuffer_append(growbuffer **gb, const char *data, int dataLen) {
|
||||||
|
int origDataLen = dataLen;
|
||||||
|
while (dataLen) {
|
||||||
|
growbuffer *buf = *gb ? (*gb)->prev : 0;
|
||||||
|
if (!buf || (buf->size == sizeof(buf->data))) {
|
||||||
|
buf = (growbuffer *)malloc(sizeof(growbuffer));
|
||||||
|
if (!buf) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
buf->size = 0;
|
||||||
|
buf->start = 0;
|
||||||
|
if (*gb && (*gb)->prev) {
|
||||||
|
buf->prev = (*gb)->prev;
|
||||||
|
buf->next = *gb;
|
||||||
|
(*gb)->prev->next = buf;
|
||||||
|
(*gb)->prev = buf;
|
||||||
|
} else {
|
||||||
|
buf->prev = buf->next = buf;
|
||||||
|
*gb = buf;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
int toCopy = (sizeof(buf->data) - buf->size);
|
||||||
|
if (toCopy > dataLen) {
|
||||||
|
toCopy = dataLen;
|
||||||
|
}
|
||||||
|
|
||||||
|
memcpy(&(buf->data[buf->size]), data, toCopy);
|
||||||
|
|
||||||
|
buf->size += toCopy, data += toCopy, dataLen -= toCopy;
|
||||||
|
}
|
||||||
|
|
||||||
|
return origDataLen;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void growbuffer_read(growbuffer **gb, int amt, int *amtReturn, char *buffer) {
|
||||||
|
*amtReturn = 0;
|
||||||
|
|
||||||
|
growbuffer *buf = *gb;
|
||||||
|
|
||||||
|
if (!buf) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
*amtReturn = (buf->size > amt) ? amt : buf->size;
|
||||||
|
|
||||||
|
memcpy(buffer, &(buf->data[buf->start]), *amtReturn);
|
||||||
|
|
||||||
|
buf->start += *amtReturn, buf->size -= *amtReturn;
|
||||||
|
|
||||||
|
if (buf->size == 0) {
|
||||||
|
if (buf->next == buf) {
|
||||||
|
*gb = 0;
|
||||||
|
} else {
|
||||||
|
*gb = buf->next;
|
||||||
|
buf->prev->next = buf->next;
|
||||||
|
buf->next->prev = buf->prev;
|
||||||
|
}
|
||||||
|
free(buf);
|
||||||
|
buf = NULL;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static void growbuffer_destroy(growbuffer *gb) {
|
||||||
|
growbuffer *start = gb;
|
||||||
|
|
||||||
|
while (gb) {
|
||||||
|
growbuffer *next = gb->next;
|
||||||
|
free(gb);
|
||||||
|
gb = (next == start) ? 0 : next;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
typedef struct put_object_callback_data {
|
||||||
|
char err_msg[128];
|
||||||
|
S3Status status;
|
||||||
|
// FILE *infile;
|
||||||
|
TdFilePtr infileFD;
|
||||||
|
growbuffer *gb;
|
||||||
|
uint64_t contentLength, originalContentLength;
|
||||||
|
uint64_t totalContentLength, totalOriginalContentLength;
|
||||||
|
int noStatus;
|
||||||
|
} put_object_callback_data;
|
||||||
|
|
||||||
|
#define MULTIPART_CHUNK_SIZE (768 << 20) // multipart is 768M
|
||||||
|
|
||||||
|
typedef struct UploadManager {
|
||||||
|
char err_msg[128];
|
||||||
|
S3Status status;
|
||||||
|
// used for initial multipart
|
||||||
|
char *upload_id;
|
||||||
|
|
||||||
|
// used for upload part object
|
||||||
|
char **etags;
|
||||||
|
int next_etags_pos;
|
||||||
|
|
||||||
|
// used for commit Upload
|
||||||
|
growbuffer *gb;
|
||||||
|
int remaining;
|
||||||
|
} UploadManager;
|
||||||
|
|
||||||
|
typedef struct list_parts_callback_data {
|
||||||
|
char err_msg[128];
|
||||||
|
S3Status status;
|
||||||
|
int isTruncated;
|
||||||
|
char nextPartNumberMarker[24];
|
||||||
|
char initiatorId[256];
|
||||||
|
char initiatorDisplayName[256];
|
||||||
|
char ownerId[256];
|
||||||
|
char ownerDisplayName[256];
|
||||||
|
char storageClass[256];
|
||||||
|
int partsCount;
|
||||||
|
int handlePartsStart;
|
||||||
|
int allDetails;
|
||||||
|
int noPrint;
|
||||||
|
UploadManager *manager;
|
||||||
|
} list_parts_callback_data;
|
||||||
|
|
||||||
|
typedef struct MultipartPartData {
|
||||||
|
char err_msg[128];
|
||||||
|
S3Status status;
|
||||||
|
put_object_callback_data put_object_data;
|
||||||
|
int seq;
|
||||||
|
UploadManager *manager;
|
||||||
|
} MultipartPartData;
|
||||||
|
|
||||||
|
static int putObjectDataCallback(int bufferSize, char *buffer, void *callbackData) {
|
||||||
|
put_object_callback_data *data = (put_object_callback_data *)callbackData;
|
||||||
|
if (data->infileFD == 0) {
|
||||||
|
MultipartPartData *mpd = (MultipartPartData *)callbackData;
|
||||||
|
data = &mpd->put_object_data;
|
||||||
|
}
|
||||||
|
|
||||||
|
int ret = 0;
|
||||||
|
|
||||||
|
if (data->contentLength) {
|
||||||
|
int toRead = ((data->contentLength > (unsigned)bufferSize) ? (unsigned)bufferSize : data->contentLength);
|
||||||
|
if (data->gb) {
|
||||||
|
growbuffer_read(&(data->gb), toRead, &ret, buffer);
|
||||||
|
} else if (data->infileFD) {
|
||||||
|
// ret = fread(buffer, 1, toRead, data->infile);
|
||||||
|
ret = taosReadFile(data->infileFD, buffer, toRead);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
data->contentLength -= ret;
|
||||||
|
data->totalContentLength -= ret;
|
||||||
|
/* log too many open files
|
||||||
|
if (data->contentLength && !data->noStatus) {
|
||||||
|
vTrace("%llu bytes remaining ", (unsigned long long)data->totalContentLength);
|
||||||
|
vTrace("(%d%% complete) ...\n", (int)(((data->totalOriginalContentLength - data->totalContentLength) * 100) /
|
||||||
|
data->totalOriginalContentLength));
|
||||||
|
}
|
||||||
|
*/
|
||||||
|
return ret;
|
||||||
|
}
|
||||||
|
|
||||||
|
S3Status initial_multipart_callback(const char *upload_id, void *callbackData) {
|
||||||
|
UploadManager *manager = (UploadManager *)callbackData;
|
||||||
|
manager->upload_id = strdup(upload_id);
|
||||||
|
return S3StatusOK;
|
||||||
|
}
|
||||||
|
|
||||||
|
S3Status MultipartResponseProperiesCallback(const S3ResponseProperties *properties, void *callbackData) {
|
||||||
|
responsePropertiesCallback(properties, callbackData);
|
||||||
|
|
||||||
|
MultipartPartData *data = (MultipartPartData *)callbackData;
|
||||||
|
int seq = data->seq;
|
||||||
|
const char *etag = properties->eTag;
|
||||||
|
data->manager->etags[seq - 1] = strdup(etag);
|
||||||
|
data->manager->next_etags_pos = seq;
|
||||||
|
return S3StatusOK;
|
||||||
|
}
|
||||||
|
|
||||||
|
static int multipartPutXmlCallback(int bufferSize, char *buffer, void *callbackData) {
|
||||||
|
UploadManager *manager = (UploadManager *)callbackData;
|
||||||
|
int ret = 0;
|
||||||
|
|
||||||
|
if (manager->remaining) {
|
||||||
|
int toRead = ((manager->remaining > bufferSize) ? bufferSize : manager->remaining);
|
||||||
|
growbuffer_read(&(manager->gb), toRead, &ret, buffer);
|
||||||
|
}
|
||||||
|
manager->remaining -= ret;
|
||||||
|
return ret;
|
||||||
|
}
|
||||||
|
/*
|
||||||
|
static S3Status listPartsCallback(int isTruncated, const char *nextPartNumberMarker, const char *initiatorId,
|
||||||
|
const char *initiatorDisplayName, const char *ownerId, const char *ownerDisplayName,
|
||||||
|
const char *storageClass, int partsCount, int handlePartsStart,
|
||||||
|
const S3ListPart *parts, void *callbackData) {
|
||||||
|
list_parts_callback_data *data = (list_parts_callback_data *)callbackData;
|
||||||
|
|
||||||
|
data->isTruncated = isTruncated;
|
||||||
|
data->handlePartsStart = handlePartsStart;
|
||||||
|
UploadManager *manager = data->manager;
|
||||||
|
|
||||||
|
if (nextPartNumberMarker) {
|
||||||
|
snprintf(data->nextPartNumberMarker, sizeof(data->nextPartNumberMarker), "%s", nextPartNumberMarker);
|
||||||
|
} else {
|
||||||
|
data->nextPartNumberMarker[0] = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (initiatorId) {
|
||||||
|
snprintf(data->initiatorId, sizeof(data->initiatorId), "%s", initiatorId);
|
||||||
|
} else {
|
||||||
|
data->initiatorId[0] = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (initiatorDisplayName) {
|
||||||
|
snprintf(data->initiatorDisplayName, sizeof(data->initiatorDisplayName), "%s", initiatorDisplayName);
|
||||||
|
} else {
|
||||||
|
data->initiatorDisplayName[0] = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (ownerId) {
|
||||||
|
snprintf(data->ownerId, sizeof(data->ownerId), "%s", ownerId);
|
||||||
|
} else {
|
||||||
|
data->ownerId[0] = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (ownerDisplayName) {
|
||||||
|
snprintf(data->ownerDisplayName, sizeof(data->ownerDisplayName), "%s", ownerDisplayName);
|
||||||
|
} else {
|
||||||
|
data->ownerDisplayName[0] = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (storageClass) {
|
||||||
|
snprintf(data->storageClass, sizeof(data->storageClass), "%s", storageClass);
|
||||||
|
} else {
|
||||||
|
data->storageClass[0] = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (partsCount && !data->partsCount && !data->noPrint) {
|
||||||
|
// printListPartsHeader();
|
||||||
|
}
|
||||||
|
|
||||||
|
int i;
|
||||||
|
for (i = 0; i < partsCount; i++) {
|
||||||
|
const S3ListPart *part = &(parts[i]);
|
||||||
|
char timebuf[256];
|
||||||
|
if (data->noPrint) {
|
||||||
|
manager->etags[handlePartsStart + i] = strdup(part->eTag);
|
||||||
|
manager->next_etags_pos++;
|
||||||
|
manager->remaining = manager->remaining - part->size;
|
||||||
|
} else {
|
||||||
|
time_t t = (time_t)part->lastModified;
|
||||||
|
strftime(timebuf, sizeof(timebuf), "%Y-%m-%dT%H:%M:%SZ", gmtime(&t));
|
||||||
|
printf("%-30s", timebuf);
|
||||||
|
printf("%-15llu", (unsigned long long)part->partNumber);
|
||||||
|
printf("%-45s", part->eTag);
|
||||||
|
printf("%-15llu\n", (unsigned long long)part->size);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
data->partsCount += partsCount;
|
||||||
|
|
||||||
|
return S3StatusOK;
|
||||||
|
}
|
||||||
|
|
||||||
|
static int try_get_parts_info(const char *bucketName, const char *key, UploadManager *manager) {
|
||||||
|
S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret,
|
||||||
|
0, awsRegionG};
|
||||||
|
|
||||||
|
S3ListPartsHandler listPartsHandler = {{&responsePropertiesCallback, &responseCompleteCallback}, &listPartsCallback};
|
||||||
|
|
||||||
|
list_parts_callback_data data;
|
||||||
|
|
||||||
|
memset(&data, 0, sizeof(list_parts_callback_data));
|
||||||
|
|
||||||
|
data.partsCount = 0;
|
||||||
|
data.allDetails = 0;
|
||||||
|
data.manager = manager;
|
||||||
|
data.noPrint = 1;
|
||||||
|
do {
|
||||||
|
data.isTruncated = 0;
|
||||||
|
do {
|
||||||
|
S3_list_parts(&bucketContext, key, data.nextPartNumberMarker, manager->upload_id, 0, 0, 0, timeoutMsG,
|
||||||
|
&listPartsHandler, &data);
|
||||||
|
} while (S3_status_is_retryable(data.status) && should_retry());
|
||||||
|
if (data.status != S3StatusOK) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
} while (data.isTruncated);
|
||||||
|
|
||||||
|
if (data.status == S3StatusOK) {
|
||||||
|
if (!data.partsCount) {
|
||||||
|
// printListMultipartHeader(data.allDetails);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
s3PrintError(__func__, data.status, data.err_msg);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
*/
|
||||||
|
int32_t s3PutObjectFromFile2(const char *file, const char *object) {
|
||||||
|
int32_t code = 0;
|
||||||
|
const char *key = object;
|
||||||
|
// const char *uploadId = 0;
|
||||||
|
const char *filename = 0;
|
||||||
|
uint64_t contentLength = 0;
|
||||||
|
const char *cacheControl = 0, *contentType = 0, *md5 = 0;
|
||||||
|
const char *contentDispositionFilename = 0, *contentEncoding = 0;
|
||||||
|
int64_t expires = -1;
|
||||||
|
S3CannedAcl cannedAcl = S3CannedAclPrivate;
|
||||||
|
int metaPropertiesCount = 0;
|
||||||
|
S3NameValue metaProperties[S3_MAX_METADATA_COUNT];
|
||||||
|
char useServerSideEncryption = 0;
|
||||||
|
int noStatus = 0;
|
||||||
|
put_object_callback_data data;
|
||||||
|
|
||||||
|
// data.infile = 0;
|
||||||
|
data.infileFD = NULL;
|
||||||
|
data.gb = 0;
|
||||||
|
data.noStatus = noStatus;
|
||||||
|
|
||||||
|
if (taosStatFile(file, &contentLength, NULL, NULL) < 0) {
|
||||||
|
vError("ERROR: %s Failed to stat file %s: ", __func__, file);
|
||||||
|
code = TAOS_SYSTEM_ERROR(errno);
|
||||||
|
return code;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!(data.infileFD = taosOpenFile(file, TD_FILE_READ))) {
|
||||||
|
vError("ERROR: %s Failed to open file %s: ", __func__, file);
|
||||||
|
code = TAOS_SYSTEM_ERROR(errno);
|
||||||
|
return code;
|
||||||
|
}
|
||||||
|
|
||||||
|
data.totalContentLength = data.totalOriginalContentLength = data.contentLength = data.originalContentLength =
|
||||||
|
contentLength;
|
||||||
|
|
||||||
|
S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret,
|
||||||
|
0, awsRegionG};
|
||||||
|
|
||||||
|
S3PutProperties putProperties = {contentType, md5,
|
||||||
|
cacheControl, contentDispositionFilename,
|
||||||
|
contentEncoding, expires,
|
||||||
|
cannedAcl, metaPropertiesCount,
|
||||||
|
metaProperties, useServerSideEncryption};
|
||||||
|
|
||||||
|
if (contentLength <= MULTIPART_CHUNK_SIZE) {
|
||||||
|
S3PutObjectHandler putObjectHandler = {{&responsePropertiesCallback, &responseCompleteCallback},
|
||||||
|
&putObjectDataCallback};
|
||||||
|
|
||||||
|
do {
|
||||||
|
S3_put_object(&bucketContext, key, contentLength, &putProperties, 0, 0, &putObjectHandler, &data);
|
||||||
|
} while (S3_status_is_retryable(data.status) && should_retry());
|
||||||
|
|
||||||
|
if (data.infileFD) {
|
||||||
|
taosCloseFile(&data.infileFD);
|
||||||
|
} else if (data.gb) {
|
||||||
|
growbuffer_destroy(data.gb);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (data.status != S3StatusOK) {
|
||||||
|
s3PrintError(__func__, data.status, data.err_msg);
|
||||||
|
code = TAOS_SYSTEM_ERROR(EIO);
|
||||||
|
} else if (data.contentLength) {
|
||||||
|
vError("ERROR: %s Failed to read remaining %llu bytes from input", __func__,
|
||||||
|
(unsigned long long)data.contentLength);
|
||||||
|
code = TAOS_SYSTEM_ERROR(EIO);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
uint64_t totalContentLength = contentLength;
|
||||||
|
uint64_t todoContentLength = contentLength;
|
||||||
|
UploadManager manager;
|
||||||
|
manager.upload_id = 0;
|
||||||
|
manager.gb = 0;
|
||||||
|
|
||||||
|
// div round up
|
||||||
|
int seq;
|
||||||
|
uint64_t chunk_size = MULTIPART_CHUNK_SIZE >> 8;
|
||||||
|
int totalSeq = ((contentLength + chunk_size - 1) / chunk_size);
|
||||||
|
|
||||||
|
MultipartPartData partData;
|
||||||
|
memset(&partData, 0, sizeof(MultipartPartData));
|
||||||
|
int partContentLength = 0;
|
||||||
|
|
||||||
|
S3MultipartInitialHandler handler = {{&responsePropertiesCallback, &responseCompleteCallback},
|
||||||
|
&initial_multipart_callback};
|
||||||
|
|
||||||
|
S3PutObjectHandler putObjectHandler = {{&MultipartResponseProperiesCallback, &responseCompleteCallback},
|
||||||
|
&putObjectDataCallback};
|
||||||
|
|
||||||
|
S3MultipartCommitHandler commit_handler = {
|
||||||
|
{&responsePropertiesCallback, &responseCompleteCallback}, &multipartPutXmlCallback, 0};
|
||||||
|
|
||||||
|
manager.etags = (char **)taosMemoryMalloc(sizeof(char *) * totalSeq);
|
||||||
|
manager.next_etags_pos = 0;
|
||||||
|
/*
|
||||||
|
if (uploadId) {
|
||||||
|
manager.upload_id = strdup(uploadId);
|
||||||
|
manager.remaining = contentLength;
|
||||||
|
if (!try_get_parts_info(tsS3BucketName, key, &manager)) {
|
||||||
|
fseek(data.infile, -(manager.remaining), 2);
|
||||||
|
taosLSeekFile(data.infileFD, -(manager.remaining), SEEK_END);
|
||||||
|
contentLength = manager.remaining;
|
||||||
|
goto upload;
|
||||||
|
} else {
|
||||||
|
goto clean;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
*/
|
||||||
|
do {
|
||||||
|
S3_initiate_multipart(&bucketContext, key, 0, &handler, 0, timeoutMsG, &manager);
|
||||||
|
} while (S3_status_is_retryable(manager.status) && should_retry());
|
||||||
|
|
||||||
|
if (manager.upload_id == 0 || manager.status != S3StatusOK) {
|
||||||
|
s3PrintError(__func__, manager.status, manager.err_msg);
|
||||||
|
code = TAOS_SYSTEM_ERROR(EIO);
|
||||||
|
goto clean;
|
||||||
|
}
|
||||||
|
|
||||||
|
upload:
|
||||||
|
todoContentLength -= chunk_size * manager.next_etags_pos;
|
||||||
|
for (seq = manager.next_etags_pos + 1; seq <= totalSeq; seq++) {
|
||||||
|
partData.manager = &manager;
|
||||||
|
partData.seq = seq;
|
||||||
|
if (partData.put_object_data.gb == NULL) {
|
||||||
|
partData.put_object_data = data;
|
||||||
|
}
|
||||||
|
partContentLength = ((contentLength > chunk_size) ? chunk_size : contentLength);
|
||||||
|
// printf("%s Part Seq %d, length=%d\n", srcSize ? "Copying" : "Sending", seq, partContentLength);
|
||||||
|
partData.put_object_data.contentLength = partContentLength;
|
||||||
|
partData.put_object_data.originalContentLength = partContentLength;
|
||||||
|
partData.put_object_data.totalContentLength = todoContentLength;
|
||||||
|
partData.put_object_data.totalOriginalContentLength = totalContentLength;
|
||||||
|
putProperties.md5 = 0;
|
||||||
|
do {
|
||||||
|
S3_upload_part(&bucketContext, key, &putProperties, &putObjectHandler, seq, manager.upload_id,
|
||||||
|
partContentLength, 0, timeoutMsG, &partData);
|
||||||
|
} while (S3_status_is_retryable(partData.status) && should_retry());
|
||||||
|
if (partData.status != S3StatusOK) {
|
||||||
|
s3PrintError(__func__, partData.status, partData.err_msg);
|
||||||
|
code = TAOS_SYSTEM_ERROR(EIO);
|
||||||
|
goto clean;
|
||||||
|
}
|
||||||
|
contentLength -= chunk_size;
|
||||||
|
todoContentLength -= chunk_size;
|
||||||
|
}
|
||||||
|
|
||||||
|
int i;
|
||||||
|
int size = 0;
|
||||||
|
size += growbuffer_append(&(manager.gb), "<CompleteMultipartUpload>", strlen("<CompleteMultipartUpload>"));
|
||||||
|
char buf[256];
|
||||||
|
int n;
|
||||||
|
for (i = 0; i < totalSeq; i++) {
|
||||||
|
n = snprintf(buf, sizeof(buf),
|
||||||
|
"<Part><PartNumber>%d</PartNumber>"
|
||||||
|
"<ETag>%s</ETag></Part>",
|
||||||
|
i + 1, manager.etags[i]);
|
||||||
|
size += growbuffer_append(&(manager.gb), buf, n);
|
||||||
|
}
|
||||||
|
size += growbuffer_append(&(manager.gb), "</CompleteMultipartUpload>", strlen("</CompleteMultipartUpload>"));
|
||||||
|
manager.remaining = size;
|
||||||
|
|
||||||
|
do {
|
||||||
|
S3_complete_multipart_upload(&bucketContext, key, &commit_handler, manager.upload_id, manager.remaining, 0,
|
||||||
|
timeoutMsG, &manager);
|
||||||
|
} while (S3_status_is_retryable(manager.status) && should_retry());
|
||||||
|
if (manager.status != S3StatusOK) {
|
||||||
|
s3PrintError(__func__, manager.status, manager.err_msg);
|
||||||
|
code = TAOS_SYSTEM_ERROR(EIO);
|
||||||
|
goto clean;
|
||||||
|
}
|
||||||
|
|
||||||
|
clean:
|
||||||
|
if (manager.upload_id) {
|
||||||
|
taosMemoryFree(manager.upload_id);
|
||||||
|
}
|
||||||
|
for (i = 0; i < manager.next_etags_pos; i++) {
|
||||||
|
taosMemoryFree(manager.etags[i]);
|
||||||
|
}
|
||||||
|
growbuffer_destroy(manager.gb);
|
||||||
|
taosMemoryFree(manager.etags);
|
||||||
|
}
|
||||||
|
|
||||||
|
return code;
|
||||||
|
}
|
||||||
|
|
||||||
|
typedef struct list_bucket_callback_data {
|
||||||
|
char err_msg[128];
|
||||||
|
S3Status status;
|
||||||
|
int isTruncated;
|
||||||
|
char nextMarker[1024];
|
||||||
|
int keyCount;
|
||||||
|
int allDetails;
|
||||||
|
SArray *objectArray;
|
||||||
|
} list_bucket_callback_data;
|
||||||
|
|
||||||
|
static S3Status listBucketCallback(int isTruncated, const char *nextMarker, int contentsCount,
|
||||||
|
const S3ListBucketContent *contents, int commonPrefixesCount,
|
||||||
|
const char **commonPrefixes, void *callbackData) {
|
||||||
|
list_bucket_callback_data *data = (list_bucket_callback_data *)callbackData;
|
||||||
|
|
||||||
|
data->isTruncated = isTruncated;
|
||||||
|
if ((!nextMarker || !nextMarker[0]) && contentsCount) {
|
||||||
|
nextMarker = contents[contentsCount - 1].key;
|
||||||
|
}
|
||||||
|
if (nextMarker) {
|
||||||
|
snprintf(data->nextMarker, sizeof(data->nextMarker), "%s", nextMarker);
|
||||||
|
} else {
|
||||||
|
data->nextMarker[0] = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (contentsCount && !data->keyCount) {
|
||||||
|
// printListBucketHeader(data->allDetails);
|
||||||
|
}
|
||||||
|
|
||||||
|
int i;
|
||||||
|
for (i = 0; i < contentsCount; ++i) {
|
||||||
|
const S3ListBucketContent *content = &(contents[i]);
|
||||||
|
// printf("%-50s", content->key);
|
||||||
|
char *object_key = strdup(content->key);
|
||||||
|
taosArrayPush(data->objectArray, &object_key);
|
||||||
|
}
|
||||||
|
data->keyCount += contentsCount;
|
||||||
|
|
||||||
|
for (i = 0; i < commonPrefixesCount; i++) {
|
||||||
|
// printf("\nCommon Prefix: %s\n", commonPrefixes[i]);
|
||||||
|
}
|
||||||
|
|
||||||
|
return S3StatusOK;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void s3FreeObjectKey(void *pItem) {
|
||||||
|
char *key = *(char **)pItem;
|
||||||
|
taosMemoryFree(key);
|
||||||
|
}
|
||||||
|
|
||||||
|
void s3DeleteObjectsByPrefix(const char *prefix) {
|
||||||
|
S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret,
|
||||||
|
0, awsRegionG};
|
||||||
|
S3ListBucketHandler listBucketHandler = {{&responsePropertiesCallback, &responseCompleteCallback},
|
||||||
|
&listBucketCallback};
|
||||||
|
|
||||||
|
const char *marker = 0, *delimiter = 0;
|
||||||
|
int maxkeys = 0, allDetails = 0;
|
||||||
|
list_bucket_callback_data data;
|
||||||
|
data.objectArray = taosArrayInit(32, POINTER_BYTES);
|
||||||
|
if (!data.objectArray) {
|
||||||
|
vError("%s: %s", __func__, "out of memoty");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (marker) {
|
||||||
|
snprintf(data.nextMarker, sizeof(data.nextMarker), "%s", marker);
|
||||||
|
} else {
|
||||||
|
data.nextMarker[0] = 0;
|
||||||
|
}
|
||||||
|
data.keyCount = 0;
|
||||||
|
data.allDetails = allDetails;
|
||||||
|
|
||||||
|
do {
|
||||||
|
data.isTruncated = 0;
|
||||||
|
do {
|
||||||
|
S3_list_bucket(&bucketContext, prefix, data.nextMarker, delimiter, maxkeys, 0, timeoutMsG, &listBucketHandler,
|
||||||
|
&data);
|
||||||
|
} while (S3_status_is_retryable(data.status) && should_retry());
|
||||||
|
if (data.status != S3StatusOK) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
} while (data.isTruncated && (!maxkeys || (data.keyCount < maxkeys)));
|
||||||
|
|
||||||
|
if (data.status == S3StatusOK) {
|
||||||
|
if (data.keyCount > 0) {
|
||||||
|
// printListBucketHeader(allDetails);
|
||||||
|
s3DeleteObjects(TARRAY_DATA(data.objectArray), TARRAY_SIZE(data.objectArray));
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
s3PrintError(__func__, data.status, data.err_msg);
|
||||||
|
}
|
||||||
|
|
||||||
|
taosArrayDestroyEx(data.objectArray, s3FreeObjectKey);
|
||||||
|
}
|
||||||
|
|
||||||
|
void s3DeleteObjects(const char *object_name[], int nobject) {
|
||||||
|
int status = 0;
|
||||||
|
S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret,
|
||||||
|
0, awsRegionG};
|
||||||
|
S3ResponseHandler responseHandler = {0, &responseCompleteCallback};
|
||||||
|
|
||||||
|
for (int i = 0; i < nobject; ++i) {
|
||||||
|
TS3SizeCBD cbd = {0};
|
||||||
|
do {
|
||||||
|
S3_delete_object(&bucketContext, object_name[i], 0, timeoutMsG, &responseHandler, &cbd);
|
||||||
|
} while (S3_status_is_retryable(cbd.status) && should_retry());
|
||||||
|
|
||||||
|
if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) {
|
||||||
|
s3PrintError(__func__, cbd.status, cbd.err_msg);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static S3Status getObjectDataCallback(int bufferSize, const char *buffer, void *callbackData) {
|
||||||
|
TS3SizeCBD *cbd = callbackData;
|
||||||
|
if (cbd->content_length != bufferSize) {
|
||||||
|
cbd->status = S3StatusAbortedByCallback;
|
||||||
|
return S3StatusAbortedByCallback;
|
||||||
|
}
|
||||||
|
|
||||||
|
char *buf = taosMemoryCalloc(1, bufferSize);
|
||||||
|
if (buf) {
|
||||||
|
memcpy(buf, buffer, bufferSize);
|
||||||
|
cbd->buf = buf;
|
||||||
|
cbd->status = S3StatusOK;
|
||||||
|
return S3StatusOK;
|
||||||
|
} else {
|
||||||
|
cbd->status = S3StatusAbortedByCallback;
|
||||||
|
return S3StatusAbortedByCallback;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size, uint8_t **ppBlock) {
|
||||||
|
int status = 0;
|
||||||
|
int64_t ifModifiedSince = -1, ifNotModifiedSince = -1;
|
||||||
|
const char *ifMatch = 0, *ifNotMatch = 0;
|
||||||
|
|
||||||
|
S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret,
|
||||||
|
0, awsRegionG};
|
||||||
|
S3GetConditions getConditions = {ifModifiedSince, ifNotModifiedSince, ifMatch, ifNotMatch};
|
||||||
|
S3GetObjectHandler getObjectHandler = {{&responsePropertiesCallback, &responseCompleteCallback},
|
||||||
|
&getObjectDataCallback};
|
||||||
|
|
||||||
|
TS3SizeCBD cbd = {0};
|
||||||
|
cbd.content_length = size;
|
||||||
|
do {
|
||||||
|
S3_get_object(&bucketContext, object_name, &getConditions, offset, size, 0, 0, &getObjectHandler, &cbd);
|
||||||
|
} while (S3_status_is_retryable(cbd.status) && should_retry());
|
||||||
|
|
||||||
|
if (cbd.status != S3StatusOK) {
|
||||||
|
vError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg);
|
||||||
|
return TAOS_SYSTEM_ERROR(EIO);
|
||||||
|
}
|
||||||
|
|
||||||
|
*ppBlock = cbd.buf;
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
long s3Size(const char *object_name) {
|
||||||
|
long size = 0;
|
||||||
|
int status = 0;
|
||||||
|
|
||||||
|
S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret,
|
||||||
|
0, awsRegionG};
|
||||||
|
|
||||||
|
S3ResponseHandler responseHandler = {&responsePropertiesCallback, &responseCompleteCallback};
|
||||||
|
|
||||||
|
TS3SizeCBD cbd = {0};
|
||||||
|
do {
|
||||||
|
S3_head_object(&bucketContext, object_name, 0, 0, &responseHandler, &cbd);
|
||||||
|
} while (S3_status_is_retryable(cbd.status) && should_retry());
|
||||||
|
|
||||||
|
if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) {
|
||||||
|
vError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg);
|
||||||
|
}
|
||||||
|
|
||||||
|
size = cbd.content_length;
|
||||||
|
|
||||||
|
return size;
|
||||||
|
}
|
||||||
|
|
||||||
|
void s3EvictCache(const char *path, long object_size) {}
|
||||||
|
|
||||||
|
#elif defined(USE_COS)
|
||||||
|
|
||||||
#ifdef USE_COS
|
|
||||||
#include "cos_api.h"
|
#include "cos_api.h"
|
||||||
#include "cos_http_io.h"
|
#include "cos_http_io.h"
|
||||||
#include "cos_log.h"
|
#include "cos_log.h"
|
||||||
|
|
|
||||||
|
|
@ -60,7 +60,7 @@ docker run \
|
||||||
-v /root/.cargo/git:/root/.cargo/git \
|
-v /root/.cargo/git:/root/.cargo/git \
|
||||||
-v /root/go/pkg/mod:/root/go/pkg/mod \
|
-v /root/go/pkg/mod:/root/go/pkg/mod \
|
||||||
-v /root/.cache/go-build:/root/.cache/go-build \
|
-v /root/.cache/go-build:/root/.cache/go-build \
|
||||||
-v /root/.cos-local.1:/root/.cos-local.1 \
|
-v /root/.cos-local.1:/root/.cos-local.2 \
|
||||||
-v ${REP_REAL_PATH}/enterprise/src/plugins/taosx/target:${REP_DIR}/enterprise/src/plugins/taosx/target \
|
-v ${REP_REAL_PATH}/enterprise/src/plugins/taosx/target:${REP_DIR}/enterprise/src/plugins/taosx/target \
|
||||||
-v ${REP_REAL_PATH}/community/tools/taosws-rs/target:${REP_DIR}/community/tools/taosws-rs/target \
|
-v ${REP_REAL_PATH}/community/tools/taosws-rs/target:${REP_DIR}/community/tools/taosws-rs/target \
|
||||||
-v ${REP_REAL_PATH}/community/contrib/cJson/:${REP_DIR}/community/contrib/cJson \
|
-v ${REP_REAL_PATH}/community/contrib/cJson/:${REP_DIR}/community/contrib/cJson \
|
||||||
|
|
@ -89,7 +89,7 @@ docker run \
|
||||||
-v /root/.cargo/git:/root/.cargo/git \
|
-v /root/.cargo/git:/root/.cargo/git \
|
||||||
-v /root/go/pkg/mod:/root/go/pkg/mod \
|
-v /root/go/pkg/mod:/root/go/pkg/mod \
|
||||||
-v /root/.cache/go-build:/root/.cache/go-build \
|
-v /root/.cache/go-build:/root/.cache/go-build \
|
||||||
-v /root/.cos-local.1:/root/.cos-local.1 \
|
-v /root/.cos-local.1:/root/.cos-local.2 \
|
||||||
-v ${REP_REAL_PATH}/enterprise/src/plugins/taosx/target:${REP_DIR}/enterprise/src/plugins/taosx/target \
|
-v ${REP_REAL_PATH}/enterprise/src/plugins/taosx/target:${REP_DIR}/enterprise/src/plugins/taosx/target \
|
||||||
-v ${REP_REAL_PATH}/community/tools/taosws-rs/target:${REP_DIR}/community/tools/taosws-rs/target \
|
-v ${REP_REAL_PATH}/community/tools/taosws-rs/target:${REP_DIR}/community/tools/taosws-rs/target \
|
||||||
-v ${REP_REAL_PATH}/community/contrib/cJson/:${REP_DIR}/community/contrib/cJson \
|
-v ${REP_REAL_PATH}/community/contrib/cJson/:${REP_DIR}/community/contrib/cJson \
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue