xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #1470 from taosdata/refact/slguan 

Refact/slguan
This commit is contained in:
hzcheng 2020-03-30 18:32:53 +08:00 committed by GitHub
parent 6824687dc9 12ccad9df5
commit f079008dd0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
59 changed files with 593 additions and 22157 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

302
CMakeLists.txt
View File

@ -1,298 +1,22 @@
CMAKE_MINIMUM_REQUIRED(VERSION 2.8) CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
PROJECT(TDengine) PROJECT(TDengine)
SET(CMAKE_C_STANDARD 11) SET(TD_CLUSTER FALSE)
SET(CMAKE_VERBOSE_MAKEFILE ON) SET(TD_ACCOUNT FALSE)
SET(TD_GRANT FALSE)
# SET(TD_COVER FALSE)
# If need to set debug options SET(TD_PAGMODE_LITE FALSE)
# 1.Generate debug version:
# mkdir debug; cd debug;
# cmake -DCMAKE_BUILD_TYPE=Debug ..
# 2.Generate release version:
# mkdir release; cd release;
# cmake -DCMAKE_BUILD_TYPE=Release ..
#
#
# If it is a Windows operating system
# 1.Use command line tool of VS2013 or higher version
# mkdir build; cd build;
# cmake -G "NMake Makefiles" ..
# nmake install
# 2.Use the VS development interface tool
# mkdir build; cd build;
# cmake -A x64 ..
# open the file named TDengine.sln
#
SET(TD_GODLL FALSE) SET(TD_GODLL FALSE)
IF (${DLLTYPE} MATCHES "go") SET(TD_MEM_CHECK FALSE)
ADD_DEFINITIONS(-D_TD_GO_DLL_)
MESSAGE(STATUS "input dll type: " ${DLLTYPE})
SET(TD_GODLL TRUE)
ENDIF ()
IF (NOT DEFINED TD_CLUSTER) SET(TD_COMMUNITY_DIR ${PROJECT_SOURCE_DIR})
MESSAGE(STATUS "Build the Lite Version") MESSAGE(STATUS "Community directory: " ${TD_COMMUNITY_DIR})
SET(TD_CLUSTER FALSE)
SET(TD_EDGE TRUE)
SET(TD_COMMUNITY_DIR ${PROJECT_SOURCE_DIR}) INCLUDE(cmake/input.inc)
MESSAGE(STATUS "Community directory: " ${TD_COMMUNITY_DIR}) INCLUDE(cmake/platform.inc)
INCLUDE(cmake/env.inc)
INCLUDE(cmake/define.inc)
# Set macro definitions according to os platform INCLUDE(cmake/install.inc)
SET(TD_LINUX_64 FALSE)
SET(TD_LINUX_32 FALSE)
SET(TD_ARM FALSE)
SET(TD_ARM_64 FALSE)
SET(TD_ARM_32 FALSE)
SET(TD_MIPS FALSE)
SET(TD_MIPS_64 FALSE)
SET(TD_MIPS_32 FALSE)
SET(TD_DARWIN_64 FALSE)
SET(TD_WINDOWS_64 FALSE)
SET(TD_PAGMODE_LITE FALSE)
IF (${PAGMODE} MATCHES "lite")
SET(TD_PAGMODE_LITE TRUE)
ENDIF ()
# if generate ARM version:
# cmake -DCPUTYPE=aarch32 .. or cmake -DCPUTYPE=aarch64
IF (${CPUTYPE} MATCHES "aarch32")
SET(TD_ARM TRUE)
SET(TD_ARM_32 TRUE)
SET(TD_PAGMODE_LITE TRUE)
ADD_DEFINITIONS(-D_TD_ARM_)
ADD_DEFINITIONS(-D_TD_ARM_32_)
ELSEIF (${CPUTYPE} MATCHES "aarch64")
SET(TD_ARM TRUE)
SET(TD_ARM_64 TRUE)
ADD_DEFINITIONS(-D_TD_ARM_)
ADD_DEFINITIONS(-D_TD_ARM_64_)
ELSEIF (${CPUTYPE} MATCHES "mips64")
SET(TD_MIPS TRUE)
SET(TD_MIPS_64 TRUE)
ADD_DEFINITIONS(-D_TD_MIPS_)
ADD_DEFINITIONS(-D_TD_MIPS_64_)
ELSEIF (${CPUTYPE} MATCHES "x64")
MESSAGE(STATUS "input cpuType: " ${CPUTYPE})
ELSEIF (${CPUTYPE} MATCHES "x86")
MESSAGE(STATUS "input cpuType: " ${CPUTYPE})
ELSE ()
MESSAGE(STATUS "input cpuType: " ${CPUTYPE})
ENDIF ()
#
# Get OS information and store in variable TD_OS_INFO.
#
execute_process(COMMAND chmod 777 ${TD_COMMUNITY_DIR}/packaging/tools/get_os.sh)
execute_process(COMMAND ${TD_COMMUNITY_DIR}/packaging/tools/get_os.sh "" OUTPUT_VARIABLE TD_OS_INFO)
MESSAGE(STATUS "The current os is " ${TD_OS_INFO})
IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
IF (${CMAKE_SIZEOF_VOID_P} MATCHES 8)
SET(TD_LINUX_64 TRUE)
SET(TD_OS_DIR ${TD_COMMUNITY_DIR}/src/os/linux)
ADD_DEFINITIONS(-D_M_X64)
MESSAGE(STATUS "The current platform is Linux 64-bit")
ELSEIF (${CMAKE_SIZEOF_VOID_P} MATCHES 4)
IF (TD_ARM)
SET(TD_LINUX_32 TRUE)
SET(TD_OS_DIR ${TD_COMMUNITY_DIR}/src/os/linux)
#ADD_DEFINITIONS(-D_M_IX86)
MESSAGE(STATUS "The current platform is Linux 32-bit")
ELSE ()
MESSAGE(FATAL_ERROR "The current platform is Linux 32-bit, but no ARM not supported yet")
EXIT ()
ENDIF ()
ELSE ()
MESSAGE(FATAL_ERROR "The current platform is Linux neither 32-bit nor 64-bit, not supported yet")
EXIT ()
ENDIF ()
ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
IF (${CMAKE_SIZEOF_VOID_P} MATCHES 8)
SET(TD_DARWIN_64 TRUE)
SET(TD_OS_DIR ${TD_COMMUNITY_DIR}/src/os/darwin)
MESSAGE(STATUS "The current platform is Darwin 64-bit")
ELSE ()
MESSAGE(FATAL_ERROR "The current platform is Darwin 32-bit, not supported yet")
EXIT ()
ENDIF ()
ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "Windows")
IF (${CMAKE_SIZEOF_VOID_P} MATCHES 8)
SET(TD_WINDOWS_64 TRUE)
SET(TD_OS_DIR ${TD_COMMUNITY_DIR}/src/os/windows)
ADD_DEFINITIONS(-D_M_X64)
MESSAGE(STATUS "The current platform is Windows 64-bit")
ELSE ()
MESSAGE(FATAL_ERROR "The current platform is Windows 32-bit, not supported yet")
EXIT ()
ENDIF ()
ELSE()
MESSAGE(FATAL_ERROR "The current platform is not Linux/Darwin/Windows, stop compile")
EXIT ()
ENDIF ()
FIND_PROGRAM(TD_MVN_INSTALLED mvn)
IF (TD_MVN_INSTALLED)
MESSAGE(STATUS "MVN is installed and JDBC will be compiled")
ELSE ()
MESSAGE(STATUS "MVN is not installed and JDBC is not compiled")
ENDIF ()
#
# debug flag
#
# ADD_DEFINITIONS(-D_CHECK_HEADER_FILE_)
IF (${MEM_CHECK} MATCHES "true")
ADD_DEFINITIONS(-DTAOS_MEM_CHECK)
ENDIF ()
IF (TD_CLUSTER)
ADD_DEFINITIONS(-DCLUSTER)
ADD_DEFINITIONS(-DTSDB_REPLICA_MAX_NUM=3)
ELSE ()
ADD_DEFINITIONS(-DLITE)
ADD_DEFINITIONS(-DTSDB_REPLICA_MAX_NUM=1)
ENDIF ()
IF (TD_LINUX_64)
SET(DEBUG_FLAGS "-O0 -DDEBUG")
SET(RELEASE_FLAGS "-O0")
IF (NOT TD_ARM)
IF (${CMAKE_CXX_COMPILER_ID} MATCHES "Clang")
SET(COMMON_FLAGS "-std=gnu99 -Wall -fPIC -malign-double -g -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
ELSE ()
SET(COMMON_FLAGS "-std=gnu99 -Wall -fPIC -malign-double -g -malign-stringops -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
ENDIF ()
ELSE ()
SET(COMMON_FLAGS "-std=gnu99 -Wall -fPIC -g -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
ENDIF ()
ADD_DEFINITIONS(-DLINUX)
ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT)
IF (${TD_OS_INFO} MATCHES "Alpine")
MESSAGE(STATUS "The current OS is Alpine, append extra flags")
SET(COMMON_FLAGS "${COMMON_FLAGS} -largp")
link_libraries(/usr/lib/libargp.a)
ADD_DEFINITIONS(-D_ALPINE)
ENDIF ()
ELSEIF (TD_LINUX_32)
IF (NOT TD_ARM)
EXIT ()
ENDIF ()
SET(DEBUG_FLAGS "-O0 -DDEBUG")
SET(RELEASE_FLAGS "-O0")
SET(COMMON_FLAGS "-std=gnu99 -Wall -fPIC -g -fsigned-char -munaligned-access -fpack-struct=8 -latomic -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
ADD_DEFINITIONS(-DLINUX)
ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT)
ADD_DEFINITIONS(-DUSE_LIBICONV)
IF (${TD_OS_INFO} MATCHES "Alpine")
MESSAGE(STATUS "The current OS is Alpine, add extra flags")
SET(COMMON_FLAGS "${COMMON_FLAGS} -largp")
link_library(/usr/lib/libargp.a)
ADD_DEFINITIONS(-D_ALPINE)
ENDIF ()
ELSEIF (TD_WINDOWS_64)
SET(CMAKE_GENERATOR "NMake Makefiles" CACHE INTERNAL "" FORCE)
IF (NOT TD_GODLL)
SET(COMMON_FLAGS "/nologo /WX- /Oi /Oy- /Gm- /EHsc /MT /GS /Gy /fp:precise /Zc:wchar_t /Zc:forScope /Gd /errorReport:prompt /analyze-")
SET(DEBUG_FLAGS "/Zi /W3 /GL")
SET(RELEASE_FLAGS "/W0 /GL")
ENDIF ()
ADD_DEFINITIONS(-DWINDOWS)
ADD_DEFINITIONS(-D__CLEANUP_C)
ADD_DEFINITIONS(-DPTW32_STATIC_LIB)
ADD_DEFINITIONS(-DPTW32_BUILD)
ADD_DEFINITIONS(-D_MBCS -D_CRT_SECURE_NO_DEPRECATE -D_CRT_NONSTDC_NO_DEPRECATE)
ELSEIF (TD_DARWIN_64)
SET(COMMON_FLAGS "-std=gnu99 -Wall -fPIC -malign-double -g -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
SET(DEBUG_FLAGS "-O0 -DDEBUG")
SET(RELEASE_FLAGS "-O0")
ADD_DEFINITIONS(-DDARWIN)
ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT)
ELSE ()
MESSAGE(FATAL_ERROR "The current platform is not support yet, stop compile")
EXIT ()
ENDIF ()
# Set compiler options
SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} ${COMMON_FLAGS} ${DEBUG_FLAGS}")
SET(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} ${COMMON_FLAGS} ${RELEASE_FLAGS}")
# Set c++ compiler options
# SET(COMMON_CXX_FLAGS "${COMMON_FLAGS} -std=c++11")
# SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${COMMON_CXX_FLAGS} ${DEBUG_FLAGS}")
# SET(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${COMMON_CXX_FLAGS} ${RELEASE_FLAGS}")
IF (${CMAKE_BUILD_TYPE} MATCHES "Debug")
MESSAGE(STATUS "Build Debug Version")
ELSEIF (${CMAKE_BUILD_TYPE} MATCHES "Release")
MESSAGE(STATUS "Build Release Version")
ELSE ()
IF (TD_WINDOWS_64)
SET(CMAKE_BUILD_TYPE "Release")
MESSAGE(STATUS "Build Release Version in Windows as default")
ELSE ()
SET(CMAKE_BUILD_TYPE "Debug")
MESSAGE(STATUS "Build Debug Version as default")
ENDIF()
ENDIF ()
#set output directory
SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/lib)
SET(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/bin)
SET(TD_TESTS_OUTPUT_DIR ${PROJECT_BINARY_DIR}/test)
MESSAGE(STATUS "Operating system dependency directory: " ${TD_OS_DIR})
MESSAGE(STATUS "Project source directory: " ${PROJECT_SOURCE_DIR})
MESSAGE(STATUS "Project binary files output path: " ${PROJECT_BINARY_DIR})
MESSAGE(STATUS "Project executable files output path: " ${EXECUTABLE_OUTPUT_PATH})
MESSAGE(STATUS "Project library files output path: " ${LIBRARY_OUTPUT_PATH})
IF (TD_LINUX_64)
SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh")
INSTALL(CODE "MESSAGE(\"make install script: ${TD_MAKE_INSTALL_SH}\")")
INSTALL(CODE "execute_process(COMMAND chmod 777 ${TD_MAKE_INSTALL_SH})")
INSTALL(CODE "execute_process(COMMAND ${TD_MAKE_INSTALL_SH} ${TD_COMMUNITY_DIR} ${PROJECT_BINARY_DIR})")
ELSEIF (TD_LINUX_32)
IF (NOT TD_ARM)
EXIT ()
ENDIF ()
SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh")
INSTALL(CODE "MESSAGE(\"make install script: ${TD_MAKE_INSTALL_SH}\")")
INSTALL(CODE "execute_process(COMMAND chmod 777 ${TD_MAKE_INSTALL_SH})")
INSTALL(CODE "execute_process(COMMAND ${TD_MAKE_INSTALL_SH} ${TD_COMMUNITY_DIR} ${PROJECT_BINARY_DIR})")
ELSEIF (TD_WINDOWS_64)
SET(CMAKE_INSTALL_PREFIX C:/TDengine)
IF (NOT TD_GODLL)
INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/go DESTINATION connector)
INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/grafana DESTINATION connector)
INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/python DESTINATION connector)
INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/tests/examples DESTINATION .)
INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/packaging/cfg DESTINATION .)
INSTALL(FILES ${TD_COMMUNITY_DIR}/src/inc/taos.h DESTINATION include)
INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.lib DESTINATION driver)
INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.exp DESTINATION driver)
INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.dll DESTINATION driver)
INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/taos.exe DESTINATION .)
#INSTALL(TARGETS taos RUNTIME DESTINATION driver)
#INSTALL(TARGETS shell RUNTIME DESTINATION .)
IF (TD_MVN_INSTALLED)
INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-1.0.2-dist.jar DESTINATION connector/jdbc)
ENDIF ()
ELSE ()
INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/libtaos.dll DESTINATION driver)
INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/libtaos.dll.a DESTINATION driver)
ENDIF ()
ELSEIF (TD_DARWIN_64)
SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh")
INSTALL(CODE "MESSAGE(\"make install script: ${TD_MAKE_INSTALL_SH}\")")
INSTALL(CODE "execute_process(COMMAND chmod 777 ${TD_MAKE_INSTALL_SH})")
INSTALL(CODE "execute_process(COMMAND ${TD_MAKE_INSTALL_SH} ${TD_COMMUNITY_DIR} ${PROJECT_BINARY_DIR} Darwin)")
ENDIF ()
ENDIF ()
ADD_SUBDIRECTORY(deps) ADD_SUBDIRECTORY(deps)
ADD_SUBDIRECTORY(src) ADD_SUBDIRECTORY(src)

26
cmake/define.inc Executable file
View File

@ -0,0 +1,26 @@
CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
PROJECT(TDengine)
IF (TD_CLUSTER)
ADD_DEFINITIONS(-D_CLUSTER)
ADD_DEFINITIONS(-DTSDB_REPLICA_MAX_NUM=3)
ELSE ()
ADD_DEFINITIONS(-DLITE)
ADD_DEFINITIONS(-DTSDB_REPLICA_MAX_NUM=1)
ENDIF ()
IF (TD_ACCOUNT)
ADD_DEFINITIONS(-D_ACCOUNT)
ENDIF ()
IF (TD_GRANT)
ADD_DEFINITIONS(-D_GRANT)
ENDIF ()
IF (TD_GODLL)
ADD_DEFINITIONS(-D_TD_GO_DLL_)
ENDIF ()
IF (TD_MEM_CHECK)
ADD_DEFINITIONS(-DTAOS_MEM_CHECK)
ENDIF ()

56
cmake/env.inc Executable file
View File

@ -0,0 +1,56 @@
CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
PROJECT(TDengine)
SET(CMAKE_C_STANDARD 11)
SET(CMAKE_VERBOSE_MAKEFILE ON)
#set output directory
SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/lib)
SET(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/bin)
SET(TD_TESTS_OUTPUT_DIR ${PROJECT_BINARY_DIR}/test)
MESSAGE(STATUS "Operating system dependency directory: " ${TD_OS_DIR})
MESSAGE(STATUS "Project source directory: " ${PROJECT_SOURCE_DIR})
MESSAGE(STATUS "Project binary files output path: " ${PROJECT_BINARY_DIR})
MESSAGE(STATUS "Project executable files output path: " ${EXECUTABLE_OUTPUT_PATH})
MESSAGE(STATUS "Project library files output path: " ${LIBRARY_OUTPUT_PATH})
FIND_PROGRAM(TD_MVN_INSTALLED mvn)
IF (TD_MVN_INSTALLED)
MESSAGE(STATUS "MVN is installed and JDBC will be compiled")
ELSE ()
MESSAGE(STATUS "MVN is not installed and JDBC is not compiled")
ENDIF ()
#
# If need to set debug options
# 1.Generate debug version:
# mkdir debug; cd debug;
# cmake -DCMAKE_BUILD_TYPE=Debug ..
# 2.Generate release version:
# mkdir release; cd release;
# cmake -DCMAKE_BUILD_TYPE=Release ..
#
# Set compiler options
SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} ${COMMON_FLAGS} ${DEBUG_FLAGS}")
SET(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} ${COMMON_FLAGS} ${RELEASE_FLAGS}")
# Set c++ compiler options
# SET(COMMON_CXX_FLAGS "${COMMON_FLAGS} -std=c++11")
# SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${COMMON_CXX_FLAGS} ${DEBUG_FLAGS}")
# SET(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${COMMON_CXX_FLAGS} ${RELEASE_FLAGS}")
IF (${CMAKE_BUILD_TYPE} MATCHES "Debug")
MESSAGE(STATUS "Build Debug Version")
ELSEIF (${CMAKE_BUILD_TYPE} MATCHES "Release")
MESSAGE(STATUS "Build Release Version")
ELSE ()
IF (TD_WINDOWS_64)
SET(CMAKE_BUILD_TYPE "Release")
MESSAGE(STATUS "Build Release Version in Windows as default")
ELSE ()
SET(CMAKE_BUILD_TYPE "Debug")
MESSAGE(STATUS "Build Debug Version as default")
ENDIF()
ENDIF ()

41
cmake/input.inc Executable file
View File

@ -0,0 +1,41 @@
CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
PROJECT(TDengine)
IF (${CLUSTER} MATCHES "true")
SET(TD_CLUSTER TRUE)
MESSAGE(STATUS "Build with cluster plugins")
ELSEIF (${CLUSTER} MATCHES "false")
SET(TD_CLUSTER FALSE)
MESSAGE(STATUS "Build without cluster plugins")
ENDIF ()
IF (${ACCOUNT} MATCHES "true")
SET(TD_ACCOUNT TRUE)
MESSAGE(STATUS "Build with account plugins")
ELSEIF (${ACCOUNT} MATCHES "false")
SET(TD_ACCOUNT FALSE)
MESSAGE(STATUS "Build without account plugins")
ENDIF ()
IF (${COVER} MATCHES "true")
SET(TD_COVER TRUE)
MESSAGE(STATUS "Build with test coverage")
ELSEIF (${COVER} MATCHES "false")
SET(TD_COVER FALSE)
MESSAGE(STATUS "Build without test coverage")
ENDIF ()
IF (${PAGMODE} MATCHES "lite")
SET(TD_PAGMODE_LITE TRUE)
MESSAGE(STATUS "Build with pagmode lite")
ENDIF ()
IF (${DLLTYPE} MATCHES "go")
SET(TD_GODLL TRUE)
MESSAGE(STATUS "input dll type: " ${DLLTYPE})
ENDIF ()
IF (${MEM_CHECK} MATCHES "true")
SET(TD_MEM_CHECK TRUE)
MESSAGE(STATUS "build with memory check")
ENDIF ()

41
cmake/install.inc Executable file
View File

@ -0,0 +1,41 @@
IF (TD_LINUX_64)
SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh")
INSTALL(CODE "MESSAGE(\"make install script: ${TD_MAKE_INSTALL_SH}\")")
INSTALL(CODE "execute_process(COMMAND chmod 777 ${TD_MAKE_INSTALL_SH})")
INSTALL(CODE "execute_process(COMMAND ${TD_MAKE_INSTALL_SH} ${TD_COMMUNITY_DIR} ${PROJECT_BINARY_DIR})")
ELSEIF (TD_LINUX_32)
IF (NOT TD_ARM)
EXIT ()
ENDIF ()
SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh")
INSTALL(CODE "MESSAGE(\"make install script: ${TD_MAKE_INSTALL_SH}\")")
INSTALL(CODE "execute_process(COMMAND chmod 777 ${TD_MAKE_INSTALL_SH})")
INSTALL(CODE "execute_process(COMMAND ${TD_MAKE_INSTALL_SH} ${TD_COMMUNITY_DIR} ${PROJECT_BINARY_DIR})")
ELSEIF (TD_WINDOWS_64)
SET(CMAKE_INSTALL_PREFIX C:/TDengine)
IF (NOT TD_GODLL)
INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/go DESTINATION connector)
INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/grafana DESTINATION connector)
INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/python DESTINATION connector)
INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/tests/examples DESTINATION .)
INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/packaging/cfg DESTINATION .)
INSTALL(FILES ${TD_COMMUNITY_DIR}/src/inc/taos.h DESTINATION include)
INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.lib DESTINATION driver)
INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.exp DESTINATION driver)
INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.dll DESTINATION driver)
INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/taos.exe DESTINATION .)
#INSTALL(TARGETS taos RUNTIME DESTINATION driver)
#INSTALL(TARGETS shell RUNTIME DESTINATION .)
IF (TD_MVN_INSTALLED)
INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-1.0.2-dist.jar DESTINATION connector/jdbc)
ENDIF ()
ELSE ()
INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/libtaos.dll DESTINATION driver)
INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/libtaos.dll.a DESTINATION driver)
ENDIF ()
ELSEIF (TD_DARWIN_64)
SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh")
INSTALL(CODE "MESSAGE(\"make install script: ${TD_MAKE_INSTALL_SH}\")")
INSTALL(CODE "execute_process(COMMAND chmod 777 ${TD_MAKE_INSTALL_SH})")
INSTALL(CODE "execute_process(COMMAND ${TD_MAKE_INSTALL_SH} ${TD_COMMUNITY_DIR} ${PROJECT_BINARY_DIR} Darwin)")
ENDIF ()

162
cmake/platform.inc Executable file
View File

@ -0,0 +1,162 @@
CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
PROJECT(TDengine)
#
# If it is a Windows operating system
# 1.Use command line tool of VS2013 or higher version
# mkdir build; cd build;
# cmake -G "NMake Makefiles" ..
# nmake install
# 2.Use the VS development interface tool
# mkdir build; cd build;
# cmake -A x64 ..
# open the file named TDengine.sln
#
# Set macro definitions according to os platform
SET(TD_LINUX_64 FALSE)
SET(TD_LINUX_32 FALSE)
SET(TD_ARM FALSE)
SET(TD_ARM_64 FALSE)
SET(TD_ARM_32 FALSE)
SET(TD_MIPS FALSE)
SET(TD_MIPS_64 FALSE)
SET(TD_MIPS_32 FALSE)
SET(TD_DARWIN_64 FALSE)
SET(TD_WINDOWS_64 FALSE)
# if generate ARM version:
# cmake -DCPUTYPE=aarch32 .. or cmake -DCPUTYPE=aarch64
IF (${CPUTYPE} MATCHES "aarch32")
SET(TD_ARM TRUE)
SET(TD_ARM_32 TRUE)
SET(TD_PAGMODE_LITE TRUE)
ADD_DEFINITIONS(-D_TD_ARM_)
ADD_DEFINITIONS(-D_TD_ARM_32_)
ELSEIF (${CPUTYPE} MATCHES "aarch64")
SET(TD_ARM TRUE)
SET(TD_ARM_64 TRUE)
ADD_DEFINITIONS(-D_TD_ARM_)
ADD_DEFINITIONS(-D_TD_ARM_64_)
ELSEIF (${CPUTYPE} MATCHES "mips64")
SET(TD_MIPS TRUE)
SET(TD_MIPS_64 TRUE)
ADD_DEFINITIONS(-D_TD_MIPS_)
ADD_DEFINITIONS(-D_TD_MIPS_64_)
ELSEIF (${CPUTYPE} MATCHES "x64")
MESSAGE(STATUS "input cpuType: " ${CPUTYPE})
ELSEIF (${CPUTYPE} MATCHES "x86")
MESSAGE(STATUS "input cpuType: " ${CPUTYPE})
ELSE ()
MESSAGE(STATUS "input cpuType: " ${CPUTYPE})
ENDIF ()
#
# Get OS information and store in variable TD_OS_INFO.
#
execute_process(COMMAND chmod 777 ${TD_COMMUNITY_DIR}/packaging/tools/get_os.sh)
execute_process(COMMAND ${TD_COMMUNITY_DIR}/packaging/tools/get_os.sh "" OUTPUT_VARIABLE TD_OS_INFO)
MESSAGE(STATUS "The current os is " ${TD_OS_INFO})
IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
IF (${CMAKE_SIZEOF_VOID_P} MATCHES 8)
SET(TD_LINUX_64 TRUE)
SET(TD_OS_DIR ${TD_COMMUNITY_DIR}/src/os/linux)
ADD_DEFINITIONS(-D_M_X64)
MESSAGE(STATUS "The current platform is Linux 64-bit")
ELSEIF (${CMAKE_SIZEOF_VOID_P} MATCHES 4)
IF (TD_ARM)
SET(TD_LINUX_32 TRUE)
SET(TD_OS_DIR ${TD_COMMUNITY_DIR}/src/os/linux)
#ADD_DEFINITIONS(-D_M_IX86)
MESSAGE(STATUS "The current platform is Linux 32-bit")
ELSE ()
MESSAGE(FATAL_ERROR "The current platform is Linux 32-bit, but no ARM not supported yet")
EXIT ()
ENDIF ()
ELSE ()
MESSAGE(FATAL_ERROR "The current platform is Linux neither 32-bit nor 64-bit, not supported yet")
EXIT ()
ENDIF ()
ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
IF (${CMAKE_SIZEOF_VOID_P} MATCHES 8)
SET(TD_DARWIN_64 TRUE)
SET(TD_OS_DIR ${TD_COMMUNITY_DIR}/src/os/darwin)
MESSAGE(STATUS "The current platform is Darwin 64-bit")
ELSE ()
MESSAGE(FATAL_ERROR "The current platform is Darwin 32-bit, not supported yet")
EXIT ()
ENDIF ()
ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "Windows")
IF (${CMAKE_SIZEOF_VOID_P} MATCHES 8)
SET(TD_WINDOWS_64 TRUE)
SET(TD_OS_DIR ${TD_COMMUNITY_DIR}/src/os/windows)
ADD_DEFINITIONS(-D_M_X64)
MESSAGE(STATUS "The current platform is Windows 64-bit")
ELSE ()
MESSAGE(FATAL_ERROR "The current platform is Windows 32-bit, not supported yet")
EXIT ()
ENDIF ()
ELSE()
MESSAGE(FATAL_ERROR "The current platform is not Linux/Darwin/Windows, stop compile")
EXIT ()
ENDIF ()
IF (TD_LINUX_64)
SET(DEBUG_FLAGS "-O0 -DDEBUG")
SET(RELEASE_FLAGS "-O0")
IF (NOT TD_ARM)
IF (${CMAKE_CXX_COMPILER_ID} MATCHES "Clang")
SET(COMMON_FLAGS "-std=gnu99 -Wall -fPIC -malign-double -g -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
ELSE ()
SET(COMMON_FLAGS "-std=gnu99 -Wall -fPIC -malign-double -g -malign-stringops -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
ENDIF ()
ELSE ()
SET(COMMON_FLAGS "-std=gnu99 -Wall -fPIC -g -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
ENDIF ()
ADD_DEFINITIONS(-DLINUX)
ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT)
IF (${TD_OS_INFO} MATCHES "Alpine")
MESSAGE(STATUS "The current OS is Alpine, append extra flags")
SET(COMMON_FLAGS "${COMMON_FLAGS} -largp")
link_libraries(/usr/lib/libargp.a)
ADD_DEFINITIONS(-D_ALPINE)
ENDIF ()
ELSEIF (TD_LINUX_32)
IF (NOT TD_ARM)
EXIT ()
ENDIF ()
SET(DEBUG_FLAGS "-O0 -DDEBUG")
SET(RELEASE_FLAGS "-O0")
SET(COMMON_FLAGS "-std=gnu99 -Wall -fPIC -g -fsigned-char -munaligned-access -fpack-struct=8 -latomic -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
ADD_DEFINITIONS(-DLINUX)
ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT)
ADD_DEFINITIONS(-DUSE_LIBICONV)
IF (${TD_OS_INFO} MATCHES "Alpine")
MESSAGE(STATUS "The current OS is Alpine, add extra flags")
SET(COMMON_FLAGS "${COMMON_FLAGS} -largp")
link_library(/usr/lib/libargp.a)
ADD_DEFINITIONS(-D_ALPINE)
ENDIF ()
ELSEIF (TD_WINDOWS_64)
SET(CMAKE_GENERATOR "NMake Makefiles" CACHE INTERNAL "" FORCE)
IF (NOT TD_GODLL)
SET(COMMON_FLAGS "/nologo /WX- /Oi /Oy- /Gm- /EHsc /MT /GS /Gy /fp:precise /Zc:wchar_t /Zc:forScope /Gd /errorReport:prompt /analyze-")
SET(DEBUG_FLAGS "/Zi /W3 /GL")
SET(RELEASE_FLAGS "/W0 /GL")
ENDIF ()
ADD_DEFINITIONS(-DWINDOWS)
ADD_DEFINITIONS(-D__CLEANUP_C)
ADD_DEFINITIONS(-DPTW32_STATIC_LIB)
ADD_DEFINITIONS(-DPTW32_BUILD)
ADD_DEFINITIONS(-D_MBCS -D_CRT_SECURE_NO_DEPRECATE -D_CRT_NONSTDC_NO_DEPRECATE)
ELSEIF (TD_DARWIN_64)
SET(COMMON_FLAGS "-std=gnu99 -Wall -fPIC -malign-double -g -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
SET(DEBUG_FLAGS "-O0 -DDEBUG")
SET(RELEASE_FLAGS "-O0")
ADD_DEFINITIONS(-DDARWIN)
ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT)
ELSE ()
MESSAGE(FATAL_ERROR "The current platform is not support yet, stop compile")
EXIT ()
ENDIF ()

13
src/dnode/CMakeLists.txt
View File

@ -15,9 +15,16 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
ADD_EXECUTABLE(taosd ${SRC}) ADD_EXECUTABLE(taosd ${SRC})
TARGET_LINK_LIBRARIES(taosd mnode taos_static monitor http tsdb) TARGET_LINK_LIBRARIES(taosd mnode taos_static monitor http tsdb)
#IF (TD_CLUSTER) IF (TD_ACCOUNT)
# TARGET_LINK_LIBRARIES(taosd dcluster) TARGET_LINK_LIBRARIES(taosd account)
#ENDIF () ENDIF ()
IF (TD_GRANT)
TARGET_LINK_LIBRARIES(taosd grant)
ENDIF ()
IF (TD_CLUSTER)
TARGET_LINK_LIBRARIES(taosd cluster)
ENDIF ()
SET(PREPARE_ENV_CMD "prepare_env_cmd") SET(PREPARE_ENV_CMD "prepare_env_cmd")
SET(PREPARE_ENV_TARGET "prepare_env_target") SET(PREPARE_ENV_TARGET "prepare_env_target")

25
src/dnode/src/dnodeMain.c
View File

@ -28,20 +28,9 @@
#include "dnodeRead.h" #include "dnodeRead.h"
#include "dnodeShell.h" #include "dnodeShell.h"
#include "dnodeWrite.h" #include "dnodeWrite.h"
#ifdef CLUSTER
#include "account.h"
#include "admin.h"
#include "balance.h"
#include "cluster.h"
#include "grant.h"
#include "mpeer.h"
#include "storage.h"
#include "vpeer.h"
#endif
static int32_t dnodeInitSystem(); static int32_t dnodeInitSystem();
static int32_t dnodeInitStorage(); static int32_t dnodeInitStorage();
static void dnodeInitPlugins();
static void dnodeCleanupStorage(); static void dnodeCleanupStorage();
static void dnodeCleanUpSystem(); static void dnodeCleanUpSystem();
static void dnodeSetRunStatus(SDnodeRunStatus status); static void dnodeSetRunStatus(SDnodeRunStatus status);
@ -51,8 +40,6 @@ static SDnodeRunStatus tsDnodeRunStatus = TSDB_DNODE_RUN_STATUS_STOPPED;
void (*dnodeParseParameterKFp)() = NULL; void (*dnodeParseParameterKFp)() = NULL;
int32_t main(int32_t argc, char *argv[]) { int32_t main(int32_t argc, char *argv[]) {
dnodeInitPlugins();
// Set global configuration file // Set global configuration file
for (int32_t i = 1; i < argc; ++i) { for (int32_t i = 1; i < argc; ++i) {
if (strcmp(argv[i], "-c") == 0) { if (strcmp(argv[i], "-c") == 0) {
@ -244,15 +231,3 @@ static int32_t dnodeInitStorage() {
} }
static void dnodeCleanupStorage() {} static void dnodeCleanupStorage() {}
static void dnodeInitPlugins() {
#ifdef CLUSTER
// acctInit();
// adminInit();
// balanceInit();
// clusterInit();
// grantInit();
// mpeerInit();
// storageInit();
#endif
}

14
src/inc/mnode.h
View File

@ -38,6 +38,10 @@ extern "C" {
#include "ttimer.h" #include "ttimer.h"
#include "tutil.h" #include "tutil.h"
struct _vg_obj;
struct _db_obj;
struct _acctObj;
typedef struct { typedef struct {
int32_t mnodeId; int32_t mnodeId;
uint32_t privateIp; uint32_t privateIp;
@ -103,8 +107,6 @@ typedef struct {
int8_t dirty; int8_t dirty;
} STableInfo; } STableInfo;
struct _vg_obj;
typedef struct SSuperTableObj { typedef struct SSuperTableObj {
STableInfo info; STableInfo info;
uint64_t uid; uint64_t uid;
@ -137,8 +139,6 @@ typedef struct {
SSuperTableObj *superTable; SSuperTableObj *superTable;
} SChildTableObj; } SChildTableObj;
struct _db_obj;
typedef struct _vg_obj { typedef struct _vg_obj {
uint32_t vgId; uint32_t vgId;
char dbName[TSDB_DB_NAME_LEN + 1]; char dbName[TSDB_DB_NAME_LEN + 1];
@ -170,10 +170,9 @@ typedef struct _db_obj {
int32_t numOfSuperTables; int32_t numOfSuperTables;
SVgObj *pHead; SVgObj *pHead;
SVgObj *pTail; SVgObj *pTail;
struct _acctObj *pAcct;
} SDbObj; } SDbObj;
struct _acctObj;
typedef struct _user_obj { typedef struct _user_obj {
char user[TSDB_USER_LEN + 1]; char user[TSDB_USER_LEN + 1];
char pass[TSDB_KEY_LEN + 1]; char pass[TSDB_KEY_LEN + 1];
@ -213,7 +212,8 @@ typedef struct _acctObj {
SAcctCfg cfg; SAcctCfg cfg;
int32_t acctId; int32_t acctId;
int64_t createdTime; int64_t createdTime;
int8_t reserved[15]; int8_t dirty;
int8_t reserved[14];
int8_t updateEnd[1]; int8_t updateEnd[1];
SAcctInfo acctInfo; SAcctInfo acctInfo;
SDbObj * pHead; SDbObj * pHead;

4
src/mnode/CMakeLists.txt
View File

@ -14,10 +14,6 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
ADD_LIBRARY(mnode ${SRC}) ADD_LIBRARY(mnode ${SRC})
TARGET_LINK_LIBRARIES(mnode trpc tutil pthread) TARGET_LINK_LIBRARIES(mnode trpc tutil pthread)
IF (TD_CLUSTER)
TARGET_LINK_LIBRARIES(mnode)
ENDIF ()
ENDIF () ENDIF ()

23
src/mnode/inc/mgmtAcct.h
View File

@ -21,18 +21,21 @@ extern "C" {
#endif #endif
#include "mnode.h" #include "mnode.h"
int32_t mgmtInitAccts(); typedef enum {
void mgmtCleanUpAccts(); TSDB_ACCT_USER,
SAcctObj *mgmtGetAcct(char *acctName); TSDB_ACCT_DB,
TSDB_ACCT_TABLE
} EAcctGrantType;
int32_t mgmtCheckUserLimit(SAcctObj *pAcct); int32_t acctInit();
int32_t mgmtCheckDbLimit(SAcctObj *pAcct); void acctCleanUp();
int32_t mgmtCheckTableLimit(SAcctObj *pAcct, int32_t numOfTimeSeries); SAcctObj *acctGetAcct(char *acctName);
int32_t acctCheck(SAcctObj *pAcct, EAcctGrantType type);
int32_t mgmtAddDbIntoAcct(SAcctObj *pAcct, SDbObj *pDb); int32_t acctAddDb(SAcctObj *pAcct, SDbObj *pDb);
int32_t mgmtRemoveDbFromAcct(SAcctObj *pAcct, SDbObj *pDb); int32_t acctRemoveDb(SAcctObj *pAcct, SDbObj *pDb);
int32_t mgmtAddUserIntoAcct(SAcctObj *pAcct, SUserObj *pUser); int32_t acctAddUser(SAcctObj *pAcct, SUserObj *pUser);
int32_t mgmtRemoveUserFromAcct(SAcctObj *pAcct, SUserObj *pUser); int32_t acctRemoveUser(SAcctObj *pAcct, SUserObj *pUser);
#ifdef __cplusplus #ifdef __cplusplus
} }

1
src/mnode/inc/mgmtDb.h
View File

@ -28,6 +28,7 @@ void mgmtCleanUpDbs();
SDbObj *mgmtGetDb(char *db); SDbObj *mgmtGetDb(char *db);
SDbObj *mgmtGetDbByTableId(char *db); SDbObj *mgmtGetDbByTableId(char *db);
bool mgmtCheckIsMonitorDB(char *db, char *monitordb); bool mgmtCheckIsMonitorDB(char *db, char *monitordb);
void mgmtDropAllDbs(SAcctObj *pAcct);
// util func // util func
void mgmtAddSuperTableIntoDb(SDbObj *pDb); void mgmtAddSuperTableIntoDb(SDbObj *pDb);

10
src/mnode/inc/mgmtDnode.h
View File

@ -24,19 +24,9 @@ extern "C" {
int32_t mgmtInitDnodes(); int32_t mgmtInitDnodes();
void mgmtCleanUpDnodes(); void mgmtCleanUpDnodes();
int32_t mgmtGetDnodesNum(); int32_t mgmtGetDnodesNum();
int32_t mgmtUpdateDnode(SDnodeObj *pDnode);
SDnodeObj* mgmtGetDnode(int32_t dnodeId); SDnodeObj* mgmtGetDnode(int32_t dnodeId);
SDnodeObj* mgmtGetDnodeByIp(uint32_t ip); SDnodeObj* mgmtGetDnodeByIp(uint32_t ip);
bool mgmtCheckDnodeInRemoveState(SDnodeObj *pDnode);
bool mgmtCheckDnodeInOfflineState(SDnodeObj *pDnode);
bool mgmtCheckModuleInDnode(SDnodeObj *pDnode, int32_t moduleType);
void mgmtSetDnodeUnRemove(SDnodeObj *pDnode);
void mgmtSetDnodeMaxVnodes(SDnodeObj *pDnode);
void mgmtCalcNumOfFreeVnodes(SDnodeObj *pDnode);
void mgmtSetDnodeVgid(SVnodeGid vnodeGid[], int32_t numOfVnodes, int32_t vgId);
void mgmtUnSetDnodeVgid(SVnodeGid vnodeGid[], int32_t numOfVnodes);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

30
src/mnode/inc/mgmtGrant.h
View File

@ -19,14 +19,30 @@
#ifdef __cplusplus #ifdef __cplusplus
"C" { "C" {
#endif #endif
#include "mnode.h"
bool mgmtCheckExpired(); typedef enum {
void mgmtAddTimeSeries(SAcctObj *pAcct, uint32_t timeSeriesNum); TSDB_GRANT_ALL,
void mgmtRestoreTimeSeries(SAcctObj *pAcct, uint32_t timeseries); TSDB_GRANT_TIME,
int32_t mgmtCheckTimeSeries(uint32_t timeseries); TSDB_GRANT_USER,
int32_t mgmtCheckUserGrant(); TSDB_GRANT_DB,
int32_t mgmtCheckDbGrant(); TSDB_GRANT_TIMESERIES,
TSDB_GRANT_DNODE,
TSDB_GRANT_ACCT,
TSDB_GRANT_STORAGE,
TSDB_GRANT_SPEED,
TSDB_GRANT_QUERY_TIME,
TSDB_GRANT_CONNS,
TSDB_GRANT_STREAMS,
TSDB_GRANT_CPU_CORES,
} EGrantType;
int32_t grantInit();
void grantCleanUp();
void grantParseParameter();
int32_t grantCheck(EGrantType grant);
void grantReset(EGrantType grant, uint64_t value);
void grantAdd(EGrantType grant, uint64_t value);
void grantRestore(EGrantType grant, uint64_t value);
#ifdef __cplusplus #ifdef __cplusplus
} }

1
src/mnode/inc/mgmtSdb.h
View File

@ -63,6 +63,7 @@ int32_t sdbUpdateRow(SSdbOperDesc *pOper);
void *sdbGetRow(void *handle, void *key); void *sdbGetRow(void *handle, void *key);
void *sdbFetchRow(void *handle, void *pNode, void **ppRow); void *sdbFetchRow(void *handle, void *pNode, void **ppRow);
int64_t sdbGetNumOfRows(void *handle); int64_t sdbGetNumOfRows(void *handle);
int64_t sdbGetId(void *handle);
uint64_t sdbGetVersion(); uint64_t sdbGetVersion();
#ifdef __cplusplus #ifdef __cplusplus

2
src/mnode/inc/mgmtUser.h
View File

@ -25,6 +25,8 @@ int32_t mgmtInitUsers();
void mgmtCleanUpUsers(); void mgmtCleanUpUsers();
SUserObj *mgmtGetUser(char *name); SUserObj *mgmtGetUser(char *name);
SUserObj *mgmtGetUserFromConn(void *pConn, bool *usePublicIp); SUserObj *mgmtGetUserFromConn(void *pConn, bool *usePublicIp);
int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass);
void mgmtDropAllUsers(SAcctObj *pAcct);
#ifdef __cplusplus #ifdef __cplusplus
} }

77
src/mnode/src/mgmtAcct.c
View File

@ -15,21 +15,29 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "os.h" #include "os.h"
#include "taoserror.h"
#include "mnode.h"
#include "mgmtAcct.h" #include "mgmtAcct.h"
#ifndef _ACCOUNT
static SAcctObj tsAcctObj; static SAcctObj tsAcctObj = {0};
int32_t (*mgmtInitAcctsFp)() = NULL; int32_t acctInit() {
void (*mgmtCleanUpAcctsFp)() = NULL; tsAcctObj.acctId = 0;
SAcctObj *(*mgmtGetAcctFp)(char *acctName) = NULL; strcpy(tsAcctObj.user, "root");
int32_t (*mgmtCheckUserLimitFp)(SAcctObj *pAcct) = NULL; return TSDB_CODE_SUCCESS;
int32_t (*mgmtCheckDbLimitFp)(SAcctObj *pAcct) = NULL; }
int32_t (*mgmtCheckTableLimitFp)(SAcctObj *pAcct, int32_t numOfTimeSeries) = NULL;
int32_t mgmtAddDbIntoAcct(SAcctObj *pAcct, SDbObj *pDb) { void acctCleanUp() {}
SAcctObj *acctGetAcct(char *acctName) { return &tsAcctObj; }
int32_t acctCheck(SAcctObj *pAcct, EAcctGrantType type) { return TSDB_CODE_SUCCESS; }
#endif
int32_t acctAddDb(SAcctObj *pAcct, SDbObj *pDb) {
pthread_mutex_lock(&pAcct->mutex); pthread_mutex_lock(&pAcct->mutex);
pDb->next = pAcct->pHead; pDb->next = pAcct->pHead;
pDb->prev = NULL; pDb->prev = NULL;
pDb->pAcct = pAcct;
if (pAcct->pHead) { if (pAcct->pHead) {
pAcct->pHead->prev = pDb; pAcct->pHead->prev = pDb;
@ -42,7 +50,7 @@ int32_t mgmtAddDbIntoAcct(SAcctObj *pAcct, SDbObj *pDb) {
return 0; return 0;
} }
int32_t mgmtRemoveDbFromAcct(SAcctObj *pAcct, SDbObj *pDb) { int32_t acctRemoveDb(SAcctObj *pAcct, SDbObj *pDb) {
pthread_mutex_lock(&pAcct->mutex); pthread_mutex_lock(&pAcct->mutex);
if (pDb->prev) { if (pDb->prev) {
pDb->prev->next = pDb->next; pDb->prev->next = pDb->next;
@ -62,7 +70,7 @@ int32_t mgmtRemoveDbFromAcct(SAcctObj *pAcct, SDbObj *pDb) {
return 0; return 0;
} }
int32_t mgmtAddUserIntoAcct(SAcctObj *pAcct, SUserObj *pUser) { int32_t acctAddUser(SAcctObj *pAcct, SUserObj *pUser) {
pthread_mutex_lock(&pAcct->mutex); pthread_mutex_lock(&pAcct->mutex);
pUser->next = pAcct->pUser; pUser->next = pAcct->pUser;
pUser->prev = NULL; pUser->prev = NULL;
@ -79,7 +87,7 @@ int32_t mgmtAddUserIntoAcct(SAcctObj *pAcct, SUserObj *pUser) {
return 0; return 0;
} }
int32_t mgmtRemoveUserFromAcct(SAcctObj *pAcct, SUserObj *pUser) { int32_t acctRemoveUser(SAcctObj *pAcct, SUserObj *pUser) {
pthread_mutex_lock(&pAcct->mutex); pthread_mutex_lock(&pAcct->mutex);
if (pUser->prev) { if (pUser->prev) {
pUser->prev->next = pUser->next; pUser->prev->next = pUser->next;
@ -98,50 +106,3 @@ int32_t mgmtRemoveUserFromAcct(SAcctObj *pAcct, SUserObj *pUser) {
return 0; return 0;
} }
int32_t mgmtInitAccts() {
if (mgmtInitAcctsFp) {
return (*mgmtInitAcctsFp)();
} else {
tsAcctObj.acctId = 0;
strcpy(tsAcctObj.user, "root");
return 0;
}
}
SAcctObj *mgmtGetAcct(char *acctName) {
if (mgmtGetAcctFp) {
return (*mgmtGetAcctFp)(acctName);
} else {
return &tsAcctObj;
}
}
void mgmtCleanUpAccts() {
if (mgmtCleanUpAcctsFp) {
(*mgmtCleanUpAcctsFp)();
}
}
int32_t mgmtCheckUserLimit(SAcctObj *pAcct) {
if (mgmtCheckUserLimitFp) {
return (*mgmtCheckUserLimitFp)(pAcct);
}
return 0;
}
int32_t mgmtCheckDbLimit(SAcctObj *pAcct) {
if (mgmtCheckDbLimitFp) {
return (*mgmtCheckDbLimitFp)(pAcct);
} else {
return 0;
}
}
int32_t mgmtCheckTableLimit(SAcctObj *pAcct, int32_t numOfTimeSeries) {
if (mgmtCheckTableLimitFp) {
return (*mgmtCheckTableLimitFp)(pAcct, numOfTimeSeries);
} else {
return 0;
}
}

26
src/mnode/src/mgmtChildTable.c
View File

@ -48,7 +48,7 @@
#include "mgmtVgroup.h" #include "mgmtVgroup.h"
#include "mgmtUser.h" #include "mgmtUser.h"
static void *tsChildTableSdb; void *tsChildTableSdb;
static int32_t tsChildTableUpdateSize; static int32_t tsChildTableUpdateSize;
static void mgmtProcessMultiTableMetaMsg(SQueuedMsg *queueMsg); static void mgmtProcessMultiTableMetaMsg(SQueuedMsg *queueMsg);
static void mgmtProcessCreateTableRsp(SRpcMsg *rpcMsg); static void mgmtProcessCreateTableRsp(SRpcMsg *rpcMsg);
@ -84,7 +84,7 @@ static int32_t mgmtChildTableActionInsert(SSdbOperDesc *pOper) {
return TSDB_CODE_INVALID_DB; return TSDB_CODE_INVALID_DB;
} }
SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); SAcctObj *pAcct = acctGetAcct(pDb->cfg.acct);
if (pAcct == NULL) { if (pAcct == NULL) {
mError("ctable:%s, account:%s not exists", pTable->info.tableId, pDb->cfg.acct); mError("ctable:%s, account:%s not exists", pTable->info.tableId, pDb->cfg.acct);
return TSDB_CODE_INVALID_ACCT; return TSDB_CODE_INVALID_ACCT;
@ -93,9 +93,11 @@ static int32_t mgmtChildTableActionInsert(SSdbOperDesc *pOper) {
if (pTable->info.type == TSDB_CHILD_TABLE) { if (pTable->info.type == TSDB_CHILD_TABLE) {
pTable->superTable = mgmtGetSuperTable(pTable->superTableId); pTable->superTable = mgmtGetSuperTable(pTable->superTableId);
pTable->superTable->numOfTables++; pTable->superTable->numOfTables++;
mgmtAddTimeSeries(pAcct, pTable->superTable->numOfColumns - 1); grantAdd(TSDB_GRANT_TIMESERIES, pTable->superTable->numOfColumns - 1);
pAcct->acctInfo.numOfTimeSeries += (pTable->superTable->numOfColumns - 1);
} else { } else {
mgmtAddTimeSeries(pAcct, pTable->numOfColumns - 1); grantAdd(TSDB_GRANT_TIMESERIES, pTable->numOfColumns - 1);
pAcct->acctInfo.numOfTimeSeries += (pTable->numOfColumns - 1);
} }
mgmtAddTableIntoDb(pDb); mgmtAddTableIntoDb(pDb);
mgmtAddTableIntoVgroup(pVgroup, pTable); mgmtAddTableIntoVgroup(pVgroup, pTable);
@ -120,17 +122,19 @@ static int32_t mgmtChildTableActionDelete(SSdbOperDesc *pOper) {
return TSDB_CODE_INVALID_DB; return TSDB_CODE_INVALID_DB;
} }
SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); SAcctObj *pAcct = acctGetAcct(pDb->cfg.acct);
if (pAcct == NULL) { if (pAcct == NULL) {
mError("ctable:%s, account:%s not exists", pTable->info.tableId, pDb->cfg.acct); mError("ctable:%s, account:%s not exists", pTable->info.tableId, pDb->cfg.acct);
return TSDB_CODE_INVALID_ACCT; return TSDB_CODE_INVALID_ACCT;
} }
if (pTable->info.type == TSDB_CHILD_TABLE) { if (pTable->info.type == TSDB_CHILD_TABLE) {
mgmtRestoreTimeSeries(pAcct, pTable->superTable->numOfColumns - 1); grantRestore(TSDB_GRANT_TIMESERIES, pTable->superTable->numOfColumns - 1);
pAcct->acctInfo.numOfTimeSeries -= (pTable->superTable->numOfColumns - 1);
pTable->superTable->numOfTables--; pTable->superTable->numOfTables--;
} else { } else {
mgmtRestoreTimeSeries(pAcct, pTable->numOfColumns - 1); grantRestore(TSDB_GRANT_TIMESERIES, pTable->numOfColumns - 1);
pAcct->acctInfo.numOfTimeSeries -= (pTable->numOfColumns - 1);
} }
mgmtRemoveTableFromDb(pDb); mgmtRemoveTableFromDb(pDb);
mgmtRemoveTableFromVgroup(pVgroup, pTable); mgmtRemoveTableFromVgroup(pVgroup, pTable);
@ -464,9 +468,9 @@ static SChildTableObj* mgmtDoCreateChildTable(SCMCreateTableMsg *pCreate, SVgObj
void mgmtCreateChildTable(SQueuedMsg *pMsg) { void mgmtCreateChildTable(SQueuedMsg *pMsg) {
SCMCreateTableMsg *pCreate = pMsg->pCont; SCMCreateTableMsg *pCreate = pMsg->pCont;
int32_t code = mgmtCheckTimeSeries(htons(pCreate->numOfColumns)); int32_t code = grantCheck(TSDB_GRANT_TIMESERIES);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
mError("table:%s, failed to create, timeseries exceed the limit", pCreate->tableId); mError("table:%s, failed to create, grant not", pCreate->tableId);
mgmtSendSimpleResp(pMsg->thandle, code); mgmtSendSimpleResp(pMsg->thandle, code);
return; return;
} }
@ -634,7 +638,7 @@ static int32_t mgmtAddNormalTableColumn(SChildTableObj *pTable, SSchema schema[]
return TSDB_CODE_APP_ERROR; return TSDB_CODE_APP_ERROR;
} }
SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); SAcctObj *pAcct = acctGetAcct(pDb->cfg.acct);
if (pAcct == NULL) { if (pAcct == NULL) {
mError("DB: %s not belongs to andy account", pDb->name); mError("DB: %s not belongs to andy account", pDb->name);
return TSDB_CODE_APP_ERROR; return TSDB_CODE_APP_ERROR;
@ -677,7 +681,7 @@ static int32_t mgmtDropNormalTableColumnByName(SChildTableObj *pTable, char *col
return TSDB_CODE_APP_ERROR; return TSDB_CODE_APP_ERROR;
} }
SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); SAcctObj *pAcct = acctGetAcct(pDb->cfg.acct);
if (pAcct == NULL) { if (pAcct == NULL) {
mError("DB: %s not belongs to any account", pDb->name); mError("DB: %s not belongs to any account", pDb->name);
return TSDB_CODE_APP_ERROR; return TSDB_CODE_APP_ERROR;

37
src/mnode/src/mgmtDb.c
View File

@ -35,7 +35,7 @@
#include "mgmtUser.h" #include "mgmtUser.h"
#include "mgmtVgroup.h" #include "mgmtVgroup.h"
static void *tsDbSdb = NULL; void * tsDbSdb = NULL;
static int32_t tsDbUpdateSize; static int32_t tsDbUpdateSize;
static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate); static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate);
@ -54,7 +54,7 @@ static int32_t mgmtDbActionDestroy(SSdbOperDesc *pOper) {
static int32_t mgmtDbActionInsert(SSdbOperDesc *pOper) { static int32_t mgmtDbActionInsert(SSdbOperDesc *pOper) {
SDbObj *pDb = pOper->pObj; SDbObj *pDb = pOper->pObj;
SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); SAcctObj *pAcct = acctGetAcct(pDb->cfg.acct);
pDb->pHead = NULL; pDb->pHead = NULL;
pDb->pTail = NULL; pDb->pTail = NULL;
@ -65,7 +65,7 @@ static int32_t mgmtDbActionInsert(SSdbOperDesc *pOper) {
pDb->numOfSuperTables = 0; pDb->numOfSuperTables = 0;
if (pAcct != NULL) { if (pAcct != NULL) {
mgmtAddDbIntoAcct(pAcct, pDb); acctAddDb(pAcct, pDb);
} }
else { else {
mError("db:%s, acct:%s info not exist in sdb", pDb->name, pDb->cfg.acct); mError("db:%s, acct:%s info not exist in sdb", pDb->name, pDb->cfg.acct);
@ -77,9 +77,9 @@ static int32_t mgmtDbActionInsert(SSdbOperDesc *pOper) {
static int32_t mgmtDbActionDelete(SSdbOperDesc *pOper) { static int32_t mgmtDbActionDelete(SSdbOperDesc *pOper) {
SDbObj *pDb = pOper->pObj; SDbObj *pDb = pOper->pObj;
SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); SAcctObj *pAcct = acctGetAcct(pDb->cfg.acct);
mgmtRemoveDbFromAcct(pAcct, pDb); acctRemoveDb(pAcct, pDb);
mgmtDropAllChildTables(pDb); mgmtDropAllChildTables(pDb);
mgmtDropAllSuperTables(pDb); mgmtDropAllSuperTables(pDb);
mgmtDropAllVgroups(pDb); mgmtDropAllVgroups(pDb);
@ -277,7 +277,7 @@ static int32_t mgmtCheckDbParams(SCMCreateDbMsg *pCreate) {
} }
static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) { static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) {
int32_t code = mgmtCheckDbLimit(pAcct); int32_t code = acctCheck(pAcct, TSDB_ACCT_DB);
if (code != 0) { if (code != 0) {
return code; return code;
} }
@ -292,7 +292,7 @@ static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) {
assert(pCreate->daysToKeep1 <= pCreate->daysToKeep2 && pCreate->daysToKeep2 <= pCreate->daysToKeep); assert(pCreate->daysToKeep1 <= pCreate->daysToKeep2 && pCreate->daysToKeep2 <= pCreate->daysToKeep);
code = mgmtCheckDbGrant(); code = grantCheck(TSDB_GRANT_DB);
if (code != 0) { if (code != 0) {
return code; return code;
} }
@ -692,7 +692,7 @@ static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg) {
pCreate->rowsInFileBlock = htonl(pCreate->rowsInFileBlock); pCreate->rowsInFileBlock = htonl(pCreate->rowsInFileBlock);
int32_t code; int32_t code;
if (mgmtCheckExpired()) { if (grantCheck(TSDB_GRANT_TIME) != TSDB_CODE_SUCCESS) {
code = TSDB_CODE_GRANT_EXPIRED; code = TSDB_CODE_GRANT_EXPIRED;
} else if (!pMsg->pUser->writeAuth) { } else if (!pMsg->pUser->writeAuth) {
code = TSDB_CODE_NO_RIGHTS; code = TSDB_CODE_NO_RIGHTS;
@ -771,7 +771,7 @@ static void mgmtProcessAlterDbMsg(SQueuedMsg *pMsg) {
SCMAlterDbMsg *pAlter = pMsg->pCont; SCMAlterDbMsg *pAlter = pMsg->pCont;
mTrace("db:%s, alter db msg is received from thandle:%p", pAlter->db, pMsg->thandle); mTrace("db:%s, alter db msg is received from thandle:%p", pAlter->db, pMsg->thandle);
if (mgmtCheckExpired()) { if (grantCheck(TSDB_GRANT_TIME) != TSDB_CODE_SUCCESS) {
mError("db:%s, failed to alter, grant expired", pAlter->db); mError("db:%s, failed to alter, grant expired", pAlter->db);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_GRANT_EXPIRED); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_GRANT_EXPIRED);
return; return;
@ -842,7 +842,7 @@ static void mgmtProcessDropDbMsg(SQueuedMsg *pMsg) {
SCMDropDbMsg *pDrop = pMsg->pCont; SCMDropDbMsg *pDrop = pMsg->pCont;
mTrace("db:%s, drop db msg is received from thandle:%p", pDrop->db, pMsg->thandle); mTrace("db:%s, drop db msg is received from thandle:%p", pDrop->db, pMsg->thandle);
if (mgmtCheckExpired()) { if (grantCheck(TSDB_GRANT_TIME) != TSDB_CODE_SUCCESS) {
mError("db:%s, failed to drop, grant expired", pDrop->db); mError("db:%s, failed to drop, grant expired", pDrop->db);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_GRANT_EXPIRED); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_GRANT_EXPIRED);
return; return;
@ -899,3 +899,20 @@ static void mgmtProcessDropDbMsg(SQueuedMsg *pMsg) {
newMsg->ahandle = pDb; newMsg->ahandle = pDb;
taosTmrReset(mgmtDropDb, 10, newMsg, tsMgmtTmr, &tmpTmr); taosTmrReset(mgmtDropDb, 10, newMsg, tsMgmtTmr, &tmpTmr);
} }
void mgmtDropAllDbs(SAcctObj *pAcct) {
int32_t numOfDbs = 0;
SDbObj *pDb = NULL;
while (1) {
void *pNode = sdbFetchRow(tsDbSdb, pNode, (void **)&pDb);
if (pDb == NULL) break;
if (pDb->pAcct == pAcct) {
mgmtSetDbDirty(pDb);
numOfDbs++;
}
}
mTrace("acct:%s, all dbs is is set dirty", pAcct->acctId, numOfDbs);
}

436
src/mnode/src/mgmtDnode.c
View File

@ -26,371 +26,32 @@
#include "mgmtUser.h" #include "mgmtUser.h"
#include "mgmtVgroup.h" #include "mgmtVgroup.h"
int32_t (*mgmtInitDnodesFp)() = NULL;
void (*mgmtCleanUpDnodesFp)() = NULL;
SDnodeObj *(*mgmtGetDnodeFp)(uint32_t ip) = NULL;
SDnodeObj *(*mgmtGetDnodeByIpFp)(int32_t dnodeId) = NULL;
int32_t (*mgmtGetDnodesNumFp)() = NULL;
int32_t (*mgmtUpdateDnodeFp)(SDnodeObj *pDnode) = NULL;
void * (*mgmtGetNextDnodeFp)(SShowObj *pShow, SDnodeObj **pDnode) = NULL;
void (*mgmtSetDnodeUnRemoveFp)(SDnodeObj *pDnode) = NULL;
static SDnodeObj tsDnodeObj = {0};
static void * mgmtGetNextDnode(SShowObj *pShow, SDnodeObj **pDnode);
static bool mgmtCheckConfigShow(SGlobalConfig *cfg);
static int32_t mgmtGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
static int32_t mgmtRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pConn);
static int32_t mgmtGetConfigMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
static int32_t mgmtRetrieveConfigs(SShowObj *pShow, char *data, int32_t rows, void *pConn);
static int32_t mgmtGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
static int32_t mgmtRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn);
static void mgmtProcessCfgDnodeMsg(SQueuedMsg *pMsg); static void mgmtProcessCfgDnodeMsg(SQueuedMsg *pMsg);
static void mgmtProcessCfgDnodeMsgRsp(SRpcMsg *rpcMsg) ; static void mgmtProcessCfgDnodeMsgRsp(SRpcMsg *rpcMsg) ;
static void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg); static void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg);
extern int32_t clusterInit();
void mgmtSetDnodeMaxVnodes(SDnodeObj *pDnode) { extern void clusterCleanUp();
int32_t maxVnodes = pDnode->numOfCores * tsNumOfVnodesPerCore; extern int32_t clusterGetDnodesNum();
extern SDnodeObj* clusterGetDnode(int32_t dnodeId);
maxVnodes = maxVnodes > TSDB_MAX_VNODES ? TSDB_MAX_VNODES : maxVnodes; extern SDnodeObj* clusterGetDnodeByIp(uint32_t ip);
maxVnodes = maxVnodes < TSDB_MIN_VNODES ? TSDB_MIN_VNODES : maxVnodes; static SDnodeObj tsDnodeObj = {0};
if (pDnode->numOfTotalVnodes == 0) {
pDnode->numOfTotalVnodes = maxVnodes;
}
if (pDnode->alternativeRole == TSDB_DNODE_ROLE_MGMT) {
pDnode->numOfTotalVnodes = 0;
}
pDnode->openVnodes = 0;
pDnode->status = TSDB_DN_STATUS_OFFLINE;
mgmtUpdateDnode(pDnode);
}
bool mgmtCheckModuleInDnode(SDnodeObj *pDnode, int32_t moduleType) {
uint32_t status = pDnode->moduleStatus & (1 << moduleType);
return status > 0;
}
int32_t mgmtGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
int32_t cols = 0;
SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL);
if (pUser == NULL) return 0;
if (strcmp(pUser->user, "root") != 0) return TSDB_CODE_NO_RIGHTS;
SSchema *pSchema = pMeta->schema;
pShow->bytes[cols] = 16;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "IP");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 10;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "module type");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 10;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "module status");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pMeta->numOfColumns = htons(cols);
pShow->numOfColumns = cols;
pShow->offset[0] = 0;
for (int32_t i = 1; i < cols; ++i) {
pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1];
}
pShow->numOfRows = 0;
SDnodeObj *pDnode = NULL;
while (1) {
pShow->pNode = mgmtGetNextDnode(pShow, (SDnodeObj **)&pDnode);
if (pDnode == NULL) break;
for (int32_t moduleType = 0; moduleType < TSDB_MOD_MAX; ++moduleType) {
if (mgmtCheckModuleInDnode(pDnode, moduleType)) {
pShow->numOfRows++;
}
}
}
pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
pShow->pNode = NULL;
return 0;
}
int32_t mgmtRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pConn) {
int32_t numOfRows = 0;
SDnodeObj *pDnode = NULL;
char * pWrite;
int32_t cols = 0;
char ipstr[20];
while (numOfRows < rows) {
pShow->pNode = mgmtGetNextDnode(pShow, (SDnodeObj **)&pDnode);
if (pDnode == NULL) break;
for (int32_t moduleType = 0; moduleType < TSDB_MOD_MAX; ++moduleType) {
if (!mgmtCheckModuleInDnode(pDnode, moduleType)) {
continue;
}
cols = 0;
tinet_ntoa(ipstr, pDnode->privateIp);
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
strcpy(pWrite, ipstr);
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
strcpy(pWrite, tsModule[moduleType].name);
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
strcpy(pWrite, taosGetDnodeStatusStr(pDnode->status) );
cols++;
numOfRows++;
}
}
pShow->numOfReads += numOfRows;
return numOfRows;
}
static int32_t mgmtGetConfigMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
int32_t cols = 0;
SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL);
if (pUser == NULL) return 0;
if (strcmp(pUser->user, "root") != 0) return TSDB_CODE_NO_RIGHTS;
SSchema *pSchema = pMeta->schema;
pShow->bytes[cols] = TSDB_CFG_OPTION_LEN;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "config name");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = TSDB_CFG_VALUE_LEN;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "config value");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pMeta->numOfColumns = htons(cols);
pShow->numOfColumns = cols;
pShow->offset[0] = 0;
for (int32_t i = 1; i < cols; ++i) pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1];
pShow->numOfRows = 0;
for (int32_t i = tsGlobalConfigNum - 1; i >= 0; --i) {
SGlobalConfig *cfg = tsGlobalConfig + i;
if (!mgmtCheckConfigShow(cfg)) continue;
pShow->numOfRows++;
}
pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
pShow->pNode = NULL;
return 0;
}
static int32_t mgmtRetrieveConfigs(SShowObj *pShow, char *data, int32_t rows, void *pConn) {
int32_t numOfRows = 0;
for (int32_t i = tsGlobalConfigNum - 1; i >= 0 && numOfRows < rows; --i) {
SGlobalConfig *cfg = tsGlobalConfig + i;
if (!mgmtCheckConfigShow(cfg)) continue;
char *pWrite;
int32_t cols = 0;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
snprintf(pWrite, TSDB_CFG_OPTION_LEN, "%s", cfg->option);
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
switch (cfg->valType) {
case TSDB_CFG_VTYPE_SHORT:
snprintf(pWrite, TSDB_CFG_VALUE_LEN, "%d", *((int16_t *)cfg->ptr));
numOfRows++;
break;
case TSDB_CFG_VTYPE_INT:
snprintf(pWrite, TSDB_CFG_VALUE_LEN, "%d", *((int32_t *)cfg->ptr));
numOfRows++;
break;
case TSDB_CFG_VTYPE_UINT:
snprintf(pWrite, TSDB_CFG_VALUE_LEN, "%d", *((uint32_t *)cfg->ptr));
numOfRows++;
break;
case TSDB_CFG_VTYPE_FLOAT:
snprintf(pWrite, TSDB_CFG_VALUE_LEN, "%f", *((float *)cfg->ptr));
numOfRows++;
break;
case TSDB_CFG_VTYPE_STRING:
case TSDB_CFG_VTYPE_IPSTR:
case TSDB_CFG_VTYPE_DIRECTORY:
snprintf(pWrite, TSDB_CFG_VALUE_LEN, "%s", (char *)cfg->ptr);
numOfRows++;
break;
default:
break;
}
}
pShow->numOfReads += numOfRows;
return numOfRows;
}
static int32_t mgmtGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
int32_t cols = 0;
SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL);
if (pUser == NULL) return 0;
if (strcmp(pUser->user, "root") != 0) return TSDB_CODE_NO_RIGHTS;
SSchema *pSchema = pMeta->schema;
pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_INT;
strcpy(pSchema[cols].name, "vnode");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 12;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "status");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 12;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "sync_status");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pMeta->numOfColumns = htons(cols);
pShow->numOfColumns = cols;
pShow->offset[0] = 0;
for (int32_t i = 1; i < cols; ++i) pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1];
SDnodeObj *pDnode = NULL;
if (pShow->payloadLen > 0 ) {
uint32_t ip = ip2uint(pShow->payload);
pDnode = mgmtGetDnodeByIp(ip);
if (NULL == pDnode) {
return TSDB_CODE_NODE_OFFLINE;
}
SVnodeLoad* pVnode;
pShow->numOfRows = 0;
for (int32_t i = 0 ; i < TSDB_MAX_VNODES; i++) {
pVnode = &pDnode->vload[i];
if (0 != pVnode->vgId) {
pShow->numOfRows++;
}
}
pShow->pNode = pDnode;
} else {
while (true) {
pShow->pNode = mgmtGetNextDnode(pShow, (SDnodeObj **)&pDnode);
if (pDnode == NULL) break;
pShow->numOfRows += pDnode->openVnodes;
if (0 == pShow->numOfRows) return TSDB_CODE_NODE_OFFLINE;
}
pShow->pNode = NULL;
}
pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
return 0;
}
static int32_t mgmtRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn) {
int32_t numOfRows = 0;
SDnodeObj *pDnode = NULL;
char * pWrite;
int32_t cols = 0;
if (0 == rows) return 0;
if (pShow->payloadLen) {
// output the vnodes info of the designated dnode. And output all vnodes of this dnode, instead of rows (max 100)
pDnode = (SDnodeObj *)(pShow->pNode);
if (pDnode != NULL) {
SVnodeLoad* pVnode;
for (int32_t i = 0 ; i < TSDB_MAX_VNODES; i++) {
pVnode = &pDnode->vload[i];
if (0 == pVnode->vgId) {
continue;
}
cols = 0;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(uint32_t *)pWrite = pVnode->vgId;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
strcpy(pWrite, taosGetVnodeStatusStr(pVnode->status));
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
strcpy(pWrite, taosGetVnodeSyncStatusStr(pVnode->syncStatus));
cols++;
numOfRows++;
}
}
} else {
// TODO: output all vnodes of all dnodes
numOfRows = 0;
}
pShow->numOfReads += numOfRows;
return numOfRows;
}
int32_t mgmtInitDnodes() { int32_t mgmtInitDnodes() {
mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_MODULE, mgmtGetModuleMeta);
mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_MODULE, mgmtRetrieveModules);
mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_CONFIGS, mgmtGetConfigMeta);
mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_CONFIGS, mgmtRetrieveConfigs);
mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_VNODES, mgmtGetVnodeMeta);
mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_VNODES, mgmtRetrieveVnodes);
mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_CONFIG_DNODE, mgmtProcessCfgDnodeMsg); mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_CONFIG_DNODE, mgmtProcessCfgDnodeMsg);
mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_CONFIG_DNODE_RSP, mgmtProcessCfgDnodeMsgRsp); mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_CONFIG_DNODE_RSP, mgmtProcessCfgDnodeMsgRsp);
mgmtAddDServerMsgHandle(TSDB_MSG_TYPE_DM_STATUS, mgmtProcessDnodeStatusMsg); mgmtAddDServerMsgHandle(TSDB_MSG_TYPE_DM_STATUS, mgmtProcessDnodeStatusMsg);
if (mgmtInitDnodesFp) { #ifdef _CLUSTER
return mgmtInitDnodesFp(); return clusterInit();
} else { #else
tsDnodeObj.dnodeId = 1; tsDnodeObj.dnodeId = 1;
tsDnodeObj.privateIp = inet_addr(tsPrivateIp); tsDnodeObj.privateIp = inet_addr(tsPrivateIp);
tsDnodeObj.publicIp = inet_addr(tsPublicIp); tsDnodeObj.publicIp = inet_addr(tsPublicIp);
tsDnodeObj.createdTime = taosGetTimestampMs(); tsDnodeObj.createdTime = taosGetTimestampMs();
tsDnodeObj.numOfTotalVnodes = tsNumOfTotalVnodes; tsDnodeObj.numOfTotalVnodes = tsNumOfTotalVnodes;
tsDnodeObj.numOfCores = (uint16_t) tsNumOfCores;
tsDnodeObj.alternativeRole = TSDB_DNODE_ROLE_ANY;
tsDnodeObj.status = TSDB_DN_STATUS_OFFLINE; tsDnodeObj.status = TSDB_DN_STATUS_OFFLINE;
tsDnodeObj.lastReboot = taosGetTimestampSec(); tsDnodeObj.lastReboot = taosGetTimestampSec();
sprintf(tsDnodeObj.dnodeName, "%d", tsDnodeObj.dnodeId); sprintf(tsDnodeObj.dnodeName, "%d", tsDnodeObj.dnodeId);
mgmtSetDnodeMaxVnodes(&tsDnodeObj);
tsDnodeObj.moduleStatus |= (1 << TSDB_MOD_MGMT); tsDnodeObj.moduleStatus |= (1 << TSDB_MOD_MGMT);
if (tsEnableHttpModule) { if (tsEnableHttpModule) {
@ -400,80 +61,41 @@ int32_t mgmtInitDnodes() {
tsDnodeObj.moduleStatus |= (1 << TSDB_MOD_MONITOR); tsDnodeObj.moduleStatus |= (1 << TSDB_MOD_MONITOR);
} }
return 0; return 0;
} #endif
} }
void mgmtCleanUpDnodes() { void mgmtCleanUpDnodes() {
if (mgmtCleanUpDnodesFp) { #ifdef _CLUSTER
(*mgmtCleanUpDnodesFp)(); clusterCleanUp();
} #endif
} }
SDnodeObj *mgmtGetDnode(int32_t dnodeId) { SDnodeObj *mgmtGetDnode(int32_t dnodeId) {
if (mgmtGetDnodeFp) { #ifdef _CLUSTER
return (*mgmtGetDnodeFp)(dnodeId); return clusterGetDnode(dnodeId);
} #else
if (dnodeId == 1) { if (dnodeId == 1) {
return &tsDnodeObj; return &tsDnodeObj;
} } else {
return NULL; return NULL;
}
#endif
} }
SDnodeObj *mgmtGetDnodeByIp(uint32_t ip) { SDnodeObj *mgmtGetDnodeByIp(uint32_t ip) {
if (mgmtGetDnodeByIpFp) { #ifdef _CLUSTER
return (*mgmtGetDnodeByIpFp)(ip); return clusterGetDnodeByIp(ip);
} #else
return &tsDnodeObj; return &tsDnodeObj;
#endif
} }
int32_t mgmtGetDnodesNum() { int32_t mgmtGetDnodesNum() {
if (mgmtGetDnodesNumFp) { #ifdef _CLUSTER
return (*mgmtGetDnodesNumFp)(); return clusterGetDnodesNum();
} else { #else
return 1; return 1;
} #endif
}
int32_t mgmtUpdateDnode(SDnodeObj *pDnode) {
if (mgmtUpdateDnodeFp) {
return (*mgmtUpdateDnodeFp)(pDnode);
} else {
return 0;
}
}
void *mgmtGetNextDnode(SShowObj *pShow, SDnodeObj **pDnode) {
if (mgmtGetNextDnodeFp) {
return (*mgmtGetNextDnodeFp)(pShow, pDnode);
} else {
if (*pDnode == NULL) {
*pDnode = &tsDnodeObj;
} else {
*pDnode = NULL;
}
}
return *pDnode;
}
void mgmtSetDnodeUnRemove(SDnodeObj *pDnode) {
if (mgmtSetDnodeUnRemoveFp) {
(*mgmtSetDnodeUnRemoveFp)(pDnode);
}
}
bool mgmtCheckConfigShow(SGlobalConfig *cfg) {
if (!(cfg->cfgType & TSDB_CFG_CTYPE_B_SHOW))
return false;
return true;
}
bool mgmtCheckDnodeInRemoveState(SDnodeObj *pDnode) {
return pDnode->lbStatus == TSDB_DN_LB_STATUS_OFFLINE_REMOVING || pDnode->lbStatus == TSDB_DN_LB_STATE_SHELL_REMOVING;
}
bool mgmtCheckDnodeInOfflineState(SDnodeObj *pDnode) {
return pDnode->status == TSDB_DN_STATUS_OFFLINE;
} }
void mgmtProcessCfgDnodeMsg(SQueuedMsg *pMsg) { void mgmtProcessCfgDnodeMsg(SQueuedMsg *pMsg) {
@ -553,14 +175,10 @@ void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) {
pDnode->numOfCores = htons(pStatus->numOfCores); pDnode->numOfCores = htons(pStatus->numOfCores);
pDnode->diskAvailable = pStatus->diskAvailable; pDnode->diskAvailable = pStatus->diskAvailable;
pDnode->alternativeRole = pStatus->alternativeRole; pDnode->alternativeRole = pStatus->alternativeRole;
if (pDnode->numOfTotalVnodes == 0) {
pDnode->numOfTotalVnodes = htons(pStatus->numOfTotalVnodes); pDnode->numOfTotalVnodes = htons(pStatus->numOfTotalVnodes);
}
if (pStatus->dnodeId == 0) { if (pStatus->dnodeId == 0) {
mTrace("dnode:%d, first access, privateIp:%s, name:%s, ", pDnode->dnodeId, taosIpStr(pDnode->privateIp), pDnode->dnodeName); mTrace("dnode:%d, first access, privateIp:%s, name:%s, ", pDnode->dnodeId, taosIpStr(pDnode->privateIp), pDnode->dnodeName);
mgmtSetDnodeMaxVnodes(pDnode);
} }
int32_t openVnodes = htons(pStatus->openVnodes); int32_t openVnodes = htons(pStatus->openVnodes);

64
src/mnode/src/mgmtGrant.c
View File

@ -14,58 +14,18 @@
*/ */
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#ifndef _GRANT
#include "os.h" #include "os.h"
#include "mgmtAcct.h" #include "taoserror.h"
#include "tlog.h"
#include "mgmtGrant.h"
int32_t (*mgmtCheckUserGrantFp)() = NULL; int32_t grantInit() { return TSDB_CODE_SUCCESS; }
int32_t (*mgmtCheckDbGrantFp)() = NULL; void grantCleanUp() {}
void (*mgmtAddTimeSeriesFp)(uint32_t timeSeriesNum) = NULL; void grantParseParameter() { mError("can't parsed parameter k"); }
void (*mgmtRestoreTimeSeriesFp)(uint32_t timeSeriesNum) = NULL; int32_t grantCheck(EGrantType grant) { return TSDB_CODE_SUCCESS; }
int32_t (*mgmtCheckTimeSeriesFp)(uint32_t timeseries) = NULL; void grantReset(EGrantType grant, uint64_t value) {}
bool (*mgmtCheckExpiredFp)() = NULL; void grantAdd(EGrantType grant, uint64_t value) {}
void grantRestore(EGrantType grant, uint64_t value) {}
int32_t mgmtCheckUserGrant() { #endif
if (mgmtCheckUserGrantFp) {
return (*mgmtCheckUserGrantFp)();
} else {
return 0;
}
}
int32_t mgmtCheckDbGrant() {
if (mgmtCheckDbGrantFp) {
return (*mgmtCheckDbGrantFp)();
} else {
return 0;
}
}
void mgmtAddTimeSeries(SAcctObj *pAcct, uint32_t timeSeriesNum) {
pAcct->acctInfo.numOfTimeSeries += timeSeriesNum;
if (mgmtAddTimeSeriesFp) {
(*mgmtAddTimeSeriesFp)(timeSeriesNum);
}
}
void mgmtRestoreTimeSeries(SAcctObj *pAcct, uint32_t timeSeriesNum) {
pAcct->acctInfo.numOfTimeSeries -= timeSeriesNum;
if (mgmtRestoreTimeSeriesFp) {
(*mgmtRestoreTimeSeriesFp)(timeSeriesNum);
}
}
int32_t mgmtCheckTimeSeries(uint32_t timeseries) {
if (mgmtCheckTimeSeriesFp) {
return (*mgmtCheckTimeSeriesFp)(timeseries);
} else {
return 0;
}
}
bool mgmtCheckExpired() {
if (mgmtCheckExpiredFp) {
return mgmtCheckExpiredFp();
} else {
return false;
}
}

11
src/mnode/src/mgmtMain.c
View File

@ -25,6 +25,7 @@
#include "mgmtDClient.h" #include "mgmtDClient.h"
#include "mgmtDnode.h" #include "mgmtDnode.h"
#include "mgmtDServer.h" #include "mgmtDServer.h"
#include "mgmtGrant.h"
#include "mgmtMnode.h" #include "mgmtMnode.h"
#include "mgmtSdb.h" #include "mgmtSdb.h"
#include "mgmtVgroup.h" #include "mgmtVgroup.h"
@ -73,11 +74,16 @@ int32_t mgmtStartSystem() {
return -1; return -1;
} }
if (mgmtInitAccts() < 0) { if (acctInit() < 0) {
mError("failed to init accts"); mError("failed to init accts");
return -1; return -1;
} }
if (grantInit() < 0) {
mError("failed to init grants");
return -1;
}
if (mgmtInitUsers() < 0) { if (mgmtInitUsers() < 0) {
mError("failed to init users"); mError("failed to init users");
return -1; return -1;
@ -138,6 +144,7 @@ void mgmtStopSystem() {
void mgmtCleanUpSystem() { void mgmtCleanUpSystem() {
mPrint("starting to clean up mgmt"); mPrint("starting to clean up mgmt");
grantCleanUp();
mgmtCleanupMnodes(); mgmtCleanupMnodes();
mgmtCleanupBalance(); mgmtCleanupBalance();
mgmtCleanUpShell(); mgmtCleanUpShell();
@ -148,7 +155,7 @@ void mgmtCleanUpSystem() {
mgmtCleanUpDbs(); mgmtCleanUpDbs();
mgmtCleanUpDnodes(); mgmtCleanUpDnodes();
mgmtCleanUpUsers(); mgmtCleanUpUsers();
mgmtCleanUpAccts(); acctCleanUp();
taosTmrCleanUp(tsMgmtTmr); taosTmrCleanUp(tsMgmtTmr);
mPrint("mgmt is cleaned up"); mPrint("mgmt is cleaned up");
} }

6
src/mnode/src/mgmtShell.c
View File

@ -137,7 +137,7 @@ static void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg) {
return; return;
} }
if (mgmtCheckExpired()) { if (grantCheck(TSDB_GRANT_TIME) != TSDB_CODE_SUCCESS) {
mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_GRANT_EXPIRED); mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_GRANT_EXPIRED);
return; return;
} }
@ -373,12 +373,12 @@ static void mgmtProcessConnectMsg(SQueuedMsg *pMsg) {
goto connect_over; goto connect_over;
} }
if (mgmtCheckExpired()) { if (grantCheck(TSDB_GRANT_TIME) != TSDB_CODE_SUCCESS) {
code = TSDB_CODE_GRANT_EXPIRED; code = TSDB_CODE_GRANT_EXPIRED;
goto connect_over; goto connect_over;
} }
SAcctObj *pAcct = mgmtGetAcct(pUser->acct); SAcctObj *pAcct = acctGetAcct(pUser->acct);
if (pAcct == NULL) { if (pAcct == NULL) {
code = TSDB_CODE_INVALID_ACCT; code = TSDB_CODE_INVALID_ACCT;
goto connect_over; goto connect_over;

8
src/mnode/src/mgmtSuperTable.c
View File

@ -253,7 +253,7 @@ static int32_t mgmtAddSuperTableTag(SSuperTableObj *pStable, SSchema schema[], i
return TSDB_CODE_APP_ERROR; return TSDB_CODE_APP_ERROR;
} }
SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); SAcctObj *pAcct = acctGetAcct(pDb->cfg.acct);
if (pAcct == NULL) { if (pAcct == NULL) {
mError("DB: %s not belongs to andy account", pDb->name); mError("DB: %s not belongs to andy account", pDb->name);
return TSDB_CODE_APP_ERROR; return TSDB_CODE_APP_ERROR;
@ -293,7 +293,7 @@ static int32_t mgmtDropSuperTableTag(SSuperTableObj *pStable, char *tagName) {
return TSDB_CODE_APP_ERROR; return TSDB_CODE_APP_ERROR;
} }
SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); SAcctObj *pAcct = acctGetAcct(pDb->cfg.acct);
if (pAcct == NULL) { if (pAcct == NULL) {
mError("DB: %s not belongs to any account", pDb->name); mError("DB: %s not belongs to any account", pDb->name);
return TSDB_CODE_APP_ERROR; return TSDB_CODE_APP_ERROR;
@ -381,7 +381,7 @@ static int32_t mgmtAddSuperTableColumn(SSuperTableObj *pStable, SSchema schema[]
return TSDB_CODE_APP_ERROR; return TSDB_CODE_APP_ERROR;
} }
SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); SAcctObj *pAcct = acctGetAcct(pDb->cfg.acct);
if (pAcct == NULL) { if (pAcct == NULL) {
mError("DB: %s not belongs to andy account", pDb->name); mError("DB: %s not belongs to andy account", pDb->name);
return TSDB_CODE_APP_ERROR; return TSDB_CODE_APP_ERROR;
@ -420,7 +420,7 @@ static int32_t mgmtDropSuperTableColumnByName(SSuperTableObj *pStable, char *col
return TSDB_CODE_APP_ERROR; return TSDB_CODE_APP_ERROR;
} }
SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); SAcctObj *pAcct = acctGetAcct(pDb->cfg.acct);
if (pAcct == NULL) { if (pAcct == NULL) {
mError("DB: %s not belongs to any account", pDb->name); mError("DB: %s not belongs to any account", pDb->name);
return TSDB_CODE_APP_ERROR; return TSDB_CODE_APP_ERROR;

48
src/mnode/src/mgmtUser.c
View File

@ -25,10 +25,9 @@
#include "mgmtShell.h" #include "mgmtShell.h"
#include "mgmtUser.h" #include "mgmtUser.h"
static void *tsUserSdb = NULL; void * tsUserSdb = NULL;
static int32_t tsUserUpdateSize = 0; static int32_t tsUserUpdateSize = 0;
static int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass);
static int32_t mgmtDropUser(SAcctObj *pAcct, char *name); static int32_t mgmtDropUser(SAcctObj *pAcct, char *name);
static int32_t mgmtUpdateUser(SUserObj *pUser); static int32_t mgmtUpdateUser(SUserObj *pUser);
static int32_t mgmtGetUserMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mgmtGetUserMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
@ -45,10 +44,10 @@ static int32_t mgmtUserActionDestroy(SSdbOperDesc *pOper) {
static int32_t mgmtUserActionInsert(SSdbOperDesc *pOper) { static int32_t mgmtUserActionInsert(SSdbOperDesc *pOper) {
SUserObj *pUser = pOper->pObj; SUserObj *pUser = pOper->pObj;
SAcctObj *pAcct = mgmtGetAcct(pUser->acct); SAcctObj *pAcct = acctGetAcct(pUser->acct);
if (pAcct != NULL) { if (pAcct != NULL) {
mgmtAddUserIntoAcct(pAcct, pUser); acctAddUser(pAcct, pUser);
} }
else { else {
mError("user:%s, acct:%s info not exist in sdb", pUser->user, pUser->acct); mError("user:%s, acct:%s info not exist in sdb", pUser->user, pUser->acct);
@ -60,9 +59,9 @@ static int32_t mgmtUserActionInsert(SSdbOperDesc *pOper) {
static int32_t mgmtUserActionDelete(SSdbOperDesc *pOper) { static int32_t mgmtUserActionDelete(SSdbOperDesc *pOper) {
SUserObj *pUser = pOper->pObj; SUserObj *pUser = pOper->pObj;
SAcctObj *pAcct = mgmtGetAcct(pUser->acct); SAcctObj *pAcct = acctGetAcct(pUser->acct);
mgmtRemoveUserFromAcct(pAcct, pUser); acctRemoveUser(pAcct, pUser);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -115,7 +114,7 @@ int32_t mgmtInitUsers() {
return -1; return -1;
} }
SAcctObj *pAcct = mgmtGetAcct("root"); SAcctObj *pAcct = acctGetAcct("root");
mgmtCreateUser(pAcct, "root", "taosdata"); mgmtCreateUser(pAcct, "root", "taosdata");
mgmtCreateUser(pAcct, "monitor", tsInternalPass); mgmtCreateUser(pAcct, "monitor", tsInternalPass);
mgmtCreateUser(pAcct, "_root", tsInternalPass); mgmtCreateUser(pAcct, "_root", tsInternalPass);
@ -155,8 +154,8 @@ static int32_t mgmtUpdateUser(SUserObj *pUser) {
return code; return code;
} }
static int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass) { int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass) {
int32_t code = mgmtCheckUserLimit(pAcct); int32_t code = acctCheck(pAcct, TSDB_ACCT_USER);
if (code != 0) { if (code != 0) {
return code; return code;
} }
@ -171,8 +170,8 @@ static int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass) {
return TSDB_CODE_USER_ALREADY_EXIST; return TSDB_CODE_USER_ALREADY_EXIST;
} }
code = mgmtCheckUserGrant(); code = grantCheck(TSDB_GRANT_USER);
if (code != 0) { if (code != TSDB_CODE_SUCCESS) {
return code; return code;
} }
@ -482,3 +481,30 @@ static void mgmtProcessDropUserMsg(SQueuedMsg *pMsg) {
mgmtSendSimpleResp(pMsg->thandle, code); mgmtSendSimpleResp(pMsg->thandle, code);
} }
void mgmtDropAllUsers(SAcctObj *pAcct) {
void *pNode = NULL;
void *pLastNode = NULL;
int32_t numOfUsers = 0;
int32_t acctNameLen = strlen(pAcct->user);
SUserObj *pUser = NULL;
while (1) {
pNode = sdbFetchRow(tsUserSdb, pNode, (void **)&pUser);
if (pUser == NULL) break;
if (strncmp(pUser->acct, pAcct->user, acctNameLen) == 0) {
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_LOCAL,
.table = tsUserSdb,
.pObj = pUser,
};
sdbDeleteRow(&oper);
pNode = pLastNode;
numOfUsers++;
continue;
}
}
mTrace("acct:%s, all users is dropped from sdb", pAcct->acctId, numOfUsers);
}

2
src/mnode/src/mgmtVgroup.c
View File

@ -32,7 +32,7 @@
#include "mgmtTable.h" #include "mgmtTable.h"
#include "mgmtVgroup.h" #include "mgmtVgroup.h"
static void *tsVgroupSdb = NULL; void *tsVgroupSdb = NULL;
static int32_t tsVgUpdateSize = 0; static int32_t tsVgUpdateSize = 0;
static int32_t mgmtGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mgmtGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);

5
src/util/CMakeLists.txt
View File

@ -9,10 +9,6 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
AUX_SOURCE_DIRECTORY(src SRC) AUX_SOURCE_DIRECTORY(src SRC)
ADD_LIBRARY(tutil ${SRC}) ADD_LIBRARY(tutil ${SRC})
TARGET_LINK_LIBRARIES(tutil thirdparty pthread os m rt) TARGET_LINK_LIBRARIES(tutil thirdparty pthread os m rt)
IF (TD_CLUSTER)
ADD_DEFINITIONS(-DUSE_LIBICONV)
TARGET_LINK_LIBRARIES(tutil iconv)
ELSE()
FIND_PATH(ICONV_INCLUDE_EXIST iconv.h /usr/include/ /usr/local/include/) FIND_PATH(ICONV_INCLUDE_EXIST iconv.h /usr/include/ /usr/local/include/)
IF (ICONV_INCLUDE_EXIST) IF (ICONV_INCLUDE_EXIST)
ADD_DEFINITIONS(-DUSE_LIBICONV) ADD_DEFINITIONS(-DUSE_LIBICONV)
@ -28,7 +24,6 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
ELSE () ELSE ()
MESSAGE(STATUS "Failed to find iconv, use default encoding method") MESSAGE(STATUS "Failed to find iconv, use default encoding method")
ENDIF () ENDIF ()
ENDIF ()
ADD_SUBDIRECTORY(tests) ADD_SUBDIRECTORY(tests)
ELSEIF (TD_WINDOWS_64) ELSEIF (TD_WINDOWS_64)

14
src/util/src/tglobalcfg.c
View File

@ -81,7 +81,7 @@ float tsRatioOfQueryThreads = 0.5;
char tsPublicIp[TSDB_IPv4ADDR_LEN] = {0}; char tsPublicIp[TSDB_IPv4ADDR_LEN] = {0};
char tsPrivateIp[TSDB_IPv4ADDR_LEN] = {0}; char tsPrivateIp[TSDB_IPv4ADDR_LEN] = {0};
short tsNumOfVnodesPerCore = 8; short tsNumOfVnodesPerCore = 8;
short tsNumOfTotalVnodes = 0; short tsNumOfTotalVnodes = TSDB_INVALID_VNODE_NUM;
short tsCheckHeaderFile = 0; short tsCheckHeaderFile = 0;
#ifdef _TD_ARM_32_ #ifdef _TD_ARM_32_
@ -189,7 +189,7 @@ int tsEnableCoreFile = 0;
int tsAnyIp = 1; int tsAnyIp = 1;
uint32_t tsPublicIpInt = 0; uint32_t tsPublicIpInt = 0;
#ifdef CLUSTER #ifdef _CLUSTER
int tsIsCluster = 1; int tsIsCluster = 1;
#else #else
int tsIsCluster = 0; int tsIsCluster = 0;
@ -960,6 +960,12 @@ bool tsReadGlobalConfig() {
tsNumOfCores = 1; tsNumOfCores = 1;
} }
if (tsNumOfTotalVnodes == TSDB_INVALID_VNODE_NUM) {
tsNumOfTotalVnodes = tsNumOfCores * tsNumOfVnodesPerCore;
tsNumOfTotalVnodes = tsNumOfTotalVnodes > TSDB_MAX_VNODES ? TSDB_MAX_VNODES : tsNumOfTotalVnodes;
tsNumOfTotalVnodes = tsNumOfTotalVnodes < TSDB_MIN_VNODES ? TSDB_MIN_VNODES : tsNumOfTotalVnodes;
}
if (strlen(tsPrivateIp) == 0) { if (strlen(tsPrivateIp) == 0) {
pError("privateIp is null"); pError("privateIp is null");
return false; return false;
@ -1052,12 +1058,12 @@ void tsPrintGlobalConfig() {
if (tscEmbedded == 0 && !(cfg->cfgType & TSDB_CFG_CTYPE_B_CLIENT)) continue; if (tscEmbedded == 0 && !(cfg->cfgType & TSDB_CFG_CTYPE_B_CLIENT)) continue;
if (cfg->cfgType & TSDB_CFG_CTYPE_B_NOT_PRINT) continue; if (cfg->cfgType & TSDB_CFG_CTYPE_B_NOT_PRINT) continue;
if (cfg->cfgType & TSDB_CFG_CTYPE_B_LITE) { if (cfg->cfgType & TSDB_CFG_CTYPE_B_LITE) {
#ifdef CLUSTER #ifdef _CLUSTER
continue; continue;
#endif #endif
} }
if (cfg->cfgType & TSDB_CFG_CTYPE_B_CLUSTER) { if (cfg->cfgType & TSDB_CFG_CTYPE_B_CLUSTER) {
#ifndef CLUSTER #ifndef _CLUSTER
continue; continue;
#endif #endif
} }

6
src/util/src/version.c
View File

@ -1,7 +1,7 @@
char version[64] = "1.6.5.4"; char version[64] = "2.0.0.0";
char compatible_version[64] = "1.6.1.0"; char compatible_version[64] = "2.0.0.0";
char gitinfo[128] = "3264067e97300c84caa61ac909d548c9ca56de6b"; char gitinfo[128] = "3264067e97300c84caa61ac909d548c9ca56de6b";
char gitinfoOfInternal[128] = "da88f4a2474737d1f9c76adcf0ff7fd0975e7342"; char gitinfoOfInternal[128] = "da88f4a2474737d1f9c76adcf0ff7fd0975e7342";
char buildinfo[512] = "Built by root at 2020-02-05 14:38"; char buildinfo[512] = "Built by root at 2020-04-01 14:38";
void libtaos_1_6_5_4_Linux_x64() {}; void libtaos_1_6_5_4_Linux_x64() {};

25
src/vnode/detail/CMakeLists.txt
View File

@ -1,25 +0,0 @@
CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
PROJECT(TDengine)
IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/dnode/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/mnode/detail/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/vnode/detail/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc)
INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
INCLUDE_DIRECTORIES(inc)
AUX_SOURCE_DIRECTORY(./src SRC)
LIST(REMOVE_ITEM SRC ./src/vnodeFileUtil.c)
LIST(REMOVE_ITEM SRC ./src/taosGrant.c)
ADD_LIBRARY(vnode ${SRC})
IF (TD_CLUSTER)
TARGET_LINK_LIBRARIES(vnode vcluster)
ELSEIF (TD_LITE)
TARGET_LINK_LIBRARIES(vnode vlite)
ENDIF ()
ENDIF ()

574
src/vnode/detail/inc/vnode.h
View File

@ -1,574 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_VNODE_H
#define TDENGINE_VNODE_H
#ifdef __cplusplus
extern "C" {
#endif
#include "os.h"
#include "tglobalcfg.h"
#include "tidpool.h"
#include "tlog.h"
#include "tmempool.h"
#include "trpc.h"
#include "tsclient.h"
#include "taosdef.h"
#include "tsocket.h"
#include "ttime.h"
#include "ttimer.h"
#include "tutil.h"
#include "vnodeCache.h"
#include "vnodeFile.h"
#include "vnodePeer.h"
#include "vnodeShell.h"
#define TSDB_FILE_HEADER_LEN 512
#define TSDB_FILE_HEADER_VERSION_SIZE 32
#define TSDB_CACHE_POS_BITS 13
#define TSDB_CACHE_POS_MASK 0x1FFF
#define TSDB_ACTION_INSERT 0
#define TSDB_ACTION_IMPORT 1
#define TSDB_ACTION_DELETE 2
#define TSDB_ACTION_UPDATE 3
#define TSDB_ACTION_MAX 4
enum _data_source {
TSDB_DATA_SOURCE_METER,
TSDB_DATA_SOURCE_VNODE,
TSDB_DATA_SOURCE_SHELL,
TSDB_DATA_SOURCE_QUEUE,
TSDB_DATA_SOURCE_LOG,
};
enum _sync_cmd {
TSDB_SYNC_CMD_FILE,
TSDB_SYNC_CMD_CACHE,
TSDB_SYNC_CMD_CREATE,
TSDB_SYNC_CMD_REMOVE,
};
typedef struct {
int64_t offset : 48;
int64_t length : 16;
} SMeterObjHeader;
typedef struct {
int64_t len;
char data[];
} SData;
#pragma pack(push, 8)
typedef struct {
SVnodeStatisticInfo vnodeStatistic;
int vnode;
SVnodeCfg cfg;
// SDiskDesc tierDisk[TSDB_MAX_TIER];
SVPeerDesc vpeers[TSDB_VNODES_SUPPORT];
SVnodePeer * peerInfo[TSDB_VNODES_SUPPORT];
char selfIndex;
char vnodeStatus;
char accessState; // Vnode access state, Readable/Writable
char syncStatus;
char commitInProcess;
pthread_t commitThread;
TSKEY firstKey; // minimum key uncommitted, it may be smaller than
// commitFirstKey
TSKEY commitFirstKey; // minimum key for a commit file, it shall be
// xxxx00000, calculated from fileId
TSKEY commitLastKey; // maximum key for a commit file, it shall be xxxx99999,
// calculated fromm fileId
int commitFileId;
TSKEY lastCreate;
TSKEY lastRemove;
TSKEY lastKey; // last key for the whole vnode, updated by every insert
// operation
uint64_t version;
int streamRole;
int numOfStreams;
void *streamTimer;
TSKEY lastKeyOnFile; // maximum key on the last file, is shall be xxxx99999
int fileId;
int badFileId;
int numOfFiles;
int maxFiles;
int maxFile1;
int maxFile2;
int nfd; // temp head file FD
int hfd; // head file FD
int lfd; // last file FD
int tfd; // temp last file FD
int dfd; // data file FD
int64_t dfSize;
int64_t lfSize;
uint64_t * fmagic; // hold magic number for each file
char cfn[TSDB_FILENAME_LEN];
char nfn[TSDB_FILENAME_LEN];
char lfn[TSDB_FILENAME_LEN]; // last file name
char tfn[TSDB_FILENAME_LEN]; // temp last file name
pthread_mutex_t vmutex;
int logFd;
char * pMem;
char * pWrite;
pthread_mutex_t logMutex;
char logFn[TSDB_FILENAME_LEN];
char logOFn[TSDB_FILENAME_LEN];
int64_t mappingSize;
int64_t mappingThreshold;
void * commitTimer;
void ** meterList;
void * pCachePool;
void * pQueue;
pthread_t thread;
int peersOnline;
int shellConns;
int meterConns;
struct _qinfo *pQInfoList;
TAOS * dbConn;
SMeterObjHeader *meterIndex;
} SVnodeObj;
#pragma pack(pop)
typedef struct SColumn {
short colId;
short bytes;
char type;
} SColumn;
typedef struct _meter_obj {
uint64_t uid;
char meterId[TSDB_TABLE_ID_LEN];
int sid;
short vnode;
short numOfColumns;
short bytesPerPoint;
short maxBytes;
int32_t pointsPerBlock;
int32_t pointsPerFileBlock;
int freePoints;
TSKEY lastKey; // updated by insert operation
TSKEY lastKeyOnFile; // last key on file, updated by commit action
TSKEY timeStamp; // delete or added time
uint64_t commitCount;
int32_t sversion;
short sqlLen;
char searchAlgorithm : 4;
char compAlgorithm : 4;
char status; // 0: ok, 1: stop stream computing
char reserved[16];
int state;
int numOfQueries;
char * pSql;
void * pStream;
void * pCache;
SColumn *schema;
} SMeterObj;
typedef struct {
char type;
char pversion; // protocol version
char action; // insert, import, delete, update
int32_t sversion; // only for insert
int32_t sid;
int32_t len;
uint64_t lastVersion; // latest version
char cont[];
} SVMsgHeader;
struct tSQLBinaryExpr;
typedef struct SColumnInfoEx {
SColumnInfo data;
int16_t colIdx;
int16_t colIdxInBuf;
/*
* 0: denotes if its is required in the first round of scan of data block
* 1: denotes if its is required in the secondary scan
*/
int16_t req[2];
} SColumnInfoEx;
struct SColumnFilterElem;
typedef bool (*__filter_func_t)(struct SColumnFilterElem *pFilter, char *val1, char *val2);
typedef struct SColumnFilterElem {
int16_t bytes; // column length
__filter_func_t fp;
SColumnFilterInfo filterInfo;
} SColumnFilterElem;
typedef struct SSingleColumnFilterInfo {
SColumnInfoEx info;
int32_t numOfFilters;
SColumnFilterElem *pFilters;
char * pData;
} SSingleColumnFilterInfo;
typedef struct SQuery {
short numOfCols;
SOrderVal order;
char keyIsMet; // if key is met, it will be set
char over;
int fileId; // only for query in file
int hfd; // only for query in file, head file handle
int dfd; // only for query in file, data file handle
int lfd; // only for query in file, last file handle
SCompBlock *pBlock; // only for query in file
SField ** pFields;
int numOfBlocks; // only for query in file
int blockBufferSize; // length of pBlock buffer
int currentSlot;
int firstSlot;
/*
* the two parameters are utilized to handle the data missing situation, caused by import operation.
* When the commit slot is the first slot, and commitPoints != 0
*/
int32_t commitSlot; // which slot is committed,
int32_t commitPoint; // starting point for next commit
int slot;
int pos;
TSKEY key;
int compBlockLen; // only for import
int64_t blockId;
TSKEY skey;
TSKEY ekey;
int64_t intervalTime;
int64_t slidingTime; // sliding time for sliding window query
char intervalTimeUnit; // interval data type, used for daytime revise
int8_t precision;
int16_t numOfOutputCols;
int16_t interpoType;
int16_t checkBufferInLoop; // check if the buffer is full during scan each block
SLimitVal limit;
int32_t rowSize;
SSqlGroupbyExpr * pGroupbyExpr;
SSqlFunctionExpr * pSelectExpr;
SColumnInfoEx * colList;
int32_t numOfFilterCols;
SSingleColumnFilterInfo *pFilterInfo;
int64_t * defaultVal;
TSKEY lastKey;
// buffer info
int64_t pointsRead; // the number of points returned
int64_t pointsToRead; // maximum number of points to read
int64_t pointsOffset; // the number of points offset to save read data
SData **sdata;
SData * tsData; // timestamp column/primary key column
} SQuery;
typedef struct {
char spi;
char encrypt;
char secret[TSDB_KEY_LEN];
char cipheringKey[TSDB_KEY_LEN];
} SConnSec;
typedef struct {
char * buffer;
char * offset;
int trans;
int bufferSize;
pthread_mutex_t qmutex;
} STranQueue;
// internal globals
extern int tsMeterSizeOnFile;
extern void * tsQueryQhandle;
extern int tsVnodePeers;
extern int tsMaxVnode;
extern int tsMaxQueues;
extern int tsOpenVnodes;
extern SVnodeObj *vnodeList;
extern void * vnodeTmrCtrl;
// read API
extern int (*vnodeSearchKeyFunc[])(char *pValue, int num, TSKEY key, int order);
void *vnodeQueryOnSingleTable(SMeterObj **pMeterObj, SSqlGroupbyExpr *pGroupbyExpr, SSqlFunctionExpr *sqlExprs,
SQueryMeterMsg *pQueryMsg, int *code);
void *vnodeQueryOnMultiMeters(SMeterObj **pMeterObj, SSqlGroupbyExpr *pGroupbyExpr, SSqlFunctionExpr *pSqlExprs,
SQueryMeterMsg *pQueryMsg, int *code);
// assistant/tool functions
SSqlGroupbyExpr *vnodeCreateGroupbyExpr(SQueryMeterMsg *pQuery, int32_t *code);
SSqlFunctionExpr *vnodeCreateSqlFunctionExpr(SQueryMeterMsg *pQuery, int32_t *code);
bool vnodeValidateExprColumnInfo(SQueryMeterMsg *pQueryMsg, SSqlFuncExprMsg *pExprMsg);
bool vnodeIsValidVnodeCfg(SVnodeCfg *pCfg);
int32_t vnodeGetResultSize(void *handle, int32_t *numOfRows);
int32_t vnodeCopyQueryResultToMsg(void *handle, char *data, int32_t numOfRows);
int64_t vnodeGetOffsetVal(void *thandle);
bool vnodeHasRemainResults(void *handle);
int vnodeRetrieveQueryResult(void *handle, int *pNum, char *argv[]);
int vnodeSaveQueryResult(void *handle, char *data, int32_t* size);
int vnodeRetrieveQueryInfo(void *handle, int *numOfRows, int *rowSize, int16_t *timePrec);
void vnodeFreeQInfo(void *, bool);
void vnodeFreeQInfoInQueue(void *param);
bool vnodeIsQInfoValid(void *param);
void vnodeDecRefCount(void *param);
void vnodeAddRefCount(void *param);
int32_t vnodeConvertQueryMeterMsg(SQueryMeterMsg *pQuery);
void vnodeQueryData(SSchedMsg *pMsg);
// meter API
int vnodeOpenMetersVnode(int vnode);
void vnodeCloseMetersVnode(int vnode);
int vnodeCreateMeterObj(SMeterObj *pNew, SConnSec *pSec);
int vnodeRemoveMeterObj(int vnode, int sid);
int vnodeInsertPoints(SMeterObj *pObj, char *cont, int contLen, char source, void *, int sversion, int *numOfPoints, TSKEY now);
int vnodeImportPoints(SMeterObj *pObj, char *cont, int contLen, char source, void *, int sversion, int *numOfPoints, TSKEY now);
int vnodeInsertBufferedPoints(int vnode);
int vnodeSaveAllMeterObjToFile(int vnode);
int vnodeSaveMeterObjToFile(SMeterObj *pObj);
int vnodeSaveVnodeCfg(int vnode, SVnodeCfg *pCfg, SVPeerDesc *pDesc);
int vnodeSaveVnodeInfo(int vnode);
// cache API
void *vnodeOpenCachePool(int vnode);
void vnodeCloseCachePool(int vnode);
void *vnodeAllocateCacheInfo(SMeterObj *pObj);
void vnodeFreeCacheInfo(SMeterObj *pObj);
void vnodeSetCommitQuery(SMeterObj *pObj, SQuery *pQuery);
int vnodeInsertPointToCache(SMeterObj *pObj, char *pData);
int vnodeQueryFromCache(SMeterObj *pObj, SQuery *pQuery);
uint64_t vnodeGetPoolCount(SVnodeObj *pVnode);
void vnodeUpdateCommitInfo(SMeterObj *pObj, int slot, int pos, uint64_t count);
void vnodeCommitOver(SVnodeObj *pVnode);
TSKEY vnodeGetFirstKey(int vnode);
int vnodeSyncRetrieveCache(int vnode, int fd);
int vnodeSyncRestoreCache(int vnode, int fd);
pthread_t vnodeCreateCommitThread(SVnodeObj *pVnode);
void vnodeCancelCommit(SVnodeObj *pVnode);
void vnodeCloseStream(SVnodeObj *pVnode);
void vnodeProcessCommitTimer(void *param, void *tmrId);
void vnodeSearchPointInCache(SMeterObj *pObj, SQuery *pQuery);
int vnodeAllocateCacheBlock(SMeterObj *pObj);
int vnodeFreeCacheBlock(SCacheBlock *pCacheBlock);
int vnodeIsCacheCommitted(SMeterObj *pObj);
// file API
int vnodeInitFile(int vnode);
int vnodeQueryFromFile(SMeterObj *pObj, SQuery *pQuery);
void *vnodeCommitToFile(void *param);
void *vnodeCommitMultiToFile(SVnodeObj *pVnode, int ssid, int esid);
int vnodeSyncRetrieveFile(int vnode, int fd, uint32_t fileId, uint64_t *fmagic);
int vnodeSyncRestoreFile(int vnode, int sfd);
int vnodeWriteBlockToFile(SMeterObj *pObj, SCompBlock *pBlock, SData *data[], SData *cdata[], int pointsRead);
int vnodeSearchPointInFile(SMeterObj *pObj, SQuery *pQuery);
int vnodeReadCompBlockToMem(SMeterObj *pObj, SQuery *pQuery, SData *sdata[]);
int vnodeOpenCommitFiles(SVnodeObj *pVnode, int noTempLast);
void vnodeCloseCommitFiles(SVnodeObj *pVnode);
int vnodeReadLastBlockToMem(SMeterObj *pObj, SCompBlock *pBlock, SData *sdata[]);
// vnode API
void vnodeUpdateStreamRole(SVnodeObj *pVnode);
int vnodeInitPeer(int numOfThreads);
void vnodeCleanUpPeer();
int vnodeOpenPeerVnode(int vnode);
void vnodeClosePeerVnode(int vnode);
void *vnodeGetMeterPeerConnection(SMeterObj *pObj, int index);
int vnodeForwardToPeer(SMeterObj *pObj, char *msg, int msgLen, char action, int sversion);
void vnodeCloseAllSyncFds(int vnode);
void vnodeConfigVPeers(int vnode, int numOfPeers, SVPeerDesc peerDesc[]);
void vnodeStartSyncProcess(SVnodeObj *pVnode);
void vnodeCancelSync(int vnode);
void vnodeListPeerStatus(char *buffer);
void vnodeCheckOwnStatus(SVnodeObj *pVnode);
int vnodeSaveMeterObjToFile(SMeterObj *pObj);
int vnodeRecoverFromPeer(SVnodeObj *pVnode, int fileId);
// vnodes API
int vnodeInitVnodes();
int vnodeInitStore();
void vnodeCleanUpVnodes();
int vnodeRemoveVnode(int vnode);
int vnodeCreateVnode(int vnode, SVnodeCfg *pCfg, SVPeerDesc *pDesc);
void vnodeOpenStreams(void *param, void *tmrId);
void vnodeCreateStream(SMeterObj *pObj);
void vnodeRemoveStream(SMeterObj *pObj);
// shell API
int vnodeInitShell();
void vnodeCleanUpShell();
int vnodeOpenShellVnode(int vnode);
void vnodeCloseShellVnode(int vnode);
// memter mgmt
int vnodeInitMeterMgmt();
void vnodeCleanUpMeterMgmt();
int vnodeOpenMeterMgmtVnode(int vnode);
int vnodeOpenMeterMgmtStoreVnode(int vnode);
void vnodeCloseMeterMgmtVnode(int vnode);
int vnodeCreateMeterMgmt(SMeterObj *pObj, SConnSec *pSec);
void vnodeRemoveMeterMgmt(SMeterObj *pObj);
SConnSec *vnodeGetMeterSec(int vnode, int sid);
int vnodeCreateMeterObjFile(int vnode);
// mgmt
void vnodeCleanUpMgmt();
int vnodeRetrieveMissedCreateMsg(int vnode, int fd, uint64_t stime);
int vnodeRestoreMissedCreateMsg(int vnode, int fd);
int vnodeRetrieveMissedRemoveMsg(int vid, int fd, uint64_t stime);
int vnodeRestoreMissedRemoveMsg(int vnode, int fd);
int vnodeProcessBufferedCreateMsgs(int vnode);
void vnodeSendVpeerCfgMsg(int vnode);
int vnodeSendMeterCfgMsg(int vnode, int sid);
int vnodeMgmtConns();
void vnodeRemoveFile(int vnode, int fileId);
// commit
int vnodeInitCommit(int vnode);
void vnodeCleanUpCommit(int vnode);
int vnodeRenewCommitLog(int vnode);
void vnodeRemoveCommitLog(int vnode);
int vnodeWriteToCommitLog(SMeterObj *pObj, char action, char *cont, int contLen, int sversion);
extern int (*vnodeProcessAction[])(SMeterObj *, char *, int, char, void *, int, int *, TSKEY);
extern int (*pCompFunc[])(const char *const input, int inputSize, const int elements, char *const output,
int outputSize, char algorithm, char *const buffer, int bufferSize);
extern int (*pDecompFunc[])(const char *const input, int compressedSize, const int elements, char *const output,
int outputSize, char algorithm, char *const buffer, int bufferSize);
// global variable and APIs provided by mgmt
extern char mgmtStatus;
extern char tsMgmtDirectory[];
extern const int16_t vnodeFileVersion;
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_VNODE_H

61
src/vnode/detail/inc/vnodeCache.h
View File

@ -1,61 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_VNODECACHE_H
#define TDENGINE_VNODECACHE_H
#ifdef __cplusplus
extern "C" {
#endif
typedef struct {
short notFree;
short numOfPoints;
int slot;
int index;
int64_t blockId;
struct _meter_obj *pMeterObj;
char * offset[];
} SCacheBlock;
typedef struct {
int64_t blocks;
int maxBlocks;
int numOfBlocks;
int unCommittedBlocks;
int32_t currentSlot;
int32_t commitSlot; // which slot is committed
int32_t commitPoint; // starting point for next commit
SCacheBlock **cacheBlocks; // cache block list, circular list
} SCacheInfo;
typedef struct {
int vnode;
char ** pMem;
int64_t freeSlot;
pthread_mutex_t vmutex;
uint64_t count; // kind of transcation ID
int64_t notFreeSlots;
int64_t threshold;
char commitInProcess;
int cacheBlockSize;
int cacheNumOfBlocks;
} SCachePool;
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_VNODECACHE_H

35
src/vnode/detail/inc/vnodeDataFilterFunc.h
View File

@ -1,35 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_VNODEDATAFILTERFUNC_H
#define TDENGINE_VNODEDATAFILTERFUNC_H
#ifdef __cplusplus
extern "C" {
#endif
#include "vnode.h"
__filter_func_t *vnodeGetRangeFilterFuncArray(int32_t type);
__filter_func_t *vnodeGetValueFilterFuncArray(int32_t type);
bool vnodeSupportPrefilter(int32_t type);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_VNODEDATAFILTERFUNC_H

99
src/vnode/detail/inc/vnodeFile.h
View File

@ -1,99 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_VNODEFILE_H
#define TDENGINE_VNODEFILE_H
#ifdef __cplusplus
extern "C" {
#endif
#include "tchecksum.h"
#define TSDB_VNODE_DELIMITER 0xF00AFA0F
typedef struct { int64_t compInfoOffset; } SCompHeader;
typedef struct {
short colId;
short bytes;
int32_t numOfNullPoints;
int32_t type : 8;
int32_t offset : 24;
int32_t len; // data length
int64_t sum;
int64_t max;
int64_t min;
int16_t maxIndex;
int16_t minIndex;
char reserved[20];
} SField;
typedef struct {
int64_t last : 1;
int64_t offset : 63;
int32_t algorithm : 8; // compression algorithm can be changed
int32_t numOfPoints : 24; // how many points have been written into this block
int32_t sversion;
int32_t len; // total length of this data block
uint16_t numOfCols;
char reserved[16];
TSKEY keyFirst; // time stamp for the first point
TSKEY keyLast; // time stamp for the last point
} SCompBlock;
typedef struct {
SCompBlock *compBlock;
SField * fields;
} SCompBlockFields;
typedef struct {
uint64_t uid;
int64_t last : 1;
int64_t numOfBlocks : 62;
uint32_t delimiter; // delimiter for recovery
TSCKSUM checksum;
SCompBlock compBlocks[]; // comp block list
} SCompInfo;
typedef struct {
int64_t tempHeadOffset;
int64_t compInfoOffset;
int64_t oldCompBlockOffset;
int64_t oldNumOfBlocks;
int64_t newNumOfBlocks;
int64_t finalNumOfBlocks;
int64_t oldCompBlockLen;
int64_t newCompBlockLen;
int64_t finalCompBlockLen;
int64_t committedPoints;
int commitSlot;
int32_t last : 1;
int32_t changed : 1;
int32_t commitPos : 30;
int64_t commitCount;
SCompBlock lastBlock;
} SMeterInfo;
typedef struct { int64_t totalStorage; } SVnodeHeadInfo;
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_VNODEFILE_H

293
src/vnode/detail/inc/vnodeQueryImpl.h
View File

@ -1,293 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_VNODEQUERYIMPL_H
#define TDENGINE_VNODEQUERYIMPL_H
#ifdef __cplusplus
extern "C" {
#endif
#include "os.h"
#include "hash.h"
#include "hashfunc.h"
#define GET_QINFO_ADDR(x) ((char*)(x)-offsetof(SQInfo, query))
#define Q_STATUS_EQUAL(p, s) (((p) & (s)) != 0)
/*
* set the output buffer page size is 16k
* The page size should be sufficient for at least one output result or intermediate result.
* Some intermediate results may be extremely large, such as top/bottom(100) query.
*/
#define DEFAULT_INTERN_BUF_SIZE 16384L
#define INIT_ALLOCATE_DISK_PAGES 60L
#define DEFAULT_DATA_FILE_MAPPING_PAGES 2L
#define DEFAULT_DATA_FILE_MMAP_WINDOW_SIZE (DEFAULT_DATA_FILE_MAPPING_PAGES * DEFAULT_INTERN_BUF_SIZE)
#define IO_ENGINE_MMAP 0
#define IO_ENGINE_SYNC 1
#define DEFAULT_IO_ENGINE IO_ENGINE_SYNC
/**
* check if the primary column is load by default, otherwise, the program will
* forced to load primary column explicitly.
*/
#define PRIMARY_TSCOL_LOADED(query) ((query)->colList[0].data.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX)
typedef enum {
/*
* the program will call this function again, if this status is set.
* used to transfer from QUERY_RESBUF_FULL
*/
QUERY_NOT_COMPLETED = 0x1u,
/*
* output buffer is full, so, the next query will be employed,
* in this case, we need to set the appropriated start scan point for
* the next query.
*
* this status is only exist in group-by clause and
* diff/add/division/multiply/ query.
*/
QUERY_RESBUF_FULL = 0x2u,
/*
* query is over
* 1. this status is used in one row result query process, e.g.,
* count/sum/first/last/
* avg...etc.
* 2. when the query range on timestamp is satisfied, it is also denoted as
* query_compeleted
*/
QUERY_COMPLETED = 0x4u,
/*
* all data has been scanned, so current search is stopped,
* At last, the function will transfer this status to QUERY_COMPLETED
*/
QUERY_NO_DATA_TO_CHECK = 0x8u,
} vnodeQueryStatus;
typedef struct SPointInterpoSupporter {
int32_t numOfCols;
char** pPrevPoint;
char** pNextPoint;
} SPointInterpoSupporter;
typedef struct SBlockInfo {
TSKEY keyFirst;
TSKEY keyLast;
int32_t numOfCols;
int32_t size;
} SBlockInfo;
typedef struct SMeterDataBlockInfoEx {
SCompBlockFields pBlock;
SMeterDataInfo* pMeterDataInfo;
int32_t blockIndex;
int32_t groupIdx; /* number of group is less than the total number of meters */
} SMeterDataBlockInfoEx;
typedef enum {
DISK_DATA_LOAD_FAILED = -0x1,
DISK_DATA_LOADED = 0x0,
DISK_DATA_DISCARDED = 0x01,
} vnodeDiskLoadStatus;
#define IS_MASTER_SCAN(runtime) (((runtime)->scanFlag & 1u) == MASTER_SCAN)
#define IS_SUPPLEMENT_SCAN(runtime) ((runtime)->scanFlag == SUPPLEMENTARY_SCAN)
#define SET_SUPPLEMENT_SCAN_FLAG(runtime) ((runtime)->scanFlag = SUPPLEMENTARY_SCAN)
#define SET_MASTER_SCAN_FLAG(runtime) ((runtime)->scanFlag = MASTER_SCAN)
typedef int (*__block_search_fn_t)(char* data, int num, int64_t key, int order);
static FORCE_INLINE SMeterObj* getMeterObj(void* hashHandle, int32_t sid) {
return *(SMeterObj**)taosHashGet(hashHandle, (const char*)&sid, sizeof(sid));
}
bool isQueryKilled(SQuery* pQuery);
bool isFixedOutputQuery(SQuery* pQuery);
bool isPointInterpoQuery(SQuery* pQuery);
bool isSumAvgRateQuery(SQuery *pQuery);
bool isTopBottomQuery(SQuery* pQuery);
bool isFirstLastRowQuery(SQuery* pQuery);
bool isTSCompQuery(SQuery* pQuery);
bool notHasQueryTimeRange(SQuery* pQuery);
bool needSupplementaryScan(SQuery* pQuery);
bool onDemandLoadDatablock(SQuery* pQuery, int16_t queryRangeSet);
void setQueryStatus(SQuery* pQuery, int8_t status);
bool doRevisedResultsByLimit(SQInfo* pQInfo);
void truncateResultByLimit(SQInfo* pQInfo, int64_t* final, int32_t* interpo);
void initCtxOutputBuf(SQueryRuntimeEnv* pRuntimeEnv);
void resetCtxOutputBuf(SQueryRuntimeEnv* pRuntimeEnv);
void forwardCtxOutputBuf(SQueryRuntimeEnv* pRuntimeEnv, int64_t output);
bool needPrimaryTimestampCol(SQuery* pQuery, SBlockInfo* pBlockInfo);
void vnodeScanAllData(SQueryRuntimeEnv* pRuntimeEnv);
int32_t vnodeQueryResultInterpolate(SQInfo* pQInfo, tFilePage** pDst, tFilePage** pDataSrc, int32_t numOfRows,
int32_t* numOfInterpo);
void copyResToQueryResultBuf(STableQuerySupportObj* pSupporter, SQuery* pQuery);
void doSkipResults(SQueryRuntimeEnv* pRuntimeEnv);
void doFinalizeResult(SQueryRuntimeEnv* pRuntimeEnv);
int64_t getNumOfResult(SQueryRuntimeEnv* pRuntimeEnv);
void forwardQueryStartPosition(SQueryRuntimeEnv* pRuntimeEnv);
bool normalizedFirstQueryRange(bool dataInDisk, bool dataInCache, STableQuerySupportObj* pSupporter,
SPointInterpoSupporter* pPointInterpSupporter, int64_t* key);
void pointInterpSupporterInit(SQuery* pQuery, SPointInterpoSupporter* pInterpoSupport);
void pointInterpSupporterDestroy(SPointInterpoSupporter* pPointInterpSupport);
void pointInterpSupporterSetData(SQInfo* pQInfo, SPointInterpoSupporter* pPointInterpSupport);
int64_t loadRequiredBlockIntoMem(SQueryRuntimeEnv* pRuntimeEnv, SPositionInfo* position);
void disableFunctForSuppleScan(STableQuerySupportObj* pSupporter, int32_t order);
void enableFunctForMasterScan(SQueryRuntimeEnv* pRuntimeEnv, int32_t order);
int32_t mergeMetersResultToOneGroups(STableQuerySupportObj* pSupporter);
void copyFromWindowResToSData(SQInfo* pQInfo, SWindowResult* result);
SBlockInfo getBlockInfo(SQueryRuntimeEnv *pRuntimeEnv);
SBlockInfo getBlockBasicInfo(SQueryRuntimeEnv *pRuntimeEnv, void* pBlock, int32_t type);
SCacheBlock* getCacheDataBlock(SMeterObj* pMeterObj, SQueryRuntimeEnv* pRuntimeEnv, int32_t slot);
void stableApplyFunctionsOnBlock(STableQuerySupportObj* pSupporter, SMeterDataInfo* pMeterDataInfo,
SBlockInfo* pBlockInfo, SField* pFields, __block_search_fn_t searchFn);
int32_t vnodeFilterQualifiedMeters(SQInfo* pQInfo, int32_t vid, tSidSet* pSidSet, SMeterDataInfo* pMeterDataInfo,
int32_t* numOfMeters, SMeterDataInfo*** pReqMeterDataInfo);
int32_t vnodeGetVnodeHeaderFileIndex(int32_t* fid, SQueryRuntimeEnv* pRuntimeEnv, int32_t order);
int32_t createDataBlocksInfoEx(SMeterDataInfo** pMeterDataInfo, int32_t numOfMeters,
SMeterDataBlockInfoEx** pDataBlockInfoEx, int32_t numOfCompBlocks,
int32_t* nAllocBlocksInfoSize, int64_t addr);
void freeMeterBlockInfoEx(SMeterDataBlockInfoEx* pDataBlockInfoEx, int32_t len);
void setExecutionContext(STableQuerySupportObj* pSupporter, SMeterQueryInfo* pMeterQueryInfo, int32_t meterIdx,
int32_t groupIdx, TSKEY nextKey);
int32_t setAdditionalInfo(STableQuerySupportObj *pSupporter, int32_t meterIdx, SMeterQueryInfo *pMeterQueryInfo);
void doGetAlignedIntervalQueryRangeImpl(SQuery* pQuery, int64_t pKey, int64_t keyFirst, int64_t keyLast,
int64_t* actualSkey, int64_t* actualEkey, int64_t* skey, int64_t* ekey);
int64_t getQueryStartPositionInCache(SQueryRuntimeEnv* pRuntimeEnv, int32_t* slot, int32_t* pos, bool ignoreQueryRange);
int32_t getDataBlocksForMeters(STableQuerySupportObj* pSupporter, SQuery* pQuery, int32_t numOfMeters,
const char* filePath, SMeterDataInfo** pMeterDataInfo, uint32_t* numOfBlocks);
int32_t LoadDatablockOnDemand(SCompBlock* pBlock, SField** pFields, uint8_t* blkStatus, SQueryRuntimeEnv* pRuntimeEnv,
int32_t fileIdx, int32_t slotIdx, __block_search_fn_t searchFn, bool onDemand);
int32_t vnodeGetHeaderFile(SQueryRuntimeEnv* pRuntimeEnv, int32_t fileIndex);
/**
* Create SMeterQueryInfo.
* The MeterQueryInfo is created one for each table during super table query
*
* @param skey
* @param ekey
* @return
*/
SMeterQueryInfo* createMeterQueryInfo(STableQuerySupportObj* pSupporter, int32_t sid, TSKEY skey, TSKEY ekey);
/**
* Destroy meter query info
* @param pMeterQInfo
* @param numOfCols
*/
void destroyMeterQueryInfo(SMeterQueryInfo* pMeterQueryInfo, int32_t numOfCols);
/**
* change the meter query info for supplement scan
* @param pMeterQueryInfo
* @param skey
* @param ekey
*/
void changeMeterQueryInfoForSuppleQuery(SQuery* pQuery, SMeterQueryInfo* pMeterQueryInfo,
TSKEY skey, TSKEY ekey);
/**
* add the new allocated disk page to meter query info
* the new allocated disk page is used to keep the intermediate (interval) results
* @param pQuery
* @param pMeterQueryInfo
* @param pSupporter
*/
tFilePage* addDataPageForMeterQueryInfo(SQuery* pQuery, SMeterQueryInfo* pMeterQueryInfo,
STableQuerySupportObj* pSupporter);
/**
* restore the query range data from SMeterQueryInfo to runtime environment
*
* @param pRuntimeEnv
* @param pMeterQueryInfo
*/
void restoreIntervalQueryRange(SQueryRuntimeEnv* pRuntimeEnv, SMeterQueryInfo* pMeterQueryInfo);
/**
* set the interval query range for the interval query, when handling a data(cache) block
*
* @param pMeterQueryInfo
* @param pSupporter
* @param key
*/
void setIntervalQueryRange(SMeterQueryInfo* pMeterQueryInfo, STableQuerySupportObj* pSupporter, int64_t key);
/**
* set the meter data information
* @param pMeterDataInfo
* @param pMeterObj current query meter object
* @param meterIdx meter index in the sid list
* @param groupId group index, which the meter is belonged to
*/
void setMeterDataInfo(SMeterDataInfo* pMeterDataInfo, SMeterObj* pMeterObj, int32_t meterIdx, int32_t groupId);
void vnodeSetTagValueInParam(tSidSet* pSidSet, SQueryRuntimeEnv* pRuntimeEnv, SMeterSidExtInfo* pMeterInfo);
void vnodeCheckIfDataExists(SQueryRuntimeEnv* pRuntimeEnv, SMeterObj* pMeterObj, bool* dataInDisk, bool* dataInCache);
void displayInterResult(SData** pdata, SQuery* pQuery, int32_t numOfRows);
void vnodePrintQueryStatistics(STableQuerySupportObj* pSupporter);
void clearTimeWindowResBuf(SQueryRuntimeEnv* pRuntimeEnv, SWindowResult* pOneOutputRes);
void copyTimeWindowResBuf(SQueryRuntimeEnv* pRuntimeEnv, SWindowResult* dst, const SWindowResult* src);
int32_t initWindowResInfo(SWindowResInfo* pWindowResInfo, SQueryRuntimeEnv* pRuntimeEnv, int32_t size,
int32_t threshold, int16_t type);
void cleanupTimeWindowInfo(SWindowResInfo* pWindowResInfo, SQueryRuntimeEnv* pRuntimeEnv);
void resetTimeWindowInfo(SQueryRuntimeEnv* pRuntimeEnv, SWindowResInfo* pWindowResInfo);
void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num);
void clearClosedTimeWindow(SQueryRuntimeEnv* pRuntimeEnv);
int32_t numOfClosedTimeWindow(SWindowResInfo* pWindowResInfo);
void closeTimeWindow(SWindowResInfo* pWindowResInfo, int32_t slot);
void closeAllTimeWindow(SWindowResInfo* pWindowResInfo);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_VNODEQUERYIMPL_H

300
src/vnode/detail/inc/vnodeRead.h
View File

@ -1,300 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_VNODEREAD_H
#define TDENGINE_VNODEREAD_H
#ifdef __cplusplus
extern "C" {
#endif
#include "os.h"
#include "qresultBuf.h"
#include "qinterpolation.h"
#include "vnodeTagMgmt.h"
/*
* use to keep the first point position, consisting of position in blk and block
* id, file id
*/
typedef struct {
int32_t pos;
int32_t slot;
int32_t fileId;
} SPositionInfo;
typedef struct SLoadDataBlockInfo {
int32_t fileListIndex; /* index of this file in files list of this vnode */
int32_t fileId;
int32_t slotIdx;
int32_t sid;
bool tsLoaded; // if timestamp column of current block is loaded or not
} SLoadDataBlockInfo;
typedef struct SLoadCompBlockInfo {
int32_t sid; /* meter sid */
int32_t fileId;
int32_t fileListIndex;
} SLoadCompBlockInfo;
/*
* the header file info for one vnode
*/
typedef struct SHeaderFileInfo {
int32_t fileID; // file id
} SHeaderFileInfo;
typedef struct SQueryCostSummary {
double cacheTimeUs;
double fileTimeUs;
int64_t numOfFiles; // opened files during query
int64_t numOfTables; // num of queries tables
int64_t numOfSeek; // number of seek operation
int64_t readDiskBlocks; // accessed disk block
int64_t skippedFileBlocks; // skipped blocks
int64_t blocksInCache; // accessed cache blocks
int64_t readField; // field size
int64_t totalFieldSize; // total read fields size
double loadFieldUs; // total elapsed time to read fields info
int64_t totalBlockSize; // read data blocks
double loadBlocksUs; // total elapsed time to read data blocks
int64_t totalGenData; // in-memory generated data
int64_t readCompInfo; // read compblock info
int64_t totalCompInfoSize; // total comp block size
double loadCompInfoUs; // total elapsed time to read comp block info
int64_t tmpBufferInDisk; // size of buffer for intermediate result
} SQueryCostSummary;
typedef struct SPosInfo {
int16_t pageId;
int16_t rowId;
} SPosInfo;
typedef struct STimeWindow {
TSKEY skey;
TSKEY ekey;
} STimeWindow;
typedef struct SWindowStatus {
bool closed;
} SWindowStatus;
typedef struct SWindowResult {
uint16_t numOfRows;
SPosInfo pos; // Position of current result in disk-based output buffer
SResultInfo* resultInfo; // For each result column, there is a resultInfo
STimeWindow window; // The time window that current result covers.
SWindowStatus status;
} SWindowResult;
/*
* header files info, avoid to iterate the directory, the data is acquired
* during in query preparation function
*/
typedef struct SQueryFilesInfo {
SHeaderFileInfo* pFileInfo;
uint32_t numOfFiles; // the total available number of files for this virtual node during query execution
int32_t current; // the memory mapped header file, NOTE: only one header file can be mmap.
int32_t vnodeId;
int32_t headerFd; // header file fd
int64_t headerFileSize;
int32_t dataFd;
int32_t lastFd;
char headerFilePath[PATH_MAX]; // current opened header file name
char dataFilePath[PATH_MAX]; // current opened data file name
char lastFilePath[PATH_MAX]; // current opened last file path
char dbFilePathPrefix[PATH_MAX];
} SQueryFilesInfo;
typedef struct SWindowResInfo {
SWindowResult* pResult; // reference to SQuerySupporter->pResult
void* hashList; // hash list for quick access
int16_t type; // data type for hash key
int32_t capacity; // max capacity
int32_t curIndex; // current start active index
int32_t size;
int64_t startTime; // start time of the first time window for sliding query
int64_t prevSKey; // previous (not completed) sliding window start key
int64_t threshold; // threshold for return completed results.
} SWindowResInfo;
typedef struct SQueryRuntimeEnv {
SPositionInfo startPos; /* the start position, used for secondary/third iteration */
SPositionInfo endPos; /* the last access position in query, served as the start pos of reversed order query */
SPositionInfo nextPos; /* start position of the next scan */
SData* colDataBuffer[TSDB_MAX_COLUMNS];
SResultInfo* resultInfo; // todo refactor to merge with SWindowResInfo
uint8_t blockStatus; // Indicate if data block is loaded, the block is first/last/internal block
int32_t unzipBufSize;
SData* primaryColBuffer;
char* unzipBuffer;
char* secondaryUnzipBuffer;
SQuery* pQuery;
SMeterObj* pMeterObj;
SQLFunctionCtx* pCtx;
SLoadDataBlockInfo loadBlockInfo; /* record current block load information */
SLoadCompBlockInfo loadCompBlockInfo; /* record current compblock information in SQuery */
SQueryFilesInfo vnodeFileInfo;
int16_t numOfRowsPerPage;
int16_t offset[TSDB_MAX_COLUMNS];
uint16_t scanFlag; // denotes reversed scan of data or not
SInterpolationInfo interpoInfo;
SData** pInterpoBuf;
SWindowResInfo windowResInfo;
STSBuf* pTSBuf;
STSCursor cur;
SQueryCostSummary summary;
bool stableQuery; // is super table query or not
SDiskbasedResultBuf* pResultBuf; // query result buffer based on blocked-wised disk file
/*
* Temporarily hold the in-memory cache block info during scan cache blocks
* Here we do not use the cache block info from pMeterObj, simple because it may change anytime
* during the query by the submit/insert handling threads.
* So we keep a copy of the support structure as well as the cache block data itself.
*/
SCacheBlock cacheBlock;
} SQueryRuntimeEnv;
/* intermediate pos during multimeter query involves interval */
typedef struct SMeterQueryInfo {
int64_t lastKey;
int64_t skey;
int64_t ekey;
int32_t numOfRes;
int16_t queryRangeSet; // denote if the query range is set, only available for interval query
int64_t tag;
STSCursor cur;
int32_t sid; // for retrieve the page id list
SWindowResInfo windowResInfo;
} SMeterQueryInfo;
typedef struct SMeterDataInfo {
uint64_t offsetInHeaderFile;
int32_t numOfBlocks;
int32_t start; // start block index
SCompBlock* pBlock;
int32_t meterOrderIdx;
SMeterObj* pMeterObj;
int32_t groupIdx; // group id in meter list
SMeterQueryInfo* pMeterQInfo;
} SMeterDataInfo;
typedef struct STableQuerySupportObj {
void* pMetersHashTable; // meter table hash list
SMeterSidExtInfo** pMeterSidExtInfo;
int32_t numOfMeters;
/*
* multimeter query resultset.
* In multimeter queries, the result is temporarily stored on this structure, instead of
* directly put result into output buffer, since we have no idea how many number of
* rows may be generated by a specific subgroup. When query on all subgroups is executed,
* the result is copy to output buffer. This attribution is not used during single meter query processing.
*/
SQueryRuntimeEnv runtimeEnv;
int64_t rawSKey;
int64_t rawEKey;
int32_t subgroupIdx;
int32_t offset; /* offset in group result set of subgroup */
tSidSet* pSidSet;
/*
* the query is executed position on which meter of the whole list.
* when the index reaches the last one of the list, it means the query is completed.
* We later may refactor to remove this attribution by using another flag to denote
* whether a multimeter query is completed or not.
*/
int32_t meterIdx;
int32_t numOfGroupResultPages;
int32_t groupResultSize;
SMeterDataInfo* pMeterDataInfo;
TSKEY* tsList;
} STableQuerySupportObj;
typedef struct _qinfo {
uint64_t signature;
int32_t refCount; // QInfo reference count, when the value is 0, it can be released safely
char user[TSDB_TABLE_ID_LEN + 1];
char sql[TSDB_SHOW_SQL_LEN];
uint8_t stream;
uint16_t port;
uint32_t ip;
uint64_t startTime;
int64_t useconds;
int killed;
struct _qinfo *prev, *next;
SQuery query;
int totalPoints;
int pointsRead;
int pointsReturned;
int pointsInterpo;
int code;
char bufIndex;
char changed;
char over;
SMeterObj* pObj;
sem_t dataReady;
STableQuerySupportObj* pTableQuerySupporter;
int (*fp)(SMeterObj*, SQuery*);
} SQInfo;
int32_t vnodeQueryTablePrepare(SQInfo* pQInfo, SMeterObj* pMeterObj, STableQuerySupportObj* pSMultiMeterObj,
void* param);
void vnodeQueryFreeQInfoEx(SQInfo* pQInfo);
bool vnodeParametersSafetyCheck(SQuery* pQuery);
int32_t vnodeSTableQueryPrepare(SQInfo* pQInfo, SQuery* pQuery, void* param);
/**
* decrease the numofQuery of each table that is queried, enable the
* remove/close operation can be executed
* @param pQInfo
*/
void vnodeDecMeterRefcnt(SQInfo* pQInfo);
/* sql query handle in dnode */
void vnodeSingleTableQuery(SSchedMsg* pMsg);
/*
* handle multi-meter query process
*/
void vnodeMultiMeterQuery(SSchedMsg* pMsg);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_VNODEREAD_H

29
src/vnode/detail/inc/vnodeShell.h
View File

@ -1,29 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_VNODESHELL_H
#define TDENGINE_VNODESHELL_H
#ifdef __cplusplus
extern "C" {
#endif
#include "os.h"
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_VNODESHELL_H

35
src/vnode/detail/inc/vnodeStore.h
View File

@ -1,35 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_VNODESTORE_H
#define TDENGINE_VNODESTORE_H
#ifdef __cplusplus
extern "C" {
#endif
void vnodeProcessDataFromVnode(SIntMsg *msg, void *tcpHandle);
void vnodeCalcOpenVnodes();
bool vnodeRemoveDataFileFromLinkFile(char* linkFile, char* de_name);
int vnodeInitInfo();
#ifdef __cplusplus
}
#endif
#endif // TDEGINE_VNODESTORE_H

27
src/vnode/detail/inc/vnodeSupertableQuery.h
View File

@ -1,27 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TBASE_MNODE_SUPER_TABLE_QUERY_H
#define TBASE_MNODE_SUPER_TABLE_QUERY_H
#include "os.h"
#include "mnode.h"
#include "qast.h"
int32_t mgmtDoJoin(SSuperTableMetaMsg* pSuperTableMetaMsg, tQueryResultset* pRes);
void mgmtReorganizeMetersInMetricMeta(SSuperTableMetaMsg* pInfo, int32_t index, tQueryResultset* pRes);
#endif

29
src/vnode/detail/inc/vnodeSystem.h
View File

@ -1,29 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_VNODESYSTEM_H
#define TDENGINE_VNODESYSTEM_H
#ifdef __cplusplus
extern "C" {
#endif
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_VNODESYSTEM_H

58
src/vnode/detail/inc/vnodeTagMgmt.h
View File

@ -1,58 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_VNODETAGMGMT_H
#define TDENGINE_VNODETAGMGMT_H
#ifdef __cplusplus
extern "C" {
#endif
/*
* @version 0.1
* @date 2018/01/02
* @author liaohj
* management of the tag value of tables
* in query, client need the vnode to aggregate results according to tags
* values,
* the grouping operation is done here.
* Note:
* 1. we implement a quick sort algorithm, may remove it later.
*/
typedef int32_t (*__ext_compar_fn_t)(const void *p1, const void *p2, void *param);
tSidSet *tSidSetCreate(struct SMeterSidExtInfo **pMeterSidExtInfo, int32_t numOfMeters, SSchema *pSchema,
int32_t numOfTags, SColIndexEx *colList, int32_t numOfOrderCols);
int32_t *calculateSubGroup(void **pSids, int32_t numOfMeters, int32_t *numOfSubset, tOrderDescriptor *pOrderDesc,
__ext_compar_fn_t compareFn);
void tSidSetDestroy(tSidSet **pSets);
void tSidSetSort(tSidSet *pSets);
int32_t meterSidComparator(const void *s1, const void *s2, void *param);
int32_t doCompare(char *f1, char *f2, int32_t type, int32_t size);
void tQSortEx(void **pMeterSids, size_t size, int32_t start, int32_t end, void *param, __ext_compar_fn_t compareFn);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_VNODETAGMGMT_H

94
src/vnode/detail/inc/vnodeUtil.h
View File

@ -1,94 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_VNODE_UTIL_H
#define TDENGINE_VNODE_UTIL_H
#ifdef __cplusplus
extern "C" {
#endif
/* get the qinfo struct address from the query struct address */
#define GET_COLUMN_BYTES(query, colidx) \
((query)->colList[(query)->pSelectExpr[colidx].pBase.colInfo.colIdxInBuf].data.bytes)
#define GET_COLUMN_TYPE(query, colidx) \
((query)->colList[(query)->pSelectExpr[colidx].pBase.colInfo.colIdxInBuf].data.type)
#define QUERY_IS_ASC_QUERY(q) (GET_FORWARD_DIRECTION_FACTOR((q)->order.order) == QUERY_ASC_FORWARD_STEP)
#define EXTRA_BYTES 2 // for possible compression deflation
#define GET_COL_DATA_POS(query, index, step) ((query)->pos + (index)*(step))
int vnodeGetEid(int days);
int vnodeCheckFileIntegrity(FILE *fp);
void vnodeCreateFileHeader(FILE *fp);
void vnodeCreateFileHeaderFd(int fd);
void vnodeGetHeadFileHeaderInfo(int fd, SVnodeHeadInfo *pHeadInfo);
void vnodeUpdateHeadFileHeader(int fd, SVnodeHeadInfo *pHeadInfo);
/**
* check if two schema is identical or not
* This function does not check if a schema is valid or not
*
* @param pSSchemaFirst
* @param numOfCols1
* @param pSSchemaSecond
* @param numOfCols2
* @return
*/
bool vnodeMeterSchemaIdentical(SColumn *pSchema1, int32_t numOfCols1, SColumn *pSchema2, int32_t numOfCols2);
/**
* free SFields in SQuery
* vnodeFreeFields must be called before free(pQuery->pBlock);
* @param pQuery
*/
void vnodeFreeFields(SQuery *pQuery);
void vnodeUpdateFilterColumnIndex(SQuery* pQuery);
void vnodeUpdateQueryColumnIndex(SQuery* pQuery, SMeterObj* pMeterObj);
int32_t vnodeCreateFilterInfo(void* pQInfo, SQuery *pQuery);
bool vnodeFilterData(SQuery* pQuery, int32_t* numOfActualRead, int32_t index);
bool vnodeDoFilterData(SQuery* pQuery, int32_t elemPos);
bool vnodeIsProjectionQuery(SSqlFunctionExpr *pExpr, int32_t numOfOutput);
int32_t vnodeIncQueryRefCount(SQueryMeterMsg *pQueryMsg, SMeterSidExtInfo **pSids, SMeterObj **pMeterObjList,
int32_t *numOfInc);
void vnodeDecQueryRefCount(SQueryMeterMsg *pQueryMsg, SMeterObj **pMeterObjList, int32_t numOfInc);
int32_t vnodeSetMeterState(SMeterObj* pMeterObj, int32_t state);
void vnodeClearMeterState(SMeterObj* pMeterObj, int32_t state);
bool vnodeIsMeterState(SMeterObj* pMeterObj, int32_t state);
void vnodeSetMeterDeleting(SMeterObj* pMeterObj);
int32_t vnodeSetMeterInsertImportStateEx(SMeterObj* pObj, int32_t st);
bool vnodeIsSafeToDeleteMeter(SVnodeObj* pVnode, int32_t sid);
void vnodeFreeColumnInfo(SColumnInfo* pColumnInfo);
bool isGroupbyNormalCol(SSqlGroupbyExpr* pExpr);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_VNODE_UTIL_H

1165
src/vnode/detail/src/vnodeCache.c
View File

File diff suppressed because it is too large Load Diff

292
src/vnode/detail/src/vnodeCommit.c
View File

@ -1,292 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _GNU_SOURCE /* See feature_test_macros(7) */
#include "os.h"
#include "taosdef.h"
#include "vnode.h"
#include "vnodeUtil.h"
#include "vnodeStatus.h"
typedef struct {
int sversion;
int sid;
int contLen;
int action:8;
int simpleCheck:24;
} SCommitHead;
int vnodeOpenCommitLog(int vnode, uint64_t firstV) {
SVnodeObj *pVnode = vnodeList + vnode;
char * fileName = pVnode->logFn;
pVnode->logFd = open(fileName, O_RDWR | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO);
if (pVnode->logFd < 0) {
dError("vid:%d, failed to open file:%s, reason:%s", vnode, fileName, strerror(errno));
return -1;
}
dTrace("vid:%d, logfd:%d, open file:%s success", vnode, pVnode->logFd, fileName);
if (posix_fallocate64(pVnode->logFd, 0, pVnode->mappingSize) != 0) {
dError("vid:%d, logfd:%d, failed to alloc file size:%d, reason:%s", vnode, pVnode->logFd, pVnode->mappingSize, strerror(errno));
perror("fallocate failed");
goto _err_log_open;
}
struct stat statbuf;
stat(fileName, &statbuf);
int64_t length = statbuf.st_size;
if (length != pVnode->mappingSize) {
dError("vid:%d, logfd:%d, alloc file size:%" PRId64 " not equal to mapping size:%" PRId64, vnode, pVnode->logFd, length,
pVnode->mappingSize);
goto _err_log_open;
}
pVnode->pMem = mmap(0, pVnode->mappingSize, PROT_WRITE | PROT_READ, MAP_SHARED, pVnode->logFd, 0);
if (pVnode->pMem == MAP_FAILED) {
dError("vid:%d, logfd:%d, failed to map file, reason:%s", vnode, pVnode->logFd, strerror(errno));
goto _err_log_open;
}
pVnode->pWrite = pVnode->pMem;
memcpy(pVnode->pWrite, &(firstV), sizeof(firstV));
pVnode->pWrite += sizeof(firstV);
return pVnode->logFd;
_err_log_open:
close(pVnode->logFd);
remove(fileName);
pVnode->logFd = -1;
return -1;
}
int vnodeRenewCommitLog(int vnode) {
SVnodeObj *pVnode = vnodeList + vnode;
char * fileName = pVnode->logFn;
char * oldName = pVnode->logOFn;
pthread_mutex_lock(&(pVnode->logMutex));
if (FD_VALID(pVnode->logFd)) {
munmap(pVnode->pMem, pVnode->mappingSize);
close(pVnode->logFd);
rename(fileName, oldName);
}
if (pVnode->cfg.commitLog) vnodeOpenCommitLog(vnode, vnodeList[vnode].version);
pthread_mutex_unlock(&(pVnode->logMutex));
return pVnode->logFd;
}
void vnodeRemoveCommitLog(int vnode) { remove(vnodeList[vnode].logOFn); }
size_t vnodeRestoreDataFromLog(int vnode, char *fileName, uint64_t *firstV) {
int fd, ret;
char * cont = NULL;
size_t totalLen = 0;
int actions = 0;
SVnodeObj *pVnode = vnodeList + vnode;
if (pVnode->meterList == NULL) {
dError("vid:%d, vnode is not initialized!!!", vnode);
return 0;
}
struct stat fstat;
if (stat(fileName, &fstat) < 0) {
dTrace("vid:%d, no log file:%s", vnode, fileName);
return 0;
}
dTrace("vid:%d, uncommitted data in file:%s, restore them ...", vnode, fileName);
fd = open(fileName, O_RDWR);
if (fd < 0) {
dError("vid:%d, failed to open:%s, reason:%s", vnode, fileName, strerror(errno));
goto _error;
}
ret = read(fd, firstV, sizeof(pVnode->version));
if (ret <= 0) {
dError("vid:%d, failed to read version", vnode);
goto _error;
}
pVnode->version = *firstV;
int32_t bufLen = TSDB_PAYLOAD_SIZE;
cont = calloc(1, bufLen);
if (cont == NULL) {
dError("vid:%d, out of memory", vnode);
goto _error;
}
TSKEY now = taosGetTimestamp(pVnode->cfg.precision);
SCommitHead head;
int simpleCheck = 0;
while (1) {
ret = read(fd, &head, sizeof(head));
if (ret < 0) goto _error;
if (ret == 0) break;
if (((head.sversion+head.sid+head.contLen+head.action) & 0xFFFFFF) != head.simpleCheck) break;
simpleCheck = head.simpleCheck;
// head.contLen validation is removed
if (head.sid >= pVnode->cfg.maxSessions || head.sid < 0 || head.action >= TSDB_ACTION_MAX) {
dError("vid, invalid commit head, sid:%d contLen:%d action:%d", head.sid, head.contLen, head.action);
} else {
if (head.contLen > 0) {
if (bufLen < head.contLen+sizeof(simpleCheck)) { // pre-allocated buffer is not enough
cont = realloc(cont, head.contLen+sizeof(simpleCheck));
bufLen = head.contLen+sizeof(simpleCheck);
}
if (read(fd, cont, head.contLen+sizeof(simpleCheck)) < 0) goto _error;
if (*(int *)(cont+head.contLen) != simpleCheck) break;
SMeterObj *pObj = pVnode->meterList[head.sid];
if (pObj == NULL) {
dError("vid:%d, sid:%d not exists, ignore data in commit log, contLen:%d action:%d",
vnode, head.sid, head.contLen, head.action);
continue;
}
if (vnodeIsMeterState(pObj, TSDB_METER_STATE_DROPPING)) {
dWarn("vid:%d sid:%d id:%s, meter is dropped, ignore data in commit log, contLen:%d action:%d",
vnode, head.sid, head.contLen, head.action);
continue;
}
int32_t numOfPoints = 0;
(*vnodeProcessAction[head.action])(pObj, cont, head.contLen, TSDB_DATA_SOURCE_LOG, NULL, head.sversion,
&numOfPoints, now);
actions++;
} else {
break;
}
}
totalLen += sizeof(head) + head.contLen + sizeof(simpleCheck);
}
tclose(fd);
tfree(cont);
dTrace("vid:%d, %d pieces of uncommitted data are restored", vnode, actions);
return totalLen;
_error:
tclose(fd);
tfree(cont);
dError("vid:%d, failed to restore %s, remove this node...", vnode, fileName);
// rename to error file for future process
char *f = NULL;
taosFileRename(fileName, "error", '/', &f);
free(f);
return -1;
}
int vnodeInitCommit(int vnode) {
size_t size = 0;
uint64_t firstV = 0;
SVnodeObj *pVnode = vnodeList + vnode;
pthread_mutex_init(&(pVnode->logMutex), NULL);
sprintf(pVnode->logFn, "%s/vnode%d/db/submit%d.log", tsDirectory, vnode, vnode);
sprintf(pVnode->logOFn, "%s/vnode%d/db/submit%d.olog", tsDirectory, vnode, vnode);
pVnode->mappingSize = ((int64_t)pVnode->cfg.cacheBlockSize) * pVnode->cfg.cacheNumOfBlocks.totalBlocks * 1.5;
pVnode->mappingThreshold = pVnode->mappingSize * 0.7;
// restore from .olog file and commit to file
size = vnodeRestoreDataFromLog(vnode, pVnode->logOFn, &firstV);
if (size < 0) return -1;
if (size > 0) {
if (pVnode->commitInProcess == 0) vnodeCommitToFile(pVnode);
remove(pVnode->logOFn);
}
// restore from .log file to cache
size = vnodeRestoreDataFromLog(vnode, pVnode->logFn, &firstV);
if (size < 0) return -1;
if (pVnode->cfg.commitLog == 0) return 0;
if (size == 0) firstV = pVnode->version;
if (vnodeOpenCommitLog(vnode, firstV) < 0) {
dError("vid:%d, commit log init failed", vnode);
return -1;
}
pVnode->pWrite += size;
dPrint("vid:%d, commit log is initialized", vnode);
return 0;
}
void vnodeCleanUpCommit(int vnode) {
SVnodeObj *pVnode = vnodeList + vnode;
if (FD_VALID(pVnode->logFd)) close(pVnode->logFd);
if (pVnode->cfg.commitLog && (pVnode->logFd > 0 && remove(pVnode->logFn) < 0)) {
dError("vid:%d, failed to remove:%s", vnode, pVnode->logFn);
taosLogError("vid:%d, failed to remove:%s", vnode, pVnode->logFn);
}
pthread_mutex_destroy(&(pVnode->logMutex));
}
int vnodeWriteToCommitLog(SMeterObj *pObj, char action, char *cont, int contLen, int sverion) {
SVnodeObj *pVnode = vnodeList + pObj->vnode;
if (pVnode->pWrite == NULL) return 0;
SCommitHead head;
head.sid = pObj->sid;
head.action = action;
head.sversion = pObj->sversion;
head.contLen = contLen;
head.simpleCheck = (head.sversion+head.sid+head.contLen+head.action) & 0xFFFFFF;
int simpleCheck = head.simpleCheck;
pthread_mutex_lock(&(pVnode->logMutex));
// 100 bytes redundant mem space
if (pVnode->mappingSize - (pVnode->pWrite - pVnode->pMem) < contLen + sizeof(SCommitHead) + sizeof(simpleCheck) + 100) {
pthread_mutex_unlock(&(pVnode->logMutex));
dTrace("vid:%d, mem mapping space is not enough, wait for commit", pObj->vnode);
vnodeProcessCommitTimer(pVnode, NULL);
return TSDB_CODE_ACTION_IN_PROGRESS;
}
char *pWrite = pVnode->pWrite;
pVnode->pWrite += sizeof(head) + contLen + sizeof(simpleCheck);
memcpy(pWrite, (char *)&head, sizeof(head));
memcpy(pWrite + sizeof(head), cont, contLen);
memcpy(pWrite + sizeof(head) + contLen, &simpleCheck, sizeof(simpleCheck));
pthread_mutex_unlock(&(pVnode->logMutex));
if (pVnode->pWrite - pVnode->pMem > pVnode->mappingThreshold) {
dTrace("vid:%d, mem mapping is close to limit, commit", pObj->vnode);
vnodeProcessCommitTimer(pVnode, NULL);
}
dTrace("vid:%d sid:%d, data is written to commit log", pObj->vnode, pObj->sid);
return 0;
}

1880
src/vnode/detail/src/vnodeFile.c
View File

File diff suppressed because it is too large Load Diff

110
src/vnode/detail/src/vnodeFile.spec.c
View File

@ -1,110 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "vnode.h"
#include "vnodeFile.h"
char* vnodeGetDiskFromHeadFile(char *headName) { return tsDirectory; }
char* vnodeGetDataDir(int vnode, int fileId) { return dataDir; }
void vnodeAdustVnodeFile(SVnodeObj *pVnode) {
// Retention policy here
int fileId = pVnode->fileId - pVnode->numOfFiles + 1;
int cfile = taosGetTimestamp(pVnode->cfg.precision)/pVnode->cfg.daysPerFile/tsMsPerDay[(uint8_t)pVnode->cfg.precision];
while (fileId <= cfile - pVnode->maxFiles) {
vnodeRemoveFile(pVnode->vnode, fileId);
pVnode->numOfFiles--;
fileId++;
}
}
int vnodeCheckNewHeaderFile(int fd, SVnodeObj *pVnode) {
SCompHeader *pHeader = NULL;
SCompBlock *pBlocks = NULL;
int blockSize = 0;
SCompInfo compInfo;
int tmsize = 0;
tmsize = sizeof(SCompHeader) * pVnode->cfg.maxSessions + sizeof(TSCKSUM);
pHeader = (SCompHeader *)malloc(tmsize);
if (pHeader == NULL) return 0;
lseek(fd, TSDB_FILE_HEADER_LEN, SEEK_SET);
if (read(fd, (void *)pHeader, tmsize) != tmsize) {
goto _broken_exit;
}
if (!taosCheckChecksumWhole((uint8_t *)pHeader, tmsize)) {
goto _broken_exit;
}
for (int sid = 0; sid < pVnode->cfg.maxSessions; sid++) {
if (pVnode->meterList == NULL) goto _correct_exit;
if (pVnode->meterList[sid] == NULL || pHeader[sid].compInfoOffset == 0) continue;
lseek(fd, pHeader[sid].compInfoOffset, SEEK_SET);
if (read(fd, (void *)(&compInfo), sizeof(SCompInfo)) != sizeof(SCompInfo)) {
goto _broken_exit;
}
if (!taosCheckChecksumWhole((uint8_t *)(&compInfo), sizeof(SCompInfo))) {
goto _broken_exit;
}
if (compInfo.uid != ((SMeterObj *)pVnode->meterList[sid])->uid) continue;
int expectedSize = sizeof(SCompBlock) * compInfo.numOfBlocks + sizeof(TSCKSUM);
if (blockSize < expectedSize) {
pBlocks = (SCompBlock *)realloc(pBlocks, expectedSize);
if (pBlocks == NULL) {
tfree(pHeader);
return 0;
}
blockSize = expectedSize;
}
if (read(fd, (void *)pBlocks, expectedSize) != expectedSize) {
dError("failed to read block part");
goto _broken_exit;
}
if (!taosCheckChecksumWhole((uint8_t *)pBlocks, expectedSize)) {
dError("block part is broken");
goto _broken_exit;
}
for (int i = 0; i < compInfo.numOfBlocks; i++) {
if (pBlocks[i].last && i != compInfo.numOfBlocks-1) {
dError("last block in middle, block:%d", i);
goto _broken_exit;
}
}
}
_correct_exit:
dPrint("vid: %d new header file %s is correct", pVnode->vnode, pVnode->nfn);
tfree(pBlocks);
tfree(pHeader);
return 0;
_broken_exit:
dError("vid: %d new header file %s is broken", pVnode->vnode, pVnode->nfn);
tfree(pBlocks);
tfree(pHeader);
return -1;
}

237
src/vnode/detail/src/vnodeFileUtil.c
View File

@ -1,237 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "vnode.h"
int vnodeCheckHeaderFile(int fd, int dfd, SVnodeCfg cfg, int mode) {
SCompHeader *pHeaders = NULL;
SVnodeCfg *pCfg = &cfg;
SCompInfo compInfo;
SCompBlock *pBlocks = NULL;
int blockSize = 0;
SField *pFields = NULL;
char *pBuf = NULL;
int size = 0;
int ret = 0;
if (fd < 0 || dfd < 0) return -1;
lseek(fd, TSDB_FILE_HEADER_LEN, SEEK_SET);
size = pCfg->maxSessions*sizeof(SCompHeader)+sizeof(TSCKSUM);
pHeaders = calloc(1, size);
if (pHeaders == NULL) {
return -1;
}
read(fd, pHeaders, size);
if (!taosCheckChecksumWhole((uint8_t *)pHeaders, size)) {
return -1;
}
for (int i = 0; i < pCfg->maxSessions; i++) {
if (pHeaders[i].compInfoOffset == 0) continue;
if (pHeaders[i].compInfoOffset < 0) {
// TODO : report error here
ret = -1;
continue;
}
lseek(fd, pHeaders[i].compInfoOffset, SEEK_SET);
read(fd, &compInfo, sizeof(SCompInfo));
if (!taosCheckChecksumWhole((uint8_t *)&compInfo, sizeof(SCompInfo))) {
// TODO : report error
ret = -1;
continue;
}
int tsize = sizeof(SCompBlock) * compInfo.numOfBlocks + sizeof(TSCKSUM);
if (tsize > blockSize) {
if (pBlocks == NULL) {
pBlocks = calloc(1, tsize);
} else {
pBlocks = realloc(pBlocks, tsize);
}
blockSize = tsize;
}
read(fd, tsize);
if (!taosCheckChecksumWhole(pBlocks, tsize)) {
// TODO: Report error
ret = -1;
continue;
}
TSKEY keyLast = 0;
for (int j = 0; j < compInfo.numOfBlocks; j++) {
SCompBlock *pBlock = pBlocks + j;
if (pBlock->last != 0 && j < compInfo.numOfBlocks-1) {
// TODO: report error
ret = -1;
break;
}
if (pBlock->offset < TSDB_FILE_HEADER_LEN) {
// TODO : report erro
ret = -1;
break;
}
if (pBlock->keyLast < pBlock->keyFirst) {
// TODO : report error
ret = -1;
break;
}
if (pBlock->keyFirst <= keyLast) {
// TODO : report error
ret = -1;
break;
}
keyLast = pBlock->keyLast;
// Check block in data
lseek(dfd, pBlock->offset, SEEK_SET);
tsize = sizeof(SField) * pBlock->numOfCols + sizeof(TSCKSUM);
pFields = realloc(pFields, tsize);
read(dfd, pFields, tsize);
if (!taosCheckChecksumWhole((uint8_t*)pFields, tsize)) {
// TODO : report error
ret = -1;
continue;
}
for (int k = 0; k < pBlock->numOfCols; k++) {
// TODO: Check pFields[k] content
pBuf = realloc(pBuf, pFields[k].len);
if (!taosCheckChecksumWhole((uint8_t *)pBuf, pFields[k].len)) {
// TODO : report error;
ret = -1;
continue;
}
}
}
}
tfree(pBuf);
tfree(pFields);
tfree(pBlocks);
tfree(pHeaders);
return ret;
}
int vnodePackDataFile(int vnode, int fileId) {
// TODO: check if it is able to pack current file
// TODO: assign value to headerFile and dataFile
char *headerFile = NULL;
char *dataFile = NULL;
char *lastFile = NULL;
SVnodeObj *pVnode = vnodeList+vnode;
SCompHeader *pHeaders = NULL;
SCompBlock *pBlocks = NULL;
int blockSize = 0;
char *pBuff = 0;
int buffSize = 0;
SCompInfo compInfo;
int size = 0;
int hfd = open(headerFile, O_RDONLY);
if (hfd < 0) {
dError("vid: %d, failed to open header file:%s\n", vnode, headerFile);
return -1;
}
int dfd = open(dataFile, O_RDONLY);
if (dfd < 0) {
dError("vid: %d, failed to open data file:%s\n", vnode, dataFile);
return -1;
}
int lfd = open(lastFile, O_RDONLY);
if (lfd < 0) {
dError("vid: %d, failed to open data file:%s\n", vnode, lastFile);
return -1;
}
lseek(hfd, TSDB_FILE_HEADER_LEN, SEEK_SET);
size = sizeof(SCompHeader)*pVnode->cfg.maxSessions+sizeof(TSCKSUM);
pHeaders = malloc(size);
if (pHeaders == NULL) goto _exit_failure;
read(hfd, pHeaders, size);
if (!taosCheckChecksumWhole((uint8_t *)pHeaders, size)) {
dError("vid: %d, header file %s is broken", vnode, headerFile);
goto _exit_failure;
}
for (size_t i = 0; i < pVnode->cfg.maxSessions; i++)
{
if (pHeaders[i].compInfoOffset <= 0) continue;
SMeterObj *pObj = (SMeterObj *)pVnode->meterList[i];
// read compInfo part
lseek(hfd, pHeaders[i].compInfoOffset, SEEK_SET);
read(hfd, &compInfo, sizeof(SCompInfo));
if (!taosCheckChecksumWhole((uint8_t *)&compInfo, sizeof(SCompInfo))) {
dError("vid: %d sid:%d fileId:%d compInfo is broken", vnode, i, fileId);
goto _exit_failure;
}
// read compBlock part
int tsize = compInfo.numOfBlocks * sizeof(SCompBlock) + sizeof(TSCKSUM);
if (tsize > blockSize) {
if (blockSize == 0) {
pBlocks = malloc(tsize);
} else {
pBlocks = realloc(pBlocks, tsize);
}
blockSize = tsize;
}
read(hfd, pBlocks, tsize);
if (!taosCheckChecksumWhole((uint8_t *)pBlocks, tsize)) {
dError("vid:%d sid:%d fileId:%d block part is broken", vnode, i, fileId);
goto _exit_failure;
}
assert(compInfo.numOfBlocks > 0);
// Loop to scan the blocks and merge block when neccessary.
tsize = sizeof(SCompInfo) + compInfo.numOfBlocks *sizeof(SCompBlock) + sizeof(TSCKSUM);
pBuff = realloc(pBuff, tsize);
SCompInfo *pInfo = (SCompInfo *)pBuff;
SCompBlock *pNBlocks = pBuff + sizeof(SCompInfo);
int nCounter = 0;
for (int j; j < compInfo.numOfBlocks; j++) {
// TODO : Check if it is the last block
// if (j == compInfo.numOfBlocks - 1) {}
if (pBlocks[j].numOfPoints + pNBlocks[nCounter].numOfPoints <= pObj->pointsPerFileBlock) {
// Merge current block to current new block
} else {
// Write new block to new data file
// pNBlocks[nCounter].
nCounter++;
}
}
}
return 0;
_exit_failure:
tfree(pHeaders);
if (hfd > 0) close(hfd);
if (dfd > 0) close(dfd);
if (lfd > 0) close(lfd);
return -1;
}

558
src/vnode/detail/src/vnodeFilterFunc.c
View File

@ -1,558 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "taosmsg.h"
#include "tsqlfunction.h"
#include "vnode.h"
#include "vnodeDataFilterFunc.h"
bool less_i8(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int8_t *)minval < pFilter->filterInfo.upperBndi);
}
bool less_i16(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int16_t *)minval < pFilter->filterInfo.upperBndi);
}
bool less_i32(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int32_t *)minval < pFilter->filterInfo.upperBndi);
}
bool less_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int64_t *)minval < pFilter->filterInfo.upperBndi);
}
bool less_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(float *)minval < pFilter->filterInfo.upperBndd);
}
bool less_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(double *)minval < pFilter->filterInfo.upperBndd);
}
//////////////////////////////////////////////////////////////////
bool large_i8(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int8_t *)maxval > pFilter->filterInfo.lowerBndi);
}
bool large_i16(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int16_t *)maxval > pFilter->filterInfo.lowerBndi);
}
bool large_i32(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int32_t *)maxval > pFilter->filterInfo.lowerBndi);
}
bool large_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int64_t *)maxval > pFilter->filterInfo.lowerBndi);
}
bool large_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(float *)maxval > pFilter->filterInfo.lowerBndd);
}
bool large_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(double *)maxval > pFilter->filterInfo.lowerBndd);
}
/////////////////////////////////////////////////////////////////////
bool lessEqual_i8(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int8_t *)minval <= pFilter->filterInfo.upperBndi);
}
bool lessEqual_i16(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int16_t *)minval <= pFilter->filterInfo.upperBndi);
}
bool lessEqual_i32(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int32_t *)minval <= pFilter->filterInfo.upperBndi);
}
bool lessEqual_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int64_t *)minval <= pFilter->filterInfo.upperBndi);
}
bool lessEqual_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(float *)minval <= pFilter->filterInfo.upperBndd);
}
bool lessEqual_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(double *)minval <= pFilter->filterInfo.upperBndd);
}
//////////////////////////////////////////////////////////////////////////
bool largeEqual_i8(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int8_t *)maxval >= pFilter->filterInfo.lowerBndi);
}
bool largeEqual_i16(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int16_t *)maxval >= pFilter->filterInfo.lowerBndi);
}
bool largeEqual_i32(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int32_t *)maxval >= pFilter->filterInfo.lowerBndi);
}
bool largeEqual_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int64_t *)maxval >= pFilter->filterInfo.lowerBndi);
}
bool largeEqual_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(float *)maxval >= pFilter->filterInfo.lowerBndd);
}
bool largeEqual_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(double *)maxval >= pFilter->filterInfo.lowerBndd);
}
////////////////////////////////////////////////////////////////////////
bool equal_i8(SColumnFilterElem *pFilter, char *minval, char *maxval) {
if (*(int8_t *)minval == *(int8_t *)maxval) {
return (*(int8_t *)minval == pFilter->filterInfo.lowerBndi);
} else { /* range filter */
assert(*(int8_t *)minval < *(int8_t *)maxval);
return *(int8_t *)minval <= pFilter->filterInfo.lowerBndi && *(int8_t *)maxval >= pFilter->filterInfo.lowerBndi;
}
}
bool equal_i16(SColumnFilterElem *pFilter, char *minval, char *maxval) {
if (*(int16_t *)minval == *(int16_t *)maxval) {
return (*(int16_t *)minval == pFilter->filterInfo.lowerBndi);
} else { /* range filter */
assert(*(int16_t *)minval < *(int16_t *)maxval);
return *(int16_t *)minval <= pFilter->filterInfo.lowerBndi && *(int16_t *)maxval >= pFilter->filterInfo.lowerBndi;
}
}
bool equal_i32(SColumnFilterElem *pFilter, char *minval, char *maxval) {
if (*(int32_t *)minval == *(int32_t *)maxval) {
return (*(int32_t *)minval == pFilter->filterInfo.lowerBndi);
} else { /* range filter */
assert(*(int32_t *)minval < *(int32_t *)maxval);
return *(int32_t *)minval <= pFilter->filterInfo.lowerBndi && *(int32_t *)maxval >= pFilter->filterInfo.lowerBndi;
}
}
bool equal_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) {
if (*(int64_t *)minval == *(int64_t *)maxval) {
return (*(int64_t *)minval == pFilter->filterInfo.lowerBndi);
} else { /* range filter */
assert(*(int64_t *)minval < *(int64_t *)maxval);
return *(int64_t *)minval <= pFilter->filterInfo.lowerBndi && *(int64_t *)maxval >= pFilter->filterInfo.lowerBndi;
}
}
bool equal_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) {
if (*(float *)minval == *(float *)maxval) {
return (fabs(*(float *)minval - pFilter->filterInfo.lowerBndd) <= FLT_EPSILON);
} else { /* range filter */
assert(*(float *)minval < *(float *)maxval);
return *(float *)minval <= pFilter->filterInfo.lowerBndd && *(float *)maxval >= pFilter->filterInfo.lowerBndd;
}
}
bool equal_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) {
if (*(double *)minval == *(double *)maxval) {
return (*(double *)minval == pFilter->filterInfo.lowerBndd);
} else { /* range filter */
assert(*(double *)minval < *(double *)maxval);
return *(double *)minval <= pFilter->filterInfo.lowerBndi && *(double *)maxval >= pFilter->filterInfo.lowerBndi;
}
}
bool equal_str(SColumnFilterElem *pFilter, char *minval, char *maxval) {
// query condition string is greater than the max length of string, not qualified data
if (pFilter->filterInfo.len > pFilter->bytes) {
return false;
}
return strncmp((char *)pFilter->filterInfo.pz, minval, pFilter->bytes) == 0;
}
bool equal_nchar(SColumnFilterElem *pFilter, char *minval, char *maxval) {
// query condition string is greater than the max length of string, not qualified data
if (pFilter->filterInfo.len > pFilter->bytes) {
return false;
}
return wcsncmp((wchar_t *)pFilter->filterInfo.pz, (wchar_t*) minval, pFilter->bytes/TSDB_NCHAR_SIZE) == 0;
}
////////////////////////////////////////////////////////////////
bool like_str(SColumnFilterElem *pFilter, char *minval, char *maxval) {
SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER;
return patternMatch((char *)pFilter->filterInfo.pz, minval, pFilter->bytes, &info) == TSDB_PATTERN_MATCH;
}
bool like_nchar(SColumnFilterElem* pFilter, char* minval, char *maxval) {
SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER;
return WCSPatternMatch((wchar_t*) pFilter->filterInfo.pz, (wchar_t*) minval, pFilter->bytes/TSDB_NCHAR_SIZE, &info) == TSDB_PATTERN_MATCH;
}
////////////////////////////////////////////////////////////////
/**
* If minval equals to maxval, it may serve as the one element filter,
* or all elements of an array are identical during pref-filter stage.
* Otherwise, it must be pre-filter of array list of elements.
*
* During pre-filter stage, if there is one element that locates in [minval, maxval],
* the filter function will return true.
*/
bool nequal_i8(SColumnFilterElem *pFilter, char *minval, char *maxval) {
if (*(int8_t *)minval == *(int8_t *)maxval) {
return (*(int8_t *)minval != pFilter->filterInfo.lowerBndi);
}
return true;
}
bool nequal_i16(SColumnFilterElem *pFilter, char *minval, char *maxval) {
if (*(int16_t *)minval == *(int16_t *)maxval) {
return (*(int16_t *)minval != pFilter->filterInfo.lowerBndi);
}
return true;
}
bool nequal_i32(SColumnFilterElem *pFilter, char *minval, char *maxval) {
if (*(int32_t *)minval == *(int32_t *)maxval) {
return (*(int32_t *)minval != pFilter->filterInfo.lowerBndi);
}
return true;
}
bool nequal_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) {
if (*(int64_t *)minval == *(int64_t *)maxval) {
return (*(int64_t *)minval != pFilter->filterInfo.lowerBndi);
}
return true;
}
bool nequal_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) {
if (*(float *)minval == *(float *)maxval) {
return (*(float *)minval != pFilter->filterInfo.lowerBndd);
}
return true;
}
bool nequal_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) {
if (*(double *)minval == *(double *)maxval) {
return (*(double *)minval != pFilter->filterInfo.lowerBndd);
}
return true;
}
bool nequal_str(SColumnFilterElem *pFilter, char *minval, char *maxval) {
if (pFilter->filterInfo.len > pFilter->bytes) {
return true;
}
return strncmp((char *)pFilter->filterInfo.pz, minval, pFilter->bytes) != 0;
}
bool nequal_nchar(SColumnFilterElem *pFilter, char* minval, char *maxval) {
if (pFilter->filterInfo.len > pFilter->bytes) {
return true;
}
return wcsncmp((wchar_t *)pFilter->filterInfo.pz, (wchar_t*)minval, pFilter->bytes/TSDB_NCHAR_SIZE) != 0;
}
////////////////////////////////////////////////////////////////
bool rangeFilter_i32_ii(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int32_t *)minval <= pFilter->filterInfo.upperBndi && *(int32_t *)maxval >= pFilter->filterInfo.lowerBndi);
}
bool rangeFilter_i32_ee(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int32_t *)minval<pFilter->filterInfo.upperBndi &&*(int32_t *)maxval> pFilter->filterInfo.lowerBndi);
}
bool rangeFilter_i32_ie(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int32_t *)minval < pFilter->filterInfo.upperBndi && *(int32_t *)maxval >= pFilter->filterInfo.lowerBndi);
}
bool rangeFilter_i32_ei(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int32_t *)minval <= pFilter->filterInfo.upperBndi && *(int32_t *)maxval > pFilter->filterInfo.lowerBndi);
}
///////////////////////////////////////////////////////////////////////////////
bool rangeFilter_i8_ii(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int8_t *)minval <= pFilter->filterInfo.upperBndi && *(int8_t *)maxval >= pFilter->filterInfo.lowerBndi);
}
bool rangeFilter_i8_ee(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int8_t *)minval<pFilter->filterInfo.upperBndi &&*(int8_t *)maxval> pFilter->filterInfo.lowerBndi);
}
bool rangeFilter_i8_ie(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int8_t *)minval < pFilter->filterInfo.upperBndi && *(int8_t *)maxval >= pFilter->filterInfo.lowerBndi);
}
bool rangeFilter_i8_ei(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int8_t *)minval <= pFilter->filterInfo.upperBndi && *(int8_t *)maxval > pFilter->filterInfo.lowerBndi);
}
/////////////////////////////////////////////////////////////////////////////////////
bool rangeFilter_i16_ii(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int16_t *)minval <= pFilter->filterInfo.upperBndi && *(int16_t *)maxval >= pFilter->filterInfo.lowerBndi);
}
bool rangeFilter_i16_ee(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int16_t *)minval<pFilter->filterInfo.upperBndi &&*(int16_t *)maxval> pFilter->filterInfo.lowerBndi);
}
bool rangeFilter_i16_ie(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int16_t *)minval < pFilter->filterInfo.upperBndi && *(int16_t *)maxval >= pFilter->filterInfo.lowerBndi);
}
bool rangeFilter_i16_ei(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int16_t *)minval <= pFilter->filterInfo.upperBndi && *(int16_t *)maxval > pFilter->filterInfo.lowerBndi);
}
////////////////////////////////////////////////////////////////////////
bool rangeFilter_i64_ii(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int64_t *)minval <= pFilter->filterInfo.upperBndi && *(int64_t *)maxval >= pFilter->filterInfo.lowerBndi);
}
bool rangeFilter_i64_ee(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int64_t *)minval<pFilter->filterInfo.upperBndi &&*(int64_t *)maxval> pFilter->filterInfo.lowerBndi);
}
bool rangeFilter_i64_ie(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int64_t *)minval < pFilter->filterInfo.upperBndi && *(int64_t *)maxval >= pFilter->filterInfo.lowerBndi);
}
bool rangeFilter_i64_ei(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int64_t *)minval <= pFilter->filterInfo.upperBndi && *(int64_t *)maxval > pFilter->filterInfo.lowerBndi);
}
////////////////////////////////////////////////////////////////////////
bool rangeFilter_ds_ii(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(float *)minval <= pFilter->filterInfo.upperBndd && *(float *)maxval >= pFilter->filterInfo.lowerBndd);
}
bool rangeFilter_ds_ee(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(float *)minval<pFilter->filterInfo.upperBndd &&*(float *)maxval> pFilter->filterInfo.lowerBndd);
}
bool rangeFilter_ds_ie(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(float *)minval < pFilter->filterInfo.upperBndd && *(float *)maxval >= pFilter->filterInfo.lowerBndd);
}
bool rangeFilter_ds_ei(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(float *)minval <= pFilter->filterInfo.upperBndd && *(float *)maxval > pFilter->filterInfo.lowerBndd);
}
//////////////////////////////////////////////////////////////////////////
bool rangeFilter_dd_ii(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(double *)minval <= pFilter->filterInfo.upperBndd && *(double *)maxval >= pFilter->filterInfo.lowerBndd);
}
bool rangeFilter_dd_ee(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(double *)minval<pFilter->filterInfo.upperBndd &&*(double *)maxval> pFilter->filterInfo.lowerBndd);
}
bool rangeFilter_dd_ie(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(double *)minval < pFilter->filterInfo.upperBndd && *(double *)maxval >= pFilter->filterInfo.lowerBndd);
}
bool rangeFilter_dd_ei(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(double *)minval <= pFilter->filterInfo.upperBndd && *(double *)maxval > pFilter->filterInfo.lowerBndd);
}
////////////////////////////////////////////////////////////////////////////
bool (*filterFunc_i8[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = {
NULL,
less_i8,
large_i8,
equal_i8,
lessEqual_i8,
largeEqual_i8,
nequal_i8,
NULL,
};
bool (*filterFunc_i16[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = {
NULL,
less_i16,
large_i16,
equal_i16,
lessEqual_i16,
largeEqual_i16,
nequal_i16,
NULL,
};
bool (*filterFunc_i32[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = {
NULL,
less_i32,
large_i32,
equal_i32,
lessEqual_i32,
largeEqual_i32,
nequal_i32,
NULL,
};
bool (*filterFunc_i64[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = {
NULL,
less_i64,
large_i64,
equal_i64,
lessEqual_i64,
largeEqual_i64,
nequal_i64,
NULL,
};
bool (*filterFunc_ds[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = {
NULL,
less_ds,
large_ds,
equal_ds,
lessEqual_ds,
largeEqual_ds,
nequal_ds,
NULL,
};
bool (*filterFunc_dd[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = {
NULL,
less_dd,
large_dd,
equal_dd,
lessEqual_dd,
largeEqual_dd,
nequal_dd,
NULL,
};
bool (*filterFunc_str[])(SColumnFilterElem* pFilter, char* minval, char *maxval) = {
NULL,
NULL,
NULL,
equal_str,
NULL,
NULL,
nequal_str,
like_str,
};
bool (*filterFunc_nchar[])(SColumnFilterElem* pFitler, char* minval, char* maxval) = {
NULL,
NULL,
NULL,
equal_nchar,
NULL,
NULL,
nequal_nchar,
like_nchar,
};
bool (*rangeFilterFunc_i8[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = {
NULL,
rangeFilter_i8_ee,
rangeFilter_i8_ie,
rangeFilter_i8_ei,
rangeFilter_i8_ii,
};
bool (*rangeFilterFunc_i16[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = {
NULL,
rangeFilter_i16_ee,
rangeFilter_i16_ie,
rangeFilter_i16_ei,
rangeFilter_i16_ii,
};
bool (*rangeFilterFunc_i32[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = {
NULL,
rangeFilter_i32_ee,
rangeFilter_i32_ie,
rangeFilter_i32_ei,
rangeFilter_i32_ii,
};
bool (*rangeFilterFunc_i64[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = {
NULL,
rangeFilter_i64_ee,
rangeFilter_i64_ie,
rangeFilter_i64_ei,
rangeFilter_i64_ii,
};
bool (*rangeFilterFunc_ds[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = {
NULL,
rangeFilter_ds_ee,
rangeFilter_ds_ie,
rangeFilter_ds_ei,
rangeFilter_ds_ii,
};
bool (*rangeFilterFunc_dd[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = {
NULL,
rangeFilter_dd_ee,
rangeFilter_dd_ie,
rangeFilter_dd_ei,
rangeFilter_dd_ii,
};
__filter_func_t* vnodeGetRangeFilterFuncArray(int32_t type) {
switch(type) {
case TSDB_DATA_TYPE_BOOL: return rangeFilterFunc_i8;
case TSDB_DATA_TYPE_TINYINT: return rangeFilterFunc_i8;
case TSDB_DATA_TYPE_SMALLINT: return rangeFilterFunc_i16;
case TSDB_DATA_TYPE_INT: return rangeFilterFunc_i32;
case TSDB_DATA_TYPE_TIMESTAMP: //timestamp uses bigint filter
case TSDB_DATA_TYPE_BIGINT: return rangeFilterFunc_i64;
case TSDB_DATA_TYPE_FLOAT: return rangeFilterFunc_ds;
case TSDB_DATA_TYPE_DOUBLE: return rangeFilterFunc_dd;
default:return NULL;
}
}
__filter_func_t* vnodeGetValueFilterFuncArray(int32_t type) {
switch(type) {
case TSDB_DATA_TYPE_BOOL: return filterFunc_i8;
case TSDB_DATA_TYPE_TINYINT: return filterFunc_i8;
case TSDB_DATA_TYPE_SMALLINT: return filterFunc_i16;
case TSDB_DATA_TYPE_INT: return filterFunc_i32;
case TSDB_DATA_TYPE_TIMESTAMP: //timestamp uses bigint filter
case TSDB_DATA_TYPE_BIGINT: return filterFunc_i64;
case TSDB_DATA_TYPE_FLOAT: return filterFunc_ds;
case TSDB_DATA_TYPE_DOUBLE: return filterFunc_dd;
case TSDB_DATA_TYPE_BINARY: return filterFunc_str;
case TSDB_DATA_TYPE_NCHAR: return filterFunc_nchar;
default: return NULL;
}
}
bool vnodeSupportPrefilter(int32_t type) { return type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR; }

1573
src/vnode/detail/src/vnodeImport.c
View File

File diff suppressed because it is too large Load Diff

825
src/vnode/detail/src/vnodeMeter.c
View File

@ -1,825 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "trpc.h"
#include "tschemautil.h"
#include "ttime.h"
#include "tutil.h"
#include "vnode.h"
#include "vnodeShell.h"
#include "vnodeUtil.h"
#include "vnodeStatus.h"
#define VALID_TIMESTAMP(key, curKey, prec) (((key) >= 0) && ((key) <= ((curKey) + 36500 * tsMsPerDay[prec])))
int tsMeterSizeOnFile;
void vnodeUpdateMeter(void *param, void *tmdId);
void vnodeRecoverMeterObjectFile(int vnode);
int (*vnodeProcessAction[])(SMeterObj *, char *, int, char, void *, int, int *, TSKEY) = {vnodeInsertPoints,
vnodeImportPoints};
void vnodeFreeMeterObj(SMeterObj *pObj) {
if (pObj == NULL) return;
dTrace("vid:%d sid:%d id:%s, meter is cleaned up", pObj->vnode, pObj->sid, pObj->meterId);
vnodeFreeCacheInfo(pObj);
if (vnodeList[pObj->vnode].meterList != NULL) {
vnodeList[pObj->vnode].meterList[pObj->sid] = NULL;
}
memset(pObj->meterId, 0, tListLen(pObj->meterId));
tfree(pObj);
}
int vnodeUpdateVnodeStatistic(FILE *fp, SVnodeObj *pVnode) {
fseek(fp, TSDB_FILE_HEADER_VERSION_SIZE, SEEK_SET);
fwrite(&(pVnode->vnodeStatistic), sizeof(SVnodeStatisticInfo), 1, fp);
return 0;
}
void vnodeUpdateVnodeFileHeader(FILE *fp, SVnodeObj *pVnode) {
fseek(fp, TSDB_FILE_HEADER_LEN * 1 / 4, SEEK_SET);
#ifdef _TD_ARM_32_
fprintf(fp, "%lld %lld %lld ", pVnode->lastCreate, pVnode->lastRemove, pVnode->version);
fprintf(fp, "%lld %d %d ", pVnode->lastKeyOnFile, pVnode->fileId, pVnode->numOfFiles);
#else
fprintf(fp, "%ld %ld %ld ", pVnode->lastCreate, pVnode->lastRemove, pVnode->version);
fprintf(fp, "%ld %d %d ", pVnode->lastKeyOnFile, pVnode->fileId, pVnode->numOfFiles);
#endif
}
int vnodeCreateMeterObjFile(int vnode) {
FILE * fp;
char fileName[TSDB_FILENAME_LEN];
int32_t size;
// SMeterObj *pObj;
sprintf(fileName, "%s/vnode%d/meterObj.v%d", tsDirectory, vnode, vnode);
fp = fopen(fileName, "w+");
if (fp == NULL) {
dError("failed to create vnode:%d file:%s, errno:%d, reason:%s", vnode, fileName, errno, strerror(errno));
if (errno == EACCES) {
return TSDB_CODE_NO_DISK_PERMISSIONS;
} else if (errno == ENOSPC) {
return TSDB_CODE_SERV_NO_DISKSPACE;
} else {
return TSDB_CODE_VG_INIT_FAILED;
}
} else {
vnodeCreateFileHeader(fp);
vnodeUpdateVnodeFileHeader(fp, vnodeList + vnode);
fseek(fp, TSDB_FILE_HEADER_LEN, SEEK_SET);
size = sizeof(SMeterObjHeader) * vnodeList[vnode].cfg.maxSessions + sizeof(TSCKSUM);
tfree(vnodeList[vnode].meterIndex);
vnodeList[vnode].meterIndex = calloc(1, size);
taosCalcChecksumAppend(0, (uint8_t *)(vnodeList[vnode].meterIndex), size);
fwrite(vnodeList[vnode].meterIndex, size, 1, fp);
fclose(fp);
}
return TSDB_CODE_SUCCESS;
}
FILE *vnodeOpenMeterObjFile(int vnode) {
FILE * fp;
char fileName[TSDB_FILENAME_LEN];
struct stat fstat;
// check if directory exists
sprintf(fileName, "%s/vnode%d", tsDirectory, vnode);
if (stat(fileName, &fstat) < 0) return NULL;
sprintf(fileName, "%s/vnode%d/meterObj.v%d", tsDirectory, vnode, vnode);
if (stat(fileName, &fstat) < 0) return NULL;
fp = fopen(fileName, "r+");
if (fp != NULL) {
if (vnodeCheckFileIntegrity(fp) < 0) {
dError("file:%s is corrupted, need to restore it first, exit program", fileName);
fclose(fp);
// todo: how to recover
exit(1);
}
} else {
dError("failed to open %s, reason:%s", fileName, strerror(errno));
}
return fp;
}
int vnodeSaveMeterObjToFile(SMeterObj *pObj) {
int64_t offset, length, new_length, new_offset;
FILE * fp;
SVnodeObj *pVnode = &vnodeList[pObj->vnode];
char * buffer = NULL;
fp = vnodeOpenMeterObjFile(pObj->vnode);
if (fp == NULL) return -1;
buffer = (char *)malloc(tsMeterSizeOnFile);
if (buffer == NULL) {
dError("Failed to allocate memory while saving meter object to file, meterId", pObj->meterId);
fclose(fp);
return -1;
}
offset = pVnode->meterIndex[pObj->sid].offset;
length = pVnode->meterIndex[pObj->sid].length;
new_length = offsetof(SMeterObj, reserved) + pObj->numOfColumns * sizeof(SColumn) + pObj->sqlLen + sizeof(TSCKSUM);
memcpy(buffer, pObj, offsetof(SMeterObj, reserved));
memcpy(buffer + offsetof(SMeterObj, reserved), pObj->schema, pObj->numOfColumns * sizeof(SColumn));
memcpy(buffer + offsetof(SMeterObj, reserved) + pObj->numOfColumns * sizeof(SColumn), pObj->pSql, pObj->sqlLen);
taosCalcChecksumAppend(0, (uint8_t *)buffer, new_length);
if (offset == 0 || length < new_length) { // New, append to file end
fseek(fp, 0, SEEK_END);
new_offset = ftell(fp);
fwrite(buffer, new_length, 1, fp);
pVnode->meterIndex[pObj->sid].offset = new_offset;
pVnode->meterIndex[pObj->sid].length = new_length;
} else if (offset < 0) { // deleted meter, append to end of file
fseek(fp, -offset, SEEK_SET);
fwrite(buffer, new_length, 1, fp);
pVnode->meterIndex[pObj->sid].offset = -offset;
pVnode->meterIndex[pObj->sid].length = new_length;
} else { // meter exists, overwrite it, offset > 0
fseek(fp, offset, SEEK_SET);
fwrite(buffer, new_length, 1, fp);
pVnode->meterIndex[pObj->sid].offset = (pObj->meterId[0] == 0) ? -offset : offset;
pVnode->meterIndex[pObj->sid].length = new_length;
}
// taosCalcChecksumAppend(0, pVnode->meterIndex, sizeof(SMeterObjHeader)*pVnode->cfg.maxSessions+sizeof(TSCKSUM));
// NOTE: no checksum, since it makes creating table slow
fseek(fp, TSDB_FILE_HEADER_LEN + sizeof(SMeterObjHeader) * pObj->sid, SEEK_SET);
fwrite(&(pVnode->meterIndex[pObj->sid]), sizeof(SMeterObjHeader), 1, fp);
// update checksum
// fseek(fp, TSDB_FILE_HEADER_LEN+sizeof(SMeterObjHeader)*(pVnode->cfg.maxSessions), SEEK_SET);
// fwrite(((char *)(pVnode->meterIndex) + sizeof(SMeterObjHeader)*(pVnode->cfg.maxSessions)), sizeof(TSCKSUM), 1, fp);
tfree(buffer);
vnodeUpdateVnodeStatistic(fp, pVnode);
vnodeUpdateVnodeFileHeader(fp, pVnode);
/* vnodeUpdateFileCheckSum(fp); */
fclose(fp);
return 0;
}
int vnodeSaveAllMeterObjToFile(int vnode) {
int64_t offset, length, new_length, new_offset;
FILE * fp;
SMeterObj *pObj;
SVnodeObj *pVnode = &vnodeList[vnode];
char * buffer = NULL;
fp = vnodeOpenMeterObjFile(vnode);
if (fp == NULL) return -1;
buffer = (char *)malloc(tsMeterSizeOnFile);
if (buffer == NULL) {
dError("Failed to allocate memory while saving all meter objects to file");
return -1;
}
for (int sid = 0; sid < pVnode->cfg.maxSessions; ++sid) {
pObj = pVnode->meterList[sid];
if (pObj == NULL) continue;
offset = pVnode->meterIndex[sid].offset;
length = pVnode->meterIndex[sid].length;
new_length = offsetof(SMeterObj, reserved) + pObj->numOfColumns * sizeof(SColumn) + pObj->sqlLen + sizeof(TSCKSUM);
memcpy(buffer, pObj, offsetof(SMeterObj, reserved));
memcpy(buffer + offsetof(SMeterObj, reserved), pObj->schema, pObj->numOfColumns * sizeof(SColumn));
memcpy(buffer + offsetof(SMeterObj, reserved) + pObj->numOfColumns * sizeof(SColumn), pObj->pSql, pObj->sqlLen);
taosCalcChecksumAppend(0, (uint8_t *)buffer, new_length);
if (offset == 0 || length > new_length) { // New, append to file end
new_offset = fseek(fp, 0, SEEK_END);
fwrite(buffer, new_length, 1, fp);
pVnode->meterIndex[sid].offset = new_offset;
pVnode->meterIndex[sid].length = new_length;
} else if (offset < 0) { // deleted meter, append to end of file
fseek(fp, -offset, SEEK_SET);
fwrite(buffer, new_length, 1, fp);
pVnode->meterIndex[sid].offset = -offset;
pVnode->meterIndex[sid].length = new_length;
} else { // meter exists, overwrite it, offset > 0
fseek(fp, offset, SEEK_SET);
fwrite(buffer, new_length, 1, fp);
pVnode->meterIndex[sid].offset = offset;
pVnode->meterIndex[sid].length = new_length;
}
}
// taosCalcChecksumAppend(0, pVnode->meterIndex, sizeof(SMeterObjHeader)*pVnode->cfg.maxSessions+sizeof(TSCKSUM));
fseek(fp, TSDB_FILE_HEADER_LEN, SEEK_SET);
fwrite(pVnode->meterIndex, sizeof(SMeterObjHeader) * pVnode->cfg.maxSessions + sizeof(TSCKSUM), 1, fp);
tfree(buffer);
vnodeUpdateVnodeStatistic(fp, pVnode);
vnodeUpdateVnodeFileHeader(fp, pVnode);
/* vnodeUpdateFileCheckSum(fp); */
fclose(fp);
return 0;
}
int vnodeSaveVnodeCfg(int vnode, SVnodeCfg *pCfg, SVPeerDesc *pDesc) {
FILE *fp;
fp = vnodeOpenMeterObjFile(vnode);
if (fp == NULL) {
dError("failed to open vnode:%d file", vnode);
return -1;
}
fseek(fp, TSDB_FILE_HEADER_LEN * 2 / 4, SEEK_SET);
fwrite(pCfg, sizeof(SVnodeCfg), 1, fp);
char temp[TSDB_FILE_HEADER_LEN / 4];
memset(temp, 0, sizeof(temp));
fseek(fp, TSDB_FILE_HEADER_LEN * 3 / 4, SEEK_SET);
fwrite(temp, sizeof(temp), 1, fp);
if (pCfg->replications >= 1) {
fseek(fp, TSDB_FILE_HEADER_LEN * 3 / 4, SEEK_SET);
fwrite(pDesc, sizeof(SVPeerDesc), pCfg->replications, fp);
}
/* vnodeUpdateFileCheckSum(fp); */
fclose(fp);
return TSDB_CODE_SUCCESS;
}
int vnodeSaveVnodeInfo(int vnode) {
FILE * fp;
SVnodeObj *pVnode = &vnodeList[vnode];
fp = vnodeOpenMeterObjFile(vnode);
if (fp == NULL) return -1;
vnodeUpdateVnodeFileHeader(fp, pVnode);
/* vnodeUpdateFileCheckSum(fp); */
fclose(fp);
return 0;
}
int vnodeRestoreMeterObj(char *buffer, int64_t length) {
SMeterObj *pSavedObj, *pObj;
int size;
pSavedObj = (SMeterObj *)buffer;
if (pSavedObj->vnode < 0 || pSavedObj->vnode >= TSDB_MAX_VNODES) {
dTrace("vid:%d is out of range, corrupted meter obj file", pSavedObj->vnode);
return -1;
}
SVnodeCfg *pCfg = &vnodeList[pSavedObj->vnode].cfg;
if (pSavedObj->sid < 0 || pSavedObj->sid >= pCfg->maxSessions) {
dTrace("vid:%d, sid:%d is larger than max:%d", pSavedObj->vnode, pSavedObj->sid, pCfg->maxSessions);
return -1;
}
if (pSavedObj->meterId[0] == 0) return TSDB_CODE_SUCCESS;
size = sizeof(SMeterObj) + pSavedObj->sqlLen + 1;
pObj = (SMeterObj *)malloc(size);
if (pObj == NULL) {
dError("vid:%d sid:%d, no memory to allocate", pSavedObj->vnode, pSavedObj->sid);
return TSDB_CODE_SERV_OUT_OF_MEMORY;
}
pObj->schema = (SColumn *)malloc(pSavedObj->numOfColumns * sizeof(SColumn));
if (NULL == pObj->schema){
dError("vid:%d sid:%d, no memory to allocate for schema", pSavedObj->vnode, pSavedObj->sid);
free(pObj);
return TSDB_CODE_SERV_OUT_OF_MEMORY;
}
memcpy(pObj, pSavedObj, offsetof(SMeterObj, reserved));
pObj->numOfQueries = 0;
pObj->pCache = vnodeAllocateCacheInfo(pObj);
if (NULL == pObj->pCache){
dError("vid:%d sid:%d, no memory to allocate for cache", pSavedObj->vnode, pSavedObj->sid);
tfree(pObj->schema);
tfree(pObj);
return TSDB_CODE_SERV_OUT_OF_MEMORY;
}
vnodeList[pSavedObj->vnode].meterList[pSavedObj->sid] = pObj;
pObj->pStream = NULL;
memcpy(pObj->schema, buffer + offsetof(SMeterObj, reserved), pSavedObj->numOfColumns * sizeof(SColumn));
pObj->state = TSDB_METER_STATE_READY;
if (pObj->sqlLen > 0)
memcpy((char *)pObj + sizeof(SMeterObj),
((char *)pSavedObj) + offsetof(SMeterObj, reserved) + sizeof(SColumn) * pSavedObj->numOfColumns,
pSavedObj->sqlLen);
pObj->pSql = (char *)pObj + sizeof(SMeterObj);
pObj->lastKey = pObj->lastKeyOnFile;
if (pObj->lastKey > vnodeList[pObj->vnode].lastKey) vnodeList[pObj->vnode].lastKey = pObj->lastKey;
// taosSetSecurityInfo(pObj->vnode, pObj->sid, pObj->meterId, pObj->spi, pObj->encrypt, pObj->secret, pObj->cipheringKey);
dTrace("vid:%d sid:%d id:%s, meter is restored, uid:%" PRIu64 "", pObj->vnode, pObj->sid, pObj->meterId, pObj->uid);
return TSDB_CODE_SUCCESS;
}
int vnodeOpenMetersVnode(int vnode) {
FILE * fp;
char * buffer;
int64_t sid;
int64_t offset, length;
SVnodeObj *pVnode = &vnodeList[vnode];
fp = vnodeOpenMeterObjFile(vnode);
if (fp == NULL) return 0;
fseek(fp, TSDB_FILE_HEADER_VERSION_SIZE, SEEK_SET);
fread(&(pVnode->vnodeStatistic), sizeof(SVnodeStatisticInfo), 1, fp);
fseek(fp, TSDB_FILE_HEADER_LEN * 1 / 4, SEEK_SET);
#ifdef _TD_ARM_32_
fscanf(fp, "%lld %lld %lld ", &(pVnode->lastCreate), &(pVnode->lastRemove), &(pVnode->version));
fscanf(fp, "%lld %d %d ", &(pVnode->lastKeyOnFile), &(pVnode->fileId), &(pVnode->numOfFiles));
#else
fscanf(fp, "%ld %ld %ld ", &(pVnode->lastCreate), &(pVnode->lastRemove), &(pVnode->version));
fscanf(fp, "%ld %d %d ", &(pVnode->lastKeyOnFile), &(pVnode->fileId), &(pVnode->numOfFiles));
#endif
fseek(fp, TSDB_FILE_HEADER_LEN * 2 / 4, SEEK_SET);
fread(&pVnode->cfg, sizeof(SVnodeCfg), 1, fp);
if (vnodeIsValidVnodeCfg(&pVnode->cfg) == false) {
dError("vid:%d, maxSessions:%d cacheBlockSize:%d replications:%d daysPerFile:%d daysToKeep:%d invalid, clear it",
vnode, pVnode->cfg.maxSessions, pVnode->cfg.cacheBlockSize, pVnode->cfg.replications,
pVnode->cfg.daysPerFile, pVnode->cfg.daysToKeep);
pVnode->cfg.maxSessions = 0; // error in vnode file
return 0;
}
fseek(fp, TSDB_FILE_HEADER_LEN * 3 / 4, SEEK_SET);
fread(&pVnode->vpeers, sizeof(SVPeerDesc), TSDB_VNODES_SUPPORT, fp);
fseek(fp, TSDB_FILE_HEADER_LEN, SEEK_SET);
tsMeterSizeOnFile = sizeof(SMeterObj) + TSDB_MAX_COLUMNS * sizeof(SColumn) + TSDB_MAX_SAVED_SQL_LEN + sizeof(TSCKSUM);
int size = sizeof(SMeterObj *) * pVnode->cfg.maxSessions;
pVnode->meterList = (void *)malloc(size);
if (pVnode->meterList == NULL) return -1;
memset(pVnode->meterList, 0, size);
size = sizeof(SMeterObjHeader) * pVnode->cfg.maxSessions + sizeof(TSCKSUM);
pVnode->meterIndex = (SMeterObjHeader *)calloc(1, size);
if (pVnode->meterIndex == NULL) {
tfree(pVnode->meterList);
return -1;
}
// Read SMeterObjHeader list from file
if (fread(pVnode->meterIndex, size, 1, fp) < 0) return -1;
// if (!taosCheckChecksumWhole(pVnode->meterIndex, size)) {
// dError("vid: %d meter obj file header is broken since checksum mismatch", vnode);
// return -1;
// }
// Read the meter object from file and recover the structure
buffer = malloc(tsMeterSizeOnFile);
memset(buffer, 0, tsMeterSizeOnFile);
for (sid = 0; sid < pVnode->cfg.maxSessions; ++sid) {
offset = pVnode->meterIndex[sid].offset;
length = pVnode->meterIndex[sid].length;
if (offset <= 0 || length <= 0) continue;
fseek(fp, offset, SEEK_SET);
if (fread(buffer, length, 1, fp) <= 0) break;
if (taosCheckChecksumWhole((uint8_t *)buffer, length)) {
vnodeRestoreMeterObj(buffer, length - sizeof(TSCKSUM));
} else {
dError("meter object file is broken since checksum mismatch, vnode: %d sid: %d, try to recover", vnode, sid);
continue;
/* vnodeRecoverMeterObjectFile(vnode); */
}
}
tfree(buffer);
fclose(fp);
return 0;
}
void vnodeCloseMetersVnode(int vnode) {
SVnodeObj *pVnode = vnodeList + vnode;
SMeterObj *pObj;
if (pVnode->meterList) {
for (int sid = 0; sid < pVnode->cfg.maxSessions; ++sid) {
pObj = pVnode->meterList[sid];
if (pObj == NULL) continue;
vnodeFreeCacheInfo(pObj);
tfree(pObj->schema);
tfree(pObj);
}
tfree(pVnode->meterList);
}
pVnode->meterList = NULL;
}
int vnodeCreateMeterObj(SMeterObj *pNew, SConnSec *pSec) {
SMeterObj *pObj;
int code;
pObj = vnodeList[pNew->vnode].meterList[pNew->sid];
code = TSDB_CODE_SUCCESS;
if (pObj && pObj->uid == pNew->uid) {
if (pObj->sversion == pNew->sversion) {
dTrace("vid:%d sid:%d id:%s sversion:%d, identical meterObj, ignore create", pNew->vnode, pNew->sid,
pNew->meterId, pNew->sversion);
return -1;
}
dTrace("vid:%d sid:%d id:%s, update schema", pNew->vnode, pNew->sid, pNew->meterId);
if (!vnodeIsMeterState(pObj, TSDB_METER_STATE_UPDATING)) vnodeUpdateMeter(pNew, NULL);
return TSDB_CODE_SUCCESS;
}
if (pObj) {
dWarn("vid:%d sid:%d id:%s, old meter is there, remove it", pNew->vnode, pNew->sid, pNew->meterId);
vnodeRemoveMeterObj(pNew->vnode, pNew->sid);
}
pNew->pCache = vnodeAllocateCacheInfo(pNew);
if (pNew->pCache == NULL) {
code = TSDB_CODE_NO_RESOURCE;
} else {
vnodeList[pNew->vnode].meterList[pNew->sid] = pNew;
pNew->state = TSDB_METER_STATE_READY;
if (pNew->timeStamp > vnodeList[pNew->vnode].lastCreate) vnodeList[pNew->vnode].lastCreate = pNew->timeStamp;
vnodeSaveMeterObjToFile(pNew);
// vnodeCreateMeterMgmt(pNew, pSec);
vnodeCreateStream(pNew);
dTrace("vid:%d, sid:%d id:%s, meterObj is created, uid:%" PRIu64 "", pNew->vnode, pNew->sid, pNew->meterId, pNew->uid);
}
return code;
}
int vnodeRemoveMeterObj(int vnode, int sid) {
SMeterObj *pObj;
if (vnode < 0 || vnode >= TSDB_MAX_VNODES) {
dError("vid:%d is out of range", vnode);
return 0;
}
SVnodeCfg *pCfg = &vnodeList[vnode].cfg;
if (sid < 0 || sid >= pCfg->maxSessions) {
dError("vid:%d, sid:%d is larger than max:%d or less than 0", vnode, sid, pCfg->maxSessions);
return 0;
}
// vnode has been closed, no meters in this vnode
if (vnodeList[vnode].meterList == NULL) return 0;
pObj = vnodeList[vnode].meterList[sid];
if (pObj == NULL) {
return TSDB_CODE_SUCCESS;
}
if (!vnodeIsSafeToDeleteMeter(&vnodeList[vnode], sid)) {
return TSDB_CODE_ACTION_IN_PROGRESS;
}
// after remove this meter, change its state to DELETED
pObj->state = TSDB_METER_STATE_DROPPED;
pObj->timeStamp = taosGetTimestampMs();
vnodeList[vnode].lastRemove = pObj->timeStamp;
vnodeRemoveStream(pObj);
vnodeSaveMeterObjToFile(pObj);
vnodeFreeMeterObj(pObj);
return 0;
}
int vnodeInsertPoints(SMeterObj *pObj, char *cont, int contLen, char source, void *param, int sversion,
int *numOfInsertPoints, TSKEY now) {
int expectedLen, i;
short numOfPoints;
SSubmitMsg *pSubmit = (SSubmitMsg *)cont;
char * pData;
TSKEY tsKey;
int points = 0;
int code = TSDB_CODE_SUCCESS;
SVnodeObj * pVnode = vnodeList + pObj->vnode;
numOfPoints = htons(pSubmit->numOfRows);
expectedLen = numOfPoints * pObj->bytesPerPoint + sizeof(pSubmit->numOfRows);
if (expectedLen != contLen) {
dError("vid:%d sid:%d id:%s, invalid submit msg length:%d, expected:%d, bytesPerPoint: %d",
pObj->vnode, pObj->sid, pObj->meterId, contLen, expectedLen, pObj->bytesPerPoint);
code = TSDB_CODE_WRONG_MSG_SIZE;
goto _over;
}
// to guarantee time stamp is the same for all vnodes
pData = pSubmit->payLoad;
tsKey = now;
if (*((TSKEY *)pData) == 0) {
for (i = 0; i < numOfPoints; ++i) {
*((TSKEY *)pData) = tsKey++;
pData += pObj->bytesPerPoint;
}
}
if (numOfPoints >= (pVnode->cfg.blocksPerMeter - 2) * pObj->pointsPerBlock) {
code = TSDB_CODE_BATCH_SIZE_TOO_BIG;
dError("vid:%d sid:%d id:%s, batch size too big, insert points:%d, it shall be smaller than:%d", pObj->vnode, pObj->sid,
pObj->meterId, numOfPoints, (pVnode->cfg.blocksPerMeter - 2) * pObj->pointsPerBlock);
return code;
}
/*
* please refer to TBASE-926, data may be lost when the cache is full
*/
if (source == TSDB_DATA_SOURCE_SHELL && pVnode->cfg.replications > 1) {
code = vnodeForwardToPeer(pObj, cont, contLen, TSDB_ACTION_INSERT, sversion);
if (code != TSDB_CODE_SUCCESS) return code;
}
SCachePool *pPool = (SCachePool *)pVnode->pCachePool;
if (pObj->freePoints < numOfPoints || pObj->freePoints < (pObj->pointsPerBlock << 1) ||
pPool->notFreeSlots > pVnode->cfg.cacheNumOfBlocks.totalBlocks - 2) {
code = TSDB_CODE_ACTION_IN_PROGRESS;
dTrace("vid:%d sid:%d id:%s, cache is full, freePoints:%d, notFreeSlots:%d", pObj->vnode, pObj->sid, pObj->meterId,
pObj->freePoints, pPool->notFreeSlots);
vnodeProcessCommitTimer(pVnode, NULL);
return code;
}
// FIXME: Here should be after the comparison of sversions.
if (pVnode->cfg.commitLog && source != TSDB_DATA_SOURCE_LOG) {
if (pVnode->logFd < 0) return TSDB_CODE_INVALID_COMMIT_LOG;
code = vnodeWriteToCommitLog(pObj, TSDB_ACTION_INSERT, cont, contLen, sversion);
if (code != TSDB_CODE_SUCCESS) return code;
}
if (pObj->sversion < sversion) {
dTrace("vid:%d sid:%d id:%s, schema is changed, new:%d old:%d", pObj->vnode, pObj->sid, pObj->meterId, sversion,
pObj->sversion);
vnodeSendMeterCfgMsg(pObj->vnode, pObj->sid);
code = TSDB_CODE_ACTION_IN_PROGRESS;
return code;
} else if (pObj->sversion > sversion) {
dTrace("vid:%d sid:%d id:%s, client schema out of date, sql is invalid. client sversion:%d vnode sversion:%d",
pObj->vnode, pObj->sid, pObj->meterId, pObj->sversion, sversion);
code = TSDB_CODE_INVALID_SQL;
return code;
}
pData = pSubmit->payLoad;
TSKEY firstKey = *((TSKEY *)pData);
TSKEY lastKey = *((TSKEY *)(pData + pObj->bytesPerPoint * (numOfPoints - 1)));
int cfid = now/pVnode->cfg.daysPerFile/tsMsPerDay[(uint8_t)pVnode->cfg.precision];
TSKEY minAllowedKey = (cfid - pVnode->maxFiles + 1)*pVnode->cfg.daysPerFile*tsMsPerDay[(uint8_t)pVnode->cfg.precision];
TSKEY maxAllowedKey = (cfid + 2)*pVnode->cfg.daysPerFile*tsMsPerDay[(uint8_t)pVnode->cfg.precision] - 2;
if (firstKey < minAllowedKey || firstKey > maxAllowedKey || lastKey < minAllowedKey || lastKey > maxAllowedKey) {
dError("vid:%d sid:%d id:%s, vnode lastKeyOnFile:%" PRId64 ", data is out of range, numOfPoints:%d firstKey:%" PRId64 " lastKey:%" PRId64 " minAllowedKey:%" PRId64 " maxAllowedKey:%" PRId64,
pObj->vnode, pObj->sid, pObj->meterId, pVnode->lastKeyOnFile, numOfPoints,firstKey, lastKey, minAllowedKey, maxAllowedKey);
return TSDB_CODE_TIMESTAMP_OUT_OF_RANGE;
}
if ((code = vnodeSetMeterInsertImportStateEx(pObj, TSDB_METER_STATE_INSERTING)) != TSDB_CODE_SUCCESS) {
goto _over;
}
for (i = 0; i < numOfPoints; ++i) { // meter will be dropped, abort current insertion
if (vnodeIsMeterState(pObj, TSDB_METER_STATE_DROPPING)) {
dWarn("vid:%d sid:%d id:%s, meter is dropped, abort insert, state:%d", pObj->vnode, pObj->sid, pObj->meterId,
pObj->state);
code = TSDB_CODE_NOT_ACTIVE_TABLE;
break;
}
if (*((TSKEY *)pData) <= pObj->lastKey) {
dWarn("vid:%d sid:%d id:%s, received key:%" PRId64 " not larger than lastKey:%" PRId64, pObj->vnode, pObj->sid, pObj->meterId,
*((TSKEY *)pData), pObj->lastKey);
pData += pObj->bytesPerPoint;
continue;
}
if (!VALID_TIMESTAMP(*((TSKEY *)pData), tsKey, (uint8_t)pVnode->cfg.precision)) {
code = TSDB_CODE_TIMESTAMP_OUT_OF_RANGE;
break;
}
if (vnodeInsertPointToCache(pObj, pData) < 0) {
code = TSDB_CODE_ACTION_IN_PROGRESS;
break;
}
pObj->lastKey = *((TSKEY *)pData);
pData += pObj->bytesPerPoint;
points++;
}
atomic_fetch_add_64(&(pVnode->vnodeStatistic.pointsWritten), points * (pObj->numOfColumns - 1));
atomic_fetch_add_64(&(pVnode->vnodeStatistic.totalStorage), points * pObj->bytesPerPoint);
pthread_mutex_lock(&(pVnode->vmutex));
if (pObj->lastKey > pVnode->lastKey) pVnode->lastKey = pObj->lastKey;
if (firstKey < pVnode->firstKey) pVnode->firstKey = firstKey;
assert(pVnode->firstKey > 0);
pVnode->version++;
pthread_mutex_unlock(&(pVnode->vmutex));
vnodeClearMeterState(pObj, TSDB_METER_STATE_INSERTING);
_over:
dTrace("vid:%d sid:%d id:%s, %d out of %d points are inserted, lastKey:%" PRId64 " source:%d, vnode total storage: %" PRId64 "",
pObj->vnode, pObj->sid, pObj->meterId, points, numOfPoints, pObj->lastKey, source,
pVnode->vnodeStatistic.totalStorage);
*numOfInsertPoints = points;
return code;
}
/**
* continue running of the function may cause the free vnode crash with high probability
* todo fix it by set flag to disable commit in any cases
*
* @param param
* @param tmrId
*/
void vnodeProcessUpdateSchemaTimer(void *param, void *tmrId) {
SMeterObj * pObj = (SMeterObj *)param;
SVnodeObj * pVnode = vnodeList + pObj->vnode;
/*
* vnode may have been dropped, check it in the first place
* if the vnode is freed, the pObj is not valid any more, the pObj->vnode is meanless
* so may be the vid should be passed into this function as a parameter?
*/
if (pVnode->meterList == NULL) {
dTrace("vnode is deleted, abort update schema");
return;
}
SCachePool *pPool = (SCachePool *)pVnode->pCachePool;
pthread_mutex_lock(&pPool->vmutex);
if (pPool->commitInProcess) {
dTrace("vid:%d sid:%d mid:%s, committing in process, commit later", pObj->vnode, pObj->sid, pObj->meterId);
if (taosTmrStart(vnodeProcessUpdateSchemaTimer, 10, pObj, vnodeTmrCtrl) == NULL) {
vnodeClearMeterState(pObj, TSDB_METER_STATE_UPDATING);
}
pthread_mutex_unlock(&pPool->vmutex);
return;
}
pPool->commitInProcess = 1;
pthread_mutex_unlock(&pPool->vmutex);
vnodeCommitMultiToFile(pVnode, pObj->sid, pObj->sid);
}
void vnodeUpdateMeter(void *param, void *tmrId) {
SMeterObj *pNew = (SMeterObj *)param;
if (pNew == NULL || pNew->vnode < 0 || pNew->sid < 0) return;
SVnodeObj* pVnode = &vnodeList[pNew->vnode];
if (pVnode->meterList == NULL) {
dTrace("vid:%d sid:%d id:%s, vnode is deleted, status:%s, abort update schema",
pNew->vnode, pNew->sid, pNew->meterId, taosGetVnodeStatusStr(vnodeList[pNew->vnode].vnodeStatus));
free(pNew->schema);
free(pNew);
return;
}
SMeterObj *pObj = pVnode->meterList[pNew->sid];
if (pObj == NULL || vnodeIsMeterState(pObj, TSDB_METER_STATE_DROPPING)) {
dTrace("vid:%d sid:%d id:%s, meter is deleted, abort update schema", pNew->vnode, pNew->sid, pNew->meterId);
free(pNew->schema);
free(pNew);
return;
}
int32_t state = vnodeSetMeterState(pObj, TSDB_METER_STATE_UPDATING);
if (state >= TSDB_METER_STATE_DROPPING) {
dError("vid:%d sid:%d id:%s, meter is deleted, failed to update, state:%d",
pObj->vnode, pObj->sid, pObj->meterId, state);
return;
}
int32_t num = 0;
pthread_mutex_lock(&pVnode->vmutex);
num = pObj->numOfQueries;
pthread_mutex_unlock(&pVnode->vmutex);
if (num > 0 || state != TSDB_METER_STATE_READY) {
// the state may have been changed by vnodeSetMeterState, recover it in the first place
vnodeClearMeterState(pObj, TSDB_METER_STATE_UPDATING);
dTrace("vid:%d sid:%d id:%s, update failed, retry later, numOfQueries:%d, state:%d",
pNew->vnode, pNew->sid, pNew->meterId, num, state);
// retry update meter in 50ms
if (taosTmrStart(vnodeUpdateMeter, 50, pNew, vnodeTmrCtrl) == NULL) {
dError("vid:%d sid:%d id:%s, failed to start update timer, no retry", pNew->vnode, pNew->sid, pNew->meterId);
free(pNew->schema);
free(pNew);
}
return;
}
// commit first
if (!vnodeIsCacheCommitted(pObj)) {
// commit data first
if (taosTmrStart(vnodeProcessUpdateSchemaTimer, 0, pObj, vnodeTmrCtrl) == NULL) {
dError("vid:%d sid:%d id:%s, failed to start commit timer", pObj->vnode, pObj->sid, pObj->meterId);
vnodeClearMeterState(pObj, TSDB_METER_STATE_UPDATING);
free(pNew->schema);
free(pNew);
return;
}
if (taosTmrStart(vnodeUpdateMeter, 50, pNew, vnodeTmrCtrl) == NULL) {
dError("vid:%d sid:%d id:%s, failed to start update timer", pNew->vnode, pNew->sid, pNew->meterId);
vnodeClearMeterState(pObj, TSDB_METER_STATE_UPDATING);
free(pNew->schema);
free(pNew);
}
dTrace("vid:%d sid:%d meterId:%s, there are data in cache, commit first, update later",
pNew->vnode, pNew->sid, pNew->meterId);
vnodeClearMeterState(pObj, TSDB_METER_STATE_UPDATING);
return;
}
strcpy(pObj->meterId, pNew->meterId);
pObj->numOfColumns = pNew->numOfColumns;
pObj->timeStamp = pNew->timeStamp;
pObj->bytesPerPoint = pNew->bytesPerPoint;
pObj->maxBytes = pNew->maxBytes;
if (pObj->timeStamp > vnodeList[pObj->vnode].lastCreate) vnodeList[pObj->vnode].lastCreate = pObj->timeStamp;
tfree(pObj->schema);
pObj->schema = pNew->schema;
vnodeFreeCacheInfo(pObj);
pObj->pCache = vnodeAllocateCacheInfo(pObj);
pObj->sversion = pNew->sversion;
vnodeSaveMeterObjToFile(pObj);
vnodeClearMeterState(pObj, TSDB_METER_STATE_UPDATING);
dTrace("vid:%d sid:%d id:%s, schema is updated, state:%d", pObj->vnode, pObj->sid, pObj->meterId, pObj->state);
free(pNew);
}
void vnodeRecoverMeterObjectFile(int vnode) {
// TODO: start the recovery process
assert(0);
}

7714
src/vnode/detail/src/vnodeQueryImpl.c
View File

File diff suppressed because it is too large Load Diff

1364
src/vnode/detail/src/vnodeQueryProcess.c
View File

File diff suppressed because it is too large Load Diff

1153
src/vnode/detail/src/vnodeRead.c
View File

File diff suppressed because it is too large Load Diff

409
src/vnode/detail/src/vnodeStore.c
View File

@ -1,409 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "dnodeSystem.h"
#include "trpc.h"
#include "ttime.h"
#include "vnode.h"
#include "vnodeStore.h"
#include "vnodeUtil.h"
#include "vnodeStatus.h"
int tsMaxVnode = -1;
int tsOpenVnodes = 0;
SVnodeObj *vnodeList = NULL;
static int vnodeInitStoreVnode(int vnode) {
SVnodeObj *pVnode = vnodeList + vnode;
pVnode->vnode = vnode;
vnodeOpenMetersVnode(vnode);
if (pVnode->cfg.maxSessions <= 0) {
return TSDB_CODE_SUCCESS;
}
pVnode->firstKey = taosGetTimestamp(pVnode->cfg.precision);
pVnode->pCachePool = vnodeOpenCachePool(vnode);
if (pVnode->pCachePool == NULL) {
dError("vid:%d, cache pool init failed.", pVnode->vnode);
return TSDB_CODE_SERV_OUT_OF_MEMORY;
}
if (vnodeInitFile(vnode) != TSDB_CODE_SUCCESS) {
dError("vid:%d, files init failed.", pVnode->vnode);
return TSDB_CODE_VG_INIT_FAILED;
}
if (vnodeInitCommit(vnode) != TSDB_CODE_SUCCESS) {
dError("vid:%d, commit init failed.", pVnode->vnode);
return TSDB_CODE_VG_INIT_FAILED;
}
pthread_mutex_init(&(pVnode->vmutex), NULL);
dPrint("vid:%d, storage initialized, version:%" PRIu64 " fileId:%d numOfFiles:%d", vnode, pVnode->version, pVnode->fileId,
pVnode->numOfFiles);
return TSDB_CODE_SUCCESS;
}
int vnodeOpenVnode(int vnode) {
int32_t code = TSDB_CODE_SUCCESS;
SVnodeObj *pVnode = vnodeList + vnode;
pVnode->vnode = vnode;
pVnode->accessState = TSDB_VN_ALL_ACCCESS;
// vnode is empty
if (pVnode->cfg.maxSessions <= 0) {
return TSDB_CODE_SUCCESS;
}
if (!(pVnode->vnodeStatus == TSDB_VN_STATUS_OFFLINE || pVnode->vnodeStatus == TSDB_VN_STATUS_CREATING)) {
dError("vid:%d, status:%s, cannot enter open operation", vnode, taosGetVnodeStatusStr(pVnode->vnodeStatus));
return TSDB_CODE_INVALID_VNODE_STATUS;
}
dPrint("vid:%d, status:%s, start to open", vnode, taosGetVnodeStatusStr(pVnode->vnodeStatus));
pthread_mutex_lock(&dmutex);
// not enough memory, abort
if ((code = vnodeOpenShellVnode(vnode)) != TSDB_CODE_SUCCESS) {
pthread_mutex_unlock(&dmutex);
return code;
}
vnodeOpenPeerVnode(vnode);
if (vnode > tsMaxVnode) tsMaxVnode = vnode;
vnodeCalcOpenVnodes();
pthread_mutex_unlock(&dmutex);
#ifndef CLUSTER
vnodeOpenStreams(pVnode, NULL);
#endif
dPrint("vid:%d, vnode is opened, openVnodes:%d, status:%s", vnode, tsOpenVnodes, taosGetVnodeStatusStr(pVnode->vnodeStatus));
return TSDB_CODE_SUCCESS;
}
static int32_t vnodeMarkAllMetersDropped(SVnodeObj* pVnode) {
if (pVnode->meterList == NULL) {
return TSDB_CODE_SUCCESS;
}
bool ready = true;
for (int sid = 0; sid < pVnode->cfg.maxSessions; ++sid) {
if (!vnodeIsSafeToDeleteMeter(pVnode, sid)) {
ready = false;
} else { // set the meter is to be deleted
SMeterObj* pObj = pVnode->meterList[sid];
if (pObj != NULL) {
pObj->state = TSDB_METER_STATE_DROPPED;
}
}
}
return ready? TSDB_CODE_SUCCESS:TSDB_CODE_ACTION_IN_PROGRESS;
}
static int vnodeCloseVnode(int vnode) {
if (vnodeList == NULL) return TSDB_CODE_SUCCESS;
SVnodeObj* pVnode = &vnodeList[vnode];
pthread_mutex_lock(&dmutex);
if (pVnode->cfg.maxSessions == 0) {
pthread_mutex_unlock(&dmutex);
return TSDB_CODE_SUCCESS;
}
if (pVnode->vnodeStatus == TSDB_VN_STATUS_DELETING) {
dPrint("vid:%d, status:%s, another thread performed delete operation", vnode, taosGetVnodeStatusStr(pVnode->vnodeStatus));
return TSDB_CODE_SUCCESS;
} else {
dPrint("vid:%d, status:%s, enter close operation", vnode, taosGetVnodeStatusStr(pVnode->vnodeStatus));
pVnode->vnodeStatus = TSDB_VN_STATUS_CLOSING;
}
// set the meter is dropped flag
if (vnodeMarkAllMetersDropped(pVnode) != TSDB_CODE_SUCCESS) {
pthread_mutex_unlock(&dmutex);
return TSDB_CODE_ACTION_IN_PROGRESS;
}
dPrint("vid:%d, status:%s, enter delete operation", vnode, taosGetVnodeStatusStr(pVnode->vnodeStatus));
pVnode->vnodeStatus = TSDB_VN_STATUS_DELETING;
vnodeCloseStream(vnodeList + vnode);
vnodeCancelCommit(vnodeList + vnode);
vnodeClosePeerVnode(vnode);
vnodeCloseMetersVnode(vnode);
vnodeCloseShellVnode(vnode);
vnodeCloseCachePool(vnode);
vnodeCleanUpCommit(vnode);
pthread_mutex_destroy(&(vnodeList[vnode].vmutex));
if (tsMaxVnode == vnode) tsMaxVnode = vnode - 1;
tfree(vnodeList[vnode].meterIndex);
pthread_mutex_unlock(&dmutex);
return TSDB_CODE_SUCCESS;
}
int vnodeCreateVnode(int vnode, SVnodeCfg *pCfg, SVPeerDesc *pDesc) {
char fileName[128];
if (vnodeList[vnode].vnodeStatus != TSDB_VN_STATUS_OFFLINE) {
dError("vid:%d, status:%s, cannot enter create operation", vnode, taosGetVnodeStatusStr(vnodeList[vnode].vnodeStatus));
return TSDB_CODE_INVALID_VNODE_STATUS;
}
vnodeList[vnode].vnodeStatus = TSDB_VN_STATUS_CREATING;
sprintf(fileName, "%s/vnode%d", tsDirectory, vnode);
if (mkdir(fileName, 0755) != 0) {
dError("failed to create vnode:%d directory:%s, errno:%d, reason:%s", vnode, fileName, errno, strerror(errno));
if (errno == EACCES) {
return TSDB_CODE_NO_DISK_PERMISSIONS;
} else if (errno == ENOSPC) {
return TSDB_CODE_SERV_NO_DISKSPACE;
} else if (errno == EEXIST) {
} else {
return TSDB_CODE_VG_INIT_FAILED;
}
}
sprintf(fileName, "%s/vnode%d/db", tsDirectory, vnode);
if (mkdir(fileName, 0755) != 0) {
dError("failed to create vnode:%d directory:%s, errno:%d, reason:%s", vnode, fileName, errno, strerror(errno));
if (errno == EACCES) {
return TSDB_CODE_NO_DISK_PERMISSIONS;
} else if (errno == ENOSPC) {
return TSDB_CODE_SERV_NO_DISKSPACE;
} else if (errno == EEXIST) {
} else {
return TSDB_CODE_VG_INIT_FAILED;
}
}
vnodeList[vnode].cfg = *pCfg;
int code = vnodeCreateMeterObjFile(vnode);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
code = vnodeSaveVnodeCfg(vnode, pCfg, pDesc);
if (code != TSDB_CODE_SUCCESS) {
return TSDB_CODE_VG_INIT_FAILED;
}
code = vnodeInitStoreVnode(vnode);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
return vnodeOpenVnode(vnode);
}
static void vnodeRemoveDataFiles(int vnode) {
char vnodeDir[TSDB_FILENAME_LEN];
char dfilePath[TSDB_FILENAME_LEN];
char linkFile[TSDB_FILENAME_LEN];
struct dirent *de = NULL;
DIR * dir = NULL;
sprintf(vnodeDir, "%s/vnode%d/db", tsDirectory, vnode);
dir = opendir(vnodeDir);
if (dir == NULL) return;
while ((de = readdir(dir)) != NULL) {
if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue;
if ((strcmp(de->d_name + strlen(de->d_name) - strlen(".head"), ".head") == 0 ||
strcmp(de->d_name + strlen(de->d_name) - strlen(".data"), ".data") == 0 ||
strcmp(de->d_name + strlen(de->d_name) - strlen(".last"), ".last") == 0) &&
(de->d_type & DT_LNK)) {
sprintf(linkFile, "%s/%s", vnodeDir, de->d_name);
if (!vnodeRemoveDataFileFromLinkFile(linkFile, de->d_name)) {
continue;
}
memset(dfilePath, 0, TSDB_FILENAME_LEN);
int tcode = readlink(linkFile, dfilePath, TSDB_FILENAME_LEN);
remove(linkFile);
if (tcode >= 0) {
remove(dfilePath);
dPrint("Data file %s is removed, link file %s", dfilePath, linkFile);
}
} else {
remove(de->d_name);
}
}
closedir(dir);
rmdir(vnodeDir);
sprintf(vnodeDir, "%s/vnode%d/meterObj.v%d", tsDirectory, vnode, vnode);
remove(vnodeDir);
sprintf(vnodeDir, "%s/vnode%d", tsDirectory, vnode);
rmdir(vnodeDir);
dPrint("vid:%d, vnode is removed, status:%s", vnode, taosGetVnodeStatusStr(vnodeList[vnode].vnodeStatus));
}
int vnodeRemoveVnode(int vnode) {
if (vnodeList == NULL) return TSDB_CODE_SUCCESS;
if (vnodeList[vnode].cfg.maxSessions > 0) {
SVnodeObj* pVnode = &vnodeList[vnode];
if (pVnode->vnodeStatus == TSDB_VN_STATUS_CREATING
|| pVnode->vnodeStatus == TSDB_VN_STATUS_OFFLINE
|| pVnode->vnodeStatus == TSDB_VN_STATUS_DELETING) {
dTrace("vid:%d, status:%s, cannot enter close/delete operation", vnode, taosGetVnodeStatusStr(pVnode->vnodeStatus));
return TSDB_CODE_ACTION_IN_PROGRESS;
} else {
int32_t ret = vnodeCloseVnode(vnode);
if (ret != TSDB_CODE_SUCCESS) {
return ret;
}
dTrace("vid:%d, status:%s, do delete operation", vnode, taosGetVnodeStatusStr(pVnode->vnodeStatus));
vnodeRemoveDataFiles(vnode);
}
} else {
dPrint("vid:%d, max sessions:%d, this vnode already dropped!!!", vnode, vnodeList[vnode].cfg.maxSessions);
vnodeList[vnode].cfg.maxSessions = 0; //reset value
vnodeCalcOpenVnodes();
}
return TSDB_CODE_SUCCESS;
}
int vnodeInitStore() {
int vnode;
int size;
size = sizeof(SVnodeObj) * TSDB_MAX_VNODES;
vnodeList = (SVnodeObj *)malloc(size);
if (vnodeList == NULL) return -1;
memset(vnodeList, 0, size);
if (vnodeInitInfo() < 0) return -1;
for (vnode = 0; vnode < TSDB_MAX_VNODES; ++vnode) {
int code = vnodeInitStoreVnode(vnode);
if (code != TSDB_CODE_SUCCESS) {
// one vnode is failed to recover from commit log, continue for remain
return -1;
}
}
return 0;
}
int vnodeInitVnodes() {
int vnode;
for (vnode = 0; vnode < TSDB_MAX_VNODES; ++vnode) {
if (vnodeOpenVnode(vnode) < 0) return -1;
}
return 0;
}
void vnodeCleanUpOneVnode(int vnode) {
static int again = 0;
if (vnodeList == NULL) return;
pthread_mutex_lock(&dmutex);
if (again) {
pthread_mutex_unlock(&dmutex);
return;
}
again = 1;
if (vnodeList[vnode].pCachePool) {
vnodeList[vnode].vnodeStatus = TSDB_VN_STATUS_OFFLINE;
vnodeClosePeerVnode(vnode);
}
pthread_mutex_unlock(&dmutex);
if (vnodeList[vnode].pCachePool) {
vnodeProcessCommitTimer(vnodeList + vnode, NULL);
while (vnodeList[vnode].commitThread != 0) {
taosMsleep(10);
}
vnodeCleanUpCommit(vnode);
}
}
void vnodeCleanUpVnodes() {
static int again = 0;
if (vnodeList == NULL) return;
pthread_mutex_lock(&dmutex);
if (again) {
pthread_mutex_unlock(&dmutex);
return;
}
again = 1;
for (int vnode = 0; vnode < TSDB_MAX_VNODES; ++vnode) {
if (vnodeList[vnode].pCachePool) {
vnodeList[vnode].vnodeStatus = TSDB_VN_STATUS_OFFLINE;
vnodeClosePeerVnode(vnode);
}
}
pthread_mutex_unlock(&dmutex);
for (int vnode = 0; vnode < TSDB_MAX_VNODES; ++vnode) {
if (vnodeList[vnode].pCachePool) {
vnodeProcessCommitTimer(vnodeList + vnode, NULL);
while (vnodeList[vnode].commitThread != 0) {
taosMsleep(10);
}
vnodeCleanUpCommit(vnode);
}
}
}
void vnodeCalcOpenVnodes() {
int openVnodes = 0;
for (int vnode = 0; vnode <= tsMaxVnode; ++vnode) {
if (vnodeList[vnode].cfg.maxSessions <= 0) continue;
openVnodes++;
}
atomic_store_32(&tsOpenVnodes, openVnodes);
}
void vnodeUpdateHeadFile(int vnode, int oldTables, int newTables) {
//todo rewrite the head file with newTables
}

22
src/vnode/detail/src/vnodeStore.spec.c
View File

@ -1,22 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include <stdbool.h>
int vnodeInitInfo() { return 0; }
bool vnodeRemoveDataFileFromLinkFile(char* linkFile, char* de_name) { return true; }

207
src/vnode/detail/src/vnodeStream.c
View File

@ -1,207 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "taosmsg.h"
#include "vnode.h"
#include "vnodeUtil.h"
#include "vnodeStatus.h"
/* static TAOS *dbConn = NULL; */
void vnodeCloseStreamCallback(void *param);
void vnodeProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) {
SMeterObj *pObj = (SMeterObj *)param;
dTrace("vid:%d sid:%d id:%s, stream result is ready", pObj->vnode, pObj->sid, pObj->meterId);
// construct data
int32_t contLen = pObj->bytesPerPoint;
char * pTemp = calloc(1, sizeof(SSubmitMsg) + pObj->bytesPerPoint + sizeof(SVMsgHeader));
SSubmitMsg *pMsg = (SSubmitMsg *)(pTemp + sizeof(SVMsgHeader));
pMsg->numOfRows = htons(1);
char ncharBuf[TSDB_MAX_BYTES_PER_ROW] = {0};
int32_t offset = 0;
for (int32_t i = 0; i < pObj->numOfColumns; ++i) {
char *dst = row[i];
if (dst == NULL) {
setNull(pMsg->payLoad + offset, pObj->schema[i].type, pObj->schema[i].bytes);
} else {
// here, we need to transfer nchar(utf8) to unicode(ucs-4)
if (pObj->schema[i].type == TSDB_DATA_TYPE_NCHAR) {
taosMbsToUcs4(row[i], pObj->schema[i].bytes, ncharBuf, TSDB_MAX_BYTES_PER_ROW);
dst = ncharBuf;
}
memcpy(pMsg->payLoad + offset, dst, pObj->schema[i].bytes);
}
offset += pObj->schema[i].bytes;
}
contLen += sizeof(SSubmitMsg);
int32_t numOfPoints = 0;
int32_t code = vnodeInsertPoints(pObj, (char *)pMsg, contLen, TSDB_DATA_SOURCE_SHELL, NULL, pObj->sversion,
&numOfPoints, taosGetTimestamp(vnodeList[pObj->vnode].cfg.precision));
if (code != TSDB_CODE_SUCCESS) {
dError("vid:%d sid:%d id:%s, failed to insert continuous query results", pObj->vnode, pObj->sid, pObj->meterId);
}
assert(numOfPoints >= 0 && numOfPoints <= 1);
tfree(pTemp);
}
static void vnodeGetDBFromMeterId(SMeterObj *pObj, char *db) {
char *st = strstr(pObj->meterId, ".");
char *end = strstr(st + 1, ".");
memcpy(db, st + 1, end - (st + 1));
}
void vnodeOpenStreams(void *param, void *tmrId) {
SVnodeObj *pVnode = (SVnodeObj *)param;
SMeterObj *pObj;
if (pVnode->streamRole == TSDB_VN_STREAM_STATUS_STOP) return;
if (pVnode->meterList == NULL) return;
taosTmrStopA(&pVnode->streamTimer);
pVnode->streamTimer = NULL;
for (int sid = 0; sid < pVnode->cfg.maxSessions; ++sid) {
pObj = pVnode->meterList[sid];
if (pObj == NULL || pObj->sqlLen == 0 || vnodeIsMeterState(pObj, TSDB_METER_STATE_DROPPING)) continue;
dTrace("vid:%d sid:%d id:%s, open stream:%s", pObj->vnode, sid, pObj->meterId, pObj->pSql);
if (pVnode->dbConn == NULL) {
char db[64] = {0};
char user[64] = {0};
vnodeGetDBFromMeterId(pObj, db);
sprintf(user, "_%s", pVnode->cfg.acct);
pVnode->dbConn = taos_connect(NULL, user, tsInternalPass, db, 0);
}
if (pVnode->dbConn == NULL) {
dError("vid:%d, failed to connect to mgmt node", pVnode->vnode);
taosTmrReset(vnodeOpenStreams, 1000, param, vnodeTmrCtrl, &pVnode->streamTimer);
return;
}
if (pObj->pStream == NULL) {
pObj->pStream = taos_open_stream(pVnode->dbConn, pObj->pSql, vnodeProcessStreamRes, pObj->lastKey, pObj,
vnodeCloseStreamCallback);
if (pObj->pStream) pVnode->numOfStreams++;
}
}
}
void vnodeCreateStream(SMeterObj *pObj) {
if (pObj->sqlLen <= 0) return;
SVnodeObj *pVnode = vnodeList + pObj->vnode;
if (pVnode->streamRole == TSDB_VN_STREAM_STATUS_STOP) return;
if (pObj->pStream) return;
dTrace("vid:%d sid:%d id:%s stream:%s is created", pObj->vnode, pObj->sid, pObj->meterId, pObj->pSql);
if (pVnode->dbConn == NULL) {
if (pVnode->streamTimer == NULL) taosTmrReset(vnodeOpenStreams, 1000, pVnode, vnodeTmrCtrl, &pVnode->streamTimer);
} else {
pObj->pStream = taos_open_stream(pVnode->dbConn, pObj->pSql, vnodeProcessStreamRes, pObj->lastKey, pObj,
vnodeCloseStreamCallback);
if (pObj->pStream) pVnode->numOfStreams++;
}
}
// Close only one stream
void vnodeRemoveStream(SMeterObj *pObj) {
SVnodeObj *pVnode = vnodeList + pObj->vnode;
if (pObj->sqlLen <= 0) return;
if (pObj->pStream) {
taos_close_stream(pObj->pStream);
pVnode->numOfStreams--;
}
pObj->pStream = NULL;
if (pVnode->numOfStreams == 0) {
taos_close(pVnode->dbConn);
pVnode->dbConn = NULL;
}
dTrace("vid:%d sid:%d id:%d stream is removed", pObj->vnode, pObj->sid, pObj->meterId);
}
// Close all streams in a vnode
void vnodeCloseStream(SVnodeObj *pVnode) {
SMeterObj *pObj;
dPrint("vid:%d, stream is closed, old role %s", pVnode->vnode, taosGetVnodeStreamStatusStr(pVnode->streamRole));
// stop stream computing
for (int sid = 0; sid < pVnode->cfg.maxSessions; ++sid) {
pObj = pVnode->meterList[sid];
if (pObj == NULL) continue;
if (pObj->sqlLen > 0 && pObj->pStream) {
taos_close_stream(pObj->pStream);
pVnode->numOfStreams--;
}
pObj->pStream = NULL;
}
}
void vnodeUpdateStreamRole(SVnodeObj *pVnode) {
/* SMeterObj *pObj; */
int newRole = (pVnode->vnodeStatus == TSDB_VN_STATUS_MASTER) ? TSDB_VN_STREAM_STATUS_START : TSDB_VN_STREAM_STATUS_STOP;
if (newRole != pVnode->streamRole) {
dPrint("vid:%d, stream role is changed from %s to %s",
pVnode->vnode, taosGetVnodeStreamStatusStr(pVnode->streamRole), taosGetVnodeStreamStatusStr(newRole));
pVnode->streamRole = newRole;
if (newRole == TSDB_VN_STREAM_STATUS_START) {
vnodeOpenStreams(pVnode, NULL);
} else {
vnodeCloseStream(pVnode);
}
} else {
dPrint("vid:%d, stream role is keep to %s", pVnode->vnode, taosGetVnodeStreamStatusStr(pVnode->streamRole));
}
}
// Callback function called from client
void vnodeCloseStreamCallback(void *param) {
SMeterObj *pTable = (SMeterObj *)param;
SVnodeObj *pVnode = NULL;
if (pTable == NULL || pTable->sqlLen == 0) return;
pVnode = vnodeList + pTable->vnode;
pTable->sqlLen = 0;
pTable->pSql = NULL;
pTable->pStream = NULL;
pVnode->numOfStreams--;
if (pVnode->numOfStreams == 0) {
taos_close(pVnode->dbConn);
pVnode->dbConn = NULL;
}
vnodeSaveMeterObjToFile(pTable);
}

874
src/vnode/detail/src/vnodeSupertableQuery.c
View File

@ -1,874 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "mnode.h"
#include "os.h"
#include "qast.h"
#include "qextbuffer.h"
#include "tschemautil.h"
#include "tsqlfunction.h"
typedef struct SSyntaxTreeFilterSupporter {
SSchema* pTagSchema;
int32_t numOfTags;
int32_t optr;
} SSyntaxTreeFilterSupporter;
typedef struct SJoinSupporter {
void** val;
void** pTabObjs;
int32_t size;
int16_t type;
int16_t colIndex;
void** qualMeterObj;
int32_t qualSize;
} SJoinSupporter;
typedef struct SMeterNameFilterSupporter {
SPatternCompareInfo info;
char* pattern;
} SMeterNameFilterSupporter;
static void tansformQueryResult(tQueryResultset* pRes);
static bool tSkipListNodeFilterCallback(const void *pNode, void *param);
static int32_t tabObjVGIDComparator(const void* pLeft, const void* pRight) {
STabObj* p1 = *(STabObj**)pLeft;
STabObj* p2 = *(STabObj**)pRight;
int32_t ret = p1->gid.vgId - p2->gid.vgId;
if (ret == 0) {
return ret;
} else {
return ret > 0 ? 1 : -1;
}
}
// monotonic inc in memory address
static int32_t tabObjPointerComparator(const void* pLeft, const void* pRight) {
int64_t ret = (*(STabObj**)(pLeft))->uid - (*(STabObj**)(pRight))->uid;
if (ret == 0) {
return 0;
} else {
return ret > 0 ? 1 : -1;
}
}
static int32_t tabObjResultComparator(const void* p1, const void* p2, void* param) {
tOrderDescriptor* pOrderDesc = (tOrderDescriptor*)param;
STabObj* pNode1 = (STabObj*)p1;
STabObj* pNode2 = (STabObj*)p2;
for (int32_t i = 0; i < pOrderDesc->orderIdx.numOfCols; ++i) {
int32_t colIdx = pOrderDesc->orderIdx.pData[i];
char* f1 = NULL;
char* f2 = NULL;
SSchema schema = {0};
if (colIdx == -1) {
f1 = pNode1->meterId;
f2 = pNode2->meterId;
schema.type = TSDB_DATA_TYPE_BINARY;
schema.bytes = TSDB_TABLE_ID_LEN;
} else {
f1 = mgmtMeterGetTag(pNode1, colIdx, NULL);
f2 = mgmtMeterGetTag(pNode2, colIdx, &schema);
SSchema* pSchema = getColumnModelSchema(pOrderDesc->pColumnModel, colIdx);
assert(schema.type == pSchema->type);
}
int32_t ret = doCompare(f1, f2, schema.type, schema.bytes);
if (ret == 0) {
continue;
} else {
return ret;
}
}
return 0;
}
/**
* update the tag order index according to the tags column index. The tags column index needs to be checked one-by-one,
* since the normal columns may be passed to server for handling the group by on status column.
*
* @param pSuperTableMetaMsg
* @param tableIndex
* @param pOrderIndexInfo
* @param numOfTags
*/
static void mgmtUpdateOrderTagColIndex(SMetricMetaMsg* pMetricMetaMsg, int32_t tableIndex, SColumnOrderInfo* pOrderIndexInfo,
int32_t numOfTags) {
SMetricMetaElemMsg* pElem = (SMetricMetaElemMsg*)((char*)pSuperTableMetaMsg + pSuperTableMetaMsg->metaElem[tableIndex]);
SColIndexEx* groupColumnList = (SColIndexEx*)((char*)pSuperTableMetaMsg + pElem->groupbyTagColumnList);
int32_t numOfGroupbyTags = 0;
for (int32_t i = 0; i < pElem->numOfGroupCols; ++i) {
if (groupColumnList[i].flag == TSDB_COL_TAG) { // ignore this column if it is not a tag column.
pOrderIndexInfo->pData[numOfGroupbyTags++] = groupColumnList[i].colIdx;
assert(groupColumnList[i].colIdx < numOfTags);
}
}
pOrderIndexInfo->numOfCols = numOfGroupbyTags;
}
// todo merge sort function with losertree used
void mgmtReorganizeMetersInMetricMeta(SSuperTableMetaMsg* pSuperTableMetaMsg, int32_t tableIndex, tQueryResultset* pRes) {
if (pRes->num <= 0) { // no result, no need to pagination
return;
}
SMetricMetaElemMsg* pElem = (SMetricMetaElemMsg*)((char*)pSuperTableMetaMsg + pSuperTableMetaMsg->metaElem[tableIndex]);
STabObj* pMetric = mgmtGetTable(pElem->meterId);
SSchema* pTagSchema = (SSchema*)(pMetric->schema + pMetric->numOfColumns * sizeof(SSchema));
/*
* To apply the group limitation and group offset, we should sort the result
* list according to the order condition
*/
tOrderDescriptor* descriptor =
(tOrderDescriptor*)calloc(1, sizeof(tOrderDescriptor) + sizeof(int32_t) * pElem->numOfGroupCols);
descriptor->pColumnModel = createColumnModel(pTagSchema, pMetric->numOfTags, 1);
descriptor->orderIdx.numOfCols = pElem->numOfGroupCols;
int32_t* startPos = NULL;
int32_t numOfSubset = 1;
mgmtUpdateOrderTagColIndex(pMetricMetaMsg, tableIndex, &descriptor->orderIdx, pMetric->numOfTags);
if (descriptor->orderIdx.numOfCols > 0) {
tQSortEx(pRes->pRes, POINTER_BYTES, 0, pRes->num - 1, descriptor, tabObjResultComparator);
startPos = calculateSubGroup(pRes->pRes, pRes->num, &numOfSubset, descriptor, tabObjResultComparator);
} else {
startPos = malloc(2 * sizeof(int32_t));
startPos[0] = 0;
startPos[1] = (int32_t)pRes->num;
}
/*
* sort the result according to vgid to ensure meters with the same vgid is
* continuous in the result list
*/
qsort(pRes->pRes, (size_t)pRes->num, POINTER_BYTES, tabObjVGIDComparator);
free(descriptor->pColumnModel);
free(descriptor);
free(startPos);
}
static void mgmtRetrieveByMeterName(tQueryResultset* pRes, char* str, STabObj* pMetric) {
const char* sep = ",";
char* pToken = NULL;
int32_t s = 4; // initial size
pRes->pRes = malloc(sizeof(char*) * s);
pRes->num = 0;
for (pToken = strsep(&str, sep); pToken != NULL; pToken = strsep(&str, sep)) {
STabObj* pMeterObj = mgmtGetTable(pToken);
if (pMeterObj == NULL) {
mWarn("metric:%s error in metric query expression, invalid meter id:%s", pMetric->meterId, pToken);
continue;
}
if (pRes->num >= s) {
s += (s >> 1); // increase 50% size
pRes->pRes = realloc(pRes->pRes, sizeof(char*) * s);
}
/* not a table created from metric, ignore */
if (pMeterObj->tableType != TSDB_TABLE_TYPE_CHILD_TABLE) {
continue;
}
/*
* queried meter not belongs to this metric, ignore, metric does not have
* uid, so compare according to meterid
*/
STabObj* parentMetric = mgmtGetTable(pMeterObj->pTagData);
if (strncasecmp(parentMetric->meterId, pMetric->meterId, TSDB_TABLE_ID_LEN) != 0 ||
(parentMetric->uid != pMetric->uid)) {
continue;
}
pRes->pRes[pRes->num++] = pMeterObj;
}
}
static bool mgmtTablenameFilterCallback(tSkipListNode* pNode, void* param) {
SMeterNameFilterSupporter* pSupporter = (SMeterNameFilterSupporter*)param;
char name[TSDB_TABLE_ID_LEN] = {0};
// pattern compare for meter name
STabObj* pMeterObj = (STabObj*)pNode->pData;
extractTableName(pMeterObj->meterId, name);
return patternMatch(pSupporter->pattern, name, TSDB_TABLE_ID_LEN, &pSupporter->info) == TSDB_PATTERN_MATCH;
}
static void mgmtRetrieveFromLikeOptr(tQueryResultset* pRes, const char* str, STabObj* pMetric) {
SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER;
SMeterNameFilterSupporter supporter = {info, (char*) str};
pRes->num =
tSkipListIterateList(pMetric->pSkipList, (tSkipListNode***)&pRes->pRes, mgmtTablenameFilterCallback, &supporter);
}
static void mgmtFilterByTableNameCond(tQueryResultset* pRes, char* condStr, int32_t len, STabObj* pMetric) {
pRes->num = 0;
if (len <= 0) {
return;
}
char* str = calloc(1, (size_t)len + 1);
memcpy(str, condStr, len);
if (strncasecmp(condStr, QUERY_COND_REL_PREFIX_IN, QUERY_COND_REL_PREFIX_IN_LEN) == 0) { // handle in expression
mgmtRetrieveByMeterName(pRes, str + QUERY_COND_REL_PREFIX_IN_LEN, pMetric);
} else { // handle like expression
assert(strncasecmp(str, QUERY_COND_REL_PREFIX_LIKE, QUERY_COND_REL_PREFIX_LIKE_LEN) == 0);
mgmtRetrieveFromLikeOptr(pRes, str + QUERY_COND_REL_PREFIX_LIKE_LEN, pMetric);
tansformQueryResult(pRes);
}
free(str);
}
UNUSED_FUNC static bool mgmtJoinFilterCallback(tSkipListNode* pNode, void* param) {
SJoinSupporter* pSupporter = (SJoinSupporter*)param;
SSchema s = {0};
char* v = mgmtTableGetTag((STabObj*)pNode->pData, pSupporter->colIndex, &s);
for (int32_t i = 0; i < pSupporter->size; ++i) {
int32_t ret = doCompare(v, pSupporter->val[i], pSupporter->type, s.bytes);
if (ret == 0) {
pSupporter->qualMeterObj[pSupporter->qualSize++] = pSupporter->pTabObjs[i];
/*
* Once a value is qualified according to the join condition, it is remove from the
* candidate list, as well as its corresponding meter object.
*
* last one does not need to move forward
*/
if (i < pSupporter->size - 1) {
memmove(pSupporter->val[i], pSupporter->val[i + 1], pSupporter->size - (i + 1));
}
pSupporter->size -= 1;
return true;
}
}
return false;
}
static void orderResult(SSuperTableMetaMsg* pSuperTableMetaMsg, tQueryResultset* pRes, int16_t colIndex, int32_t tableIndex) {
SMetricMetaElemMsg* pElem = (SMetricMetaElemMsg*)((char*)pSuperTableMetaMsg + pSuperTableMetaMsg->metaElem[tableIndex]);
tOrderDescriptor* descriptor =
(tOrderDescriptor*)calloc(1, sizeof(tOrderDescriptor) + sizeof(int32_t) * 1); // only one column for join
STabObj* pMetric = mgmtGetTable(pElem->meterId);
SSchema* pTagSchema = (SSchema*)(pMetric->schema + pMetric->numOfColumns * sizeof(SSchema));
descriptor->pColumnModel = createColumnModel(pTagSchema, pMetric->numOfTags, 1);
descriptor->orderIdx.pData[0] = colIndex;
descriptor->orderIdx.numOfCols = 1;
// sort results list
tQSortEx(pRes->pRes, POINTER_BYTES, 0, pRes->num - 1, descriptor, tabObjResultComparator);
free(descriptor->pColumnModel);
free(descriptor);
}
// check for duplicate join tags
static int32_t mgmtCheckForDuplicateTagValue(tQueryResultset* pRes, int32_t index, int32_t tagCol) {
SSchema s = {0};
for (int32_t k = 1; k < pRes[index].num; ++k) {
STabObj* pObj1 = pRes[index].pRes[k - 1];
STabObj* pObj2 = pRes[index].pRes[k];
char* val1 = mgmtTableGetTag(pObj1, tagCol, &s);
char* val2 = mgmtTableGetTag(pObj2, tagCol, NULL);
if (doCompare(val1, val2, s.type, s.bytes) == 0) {
return TSDB_CODE_DUPLICATE_TAGS;
}
}
return TSDB_CODE_SUCCESS;
}
int32_t mgmtDoJoin(SSuperTableMetaMsg* pSuperTableMetaMsg, tQueryResultset* pRes) {
if (pSuperTableMetaMsg->numOfMeters == 1) {
return TSDB_CODE_SUCCESS;
}
bool allEmpty = false;
for (int32_t i = 0; i < pSuperTableMetaMsg->numOfMeters; ++i) {
if (pRes[i].num == 0) { // all results are empty if one of them is empty
allEmpty = true;
break;
}
}
if (allEmpty) {
for (int32_t i = 0; i < pSuperTableMetaMsg->numOfMeters; ++i) {
pRes[i].num = 0;
tfree(pRes[i].pRes);
}
return TSDB_CODE_SUCCESS;
}
char* cond = (char*)pSuperTableMetaMsg + pSuperTableMetaMsg->join;
char left[TSDB_TABLE_ID_LEN + 1] = {0};
strcpy(left, cond);
int16_t leftTagColIndex = *(int16_t*)(cond + TSDB_TABLE_ID_LEN);
char right[TSDB_TABLE_ID_LEN + 1] = {0};
strcpy(right, cond + TSDB_TABLE_ID_LEN + sizeof(int16_t));
int16_t rightTagColIndex = *(int16_t*)(cond + TSDB_TABLE_ID_LEN * 2 + sizeof(int16_t));
STabObj* pLeftMetric = mgmtGetTable(left);
STabObj* pRightMetric = mgmtGetTable(right);
// decide the pRes belongs to
int32_t leftIndex = 0;
int32_t rightIndex = 0;
for (int32_t i = 0; i < pSuperTableMetaMsg->numOfMeters; ++i) {
STabObj* pObj = (STabObj*)pRes[i].pRes[0];
STabObj* pMetric1 = mgmtGetTable(pObj->pTagData);
if (pMetric1 == pLeftMetric) {
leftIndex = i;
} else if (pMetric1 == pRightMetric) {
rightIndex = i;
}
}
orderResult(pSuperTableMetaMsg, &pRes[leftIndex], leftTagColIndex, leftIndex);
orderResult(pSuperTableMetaMsg, &pRes[rightIndex], rightTagColIndex, rightIndex);
int32_t i = 0;
int32_t j = 0;
SSchema s = {0};
int32_t res = 0;
// check for duplicated tag values
int32_t ret1 = mgmtCheckForDuplicateTagValue(pRes, leftIndex, leftTagColIndex);
int32_t ret2 = mgmtCheckForDuplicateTagValue(pRes, rightIndex, rightTagColIndex);
if (ret1 != TSDB_CODE_SUCCESS || ret2 != TSDB_CODE_SUCCESS) {
return ret1;
}
while (i < pRes[leftIndex].num && j < pRes[rightIndex].num) {
STabObj* pLeftObj = pRes[leftIndex].pRes[i];
STabObj* pRightObj = pRes[rightIndex].pRes[j];
char* v1 = mgmtTableGetTag(pLeftObj, leftTagColIndex, &s);
char* v2 = mgmtTableGetTag(pRightObj, rightTagColIndex, NULL);
int32_t ret = doCompare(v1, v2, s.type, s.bytes);
if (ret == 0) { // qualified
pRes[leftIndex].pRes[res] = pRes[leftIndex].pRes[i++];
pRes[rightIndex].pRes[res] = pRes[rightIndex].pRes[j++];
res++;
} else if (ret < 0) {
i++;
} else {
j++;
}
}
pRes[leftIndex].num = res;
pRes[rightIndex].num = res;
return TSDB_CODE_SUCCESS;
}
/**
* convert the result pointer to STabObj instead of tSkipListNode
* @param pRes
*/
static void tansformQueryResult(tQueryResultset* pRes) {
if (pRes == NULL || pRes->num == 0) {
return;
}
for (int32_t i = 0; i < pRes->num; ++i) {
pRes->pRes[i] = ((tSkipListNode*)(pRes->pRes[i]))->pData;
}
}
static tQueryResultset* doNestedLoopIntersect(tQueryResultset* pRes1, tQueryResultset* pRes2) {
int32_t num = 0;
void** pResult = pRes1->pRes;
for (int32_t i = 0; i < pRes1->num; ++i) {
for (int32_t j = 0; j < pRes2->num; ++j) {
if (pRes1->pRes[i] == pRes2->pRes[j]) {
pResult[num++] = pRes1->pRes[i];
break;
}
}
}
tQueryResultClean(pRes2);
memset(pRes1->pRes + num, 0, sizeof(void*) * (pRes1->num - num));
pRes1->num = num;
return pRes1;
}
static tQueryResultset* doSortIntersect(tQueryResultset* pRes1, tQueryResultset* pRes2) {
size_t sizePtr = sizeof(void *);
qsort(pRes1->pRes, pRes1->num, sizePtr, tabObjPointerComparator);
qsort(pRes2->pRes, pRes2->num, sizePtr, tabObjPointerComparator);
int32_t i = 0;
int32_t j = 0;
int32_t num = 0;
while (i < pRes1->num && j < pRes2->num) {
if (pRes1->pRes[i] == pRes2->pRes[j]) {
j++;
pRes1->pRes[num++] = pRes1->pRes[i++];
} else if (pRes1->pRes[i] < pRes2->pRes[j]) {
i++;
} else {
j++;
}
}
tQueryResultClean(pRes2);
memset(pRes1->pRes + num, 0, sizeof(void*) * (pRes1->num - num));
pRes1->num = num;
return pRes1;
}
static void queryResultIntersect(tQueryResultset* pFinalRes, tQueryResultset* pRes) {
const int32_t NUM_OF_RES_THRESHOLD = 20;
// for small result, use nested loop join
if (pFinalRes->num <= NUM_OF_RES_THRESHOLD && pRes->num <= NUM_OF_RES_THRESHOLD) {
doNestedLoopIntersect(pFinalRes, pRes);
} else { // for larger result, sort merge is employed
doSortIntersect(pFinalRes, pRes);
}
}
static void queryResultUnion(tQueryResultset* pFinalRes, tQueryResultset* pRes) {
if (pRes->num == 0) {
tQueryResultClean(pRes);
return;
}
int32_t total = pFinalRes->num + pRes->num;
void* tmp = realloc(pFinalRes->pRes, total * POINTER_BYTES);
if (tmp == NULL) {
return;
}
pFinalRes->pRes = tmp;
memcpy(&pFinalRes->pRes[pFinalRes->num], pRes->pRes, POINTER_BYTES * pRes->num);
qsort(pFinalRes->pRes, total, POINTER_BYTES, tabObjPointerComparator);
int32_t num = 1;
for (int32_t i = 1; i < total; ++i) {
if (pFinalRes->pRes[i] != pFinalRes->pRes[i - 1]) {
pFinalRes->pRes[num++] = pFinalRes->pRes[i];
}
}
if (num < total) {
memset(&pFinalRes->pRes[num], 0, POINTER_BYTES * (total - num));
}
pFinalRes->num = num;
tQueryResultClean(pRes);
}
static int32_t compareIntVal(const void* pLeft, const void* pRight) {
DEFAULT_COMP(GET_INT64_VAL(pLeft), GET_INT64_VAL(pRight));
}
static int32_t compareIntDoubleVal(const void* pLeft, const void* pRight) {
DEFAULT_COMP(GET_INT64_VAL(pLeft), GET_DOUBLE_VAL(pRight));
}
static int32_t compareDoubleVal(const void* pLeft, const void* pRight) {
DEFAULT_COMP(GET_DOUBLE_VAL(pLeft), GET_DOUBLE_VAL(pRight));
}
static int32_t compareDoubleIntVal(const void* pLeft, const void* pRight) {
double ret = (*(double*)pLeft) - (*(int64_t*)pRight);
if (fabs(ret) < DBL_EPSILON) {
return 0;
} else {
return ret > 0 ? 1 : -1;
}
}
static int32_t compareStrVal(const void* pLeft, const void* pRight) {
int32_t ret = strcmp(pLeft, pRight);
if (ret == 0) {
return 0;
} else {
return ret > 0 ? 1 : -1;
}
}
static int32_t compareWStrVal(const void* pLeft, const void* pRight) {
int32_t ret = wcscmp(pLeft, pRight);
if (ret == 0) {
return 0;
} else {
return ret > 0 ? 1 : -1;
}
}
static int32_t compareStrPatternComp(const void* pLeft, const void* pRight) {
SPatternCompareInfo pInfo = {'%', '_'};
const char* pattern = pRight;
const char* str = pLeft;
int32_t ret = patternMatch(pattern, str, strlen(str), &pInfo);
return (ret == TSDB_PATTERN_MATCH) ? 0 : 1;
}
static int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) {
SPatternCompareInfo pInfo = {'%', '_'};
const wchar_t* pattern = pRight;
const wchar_t* str = pLeft;
int32_t ret = WCSPatternMatch(pattern, str, wcslen(str), &pInfo);
return (ret == TSDB_PATTERN_MATCH) ? 0 : 1;
}
static __compar_fn_t getFilterComparator(int32_t type, int32_t filterType, int32_t optr) {
__compar_fn_t comparator = NULL;
switch (type) {
case TSDB_DATA_TYPE_TINYINT:
case TSDB_DATA_TYPE_SMALLINT:
case TSDB_DATA_TYPE_INT:
case TSDB_DATA_TYPE_BIGINT:
case TSDB_DATA_TYPE_BOOL: {
if (filterType >= TSDB_DATA_TYPE_BOOL && filterType <= TSDB_DATA_TYPE_BIGINT) {
comparator = compareIntVal;
} else if (filterType >= TSDB_DATA_TYPE_FLOAT && filterType <= TSDB_DATA_TYPE_DOUBLE) {
comparator = compareIntDoubleVal;
}
break;
}
case TSDB_DATA_TYPE_FLOAT:
case TSDB_DATA_TYPE_DOUBLE: {
if (filterType >= TSDB_DATA_TYPE_BOOL && filterType <= TSDB_DATA_TYPE_BIGINT) {
comparator = compareDoubleIntVal;
} else if (filterType >= TSDB_DATA_TYPE_FLOAT && filterType <= TSDB_DATA_TYPE_DOUBLE) {
comparator = compareDoubleVal;
}
break;
}
case TSDB_DATA_TYPE_BINARY: {
assert(filterType == TSDB_DATA_TYPE_BINARY);
if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */
comparator = compareStrPatternComp;
} else { /* normal relational comparator */
comparator = compareStrVal;
}
break;
}
case TSDB_DATA_TYPE_NCHAR: {
assert(filterType == TSDB_DATA_TYPE_NCHAR);
if (optr == TSDB_RELATION_LIKE) {
comparator = compareWStrPatternComp;
} else {
comparator = compareWStrVal;
}
break;
}
default:
comparator = compareIntVal;
break;
}
return comparator;
}
static void getTagColumnInfo(SSyntaxTreeFilterSupporter* pSupporter, SSchema* pSchema, int32_t* index,
int32_t* offset) {
*index = 0;
*offset = 0;
// filter on table name(TBNAME)
if (strcasecmp(pSchema->name, TSQL_TBNAME_L) == 0) {
*index = TSDB_TBNAME_COLUMN_INDEX;
*offset = TSDB_TBNAME_COLUMN_INDEX;
return;
}
while ((*index) < pSupporter->numOfTags) {
if (pSupporter->pTagSchema[*index].bytes == pSchema->bytes &&
pSupporter->pTagSchema[*index].type == pSchema->type &&
strcmp(pSupporter->pTagSchema[*index].name, pSchema->name) == 0) {
break;
} else {
(*offset) += pSupporter->pTagSchema[(*index)++].bytes;
}
}
}
void filterPrepare(void* expr, void* param) {
tSQLBinaryExpr *pExpr = (tSQLBinaryExpr*) expr;
if (pExpr->info != NULL) {
return;
}
int32_t i = 0, offset = 0;
pExpr->info = calloc(1, sizeof(tQueryInfo));
tQueryInfo* pInfo = pExpr->info;
SSyntaxTreeFilterSupporter* pSupporter = (SSyntaxTreeFilterSupporter*)param;
tVariant* pCond = pExpr->pRight->pVal;
SSchema* pSchema = pExpr->pLeft->pSchema;
getTagColumnInfo(pSupporter, pSchema, &i, &offset);
assert((i >= 0 && i < TSDB_MAX_TAGS) || (i == TSDB_TBNAME_COLUMN_INDEX));
assert((offset >= 0 && offset < TSDB_MAX_TAGS_LEN) || (offset == TSDB_TBNAME_COLUMN_INDEX));
pInfo->sch = *pSchema;
pInfo->colIdx = i;
pInfo->optr = pExpr->nSQLBinaryOptr;
pInfo->offset = offset;
pInfo->compare = getFilterComparator(pSchema->type, pCond->nType, pInfo->optr);
tVariantAssign(&pInfo->q, pCond);
tVariantTypeSetType(&pInfo->q, pInfo->sch.type);
}
void tSQLListTraverseDestroyInfo(void* param) {
if (param == NULL) {
return;
}
tQueryInfo* pInfo = (tQueryInfo*)param;
tVariantDestroy(&(pInfo->q));
free(param);
}
static int32_t mgmtFilterMeterByIndex(STabObj* pMetric, tQueryResultset* pRes, char* pCond, int32_t condLen) {
SSchema* pTagSchema = (SSchema*)(pMetric->schema + pMetric->numOfColumns * sizeof(SSchema));
tSQLBinaryExpr* pExpr = NULL;
tSQLBinaryExprFromString(&pExpr, pTagSchema, pMetric->numOfTags, pCond, condLen);
// failed to build expression, no result, return immediately
if (pExpr == NULL) {
mError("metric:%s, no result returned, error in super table query expression:%s", pMetric->meterId, pCond);
tfree(pCond);
return TSDB_CODE_OPS_NOT_SUPPORT;
} else { // query according to the binary expression
SSyntaxTreeFilterSupporter s = {.pTagSchema = pTagSchema, .numOfTags = pMetric->numOfTags};
SBinaryFilterSupp supp = {.fp = (__result_filter_fn_t)tSkipListNodeFilterCallback,
.setupInfoFn = (__do_filter_suppl_fn_t)filterPrepare,
.pExtInfo = &s};
// tSQLBinaryExprTraverse(pExpr, pMetric->pSkipList, pRes, &supp);
tSQLBinaryExprDestroy(&pExpr, tSQLListTraverseDestroyInfo);
}
tansformQueryResult(pRes);
return TSDB_CODE_SUCCESS;
}
int32_t mgmtRetrieveMetersFromSuperTable(SSuperTableMetaMsg* pMsg, int32_t tableIndex, tQueryResultset* pRes) {
SMetricMetaElemMsg* pElem = (SMetricMetaElemMsg*)((char*)pMsg + pMsg->metaElem[tableIndex]);
STabObj* pMetric = mgmtGetTable(pElem->meterId);
char* pCond = NULL;
char* tmpTableNameCond = NULL;
// no table created in accordance with this metric.
if (pMetric->pSkipList == NULL || pMetric->pSkipList->nSize == 0) {
assert(pMetric->numOfMeters == 0);
return TSDB_CODE_SUCCESS;
}
char* pQueryCond = (char*)pMsg + pElem->cond;
int32_t condLen = pElem->condLen;
// transfer the unicode string to mbs binary expression
if (condLen > 0) {
pCond = calloc(1, (condLen + 1) * TSDB_NCHAR_SIZE);
taosUcs4ToMbs(pQueryCond, condLen * TSDB_NCHAR_SIZE, pCond);
condLen = strlen(pCond) + 1;
mTrace("metric:%s len:%d, metric query condition:%s", pMetric->meterId, condLen, pCond);
}
char* tablenameCond = (char*)pMsg + pElem->tableCond;
if (pElem->tableCondLen > 0) {
tmpTableNameCond = calloc(1, pElem->tableCondLen + 1);
strncpy(tmpTableNameCond, tablenameCond, pElem->tableCondLen);
mTrace("metric:%s rel:%d, len:%d, table name cond:%s", pMetric->meterId, pElem->rel, pElem->tableCondLen,
tmpTableNameCond);
}
if (pElem->tableCondLen > 0 || condLen > 0) {
mgmtFilterByTableNameCond(pRes, tmpTableNameCond, pElem->tableCondLen, pMetric);
bool noNextCal = (pRes->num == 0 && pElem->rel == TSDB_RELATION_AND); // no need to calculate next result
if (!noNextCal && condLen > 0) {
tQueryResultset filterRes = {0};
int32_t ret = mgmtFilterMeterByIndex(pMetric, &filterRes, pCond, condLen);
if (ret != TSDB_CODE_SUCCESS) {
tfree(pCond);
tfree(tmpTableNameCond);
return ret;
}
// union or intersect of two results
assert(pElem->rel == TSDB_RELATION_AND || pElem->rel == TSDB_RELATION_OR);
if (pElem->rel == TSDB_RELATION_AND) {
if (filterRes.num == 0 || pRes->num == 0) { // intersect two sets
tQueryResultClean(pRes);
} else {
queryResultIntersect(pRes, &filterRes);
}
} else { // union two sets
queryResultUnion(pRes, &filterRes);
}
tQueryResultClean(&filterRes);
}
} else {
mTrace("metric:%s retrieve all meter, no query condition", pMetric->meterId);
pRes->num = tSkipListIterateList(pMetric->pSkipList, (tSkipListNode***)&pRes->pRes, NULL, NULL);
tansformQueryResult(pRes);
}
tfree(pCond);
tfree(tmpTableNameCond);
mTrace("metric:%s numOfRes:%d", pMetric->meterId, pRes->num);
return TSDB_CODE_SUCCESS;
}
// todo refactor!!!!!
static char* getTagValueFromMeter(STabObj* pTable, int32_t offset, int32_t len, char* param) {
if (offset == TSDB_TBNAME_COLUMN_INDEX) {
extractTableName(pTable->meterId, param);
} else {
char* tags = pTable->pTagData + offset + TSDB_TABLE_ID_LEN; // tag start position
memcpy(param, tags, len); // make sure the value is null-terminated string
}
return param;
}
bool tSkipListNodeFilterCallback(const void* pNode, void* param) {
tQueryInfo* pInfo = (tQueryInfo*)param;
STabObj* pTable = (STabObj*)(((tSkipListNode*)pNode)->pData);
char buf[TSDB_MAX_TAGS_LEN] = {0};
char* val = getTagValueFromMeter(pTable, pInfo->offset, pInfo->sch.bytes, buf);
int8_t type = pInfo->sch.type;
int32_t ret = 0;
if (pInfo->q.nType == TSDB_DATA_TYPE_BINARY || pInfo->q.nType == TSDB_DATA_TYPE_NCHAR) {
ret = pInfo->compare(val, pInfo->q.pz);
} else {
tVariant t = {0};
tVariantCreateFromBinary(&t, val, (uint32_t) pInfo->sch.bytes, type);
ret = pInfo->compare(&t.i64Key, &pInfo->q.i64Key);
}
switch (pInfo->optr) {
case TSDB_RELATION_EQUAL: {
return ret == 0;
}
case TSDB_RELATION_NOT_EQUAL: {
return ret != 0;
}
case TSDB_RELATION_LARGE_EQUAL: {
return ret >= 0;
}
case TSDB_RELATION_LARGE: {
return ret > 0;
}
case TSDB_RELATION_LESS_EQUAL: {
return ret <= 0;
}
case TSDB_RELATION_LESS: {
return ret < 0;
}
case TSDB_RELATION_LIKE: {
return ret == 0;
}
default:
assert(false);
}
return true;
}

391
src/vnode/detail/src/vnodeTagMgmt.c
View File

@ -1,391 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "qast.h"
#include "qextbuffer.h"
#include "taosdef.h"
#include "taosmsg.h"
#include "tlog.h"
#include "tutil.h"
#include "vnodeTagMgmt.h"
#define GET_TAG_VAL_POINTER(s, col, sc, t) ((t *)(&((s)->tags[getColumnModelOffset(sc, col)])))
#define GET_TAG_VAL(s, col, sc, t) (*GET_TAG_VAL_POINTER(s, col, sc, t))
static void tTagsPrints(SMeterSidExtInfo *pMeterInfo, SColumnModel *pSchema, SColumnOrderInfo *pOrder);
static void tSidSetDisplay(tSidSet *pSets);
//todo merge with losertree_compar/ext_comp
int32_t doCompare(char* f1, char* f2, int32_t type, int32_t size) {
switch (type) {
case TSDB_DATA_TYPE_INT: DEFAULT_COMP(GET_INT32_VAL(f1), GET_INT32_VAL(f2));
case TSDB_DATA_TYPE_DOUBLE: DEFAULT_COMP(GET_DOUBLE_VAL(f1), GET_DOUBLE_VAL(f2));
case TSDB_DATA_TYPE_FLOAT: DEFAULT_COMP(GET_FLOAT_VAL(f1), GET_FLOAT_VAL(f2));
case TSDB_DATA_TYPE_BIGINT: DEFAULT_COMP(GET_INT64_VAL(f1), GET_INT64_VAL(f2));
case TSDB_DATA_TYPE_SMALLINT: DEFAULT_COMP(GET_INT16_VAL(f1), GET_INT16_VAL(f2));
case TSDB_DATA_TYPE_TINYINT:
case TSDB_DATA_TYPE_BOOL: DEFAULT_COMP(GET_INT8_VAL(f1), GET_INT8_VAL(f2));
case TSDB_DATA_TYPE_NCHAR: {
int32_t ret = wcsncmp((wchar_t*) f1, (wchar_t*) f2, size/TSDB_NCHAR_SIZE);
if (ret == 0) {
return ret;
}
return (ret < 0) ? -1 : 1;
}
default: {
int32_t ret = strncmp(f1, f2, (size_t)size);
if (ret == 0) {
return ret;
}
return (ret < 0) ? -1 : 1;
}
}
}
int32_t meterSidComparator(const void *p1, const void *p2, void *param) {
tOrderDescriptor *pOrderDesc = (tOrderDescriptor *)param;
SMeterSidExtInfo *s1 = (SMeterSidExtInfo *)p1;
SMeterSidExtInfo *s2 = (SMeterSidExtInfo *)p2;
for (int32_t i = 0; i < pOrderDesc->orderIdx.numOfCols; ++i) {
int32_t colIdx = pOrderDesc->orderIdx.pData[i];
char * f1 = NULL;
char * f2 = NULL;
int32_t type = 0;
int32_t bytes = 0;
if (colIdx == -1) {
f1 = s1->tags;
f2 = s2->tags;
type = TSDB_DATA_TYPE_BINARY;
bytes = TSDB_METER_NAME_LEN;
} else {
f1 = GET_TAG_VAL_POINTER(s1, colIdx, pOrderDesc->pColumnModel, char);
f2 = GET_TAG_VAL_POINTER(s2, colIdx, pOrderDesc->pColumnModel, char);
SSchema *pSchema = getColumnModelSchema(pOrderDesc->pColumnModel, colIdx);
type = pSchema->type;
bytes = pSchema->bytes;
}
int32_t ret = doCompare(f1, f2, type, bytes);
if (ret == 0) {
continue;
} else {
return ret;
}
}
return 0;
}
static void median(void **pMeterSids, size_t size, int32_t s1, int32_t s2, tOrderDescriptor *pOrderDesc,
__ext_compar_fn_t compareFn) {
int32_t midIdx = ((s2 - s1) >> 1) + s1;
if (compareFn(pMeterSids[midIdx], pMeterSids[s1], pOrderDesc) == 1) {
tsDataSwap(&pMeterSids[midIdx], &pMeterSids[s1], TSDB_DATA_TYPE_BINARY, size);
}
if (compareFn(pMeterSids[midIdx], pMeterSids[s2], pOrderDesc) == 1) {
tsDataSwap(&pMeterSids[midIdx], &pMeterSids[s1], TSDB_DATA_TYPE_BINARY, size);
tsDataSwap(&pMeterSids[midIdx], &pMeterSids[s2], TSDB_DATA_TYPE_BINARY, size);
} else if (compareFn(pMeterSids[s1], pMeterSids[s2], pOrderDesc) == 1) {
tsDataSwap(&pMeterSids[s1], &pMeterSids[s2], TSDB_DATA_TYPE_BINARY, size);
}
assert(compareFn(pMeterSids[midIdx], pMeterSids[s1], pOrderDesc) <= 0 &&
compareFn(pMeterSids[s1], pMeterSids[s2], pOrderDesc) <= 0);
#ifdef _DEBUG_VIEW
tTagsPrints(pMeterSids[s1], pOrderDesc->pColumnModel, &pOrderDesc->orderIdx);
tTagsPrints(pMeterSids[midIdx], pOrderDesc->pColumnModel, &pOrderDesc->orderIdx);
tTagsPrints(pMeterSids[s2], pOrderDesc->pColumnModel, &pOrderDesc->orderIdx);
#endif
}
static void tInsertSort(void **pMeterSids, size_t size, int32_t startPos, int32_t endPos, void *param,
__ext_compar_fn_t compareFn) {
for (int32_t i = startPos + 1; i <= endPos; ++i) {
for (int32_t j = i; j > startPos; --j) {
if (compareFn(pMeterSids[j], pMeterSids[j - 1], param) == -1) {
tsDataSwap(&pMeterSids[j], &pMeterSids[j - 1], TSDB_DATA_TYPE_BINARY, size);
} else {
break;
}
}
}
}
void tQSortEx(void **pMeterSids, size_t size, int32_t start, int32_t end, void *param, __ext_compar_fn_t compareFn) {
tOrderDescriptor *pOrderDesc = (tOrderDescriptor *)param;
// short array sort, incur another sort procedure instead of quick sort process
if (end - start + 1 <= 8) {
tInsertSort(pMeterSids, size, start, end, pOrderDesc, compareFn);
return;
}
median(pMeterSids, size, start, end, pOrderDesc, compareFn);
int32_t s = start, e = end;
int32_t endRightS = end, startLeftS = start;
while (s < e) {
while (e > s) {
int32_t ret = compareFn(pMeterSids[e], pMeterSids[s], pOrderDesc);
if (ret < 0) {
break;
}
/*
* move the data that equals to pivotal value to the right end of the list
*/
if (ret == 0 && e != endRightS) {
tsDataSwap(&pMeterSids[e], &pMeterSids[endRightS--], TSDB_DATA_TYPE_BINARY, size);
}
e--;
}
if (e != s) {
tsDataSwap(&pMeterSids[e], &pMeterSids[s], TSDB_DATA_TYPE_BINARY, size);
}
while (s < e) {
int32_t ret = compareFn(pMeterSids[s], pMeterSids[e], pOrderDesc);
if (ret > 0) {
break;
}
if (ret == 0 && s != startLeftS) {
tsDataSwap(&pMeterSids[s], &pMeterSids[startLeftS++], TSDB_DATA_TYPE_BINARY, size);
}
s++;
}
if (e != s) {
tsDataSwap(&pMeterSids[s], &pMeterSids[e], TSDB_DATA_TYPE_BINARY, size);
}
}
int32_t rightPartStart = e + 1;
if (endRightS != end && e < end) {
int32_t left = rightPartStart;
int32_t right = end;
while (right > endRightS && left <= endRightS) {
tsDataSwap(&pMeterSids[left++], &pMeterSids[right--], TSDB_DATA_TYPE_BINARY, size);
}
rightPartStart += (end - endRightS);
}
int32_t leftPartEnd = e - 1;
if (startLeftS != end && s > start) {
int32_t left = start;
int32_t right = leftPartEnd;
while (left < startLeftS && right >= startLeftS) {
tsDataSwap(&pMeterSids[left++], &pMeterSids[right--], TSDB_DATA_TYPE_BINARY, size);
}
leftPartEnd -= (startLeftS - start);
}
if (leftPartEnd > start) {
tQSortEx(pMeterSids, size, start, leftPartEnd, pOrderDesc, compareFn);
}
if (rightPartStart < end) {
tQSortEx(pMeterSids, size, rightPartStart, end, pOrderDesc, compareFn);
}
}
int32_t *calculateSubGroup(void **pSids, int32_t numOfMeters, int32_t *numOfSubset, tOrderDescriptor *pOrderDesc,
__ext_compar_fn_t compareFn) {
int32_t *starterPos = (int32_t *)malloc((numOfMeters + 1) * sizeof(int32_t)); // add additional buffer
starterPos[0] = 0;
*numOfSubset = 1;
for (int32_t i = 1; i < numOfMeters; ++i) {
int32_t ret = compareFn(pSids[i - 1], pSids[i], pOrderDesc);
if (ret != 0) {
assert(ret == -1);
starterPos[(*numOfSubset)++] = i;
}
}
starterPos[*numOfSubset] = numOfMeters;
assert(*numOfSubset <= numOfMeters);
return starterPos;
}
tSidSet *tSidSetCreate(struct SMeterSidExtInfo **pMeterSidExtInfo, int32_t numOfMeters, SSchema *pSchema,
int32_t numOfTags, SColIndexEx *colList, int32_t numOfCols) {
tSidSet *pSidSet = (tSidSet *)calloc(1, sizeof(tSidSet) + numOfCols * sizeof(int16_t));
if (pSidSet == NULL) {
return NULL;
}
pSidSet->numOfSids = numOfMeters;
pSidSet->pSids = pMeterSidExtInfo;
pSidSet->pColumnModel = createColumnModel(pSchema, numOfTags, 1);
pSidSet->orderIdx.numOfCols = numOfCols;
/*
* in case of "group by tbname,normal_col", the normal_col is ignored
*/
int32_t numOfTagCols = 0;
for(int32_t i = 0; i < numOfCols; ++i) {
if (colList[i].flag == TSDB_COL_TAG) {
pSidSet->orderIdx.pData[numOfTagCols++] = colList[i].colIdx;
}
}
pSidSet->orderIdx.numOfCols = numOfTagCols;
pSidSet->starterPos = NULL;
return pSidSet;
}
void tSidSetDestroy(tSidSet **pSets) {
if ((*pSets) != NULL) {
tfree((*pSets)->starterPos);
tfree((*pSets)->pColumnModel)(*pSets)->pSids = NULL;
tfree(*pSets);
}
}
void tTagsPrints(SMeterSidExtInfo *pMeterInfo, SColumnModel *pSchema, SColumnOrderInfo *pOrder) {
if (pSchema == NULL) {
return;
}
printf("sid: %-5d tags(", pMeterInfo->sid);
for (int32_t i = 0; i < pOrder->numOfCols; ++i) {
int32_t colIndex = pOrder->pData[i];
// it is the tbname column
if (colIndex == -1) {
printf("%s, ", pMeterInfo->tags);
continue;
}
SSchema* s = getColumnModelSchema(pSchema, colIndex);
switch (s->type) {
case TSDB_DATA_TYPE_INT:
printf("%d, ", GET_TAG_VAL(pMeterInfo, colIndex, pSchema, int32_t));
break;
case TSDB_DATA_TYPE_DOUBLE:
printf("%lf, ", GET_TAG_VAL(pMeterInfo, colIndex, pSchema, double));
break;
case TSDB_DATA_TYPE_FLOAT:
printf("%f, ", GET_TAG_VAL(pMeterInfo, colIndex, pSchema, float));
break;
case TSDB_DATA_TYPE_BIGINT:
printf("%" PRId64 ", ", GET_TAG_VAL(pMeterInfo, colIndex, pSchema, int64_t));
break;
case TSDB_DATA_TYPE_SMALLINT:
printf("%d, ", GET_TAG_VAL(pMeterInfo, colIndex, pSchema, int16_t));
break;
case TSDB_DATA_TYPE_TINYINT:
printf("%d, ", GET_TAG_VAL(pMeterInfo, colIndex, pSchema, int8_t));
break;
case TSDB_DATA_TYPE_BINARY:
printf("%s, ", GET_TAG_VAL_POINTER(pMeterInfo, colIndex, pSchema, char));
break;
case TSDB_DATA_TYPE_NCHAR: {
char *data = GET_TAG_VAL_POINTER(pMeterInfo, colIndex, pSchema, char);
char buffer[512] = {0};
taosUcs4ToMbs(data, s->bytes, buffer);
printf("%s, ", buffer);
break;
}
case TSDB_DATA_TYPE_BOOL:
printf("%d, ", GET_TAG_VAL(pMeterInfo, colIndex, pSchema, int8_t));
break;
default:
assert(false);
}
}
printf(")\n");
}
/*
* display all the subset groups for debug purpose only
*/
static void UNUSED_FUNC tSidSetDisplay(tSidSet *pSets) {
printf("%d meters.\n", pSets->numOfSids);
for (int32_t i = 0; i < pSets->numOfSids; ++i) {
printf("%d\t", pSets->pSids[i]->sid);
}
printf("\n");
printf("total number of subset group is: %d\n", pSets->numOfSubSet);
for (int32_t i = 0; i < pSets->numOfSubSet; ++i) {
int32_t s = pSets->starterPos[i];
int32_t e = pSets->starterPos[i + 1];
printf("the %d-th subgroup: \n", i + 1);
for (int32_t j = s; j < e; ++j) {
tTagsPrints(pSets->pSids[j], pSets->pColumnModel, &pSets->orderIdx);
}
}
}
void tSidSetSort(tSidSet *pSets) {
pTrace("number of meters in sort: %d", pSets->numOfSids);
SColumnOrderInfo *pOrderIdx = &pSets->orderIdx;
if (pOrderIdx->numOfCols == 0 || pSets->numOfSids <= 1 || pSets->pColumnModel == NULL) { // no group by tags clause
pSets->numOfSubSet = 1;
pSets->starterPos = (int32_t *)malloc(sizeof(int32_t) * (pSets->numOfSubSet + 1));
pSets->starterPos[0] = 0;
pSets->starterPos[1] = pSets->numOfSids;
pTrace("all meters belong to one subgroup, no need to subgrouping ops");
#ifdef _DEBUG_VIEW
tSidSetDisplay(pSets);
#endif
} else {
tOrderDescriptor *descriptor =
(tOrderDescriptor *)calloc(1, sizeof(tOrderDescriptor) + sizeof(int16_t) * pSets->orderIdx.numOfCols);
descriptor->pColumnModel = pSets->pColumnModel;
descriptor->orderIdx = pSets->orderIdx;
memcpy(descriptor->orderIdx.pData, pOrderIdx->pData, sizeof(int16_t) * pSets->orderIdx.numOfCols);
tQSortEx((void **)pSets->pSids, POINTER_BYTES, 0, pSets->numOfSids - 1, descriptor, meterSidComparator);
pSets->starterPos =
calculateSubGroup((void **)pSets->pSids, pSets->numOfSids, &pSets->numOfSubSet, descriptor, meterSidComparator);
#ifdef _DEBUG_VIEW
tSidSetDisplay(pSets);
#endif
tfree(descriptor);
}
}

766
src/vnode/detail/src/vnodeUtil.c
View File

@ -1,766 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "qast.h"
#include "tscUtil.h"
#include "tschemautil.h"
#include "vnode.h"
#include "vnodeDataFilterFunc.h"
#include "vnodeStatus.h"
#include "vnodeUtil.h"
int vnodeCheckFileIntegrity(FILE* fp) {
/*
int savedSessions, savedMeterSize;
fseek(fp, TSDB_FILE_HEADER_LEN/3, SEEK_SET);
fscanf(fp, "%d %d", &savedSessions, &savedMeterSize);
if ( (savedSessions != tsSessionsPerVnode) || (savedMeterSize != tsMeterSizeOnFile) ) {
dError("file structure is changed");
return -1;
}
uint64_t checkSum = 0, savedCheckSum=0;
checkSum = taosGetCheckSum(fp, TSDB_FILE_HEADER_LEN);
fseek(fp, TSDB_FILE_HEADER_LEN - cksumsize, SEEK_SET);
fread(&savedCheckSum, cksumsize, 1, fp);
if ( savedCheckSum != checkSum ) {
dError("check sum is not matched:0x%x 0x%x", checkSum, savedCheckSum);
return -1;
}
*/
return 0;
}
void vnodeCreateFileHeaderFd(int fd) {
char temp[TSDB_FILE_HEADER_LEN / 4];
int lineLen;
lineLen = sizeof(temp);
// write the first line`
memset(temp, 0, lineLen);
*(int16_t*)temp = vnodeFileVersion;
sprintf(temp + sizeof(int16_t), "tsdb version: %s\n", version);
/* *((int16_t *)(temp + TSDB_FILE_HEADER_LEN/8)) = vnodeFileVersion; */
lseek(fd, 0, SEEK_SET);
twrite(fd, temp, lineLen);
// second line
memset(temp, 0, lineLen);
twrite(fd, temp, lineLen);
// the third/forth line is the dynamic info
memset(temp, 0, lineLen);
twrite(fd, temp, lineLen);
twrite(fd, temp, lineLen);
}
void vnodeGetHeadFileHeaderInfo(int fd, SVnodeHeadInfo* pHeadInfo) {
lseek(fd, TSDB_FILE_HEADER_LEN / 4, SEEK_SET);
read(fd, pHeadInfo, sizeof(SVnodeHeadInfo));
}
void vnodeUpdateHeadFileHeader(int fd, SVnodeHeadInfo* pHeadInfo) {
lseek(fd, TSDB_FILE_HEADER_LEN / 4, SEEK_SET);
twrite(fd, pHeadInfo, sizeof(SVnodeHeadInfo));
}
void vnodeCreateFileHeader(FILE* fp) {
char temp[TSDB_FILE_HEADER_LEN / 4];
int lineLen;
lineLen = sizeof(temp);
// write the first line
memset(temp, 0, lineLen);
*(int16_t*)temp = vnodeFileVersion;
sprintf(temp + sizeof(int16_t), "tsdb version: %s\n", version);
/* *((int16_t *)(temp + TSDB_FILE_HEADER_LEN/8)) = vnodeFileVersion; */
fseek(fp, 0, SEEK_SET);
fwrite(temp, lineLen, 1, fp);
// second line
memset(temp, 0, lineLen);
fwrite(temp, lineLen, 1, fp);
// the third line is the dynamic info
memset(temp, 0, lineLen);
fwrite(temp, lineLen, 1, fp);
fwrite(temp, lineLen, 1, fp);
}
SSqlGroupbyExpr* vnodeCreateGroupbyExpr(SQueryMeterMsg* pQueryMsg, int32_t* code) {
if (pQueryMsg->numOfGroupCols == 0) {
return NULL;
}
// using group by tag columns
SSqlGroupbyExpr* pGroupbyExpr =
(SSqlGroupbyExpr*)malloc(sizeof(SSqlGroupbyExpr) + pQueryMsg->numOfGroupCols * sizeof(SColIndexEx));
if (pGroupbyExpr == NULL) {
*code = TSDB_CODE_SERV_OUT_OF_MEMORY;
return NULL;
}
SColIndexEx* pGroupbyColInfo = (SColIndexEx*)pQueryMsg->groupbyTagIds;
pGroupbyExpr->numOfGroupCols = pQueryMsg->numOfGroupCols;
pGroupbyExpr->orderType = pQueryMsg->orderType;
pGroupbyExpr->orderIndex = pQueryMsg->orderByIdx;
memcpy(pGroupbyExpr->columnInfo, pGroupbyColInfo, sizeof(SColIndexEx) * pGroupbyExpr->numOfGroupCols);
// TODO: update the colIndexInBuf for each column in group by clause
return pGroupbyExpr;
}
static SSchema* toSchema(SQueryMeterMsg* pQuery, SColumnInfo* pCols, int32_t numOfCols) {
char* start = (char*)pQuery->colNameList;
char* end = start;
SSchema* pSchema = calloc(1, sizeof(SSchema) * numOfCols);
for (int32_t i = 0; i < numOfCols; ++i) {
pSchema[i].type = pCols[i].type;
pSchema[i].bytes = pCols[i].bytes;
pSchema[i].colId = pCols[i].colId;
end = strstr(start, ",");
memcpy(pSchema[i].name, start, end - start);
start = end + 1;
}
return pSchema;
}
static int32_t id_compar(const void* left, const void* right) {
DEFAULT_COMP(GET_INT16_VAL(left), GET_INT16_VAL(right));
}
static int32_t vnodeBuildExprFromArithmeticStr(SSqlFunctionExpr* pExpr, SQueryMeterMsg* pQueryMsg) {
SSqlBinaryExprInfo* pBinaryExprInfo = &pExpr->pBinExprInfo;
SColumnInfo* pColMsg = pQueryMsg->colList;
tSQLBinaryExpr* pBinExpr = NULL;
SSchema* pSchema = toSchema(pQueryMsg, pColMsg, pQueryMsg->numOfCols);
dTrace("qmsg:%p create binary expr from string:%s", pQueryMsg, pExpr->pBase.arg[0].argValue.pz);
tSQLBinaryExprFromString(&pBinExpr, pSchema, pQueryMsg->numOfCols, pExpr->pBase.arg[0].argValue.pz);
if (pBinExpr == NULL) {
dError("qmsg:%p failed to create arithmetic expression string from:%s", pQueryMsg, pExpr->pBase.arg[0].argValue.pz);
return TSDB_CODE_APP_ERROR;
}
pBinaryExprInfo->pBinExpr = pBinExpr;
int32_t num = 0;
int16_t ids[TSDB_MAX_COLUMNS] = {0};
tSQLBinaryExprTrv(pBinExpr, &num, ids);
qsort(ids, num, sizeof(int16_t), id_compar);
int32_t i = 0, j = 0;
while (i < num && j < num) {
if (ids[i] == ids[j]) {
j++;
} else {
ids[++i] = ids[j++];
}
}
assert(i <= num);
// there may be duplicated referenced columns.
num = i + 1;
pBinaryExprInfo->pReqColumns = malloc(sizeof(SColIndexEx) * num);
for (int32_t k = 0; k < num; ++k) {
SColIndexEx* pColIndex = &pBinaryExprInfo->pReqColumns[k];
pColIndex->colId = ids[k];
}
pBinaryExprInfo->numOfCols = num;
free(pSchema);
return TSDB_CODE_SUCCESS;
}
static int32_t getColumnIndexInSource(SQueryMeterMsg* pQueryMsg, SSqlFuncExprMsg* pExprMsg) {
int32_t j = 0;
while(j < pQueryMsg->numOfCols) {
if (pExprMsg->colInfo.colId == pQueryMsg->colList[j].colId) {
break;
}
j += 1;
}
return j;
}
bool vnodeValidateExprColumnInfo(SQueryMeterMsg* pQueryMsg, SSqlFuncExprMsg* pExprMsg) {
int32_t j = getColumnIndexInSource(pQueryMsg, pExprMsg);
return j < pQueryMsg->numOfCols;
}
SSqlFunctionExpr* vnodeCreateSqlFunctionExpr(SQueryMeterMsg* pQueryMsg, int32_t* code) {
SSqlFunctionExpr* pExprs = (SSqlFunctionExpr*)calloc(1, sizeof(SSqlFunctionExpr) * pQueryMsg->numOfOutputCols);
if (pExprs == NULL) {
tfree(pQueryMsg->pSqlFuncExprs);
*code = TSDB_CODE_SERV_OUT_OF_MEMORY;
return NULL;
}
bool isSuperTable = QUERY_IS_STABLE_QUERY(pQueryMsg->queryType);
int16_t tagLen = 0;
SSchema* pTagSchema = (SSchema*)pQueryMsg->pTagSchema;
for (int32_t i = 0; i < pQueryMsg->numOfOutputCols; ++i) {
pExprs[i].pBase = *((SSqlFuncExprMsg**)pQueryMsg->pSqlFuncExprs)[i];
pExprs[i].resBytes = 0;
int16_t type = 0;
int16_t bytes = 0;
SColIndexEx* pColumnIndexExInfo = &pExprs[i].pBase.colInfo;
// tag column schema is kept in pQueryMsg->pColumnModel
if (TSDB_COL_IS_TAG(pColumnIndexExInfo->flag)) {
if (pColumnIndexExInfo->colIdx >= pQueryMsg->numOfTagsCols) {
*code = TSDB_CODE_INVALID_QUERY_MSG;
tfree(pExprs);
return NULL;
}
type = pTagSchema[pColumnIndexExInfo->colIdx].type;
bytes = pTagSchema[pColumnIndexExInfo->colIdx].bytes;
} else { // parse the arithmetic expression
if (pExprs[i].pBase.functionId == TSDB_FUNC_ARITHM) {
*code = vnodeBuildExprFromArithmeticStr(&pExprs[i], pQueryMsg);
if (*code != TSDB_CODE_SUCCESS) {
tfree(pExprs);
return NULL;
}
type = TSDB_DATA_TYPE_DOUBLE;
bytes = tDataTypeDesc[type].nSize;
} else { // parse the normal column
int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].pBase);
assert(j < pQueryMsg->numOfCols);
SColumnInfo* pCol = &pQueryMsg->colList[j];
type = pCol->type;
bytes = pCol->bytes;
}
}
int32_t param = pExprs[i].pBase.arg[0].argValue.i64;
if (getResultDataInfo(type, bytes, pExprs[i].pBase.functionId, param, &pExprs[i].resType, &pExprs[i].resBytes,
&pExprs[i].interResBytes, 0, isSuperTable) != TSDB_CODE_SUCCESS) {
*code = TSDB_CODE_INVALID_QUERY_MSG;
return NULL;
}
if (pExprs[i].pBase.functionId == TSDB_FUNC_TAG_DUMMY || pExprs[i].pBase.functionId == TSDB_FUNC_TS_DUMMY) {
tagLen += pExprs[i].resBytes;
}
assert(isValidDataType(pExprs[i].resType, pExprs[i].resBytes));
}
//get the correct result size for top/bottom query, according to the number of tags columns in selection clause
// TODO refactor
for(int32_t i = 0; i < pQueryMsg->numOfOutputCols; ++i) {
pExprs[i].pBase = *((SSqlFuncExprMsg**)pQueryMsg->pSqlFuncExprs)[i];
int16_t functId = pExprs[i].pBase.functionId;
if (functId == TSDB_FUNC_TOP || functId == TSDB_FUNC_BOTTOM) {
int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].pBase);
assert(j < pQueryMsg->numOfCols);
SColumnInfo* pCol = &pQueryMsg->colList[j];
int16_t type = pCol->type;
int16_t bytes = pCol->bytes;
int32_t ret = getResultDataInfo(type, bytes, pExprs[i].pBase.functionId, pExprs[i].pBase.arg[0].argValue.i64,
&pExprs[i].resType, &pExprs[i].resBytes, &pExprs[i].interResBytes, tagLen, isSuperTable);
assert(ret == TSDB_CODE_SUCCESS);
}
}
tfree(pQueryMsg->pSqlFuncExprs);
return pExprs;
}
bool vnodeIsValidVnodeCfg(SVnodeCfg* pCfg) {
if (pCfg == NULL) return false;
if (pCfg->maxSessions <= 0 || pCfg->cacheBlockSize <= 0 || pCfg->replications <= 0 || pCfg->replications > 20 ||
pCfg->daysPerFile <= 0 || pCfg->daysToKeep <= 0) {
return false;
}
return true;
}
/**
* compare if schema of two tables are identical.
* when multi-table query is issued, the schemas of all requested tables
* should be identical. Otherwise,query process will abort.
*/
bool vnodeMeterSchemaIdentical(SColumn* pSchema1, int32_t numOfCols1, SColumn* pSchema2, int32_t numOfCols2) {
if (!VALIDNUMOFCOLS(numOfCols1) || !VALIDNUMOFCOLS(numOfCols2) || numOfCols1 != numOfCols2) {
return false;
}
return memcmp((char*)pSchema1, (char*)pSchema2, sizeof(SColumn) * numOfCols1) == 0;
}
void vnodeFreeFields(SQuery* pQuery) {
if (pQuery == NULL || pQuery->pFields == NULL) {
return;
}
for (int32_t i = 0; i < pQuery->numOfBlocks; ++i) {
tfree(pQuery->pFields[i]);
}
/*
* pQuery->pFields does not need to be released, it is allocated at the last part of pBlock
* so free(pBlock) can release this memory at the same time.
*/
pQuery->pFields = NULL;
pQuery->numOfBlocks = 0;
}
void vnodeUpdateFilterColumnIndex(SQuery* pQuery) {
for (int32_t i = 0; i < pQuery->numOfFilterCols; ++i) {
for (int16_t j = 0; j < pQuery->numOfCols; ++j) {
if (pQuery->pFilterInfo[i].info.data.colId == pQuery->colList[j].data.colId) {
pQuery->pFilterInfo[i].info.colIdx = pQuery->colList[j].colIdx;
pQuery->pFilterInfo[i].info.colIdxInBuf = pQuery->colList[j].colIdxInBuf;
// supplementary scan is also require this column
pQuery->colList[j].req[1] = 1;
break;
}
}
}
// set the column index in buffer for arithmetic operation
if (pQuery->pSelectExpr == NULL) {
return;
}
for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) {
SSqlBinaryExprInfo* pBinExprInfo = &pQuery->pSelectExpr[i].pBinExprInfo;
if (pBinExprInfo->pBinExpr == NULL) {
continue;
}
for (int16_t j = 0; j < pBinExprInfo->numOfCols; ++j) {
for (int32_t k = 0; k < pQuery->numOfCols; ++k) {
if (pBinExprInfo->pReqColumns[j].colId == pQuery->colList[k].data.colId) {
pBinExprInfo->pReqColumns[j].colIdxInBuf = pQuery->colList[k].colIdxInBuf;
assert(pQuery->colList[k].colIdxInBuf == k);
break;
}
}
}
}
}
// TODO support k<12 and k<>9
int32_t vnodeCreateFilterInfo(void* pQInfo, SQuery* pQuery) {
for (int32_t i = 0; i < pQuery->numOfCols; ++i) {
if (pQuery->colList[i].data.numOfFilters > 0) {
pQuery->numOfFilterCols++;
}
}
if (pQuery->numOfFilterCols == 0) {
return TSDB_CODE_SUCCESS;
}
pQuery->pFilterInfo = calloc(1, sizeof(SSingleColumnFilterInfo) * pQuery->numOfFilterCols);
for (int32_t i = 0, j = 0; i < pQuery->numOfCols; ++i) {
if (pQuery->colList[i].data.numOfFilters > 0) {
SSingleColumnFilterInfo* pFilterInfo = &pQuery->pFilterInfo[j];
memcpy(&pFilterInfo->info, &pQuery->colList[i], sizeof(SColumnInfoEx));
pFilterInfo->info.data.filters = NULL;
pFilterInfo->numOfFilters = pQuery->colList[i].data.numOfFilters;
pFilterInfo->pFilters = calloc(pFilterInfo->numOfFilters, sizeof(SColumnFilterElem));
for(int32_t f = 0; f < pFilterInfo->numOfFilters; ++f) {
SColumnFilterElem *pSingleColFilter = &pFilterInfo->pFilters[f];
pSingleColFilter->filterInfo = pQuery->colList[i].data.filters[f];
int32_t lower = pSingleColFilter->filterInfo.lowerRelOptr;
int32_t upper = pSingleColFilter->filterInfo.upperRelOptr;
if (lower == TSDB_RELATION_INVALID && upper == TSDB_RELATION_INVALID) {
dError("QInfo:%p invalid filter info", pQInfo);
return TSDB_CODE_INVALID_QUERY_MSG;
}
int16_t type = pQuery->colList[i].data.type;
int16_t bytes = pQuery->colList[i].data.bytes;
__filter_func_t *rangeFilterArray = vnodeGetRangeFilterFuncArray(type);
__filter_func_t *filterArray = vnodeGetValueFilterFuncArray(type);
if (rangeFilterArray == NULL && filterArray == NULL) {
dError("QInfo:%p failed to get filter function, invalid data type:%d", pQInfo, type);
return TSDB_CODE_INVALID_QUERY_MSG;
}
if ((lower == TSDB_RELATION_LARGE_EQUAL || lower == TSDB_RELATION_LARGE) &&
(upper == TSDB_RELATION_LESS_EQUAL || upper == TSDB_RELATION_LESS)) {
if (lower == TSDB_RELATION_LARGE_EQUAL) {
if (upper == TSDB_RELATION_LESS_EQUAL) {
pSingleColFilter->fp = rangeFilterArray[4];
} else {
pSingleColFilter->fp = rangeFilterArray[2];
}
} else {
if (upper == TSDB_RELATION_LESS_EQUAL) {
pSingleColFilter->fp = rangeFilterArray[3];
} else {
pSingleColFilter->fp = rangeFilterArray[1];
}
}
} else { // set callback filter function
if (lower != TSDB_RELATION_INVALID) {
pSingleColFilter->fp = filterArray[lower];
if (upper != TSDB_RELATION_INVALID) {
dError("pQInfo:%p failed to get filter function, invalid filter condition", pQInfo, type);
return TSDB_CODE_INVALID_QUERY_MSG;
}
} else {
pSingleColFilter->fp = filterArray[upper];
}
}
assert (pSingleColFilter->fp != NULL);
pSingleColFilter->bytes = bytes;
}
j++;
}
}
return TSDB_CODE_SUCCESS;
}
bool vnodeDoFilterData(SQuery* pQuery, int32_t elemPos) {
for (int32_t k = 0; k < pQuery->numOfFilterCols; ++k) {
SSingleColumnFilterInfo *pFilterInfo = &pQuery->pFilterInfo[k];
char* pElem = pFilterInfo->pData + pFilterInfo->info.data.bytes * elemPos;
if(isNull(pElem, pFilterInfo->info.data.type)) {
return false;
}
int32_t num = pFilterInfo->numOfFilters;
bool qualified = false;
for(int32_t j = 0; j < num; ++j) {
SColumnFilterElem* pFilterElem = &pFilterInfo->pFilters[j];
if (pFilterElem->fp(pFilterElem, pElem, pElem)) {
qualified = true;
break;
}
}
if (!qualified) {
return false;
}
}
return true;
}
bool vnodeFilterData(SQuery* pQuery, int32_t* numOfActualRead, int32_t index) {
(*numOfActualRead)++;
if (!vnodeDoFilterData(pQuery, index)) {
return false;
}
if (pQuery->limit.offset > 0) {
pQuery->limit.offset--; // ignore this qualified row
return false;
}
return true;
}
bool vnodeIsProjectionQuery(SSqlFunctionExpr* pExpr, int32_t numOfOutput) {
for (int32_t i = 0; i < numOfOutput; ++i) {
if (pExpr[i].pBase.functionId != TSDB_FUNC_PRJ) {
return false;
}
}
return true;
}
/*
* the pTable->state may be changed by vnodeIsSafeToDeleteMeter and import/update processor, the check of
* the state will not always be correct.
*
* The import/update/deleting is actually blocked by current query processing if the check of meter state is
* passed, but later queries are denied.
*
* 1. vnodeIsSafeToDelete will wait for this complete, since it also use the vmutex to check the numOfQueries
* 2. import will check the numOfQueries again after setting state to be TSDB_METER_STATE_IMPORTING, while the
* vmutex is also used.
* 3. insert has nothing to do with the query processing.
*/
int32_t vnodeIncQueryRefCount(SQueryMeterMsg* pQueryMsg, SMeterSidExtInfo** pSids, SMeterObj** pMeterObjList,
int32_t* numOfIncTables) {
SVnodeObj* pVnode = &vnodeList[pQueryMsg->vnode];
int32_t num = 0;
int32_t index = 0;
int32_t code = TSDB_CODE_SUCCESS;
for (int32_t i = 0; i < pQueryMsg->numOfSids; ++i) {
SMeterObj* pTable = pVnode->meterList[pSids[i]->sid];
/*
* If table is missing or is in dropping status, config it from management node, and ignore it
* during query processing. The error code of TSDB_CODE_NOT_ACTIVE_TABLE will never return to client.
* The missing table needs to be removed from pSids list
*/
if (pTable == NULL || vnodeIsMeterState(pTable, TSDB_METER_STATE_DROPPING)) {
dWarn("qmsg:%p, vid:%d sid:%d, not there or will be dropped, ignore this table in query", pQueryMsg,
pQueryMsg->vnode, pSids[i]->sid);
vnodeSendMeterCfgMsg(pQueryMsg->vnode, pSids[i]->sid);
continue;
} else if (pTable->uid != pSids[i]->uid || pTable->sid != pSids[i]->sid) {
code = TSDB_CODE_TABLE_ID_MISMATCH;
dError("qmsg:%p, vid:%d sid:%d id:%s uid:%" PRIu64 ", id mismatch. sid:%d uid:%" PRId64 " in msg", pQueryMsg,
pQueryMsg->vnode, pTable->sid, pTable->meterId, pTable->uid, pSids[i]->sid, pSids[i]->uid);
vnodeSendMeterCfgMsg(pQueryMsg->vnode, pSids[i]->sid);
continue;
} else if (pTable->state > TSDB_METER_STATE_INSERTING) { //update or import
code = TSDB_CODE_ACTION_IN_PROGRESS;
dTrace("qmsg:%p, vid:%d sid:%d id:%s, it is in state:%s, wait!", pQueryMsg, pQueryMsg->vnode, pSids[i]->sid,
pTable->meterId, taosGetTableStatusStr(pTable->state));
continue;
}
/*
* vnodeIsSafeToDeleteMeter will wait for this function complete, and then it can
* check if the numOfQueries is 0 or not.
*/
pMeterObjList[(*numOfIncTables)++] = pTable;
atomic_fetch_add_32(&pTable->numOfQueries, 1);
pSids[index++] = pSids[i];
// output for meter more than one query executed
if (pTable->numOfQueries > 1) {
dTrace("qmsg:%p, vid:%d sid:%d id:%s, inc query ref, numOfQueries:%d", pQueryMsg, pTable->vnode, pTable->sid,
pTable->meterId, pTable->numOfQueries);
num++;
}
}
dTrace("qmsg:%p, query meters: %d, inc query ref %d, numOfQueries on %d meters are 1, queried meters:%d after "
"filter missing meters", pQueryMsg, pQueryMsg->numOfSids, *numOfIncTables, (*numOfIncTables) - num, index);
assert(pQueryMsg->numOfSids >= (*numOfIncTables) && pQueryMsg->numOfSids >= index);
pQueryMsg->numOfSids = index;
return code;
}
void vnodeDecQueryRefCount(SQueryMeterMsg* pQueryMsg, SMeterObj** pMeterObjList, int32_t numOfIncTables) {
int32_t num = 0;
for (int32_t i = 0; i < numOfIncTables; ++i) {
SMeterObj* pTable = pMeterObjList[i];
if (pTable != NULL) { // here, do not need to lock to perform operations
atomic_fetch_sub_32(&pTable->numOfQueries, 1);
if (pTable->numOfQueries > 0) {
dTrace("qmsg:%p, vid:%d sid:%d id:%s dec query ref, numOfQueries:%d", pQueryMsg, pTable->vnode, pTable->sid,
pTable->meterId, pTable->numOfQueries);
num++;
}
}
}
dTrace("qmsg:%p, dec query ref for %d meters, numOfQueries on %d meters are 0", pQueryMsg, numOfIncTables, numOfIncTables - num);
}
void vnodeUpdateQueryColumnIndex(SQuery* pQuery, SMeterObj* pMeterObj) {
if (pQuery == NULL || pMeterObj == NULL) {
return;
}
int32_t i = 0, j = 0;
while (i < pQuery->numOfCols && j < pMeterObj->numOfColumns) {
if (pQuery->colList[i].data.colId == pMeterObj->schema[j].colId) {
pQuery->colList[i++].colIdx = (int16_t)j++;
} else if (pQuery->colList[i].data.colId < pMeterObj->schema[j].colId) {
pQuery->colList[i++].colIdx = -1;
} else if (pQuery->colList[i].data.colId > pMeterObj->schema[j].colId) {
j++;
}
}
while (i < pQuery->numOfCols) {
pQuery->colList[i++].colIdx = -1; // not such column in current meter
}
// sql expression has not been created yet
if (pQuery->pSelectExpr == NULL) {
return;
}
for(int32_t k = 0; k < pQuery->numOfOutputCols; ++k) {
SSqlFuncExprMsg* pSqlExprMsg = &pQuery->pSelectExpr[k].pBase;
if (pSqlExprMsg->functionId == TSDB_FUNC_ARITHM || pSqlExprMsg->colInfo.flag == TSDB_COL_TAG) {
continue;
}
SColIndexEx* pColIndexEx = &pSqlExprMsg->colInfo;
for(int32_t f = 0; f < pQuery->numOfCols; ++f) {
if (pColIndexEx->colId == pQuery->colList[f].data.colId) {
pColIndexEx->colIdx = pQuery->colList[f].colIdx;
break;
}
}
}
}
int32_t vnodeSetMeterState(SMeterObj* pMeterObj, int32_t state) {
return atomic_val_compare_exchange_32(&pMeterObj->state, TSDB_METER_STATE_READY, state);
}
void vnodeClearMeterState(SMeterObj* pMeterObj, int32_t state) {
pMeterObj->state &= (~state);
}
bool vnodeIsMeterState(SMeterObj* pMeterObj, int32_t state) {
if (state == TSDB_METER_STATE_READY) {
return pMeterObj->state == TSDB_METER_STATE_READY;
} else if (state == TSDB_METER_STATE_DROPPING) {
return pMeterObj->state >= state;
} else {
return (((pMeterObj->state) & state) == state);
}
}
void vnodeSetMeterDeleting(SMeterObj* pMeterObj) {
if (pMeterObj == NULL) {
return;
}
pMeterObj->state |= TSDB_METER_STATE_DROPPING;
}
int32_t vnodeSetMeterInsertImportStateEx(SMeterObj* pObj, int32_t st) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t state = vnodeSetMeterState(pObj, st);
if (state != TSDB_METER_STATE_READY) {//return to denote import is not performed
if (vnodeIsMeterState(pObj, TSDB_METER_STATE_DROPPING)) {
dTrace("vid:%d sid:%d id:%s, meter is deleted, state:%d", pObj->vnode, pObj->sid, pObj->meterId,
pObj->state);
code = TSDB_CODE_NOT_ACTIVE_TABLE;
} else {// waiting for 300ms by default and try again
dTrace("vid:%d sid:%d id:%s, try submit again since in state:%d", pObj->vnode, pObj->sid,
pObj->meterId, pObj->state);
code = TSDB_CODE_ACTION_IN_PROGRESS;
}
}
return code;
}
bool vnodeIsSafeToDeleteMeter(SVnodeObj* pVnode, int32_t sid) {
SMeterObj* pObj = pVnode->meterList[sid];
if (pObj == NULL || vnodeIsMeterState(pObj, TSDB_METER_STATE_DROPPED)) {
return true;
}
int32_t prev = vnodeSetMeterState(pObj, TSDB_METER_STATE_DROPPING);
/*
* if the meter is not in ready/deleting state, it must be in insert/import/update,
* set the deleting state and wait the procedure to be completed
*/
if (prev != TSDB_METER_STATE_READY && prev < TSDB_METER_STATE_DROPPING) {
vnodeSetMeterDeleting(pObj);
dWarn("vid:%d sid:%d id:%s, can not be deleted, state:%d, wait", pObj->vnode, pObj->sid, pObj->meterId, prev);
return false;
}
bool ready = true;
/*
* the query will be stopped ASAP, since the state of meter is set to TSDB_METER_STATE_DROPPING,
* and new query will abort since the meter is deleted.
*/
pthread_mutex_lock(&pVnode->vmutex);
if (pObj->numOfQueries > 0) {
dWarn("vid:%d sid:%d id:%s %d queries executing on it, wait query to be finished",
pObj->vnode, pObj->sid, pObj->meterId, pObj->numOfQueries);
ready = false;
}
pthread_mutex_unlock(&pVnode->vmutex);
return ready;
}
void vnodeFreeColumnInfo(SColumnInfo* pColumnInfo) {
if (pColumnInfo == NULL) {
return;
}
if (pColumnInfo->numOfFilters > 0) {
if (pColumnInfo->type == TSDB_DATA_TYPE_BINARY) {
for (int32_t i = 0; i < pColumnInfo->numOfFilters; ++i) {
tfree(pColumnInfo->filters[i].pz);
pColumnInfo->filters[i].len = 0;
}
}
tfree(pColumnInfo->filters);
}
}