xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '2.0' into feature/2.0tsdb

This commit is contained in:
hzcheng 2020-03-30 22:54:46 +08:00
parent 9fee498315 f079008dd0
commit f2a36718e5
59 changed files with 593 additions and 22157 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

302
CMakeLists.txt
View File

@ -1,298 +1,22 @@
CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
PROJECT(TDengine)
SET(CMAKE_C_STANDARD 11)
SET(CMAKE_VERBOSE_MAKEFILE ON)
#
# If need to set debug options
# 1.Generate debug version:
# mkdir debug; cd debug;
# cmake -DCMAKE_BUILD_TYPE=Debug ..
# 2.Generate release version:
# mkdir release; cd release;
# cmake -DCMAKE_BUILD_TYPE=Release ..
#
#
# If it is a Windows operating system
# 1.Use command line tool of VS2013 or higher version
# mkdir build; cd build;
# cmake -G "NMake Makefiles" ..
# nmake install
# 2.Use the VS development interface tool
# mkdir build; cd build;
# cmake -A x64 ..
# open the file named TDengine.sln
#
SET(TD_CLUSTER FALSE)
SET(TD_ACCOUNT FALSE)
SET(TD_GRANT FALSE)
SET(TD_COVER FALSE)
SET(TD_PAGMODE_LITE FALSE)
SET(TD_GODLL FALSE)
IF (${DLLTYPE} MATCHES "go")
ADD_DEFINITIONS(-D_TD_GO_DLL_)
MESSAGE(STATUS "input dll type: " ${DLLTYPE})
SET(TD_GODLL TRUE)
ENDIF ()
SET(TD_MEM_CHECK FALSE)
IF (NOT DEFINED TD_CLUSTER)
MESSAGE(STATUS "Build the Lite Version")
SET(TD_CLUSTER FALSE)
SET(TD_EDGE TRUE)
SET(TD_COMMUNITY_DIR ${PROJECT_SOURCE_DIR})
MESSAGE(STATUS "Community directory: " ${TD_COMMUNITY_DIR})
SET(TD_COMMUNITY_DIR ${PROJECT_SOURCE_DIR})
MESSAGE(STATUS "Community directory: " ${TD_COMMUNITY_DIR})
# Set macro definitions according to os platform
SET(TD_LINUX_64 FALSE)
SET(TD_LINUX_32 FALSE)
SET(TD_ARM FALSE)
SET(TD_ARM_64 FALSE)
SET(TD_ARM_32 FALSE)
SET(TD_MIPS FALSE)
SET(TD_MIPS_64 FALSE)
SET(TD_MIPS_32 FALSE)
SET(TD_DARWIN_64 FALSE)
SET(TD_WINDOWS_64 FALSE)
SET(TD_PAGMODE_LITE FALSE)
IF (${PAGMODE} MATCHES "lite")
SET(TD_PAGMODE_LITE TRUE)
ENDIF ()
# if generate ARM version:
# cmake -DCPUTYPE=aarch32 .. or cmake -DCPUTYPE=aarch64
IF (${CPUTYPE} MATCHES "aarch32")
SET(TD_ARM TRUE)
SET(TD_ARM_32 TRUE)
SET(TD_PAGMODE_LITE TRUE)
ADD_DEFINITIONS(-D_TD_ARM_)
ADD_DEFINITIONS(-D_TD_ARM_32_)
ELSEIF (${CPUTYPE} MATCHES "aarch64")
SET(TD_ARM TRUE)
SET(TD_ARM_64 TRUE)
ADD_DEFINITIONS(-D_TD_ARM_)
ADD_DEFINITIONS(-D_TD_ARM_64_)
ELSEIF (${CPUTYPE} MATCHES "mips64")
SET(TD_MIPS TRUE)
SET(TD_MIPS_64 TRUE)
ADD_DEFINITIONS(-D_TD_MIPS_)
ADD_DEFINITIONS(-D_TD_MIPS_64_)
ELSEIF (${CPUTYPE} MATCHES "x64")
MESSAGE(STATUS "input cpuType: " ${CPUTYPE})
ELSEIF (${CPUTYPE} MATCHES "x86")
MESSAGE(STATUS "input cpuType: " ${CPUTYPE})
ELSE ()
MESSAGE(STATUS "input cpuType: " ${CPUTYPE})
ENDIF ()
#
# Get OS information and store in variable TD_OS_INFO.
#
execute_process(COMMAND chmod 777 ${TD_COMMUNITY_DIR}/packaging/tools/get_os.sh)
execute_process(COMMAND ${TD_COMMUNITY_DIR}/packaging/tools/get_os.sh "" OUTPUT_VARIABLE TD_OS_INFO)
MESSAGE(STATUS "The current os is " ${TD_OS_INFO})
IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
IF (${CMAKE_SIZEOF_VOID_P} MATCHES 8)
SET(TD_LINUX_64 TRUE)
SET(TD_OS_DIR ${TD_COMMUNITY_DIR}/src/os/linux)
ADD_DEFINITIONS(-D_M_X64)
MESSAGE(STATUS "The current platform is Linux 64-bit")
ELSEIF (${CMAKE_SIZEOF_VOID_P} MATCHES 4)
IF (TD_ARM)
SET(TD_LINUX_32 TRUE)
SET(TD_OS_DIR ${TD_COMMUNITY_DIR}/src/os/linux)
#ADD_DEFINITIONS(-D_M_IX86)
MESSAGE(STATUS "The current platform is Linux 32-bit")
ELSE ()
MESSAGE(FATAL_ERROR "The current platform is Linux 32-bit, but no ARM not supported yet")
EXIT ()
ENDIF ()
ELSE ()
MESSAGE(FATAL_ERROR "The current platform is Linux neither 32-bit nor 64-bit, not supported yet")
EXIT ()
ENDIF ()
ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
IF (${CMAKE_SIZEOF_VOID_P} MATCHES 8)
SET(TD_DARWIN_64 TRUE)
SET(TD_OS_DIR ${TD_COMMUNITY_DIR}/src/os/darwin)
MESSAGE(STATUS "The current platform is Darwin 64-bit")
ELSE ()
MESSAGE(FATAL_ERROR "The current platform is Darwin 32-bit, not supported yet")
EXIT ()
ENDIF ()
ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "Windows")
IF (${CMAKE_SIZEOF_VOID_P} MATCHES 8)
SET(TD_WINDOWS_64 TRUE)
SET(TD_OS_DIR ${TD_COMMUNITY_DIR}/src/os/windows)
ADD_DEFINITIONS(-D_M_X64)
MESSAGE(STATUS "The current platform is Windows 64-bit")
ELSE ()
MESSAGE(FATAL_ERROR "The current platform is Windows 32-bit, not supported yet")
EXIT ()
ENDIF ()
ELSE()
MESSAGE(FATAL_ERROR "The current platform is not Linux/Darwin/Windows, stop compile")
EXIT ()
ENDIF ()
FIND_PROGRAM(TD_MVN_INSTALLED mvn)
IF (TD_MVN_INSTALLED)
MESSAGE(STATUS "MVN is installed and JDBC will be compiled")
ELSE ()
MESSAGE(STATUS "MVN is not installed and JDBC is not compiled")
ENDIF ()
#
# debug flag
#
# ADD_DEFINITIONS(-D_CHECK_HEADER_FILE_)
IF (${MEM_CHECK} MATCHES "true")
ADD_DEFINITIONS(-DTAOS_MEM_CHECK)
ENDIF ()
IF (TD_CLUSTER)
ADD_DEFINITIONS(-DCLUSTER)
ADD_DEFINITIONS(-DTSDB_REPLICA_MAX_NUM=3)
ELSE ()
ADD_DEFINITIONS(-DLITE)
ADD_DEFINITIONS(-DTSDB_REPLICA_MAX_NUM=1)
ENDIF ()
IF (TD_LINUX_64)
SET(DEBUG_FLAGS "-O0 -DDEBUG")
SET(RELEASE_FLAGS "-O0")
IF (NOT TD_ARM)
IF (${CMAKE_CXX_COMPILER_ID} MATCHES "Clang")
SET(COMMON_FLAGS "-std=gnu99 -Wall -fPIC -malign-double -g -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
ELSE ()
SET(COMMON_FLAGS "-std=gnu99 -Wall -fPIC -malign-double -g -malign-stringops -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
ENDIF ()
ELSE ()
SET(COMMON_FLAGS "-std=gnu99 -Wall -fPIC -g -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
ENDIF ()
ADD_DEFINITIONS(-DLINUX)
ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT)
IF (${TD_OS_INFO} MATCHES "Alpine")
MESSAGE(STATUS "The current OS is Alpine, append extra flags")
SET(COMMON_FLAGS "${COMMON_FLAGS} -largp")
link_libraries(/usr/lib/libargp.a)
ADD_DEFINITIONS(-D_ALPINE)
ENDIF ()
ELSEIF (TD_LINUX_32)
IF (NOT TD_ARM)
EXIT ()
ENDIF ()
SET(DEBUG_FLAGS "-O0 -DDEBUG")
SET(RELEASE_FLAGS "-O0")
SET(COMMON_FLAGS "-std=gnu99 -Wall -fPIC -g -fsigned-char -munaligned-access -fpack-struct=8 -latomic -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
ADD_DEFINITIONS(-DLINUX)
ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT)
ADD_DEFINITIONS(-DUSE_LIBICONV)
IF (${TD_OS_INFO} MATCHES "Alpine")
MESSAGE(STATUS "The current OS is Alpine, add extra flags")
SET(COMMON_FLAGS "${COMMON_FLAGS} -largp")
link_library(/usr/lib/libargp.a)
ADD_DEFINITIONS(-D_ALPINE)
ENDIF ()
ELSEIF (TD_WINDOWS_64)
SET(CMAKE_GENERATOR "NMake Makefiles" CACHE INTERNAL "" FORCE)
IF (NOT TD_GODLL)
SET(COMMON_FLAGS "/nologo /WX- /Oi /Oy- /Gm- /EHsc /MT /GS /Gy /fp:precise /Zc:wchar_t /Zc:forScope /Gd /errorReport:prompt /analyze-")
SET(DEBUG_FLAGS "/Zi /W3 /GL")
SET(RELEASE_FLAGS "/W0 /GL")
ENDIF ()
ADD_DEFINITIONS(-DWINDOWS)
ADD_DEFINITIONS(-D__CLEANUP_C)
ADD_DEFINITIONS(-DPTW32_STATIC_LIB)
ADD_DEFINITIONS(-DPTW32_BUILD)
ADD_DEFINITIONS(-D_MBCS -D_CRT_SECURE_NO_DEPRECATE -D_CRT_NONSTDC_NO_DEPRECATE)
ELSEIF (TD_DARWIN_64)
SET(COMMON_FLAGS "-std=gnu99 -Wall -fPIC -malign-double -g -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
SET(DEBUG_FLAGS "-O0 -DDEBUG")
SET(RELEASE_FLAGS "-O0")
ADD_DEFINITIONS(-DDARWIN)
ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT)
ELSE ()
MESSAGE(FATAL_ERROR "The current platform is not support yet, stop compile")
EXIT ()
ENDIF ()
# Set compiler options
SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} ${COMMON_FLAGS} ${DEBUG_FLAGS}")
SET(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} ${COMMON_FLAGS} ${RELEASE_FLAGS}")
# Set c++ compiler options
# SET(COMMON_CXX_FLAGS "${COMMON_FLAGS} -std=c++11")
# SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${COMMON_CXX_FLAGS} ${DEBUG_FLAGS}")
# SET(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${COMMON_CXX_FLAGS} ${RELEASE_FLAGS}")
IF (${CMAKE_BUILD_TYPE} MATCHES "Debug")
MESSAGE(STATUS "Build Debug Version")
ELSEIF (${CMAKE_BUILD_TYPE} MATCHES "Release")
MESSAGE(STATUS "Build Release Version")
ELSE ()
IF (TD_WINDOWS_64)
SET(CMAKE_BUILD_TYPE "Release")
MESSAGE(STATUS "Build Release Version in Windows as default")
ELSE ()
SET(CMAKE_BUILD_TYPE "Debug")
MESSAGE(STATUS "Build Debug Version as default")
ENDIF()
ENDIF ()
#set output directory
SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/lib)
SET(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/bin)
SET(TD_TESTS_OUTPUT_DIR ${PROJECT_BINARY_DIR}/test)
MESSAGE(STATUS "Operating system dependency directory: " ${TD_OS_DIR})
MESSAGE(STATUS "Project source directory: " ${PROJECT_SOURCE_DIR})
MESSAGE(STATUS "Project binary files output path: " ${PROJECT_BINARY_DIR})
MESSAGE(STATUS "Project executable files output path: " ${EXECUTABLE_OUTPUT_PATH})
MESSAGE(STATUS "Project library files output path: " ${LIBRARY_OUTPUT_PATH})
IF (TD_LINUX_64)
SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh")
INSTALL(CODE "MESSAGE(\"make install script: ${TD_MAKE_INSTALL_SH}\")")
INSTALL(CODE "execute_process(COMMAND chmod 777 ${TD_MAKE_INSTALL_SH})")
INSTALL(CODE "execute_process(COMMAND ${TD_MAKE_INSTALL_SH} ${TD_COMMUNITY_DIR} ${PROJECT_BINARY_DIR})")
ELSEIF (TD_LINUX_32)
IF (NOT TD_ARM)
EXIT ()
ENDIF ()
SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh")
INSTALL(CODE "MESSAGE(\"make install script: ${TD_MAKE_INSTALL_SH}\")")
INSTALL(CODE "execute_process(COMMAND chmod 777 ${TD_MAKE_INSTALL_SH})")
INSTALL(CODE "execute_process(COMMAND ${TD_MAKE_INSTALL_SH} ${TD_COMMUNITY_DIR} ${PROJECT_BINARY_DIR})")
ELSEIF (TD_WINDOWS_64)
SET(CMAKE_INSTALL_PREFIX C:/TDengine)
IF (NOT TD_GODLL)
INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/go DESTINATION connector)
INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/grafana DESTINATION connector)
INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/python DESTINATION connector)
INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/tests/examples DESTINATION .)
INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/packaging/cfg DESTINATION .)
INSTALL(FILES ${TD_COMMUNITY_DIR}/src/inc/taos.h DESTINATION include)
INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.lib DESTINATION driver)
INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.exp DESTINATION driver)
INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.dll DESTINATION driver)
INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/taos.exe DESTINATION .)
#INSTALL(TARGETS taos RUNTIME DESTINATION driver)
#INSTALL(TARGETS shell RUNTIME DESTINATION .)
IF (TD_MVN_INSTALLED)
INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-1.0.2-dist.jar DESTINATION connector/jdbc)
ENDIF ()
ELSE ()
INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/libtaos.dll DESTINATION driver)
INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/libtaos.dll.a DESTINATION driver)
ENDIF ()
ELSEIF (TD_DARWIN_64)
SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh")
INSTALL(CODE "MESSAGE(\"make install script: ${TD_MAKE_INSTALL_SH}\")")
INSTALL(CODE "execute_process(COMMAND chmod 777 ${TD_MAKE_INSTALL_SH})")
INSTALL(CODE "execute_process(COMMAND ${TD_MAKE_INSTALL_SH} ${TD_COMMUNITY_DIR} ${PROJECT_BINARY_DIR} Darwin)")
ENDIF ()
ENDIF ()
INCLUDE(cmake/input.inc)
INCLUDE(cmake/platform.inc)
INCLUDE(cmake/env.inc)
INCLUDE(cmake/define.inc)
INCLUDE(cmake/install.inc)
ADD_SUBDIRECTORY(deps)
ADD_SUBDIRECTORY(src)

26
cmake/define.inc Executable file
View File

@ -0,0 +1,26 @@
CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
PROJECT(TDengine)
IF (TD_CLUSTER)
ADD_DEFINITIONS(-D_CLUSTER)
ADD_DEFINITIONS(-DTSDB_REPLICA_MAX_NUM=3)
ELSE ()
ADD_DEFINITIONS(-DLITE)
ADD_DEFINITIONS(-DTSDB_REPLICA_MAX_NUM=1)
ENDIF ()
IF (TD_ACCOUNT)
ADD_DEFINITIONS(-D_ACCOUNT)
ENDIF ()
IF (TD_GRANT)
ADD_DEFINITIONS(-D_GRANT)
ENDIF ()
IF (TD_GODLL)
ADD_DEFINITIONS(-D_TD_GO_DLL_)
ENDIF ()
IF (TD_MEM_CHECK)
ADD_DEFINITIONS(-DTAOS_MEM_CHECK)
ENDIF ()

56
cmake/env.inc Executable file
View File

@ -0,0 +1,56 @@
CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
PROJECT(TDengine)
SET(CMAKE_C_STANDARD 11)
SET(CMAKE_VERBOSE_MAKEFILE ON)
#set output directory
SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/lib)
SET(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/bin)
SET(TD_TESTS_OUTPUT_DIR ${PROJECT_BINARY_DIR}/test)
MESSAGE(STATUS "Operating system dependency directory: " ${TD_OS_DIR})
MESSAGE(STATUS "Project source directory: " ${PROJECT_SOURCE_DIR})
MESSAGE(STATUS "Project binary files output path: " ${PROJECT_BINARY_DIR})
MESSAGE(STATUS "Project executable files output path: " ${EXECUTABLE_OUTPUT_PATH})
MESSAGE(STATUS "Project library files output path: " ${LIBRARY_OUTPUT_PATH})
FIND_PROGRAM(TD_MVN_INSTALLED mvn)
IF (TD_MVN_INSTALLED)
MESSAGE(STATUS "MVN is installed and JDBC will be compiled")
ELSE ()
MESSAGE(STATUS "MVN is not installed and JDBC is not compiled")
ENDIF ()
#
# If need to set debug options
# 1.Generate debug version:
# mkdir debug; cd debug;
# cmake -DCMAKE_BUILD_TYPE=Debug ..
# 2.Generate release version:
# mkdir release; cd release;
# cmake -DCMAKE_BUILD_TYPE=Release ..
#
# Set compiler options
SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} ${COMMON_FLAGS} ${DEBUG_FLAGS}")
SET(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} ${COMMON_FLAGS} ${RELEASE_FLAGS}")
# Set c++ compiler options
# SET(COMMON_CXX_FLAGS "${COMMON_FLAGS} -std=c++11")
# SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${COMMON_CXX_FLAGS} ${DEBUG_FLAGS}")
# SET(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${COMMON_CXX_FLAGS} ${RELEASE_FLAGS}")
IF (${CMAKE_BUILD_TYPE} MATCHES "Debug")
MESSAGE(STATUS "Build Debug Version")
ELSEIF (${CMAKE_BUILD_TYPE} MATCHES "Release")
MESSAGE(STATUS "Build Release Version")
ELSE ()
IF (TD_WINDOWS_64)
SET(CMAKE_BUILD_TYPE "Release")
MESSAGE(STATUS "Build Release Version in Windows as default")
ELSE ()
SET(CMAKE_BUILD_TYPE "Debug")
MESSAGE(STATUS "Build Debug Version as default")
ENDIF()
ENDIF ()

41
cmake/input.inc Executable file
View File

@ -0,0 +1,41 @@
CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
PROJECT(TDengine)
IF (${CLUSTER} MATCHES "true")
SET(TD_CLUSTER TRUE)
MESSAGE(STATUS "Build with cluster plugins")
ELSEIF (${CLUSTER} MATCHES "false")
SET(TD_CLUSTER FALSE)
MESSAGE(STATUS "Build without cluster plugins")
ENDIF ()
IF (${ACCOUNT} MATCHES "true")
SET(TD_ACCOUNT TRUE)
MESSAGE(STATUS "Build with account plugins")
ELSEIF (${ACCOUNT} MATCHES "false")
SET(TD_ACCOUNT FALSE)
MESSAGE(STATUS "Build without account plugins")
ENDIF ()
IF (${COVER} MATCHES "true")
SET(TD_COVER TRUE)
MESSAGE(STATUS "Build with test coverage")
ELSEIF (${COVER} MATCHES "false")
SET(TD_COVER FALSE)
MESSAGE(STATUS "Build without test coverage")
ENDIF ()
IF (${PAGMODE} MATCHES "lite")
SET(TD_PAGMODE_LITE TRUE)
MESSAGE(STATUS "Build with pagmode lite")
ENDIF ()
IF (${DLLTYPE} MATCHES "go")
SET(TD_GODLL TRUE)
MESSAGE(STATUS "input dll type: " ${DLLTYPE})
ENDIF ()
IF (${MEM_CHECK} MATCHES "true")
SET(TD_MEM_CHECK TRUE)
MESSAGE(STATUS "build with memory check")
ENDIF ()

41
cmake/install.inc Executable file
View File

@ -0,0 +1,41 @@
IF (TD_LINUX_64)
SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh")
INSTALL(CODE "MESSAGE(\"make install script: ${TD_MAKE_INSTALL_SH}\")")
INSTALL(CODE "execute_process(COMMAND chmod 777 ${TD_MAKE_INSTALL_SH})")
INSTALL(CODE "execute_process(COMMAND ${TD_MAKE_INSTALL_SH} ${TD_COMMUNITY_DIR} ${PROJECT_BINARY_DIR})")
ELSEIF (TD_LINUX_32)
IF (NOT TD_ARM)
EXIT ()
ENDIF ()
SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh")
INSTALL(CODE "MESSAGE(\"make install script: ${TD_MAKE_INSTALL_SH}\")")
INSTALL(CODE "execute_process(COMMAND chmod 777 ${TD_MAKE_INSTALL_SH})")
INSTALL(CODE "execute_process(COMMAND ${TD_MAKE_INSTALL_SH} ${TD_COMMUNITY_DIR} ${PROJECT_BINARY_DIR})")
ELSEIF (TD_WINDOWS_64)
SET(CMAKE_INSTALL_PREFIX C:/TDengine)
IF (NOT TD_GODLL)
INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/go DESTINATION connector)
INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/grafana DESTINATION connector)
INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/python DESTINATION connector)
INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/tests/examples DESTINATION .)
INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/packaging/cfg DESTINATION .)
INSTALL(FILES ${TD_COMMUNITY_DIR}/src/inc/taos.h DESTINATION include)
INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.lib DESTINATION driver)
INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.exp DESTINATION driver)
INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.dll DESTINATION driver)
INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/taos.exe DESTINATION .)
#INSTALL(TARGETS taos RUNTIME DESTINATION driver)
#INSTALL(TARGETS shell RUNTIME DESTINATION .)
IF (TD_MVN_INSTALLED)
INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-1.0.2-dist.jar DESTINATION connector/jdbc)
ENDIF ()
ELSE ()
INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/libtaos.dll DESTINATION driver)
INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/libtaos.dll.a DESTINATION driver)
ENDIF ()
ELSEIF (TD_DARWIN_64)
SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh")
INSTALL(CODE "MESSAGE(\"make install script: ${TD_MAKE_INSTALL_SH}\")")
INSTALL(CODE "execute_process(COMMAND chmod 777 ${TD_MAKE_INSTALL_SH})")
INSTALL(CODE "execute_process(COMMAND ${TD_MAKE_INSTALL_SH} ${TD_COMMUNITY_DIR} ${PROJECT_BINARY_DIR} Darwin)")
ENDIF ()

162
cmake/platform.inc Executable file
View File

@ -0,0 +1,162 @@
CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
PROJECT(TDengine)
#
# If it is a Windows operating system
# 1.Use command line tool of VS2013 or higher version
# mkdir build; cd build;
# cmake -G "NMake Makefiles" ..
# nmake install
# 2.Use the VS development interface tool
# mkdir build; cd build;
# cmake -A x64 ..
# open the file named TDengine.sln
#
# Set macro definitions according to os platform
SET(TD_LINUX_64 FALSE)
SET(TD_LINUX_32 FALSE)
SET(TD_ARM FALSE)
SET(TD_ARM_64 FALSE)
SET(TD_ARM_32 FALSE)
SET(TD_MIPS FALSE)
SET(TD_MIPS_64 FALSE)
SET(TD_MIPS_32 FALSE)
SET(TD_DARWIN_64 FALSE)
SET(TD_WINDOWS_64 FALSE)
# if generate ARM version:
# cmake -DCPUTYPE=aarch32 .. or cmake -DCPUTYPE=aarch64
IF (${CPUTYPE} MATCHES "aarch32")
SET(TD_ARM TRUE)
SET(TD_ARM_32 TRUE)
SET(TD_PAGMODE_LITE TRUE)
ADD_DEFINITIONS(-D_TD_ARM_)
ADD_DEFINITIONS(-D_TD_ARM_32_)
ELSEIF (${CPUTYPE} MATCHES "aarch64")
SET(TD_ARM TRUE)
SET(TD_ARM_64 TRUE)
ADD_DEFINITIONS(-D_TD_ARM_)
ADD_DEFINITIONS(-D_TD_ARM_64_)
ELSEIF (${CPUTYPE} MATCHES "mips64")
SET(TD_MIPS TRUE)
SET(TD_MIPS_64 TRUE)
ADD_DEFINITIONS(-D_TD_MIPS_)
ADD_DEFINITIONS(-D_TD_MIPS_64_)
ELSEIF (${CPUTYPE} MATCHES "x64")
MESSAGE(STATUS "input cpuType: " ${CPUTYPE})
ELSEIF (${CPUTYPE} MATCHES "x86")
MESSAGE(STATUS "input cpuType: " ${CPUTYPE})
ELSE ()
MESSAGE(STATUS "input cpuType: " ${CPUTYPE})
ENDIF ()
#
# Get OS information and store in variable TD_OS_INFO.
#
execute_process(COMMAND chmod 777 ${TD_COMMUNITY_DIR}/packaging/tools/get_os.sh)
execute_process(COMMAND ${TD_COMMUNITY_DIR}/packaging/tools/get_os.sh "" OUTPUT_VARIABLE TD_OS_INFO)
MESSAGE(STATUS "The current os is " ${TD_OS_INFO})
IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
IF (${CMAKE_SIZEOF_VOID_P} MATCHES 8)
SET(TD_LINUX_64 TRUE)
SET(TD_OS_DIR ${TD_COMMUNITY_DIR}/src/os/linux)
ADD_DEFINITIONS(-D_M_X64)
MESSAGE(STATUS "The current platform is Linux 64-bit")
ELSEIF (${CMAKE_SIZEOF_VOID_P} MATCHES 4)
IF (TD_ARM)
SET(TD_LINUX_32 TRUE)
SET(TD_OS_DIR ${TD_COMMUNITY_DIR}/src/os/linux)
#ADD_DEFINITIONS(-D_M_IX86)
MESSAGE(STATUS "The current platform is Linux 32-bit")
ELSE ()
MESSAGE(FATAL_ERROR "The current platform is Linux 32-bit, but no ARM not supported yet")
EXIT ()
ENDIF ()
ELSE ()
MESSAGE(FATAL_ERROR "The current platform is Linux neither 32-bit nor 64-bit, not supported yet")
EXIT ()
ENDIF ()
ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
IF (${CMAKE_SIZEOF_VOID_P} MATCHES 8)
SET(TD_DARWIN_64 TRUE)
SET(TD_OS_DIR ${TD_COMMUNITY_DIR}/src/os/darwin)
MESSAGE(STATUS "The current platform is Darwin 64-bit")
ELSE ()
MESSAGE(FATAL_ERROR "The current platform is Darwin 32-bit, not supported yet")
EXIT ()
ENDIF ()
ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "Windows")
IF (${CMAKE_SIZEOF_VOID_P} MATCHES 8)
SET(TD_WINDOWS_64 TRUE)
SET(TD_OS_DIR ${TD_COMMUNITY_DIR}/src/os/windows)
ADD_DEFINITIONS(-D_M_X64)
MESSAGE(STATUS "The current platform is Windows 64-bit")
ELSE ()
MESSAGE(FATAL_ERROR "The current platform is Windows 32-bit, not supported yet")
EXIT ()
ENDIF ()
ELSE()
MESSAGE(FATAL_ERROR "The current platform is not Linux/Darwin/Windows, stop compile")
EXIT ()
ENDIF ()
IF (TD_LINUX_64)
SET(DEBUG_FLAGS "-O0 -DDEBUG")
SET(RELEASE_FLAGS "-O0")
IF (NOT TD_ARM)
IF (${CMAKE_CXX_COMPILER_ID} MATCHES "Clang")
SET(COMMON_FLAGS "-std=gnu99 -Wall -fPIC -malign-double -g -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
ELSE ()
SET(COMMON_FLAGS "-std=gnu99 -Wall -fPIC -malign-double -g -malign-stringops -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
ENDIF ()
ELSE ()
SET(COMMON_FLAGS "-std=gnu99 -Wall -fPIC -g -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
ENDIF ()
ADD_DEFINITIONS(-DLINUX)
ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT)
IF (${TD_OS_INFO} MATCHES "Alpine")
MESSAGE(STATUS "The current OS is Alpine, append extra flags")
SET(COMMON_FLAGS "${COMMON_FLAGS} -largp")
link_libraries(/usr/lib/libargp.a)
ADD_DEFINITIONS(-D_ALPINE)
ENDIF ()
ELSEIF (TD_LINUX_32)
IF (NOT TD_ARM)
EXIT ()
ENDIF ()
SET(DEBUG_FLAGS "-O0 -DDEBUG")
SET(RELEASE_FLAGS "-O0")
SET(COMMON_FLAGS "-std=gnu99 -Wall -fPIC -g -fsigned-char -munaligned-access -fpack-struct=8 -latomic -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
ADD_DEFINITIONS(-DLINUX)
ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT)
ADD_DEFINITIONS(-DUSE_LIBICONV)
IF (${TD_OS_INFO} MATCHES "Alpine")
MESSAGE(STATUS "The current OS is Alpine, add extra flags")
SET(COMMON_FLAGS "${COMMON_FLAGS} -largp")
link_library(/usr/lib/libargp.a)
ADD_DEFINITIONS(-D_ALPINE)
ENDIF ()
ELSEIF (TD_WINDOWS_64)
SET(CMAKE_GENERATOR "NMake Makefiles" CACHE INTERNAL "" FORCE)
IF (NOT TD_GODLL)
SET(COMMON_FLAGS "/nologo /WX- /Oi /Oy- /Gm- /EHsc /MT /GS /Gy /fp:precise /Zc:wchar_t /Zc:forScope /Gd /errorReport:prompt /analyze-")
SET(DEBUG_FLAGS "/Zi /W3 /GL")
SET(RELEASE_FLAGS "/W0 /GL")
ENDIF ()
ADD_DEFINITIONS(-DWINDOWS)
ADD_DEFINITIONS(-D__CLEANUP_C)
ADD_DEFINITIONS(-DPTW32_STATIC_LIB)
ADD_DEFINITIONS(-DPTW32_BUILD)
ADD_DEFINITIONS(-D_MBCS -D_CRT_SECURE_NO_DEPRECATE -D_CRT_NONSTDC_NO_DEPRECATE)
ELSEIF (TD_DARWIN_64)
SET(COMMON_FLAGS "-std=gnu99 -Wall -fPIC -malign-double -g -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
SET(DEBUG_FLAGS "-O0 -DDEBUG")
SET(RELEASE_FLAGS "-O0")
ADD_DEFINITIONS(-DDARWIN)
ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT)
ELSE ()
MESSAGE(FATAL_ERROR "The current platform is not support yet, stop compile")
EXIT ()
ENDIF ()

13
src/dnode/CMakeLists.txt
View File

@ -15,9 +15,16 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
ADD_EXECUTABLE(taosd ${SRC})
TARGET_LINK_LIBRARIES(taosd mnode taos_static monitor http tsdb)
#IF (TD_CLUSTER)
# TARGET_LINK_LIBRARIES(taosd dcluster)
#ENDIF ()
IF (TD_ACCOUNT)
TARGET_LINK_LIBRARIES(taosd account)
ENDIF ()
IF (TD_GRANT)
TARGET_LINK_LIBRARIES(taosd grant)
ENDIF ()
IF (TD_CLUSTER)
TARGET_LINK_LIBRARIES(taosd cluster)
ENDIF ()
SET(PREPARE_ENV_CMD "prepare_env_cmd")
SET(PREPARE_ENV_TARGET "prepare_env_target")

25
src/dnode/src/dnodeMain.c
View File

@ -28,20 +28,9 @@
#include "dnodeRead.h"
#include "dnodeShell.h"
#include "dnodeWrite.h"
#ifdef CLUSTER
#include "account.h"
#include "admin.h"
#include "balance.h"
#include "cluster.h"
#include "grant.h"
#include "mpeer.h"
#include "storage.h"
#include "vpeer.h"
#endif
static int32_t dnodeInitSystem();
static int32_t dnodeInitStorage();
static void dnodeInitPlugins();
static void dnodeCleanupStorage();
static void dnodeCleanUpSystem();
static void dnodeSetRunStatus(SDnodeRunStatus status);
@ -51,8 +40,6 @@ static SDnodeRunStatus tsDnodeRunStatus = TSDB_DNODE_RUN_STATUS_STOPPED;
void (*dnodeParseParameterKFp)() = NULL;
int32_t main(int32_t argc, char *argv[]) {
dnodeInitPlugins();
// Set global configuration file
for (int32_t i = 1; i < argc; ++i) {
if (strcmp(argv[i], "-c") == 0) {
@ -244,15 +231,3 @@ static int32_t dnodeInitStorage() {
}
static void dnodeCleanupStorage() {}
static void dnodeInitPlugins() {
#ifdef CLUSTER
// acctInit();
// adminInit();
// balanceInit();
// clusterInit();
// grantInit();
// mpeerInit();
// storageInit();
#endif
}

14
src/inc/mnode.h
View File

@ -38,6 +38,10 @@ extern "C" {
#include "ttimer.h"
#include "tutil.h"
struct _vg_obj;
struct _db_obj;
struct _acctObj;
typedef struct {
int32_t mnodeId;
uint32_t privateIp;
@ -103,8 +107,6 @@ typedef struct {
int8_t dirty;
} STableInfo;
struct _vg_obj;
typedef struct SSuperTableObj {
STableInfo info;
uint64_t uid;
@ -137,8 +139,6 @@ typedef struct {
SSuperTableObj *superTable;
} SChildTableObj;
struct _db_obj;
typedef struct _vg_obj {
uint32_t vgId;
char dbName[TSDB_DB_NAME_LEN + 1];
@ -170,10 +170,9 @@ typedef struct _db_obj {
int32_t numOfSuperTables;
SVgObj *pHead;
SVgObj *pTail;
struct _acctObj *pAcct;
} SDbObj;
struct _acctObj;
typedef struct _user_obj {
char user[TSDB_USER_LEN + 1];
char pass[TSDB_KEY_LEN + 1];
@ -213,7 +212,8 @@ typedef struct _acctObj {
SAcctCfg cfg;
int32_t acctId;
int64_t createdTime;
int8_t reserved[15];
int8_t dirty;
int8_t reserved[14];
int8_t updateEnd[1];
SAcctInfo acctInfo;
SDbObj * pHead;

4
src/mnode/CMakeLists.txt
View File

@ -14,10 +14,6 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
ADD_LIBRARY(mnode ${SRC})
TARGET_LINK_LIBRARIES(mnode trpc tutil pthread)
IF (TD_CLUSTER)
TARGET_LINK_LIBRARIES(mnode)
ENDIF ()
ENDIF ()

23
src/mnode/inc/mgmtAcct.h
View File

@ -21,18 +21,21 @@ extern "C" {
#endif
#include "mnode.h"
int32_t mgmtInitAccts();
void mgmtCleanUpAccts();
SAcctObj *mgmtGetAcct(char *acctName);
typedef enum {
TSDB_ACCT_USER,
TSDB_ACCT_DB,
TSDB_ACCT_TABLE
} EAcctGrantType;
int32_t mgmtCheckUserLimit(SAcctObj *pAcct);
int32_t mgmtCheckDbLimit(SAcctObj *pAcct);
int32_t mgmtCheckTableLimit(SAcctObj *pAcct, int32_t numOfTimeSeries);
int32_t acctInit();
void acctCleanUp();
SAcctObj *acctGetAcct(char *acctName);
int32_t acctCheck(SAcctObj *pAcct, EAcctGrantType type);
int32_t mgmtAddDbIntoAcct(SAcctObj *pAcct, SDbObj *pDb);
int32_t mgmtRemoveDbFromAcct(SAcctObj *pAcct, SDbObj *pDb);
int32_t mgmtAddUserIntoAcct(SAcctObj *pAcct, SUserObj *pUser);
int32_t mgmtRemoveUserFromAcct(SAcctObj *pAcct, SUserObj *pUser);
int32_t acctAddDb(SAcctObj *pAcct, SDbObj *pDb);
int32_t acctRemoveDb(SAcctObj *pAcct, SDbObj *pDb);
int32_t acctAddUser(SAcctObj *pAcct, SUserObj *pUser);
int32_t acctRemoveUser(SAcctObj *pAcct, SUserObj *pUser);
#ifdef __cplusplus
}

1
src/mnode/inc/mgmtDb.h
View File

@ -28,6 +28,7 @@ void mgmtCleanUpDbs();
SDbObj *mgmtGetDb(char *db);
SDbObj *mgmtGetDbByTableId(char *db);
bool mgmtCheckIsMonitorDB(char *db, char *monitordb);
void mgmtDropAllDbs(SAcctObj *pAcct);
// util func
void mgmtAddSuperTableIntoDb(SDbObj *pDb);

10
src/mnode/inc/mgmtDnode.h
View File

@ -24,19 +24,9 @@ extern "C" {
int32_t mgmtInitDnodes();
void mgmtCleanUpDnodes();
int32_t mgmtGetDnodesNum();
int32_t mgmtUpdateDnode(SDnodeObj *pDnode);
SDnodeObj* mgmtGetDnode(int32_t dnodeId);
SDnodeObj* mgmtGetDnodeByIp(uint32_t ip);
bool mgmtCheckDnodeInRemoveState(SDnodeObj *pDnode);
bool mgmtCheckDnodeInOfflineState(SDnodeObj *pDnode);
bool mgmtCheckModuleInDnode(SDnodeObj *pDnode, int32_t moduleType);
void mgmtSetDnodeUnRemove(SDnodeObj *pDnode);
void mgmtSetDnodeMaxVnodes(SDnodeObj *pDnode);
void mgmtCalcNumOfFreeVnodes(SDnodeObj *pDnode);
void mgmtSetDnodeVgid(SVnodeGid vnodeGid[], int32_t numOfVnodes, int32_t vgId);
void mgmtUnSetDnodeVgid(SVnodeGid vnodeGid[], int32_t numOfVnodes);
#ifdef __cplusplus
}
#endif

30
src/mnode/inc/mgmtGrant.h
View File

@ -19,14 +19,30 @@
#ifdef __cplusplus
"C" {
#endif
#include "mnode.h"
bool mgmtCheckExpired();
void mgmtAddTimeSeries(SAcctObj *pAcct, uint32_t timeSeriesNum);
void mgmtRestoreTimeSeries(SAcctObj *pAcct, uint32_t timeseries);
int32_t mgmtCheckTimeSeries(uint32_t timeseries);
int32_t mgmtCheckUserGrant();
int32_t mgmtCheckDbGrant();
typedef enum {
TSDB_GRANT_ALL,
TSDB_GRANT_TIME,
TSDB_GRANT_USER,
TSDB_GRANT_DB,
TSDB_GRANT_TIMESERIES,
TSDB_GRANT_DNODE,
TSDB_GRANT_ACCT,
TSDB_GRANT_STORAGE,
TSDB_GRANT_SPEED,
TSDB_GRANT_QUERY_TIME,
TSDB_GRANT_CONNS,
TSDB_GRANT_STREAMS,
TSDB_GRANT_CPU_CORES,
} EGrantType;
int32_t grantInit();
void grantCleanUp();
void grantParseParameter();
int32_t grantCheck(EGrantType grant);
void grantReset(EGrantType grant, uint64_t value);
void grantAdd(EGrantType grant, uint64_t value);
void grantRestore(EGrantType grant, uint64_t value);
#ifdef __cplusplus
}

1
src/mnode/inc/mgmtSdb.h
View File

@ -63,6 +63,7 @@ int32_t sdbUpdateRow(SSdbOperDesc *pOper);
void *sdbGetRow(void *handle, void *key);
void *sdbFetchRow(void *handle, void *pNode, void **ppRow);
int64_t sdbGetNumOfRows(void *handle);
int64_t sdbGetId(void *handle);
uint64_t sdbGetVersion();
#ifdef __cplusplus

2
src/mnode/inc/mgmtUser.h
View File

@ -25,6 +25,8 @@ int32_t mgmtInitUsers();
void mgmtCleanUpUsers();
SUserObj *mgmtGetUser(char *name);
SUserObj *mgmtGetUserFromConn(void *pConn, bool *usePublicIp);
int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass);
void mgmtDropAllUsers(SAcctObj *pAcct);
#ifdef __cplusplus
}

79
src/mnode/src/mgmtAcct.c
View File

@ -15,21 +15,29 @@
#define _DEFAULT_SOURCE
#include "os.h"
#include "taoserror.h"
#include "mnode.h"
#include "mgmtAcct.h"
#ifndef _ACCOUNT
static SAcctObj tsAcctObj;
static SAcctObj tsAcctObj = {0};
int32_t (*mgmtInitAcctsFp)() = NULL;
void (*mgmtCleanUpAcctsFp)() = NULL;
SAcctObj *(*mgmtGetAcctFp)(char *acctName) = NULL;
int32_t (*mgmtCheckUserLimitFp)(SAcctObj *pAcct) = NULL;
int32_t (*mgmtCheckDbLimitFp)(SAcctObj *pAcct) = NULL;
int32_t (*mgmtCheckTableLimitFp)(SAcctObj *pAcct, int32_t numOfTimeSeries) = NULL;
int32_t acctInit() {
tsAcctObj.acctId = 0;
strcpy(tsAcctObj.user, "root");
return TSDB_CODE_SUCCESS;
}
int32_t mgmtAddDbIntoAcct(SAcctObj *pAcct, SDbObj *pDb) {
void acctCleanUp() {}
SAcctObj *acctGetAcct(char *acctName) { return &tsAcctObj; }
int32_t acctCheck(SAcctObj *pAcct, EAcctGrantType type) { return TSDB_CODE_SUCCESS; }
#endif
int32_t acctAddDb(SAcctObj *pAcct, SDbObj *pDb) {
pthread_mutex_lock(&pAcct->mutex);
pDb->next = pAcct->pHead;
pDb->prev = NULL;
pDb->pAcct = pAcct;
if (pAcct->pHead) {
pAcct->pHead->prev = pDb;
@ -42,7 +50,7 @@ int32_t mgmtAddDbIntoAcct(SAcctObj *pAcct, SDbObj *pDb) {
return 0;
}
int32_t mgmtRemoveDbFromAcct(SAcctObj *pAcct, SDbObj *pDb) {
int32_t acctRemoveDb(SAcctObj *pAcct, SDbObj *pDb) {
pthread_mutex_lock(&pAcct->mutex);
if (pDb->prev) {
pDb->prev->next = pDb->next;
@ -62,7 +70,7 @@ int32_t mgmtRemoveDbFromAcct(SAcctObj *pAcct, SDbObj *pDb) {
return 0;
}
int32_t mgmtAddUserIntoAcct(SAcctObj *pAcct, SUserObj *pUser) {
int32_t acctAddUser(SAcctObj *pAcct, SUserObj *pUser) {
pthread_mutex_lock(&pAcct->mutex);
pUser->next = pAcct->pUser;
pUser->prev = NULL;
@ -79,7 +87,7 @@ int32_t mgmtAddUserIntoAcct(SAcctObj *pAcct, SUserObj *pUser) {
return 0;
}
int32_t mgmtRemoveUserFromAcct(SAcctObj *pAcct, SUserObj *pUser) {
int32_t acctRemoveUser(SAcctObj *pAcct, SUserObj *pUser) {
pthread_mutex_lock(&pAcct->mutex);
if (pUser->prev) {
pUser->prev->next = pUser->next;
@ -88,7 +96,7 @@ int32_t mgmtRemoveUserFromAcct(SAcctObj *pAcct, SUserObj *pUser) {
if (pUser->next) {
pUser->next->prev = pUser->prev;
}
if (pUser->prev == NULL) {
pAcct->pUser = pUser->next;
}
@ -97,51 +105,4 @@ int32_t mgmtRemoveUserFromAcct(SAcctObj *pAcct, SUserObj *pUser) {
pthread_mutex_unlock(&pAcct->mutex);
return 0;
}
int32_t mgmtInitAccts() {
if (mgmtInitAcctsFp) {
return (*mgmtInitAcctsFp)();
} else {
tsAcctObj.acctId = 0;
strcpy(tsAcctObj.user, "root");
return 0;
}
}
SAcctObj *mgmtGetAcct(char *acctName) {
if (mgmtGetAcctFp) {
return (*mgmtGetAcctFp)(acctName);
} else {
return &tsAcctObj;
}
}
void mgmtCleanUpAccts() {
if (mgmtCleanUpAcctsFp) {
(*mgmtCleanUpAcctsFp)();
}
}
int32_t mgmtCheckUserLimit(SAcctObj *pAcct) {
if (mgmtCheckUserLimitFp) {
return (*mgmtCheckUserLimitFp)(pAcct);
}
return 0;
}
int32_t mgmtCheckDbLimit(SAcctObj *pAcct) {
if (mgmtCheckDbLimitFp) {
return (*mgmtCheckDbLimitFp)(pAcct);
} else {
return 0;
}
}
int32_t mgmtCheckTableLimit(SAcctObj *pAcct, int32_t numOfTimeSeries) {
if (mgmtCheckTableLimitFp) {
return (*mgmtCheckTableLimitFp)(pAcct, numOfTimeSeries);
} else {
return 0;
}
}

26
src/mnode/src/mgmtChildTable.c
View File

@ -48,7 +48,7 @@
#include "mgmtVgroup.h"
#include "mgmtUser.h"
static void *tsChildTableSdb;
void *tsChildTableSdb;
static int32_t tsChildTableUpdateSize;
static void mgmtProcessMultiTableMetaMsg(SQueuedMsg *queueMsg);
static void mgmtProcessCreateTableRsp(SRpcMsg *rpcMsg);
@ -84,7 +84,7 @@ static int32_t mgmtChildTableActionInsert(SSdbOperDesc *pOper) {
return TSDB_CODE_INVALID_DB;
}
SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct);
SAcctObj *pAcct = acctGetAcct(pDb->cfg.acct);
if (pAcct == NULL) {
mError("ctable:%s, account:%s not exists", pTable->info.tableId, pDb->cfg.acct);
return TSDB_CODE_INVALID_ACCT;
@ -93,9 +93,11 @@ static int32_t mgmtChildTableActionInsert(SSdbOperDesc *pOper) {
if (pTable->info.type == TSDB_CHILD_TABLE) {
pTable->superTable = mgmtGetSuperTable(pTable->superTableId);
pTable->superTable->numOfTables++;
mgmtAddTimeSeries(pAcct, pTable->superTable->numOfColumns - 1);
grantAdd(TSDB_GRANT_TIMESERIES, pTable->superTable->numOfColumns - 1);
pAcct->acctInfo.numOfTimeSeries += (pTable->superTable->numOfColumns - 1);
} else {
mgmtAddTimeSeries(pAcct, pTable->numOfColumns - 1);
grantAdd(TSDB_GRANT_TIMESERIES, pTable->numOfColumns - 1);
pAcct->acctInfo.numOfTimeSeries += (pTable->numOfColumns - 1);
}
mgmtAddTableIntoDb(pDb);
mgmtAddTableIntoVgroup(pVgroup, pTable);
@ -120,17 +122,19 @@ static int32_t mgmtChildTableActionDelete(SSdbOperDesc *pOper) {
return TSDB_CODE_INVALID_DB;
}
SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct);
SAcctObj *pAcct = acctGetAcct(pDb->cfg.acct);
if (pAcct == NULL) {
mError("ctable:%s, account:%s not exists", pTable->info.tableId, pDb->cfg.acct);
return TSDB_CODE_INVALID_ACCT;
}
if (pTable->info.type == TSDB_CHILD_TABLE) {
mgmtRestoreTimeSeries(pAcct, pTable->superTable->numOfColumns - 1);
grantRestore(TSDB_GRANT_TIMESERIES, pTable->superTable->numOfColumns - 1);
pAcct->acctInfo.numOfTimeSeries -= (pTable->superTable->numOfColumns - 1);
pTable->superTable->numOfTables--;
} else {
mgmtRestoreTimeSeries(pAcct, pTable->numOfColumns - 1);
grantRestore(TSDB_GRANT_TIMESERIES, pTable->numOfColumns - 1);
pAcct->acctInfo.numOfTimeSeries -= (pTable->numOfColumns - 1);
}
mgmtRemoveTableFromDb(pDb);
mgmtRemoveTableFromVgroup(pVgroup, pTable);
@ -464,9 +468,9 @@ static SChildTableObj* mgmtDoCreateChildTable(SCMCreateTableMsg *pCreate, SVgObj
void mgmtCreateChildTable(SQueuedMsg *pMsg) {
SCMCreateTableMsg *pCreate = pMsg->pCont;
int32_t code = mgmtCheckTimeSeries(htons(pCreate->numOfColumns));
int32_t code = grantCheck(TSDB_GRANT_TIMESERIES);
if (code != TSDB_CODE_SUCCESS) {
mError("table:%s, failed to create, timeseries exceed the limit", pCreate->tableId);
mError("table:%s, failed to create, grant not", pCreate->tableId);
mgmtSendSimpleResp(pMsg->thandle, code);
return;
}
@ -634,7 +638,7 @@ static int32_t mgmtAddNormalTableColumn(SChildTableObj *pTable, SSchema schema[]
return TSDB_CODE_APP_ERROR;
}
SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct);
SAcctObj *pAcct = acctGetAcct(pDb->cfg.acct);
if (pAcct == NULL) {
mError("DB: %s not belongs to andy account", pDb->name);
return TSDB_CODE_APP_ERROR;
@ -677,7 +681,7 @@ static int32_t mgmtDropNormalTableColumnByName(SChildTableObj *pTable, char *col
return TSDB_CODE_APP_ERROR;
}
SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct);
SAcctObj *pAcct = acctGetAcct(pDb->cfg.acct);
if (pAcct == NULL) {
mError("DB: %s not belongs to any account", pDb->name);
return TSDB_CODE_APP_ERROR;

37
src/mnode/src/mgmtDb.c
View File

@ -35,7 +35,7 @@
#include "mgmtUser.h"
#include "mgmtVgroup.h"
static void *tsDbSdb = NULL;
void * tsDbSdb = NULL;
static int32_t tsDbUpdateSize;
static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate);
@ -54,7 +54,7 @@ static int32_t mgmtDbActionDestroy(SSdbOperDesc *pOper) {
static int32_t mgmtDbActionInsert(SSdbOperDesc *pOper) {
SDbObj *pDb = pOper->pObj;
SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct);
SAcctObj *pAcct = acctGetAcct(pDb->cfg.acct);
pDb->pHead = NULL;
pDb->pTail = NULL;
@ -65,7 +65,7 @@ static int32_t mgmtDbActionInsert(SSdbOperDesc *pOper) {
pDb->numOfSuperTables = 0;
if (pAcct != NULL) {
mgmtAddDbIntoAcct(pAcct, pDb);
acctAddDb(pAcct, pDb);
}
else {
mError("db:%s, acct:%s info not exist in sdb", pDb->name, pDb->cfg.acct);
@ -77,9 +77,9 @@ static int32_t mgmtDbActionInsert(SSdbOperDesc *pOper) {
static int32_t mgmtDbActionDelete(SSdbOperDesc *pOper) {
SDbObj *pDb = pOper->pObj;
SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct);
SAcctObj *pAcct = acctGetAcct(pDb->cfg.acct);
mgmtRemoveDbFromAcct(pAcct, pDb);
acctRemoveDb(pAcct, pDb);
mgmtDropAllChildTables(pDb);
mgmtDropAllSuperTables(pDb);
mgmtDropAllVgroups(pDb);
@ -277,7 +277,7 @@ static int32_t mgmtCheckDbParams(SCMCreateDbMsg *pCreate) {
}
static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) {
int32_t code = mgmtCheckDbLimit(pAcct);
int32_t code = acctCheck(pAcct, TSDB_ACCT_DB);
if (code != 0) {
return code;
}
@ -292,7 +292,7 @@ static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) {
assert(pCreate->daysToKeep1 <= pCreate->daysToKeep2 && pCreate->daysToKeep2 <= pCreate->daysToKeep);
code = mgmtCheckDbGrant();
code = grantCheck(TSDB_GRANT_DB);
if (code != 0) {
return code;
}
@ -692,7 +692,7 @@ static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg) {
pCreate->rowsInFileBlock = htonl(pCreate->rowsInFileBlock);
int32_t code;
if (mgmtCheckExpired()) {
if (grantCheck(TSDB_GRANT_TIME) != TSDB_CODE_SUCCESS) {
code = TSDB_CODE_GRANT_EXPIRED;
} else if (!pMsg->pUser->writeAuth) {
code = TSDB_CODE_NO_RIGHTS;
@ -771,7 +771,7 @@ static void mgmtProcessAlterDbMsg(SQueuedMsg *pMsg) {
SCMAlterDbMsg *pAlter = pMsg->pCont;
mTrace("db:%s, alter db msg is received from thandle:%p", pAlter->db, pMsg->thandle);
if (mgmtCheckExpired()) {
if (grantCheck(TSDB_GRANT_TIME) != TSDB_CODE_SUCCESS) {
mError("db:%s, failed to alter, grant expired", pAlter->db);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_GRANT_EXPIRED);
return;
@ -842,7 +842,7 @@ static void mgmtProcessDropDbMsg(SQueuedMsg *pMsg) {
SCMDropDbMsg *pDrop = pMsg->pCont;
mTrace("db:%s, drop db msg is received from thandle:%p", pDrop->db, pMsg->thandle);
if (mgmtCheckExpired()) {
if (grantCheck(TSDB_GRANT_TIME) != TSDB_CODE_SUCCESS) {
mError("db:%s, failed to drop, grant expired", pDrop->db);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_GRANT_EXPIRED);
return;
@ -899,3 +899,20 @@ static void mgmtProcessDropDbMsg(SQueuedMsg *pMsg) {
newMsg->ahandle = pDb;
taosTmrReset(mgmtDropDb, 10, newMsg, tsMgmtTmr, &tmpTmr);
}
void mgmtDropAllDbs(SAcctObj *pAcct) {
int32_t numOfDbs = 0;
SDbObj *pDb = NULL;
while (1) {
void *pNode = sdbFetchRow(tsDbSdb, pNode, (void **)&pDb);
if (pDb == NULL) break;
if (pDb->pAcct == pAcct) {
mgmtSetDbDirty(pDb);
numOfDbs++;
}
}
mTrace("acct:%s, all dbs is is set dirty", pAcct->acctId, numOfDbs);
}

470
src/mnode/src/mgmtDnode.c
View File

@ -26,454 +26,76 @@
#include "mgmtUser.h"
#include "mgmtVgroup.h"
int32_t (*mgmtInitDnodesFp)() = NULL;
void (*mgmtCleanUpDnodesFp)() = NULL;
SDnodeObj *(*mgmtGetDnodeFp)(uint32_t ip) = NULL;
SDnodeObj *(*mgmtGetDnodeByIpFp)(int32_t dnodeId) = NULL;
int32_t (*mgmtGetDnodesNumFp)() = NULL;
int32_t (*mgmtUpdateDnodeFp)(SDnodeObj *pDnode) = NULL;
void * (*mgmtGetNextDnodeFp)(SShowObj *pShow, SDnodeObj **pDnode) = NULL;
void (*mgmtSetDnodeUnRemoveFp)(SDnodeObj *pDnode) = NULL;
static SDnodeObj tsDnodeObj = {0};
static void * mgmtGetNextDnode(SShowObj *pShow, SDnodeObj **pDnode);
static bool mgmtCheckConfigShow(SGlobalConfig *cfg);
static int32_t mgmtGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
static int32_t mgmtRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pConn);
static int32_t mgmtGetConfigMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
static int32_t mgmtRetrieveConfigs(SShowObj *pShow, char *data, int32_t rows, void *pConn);
static int32_t mgmtGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
static int32_t mgmtRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn);
static void mgmtProcessCfgDnodeMsg(SQueuedMsg *pMsg);
static void mgmtProcessCfgDnodeMsgRsp(SRpcMsg *rpcMsg) ;
static void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg);
void mgmtSetDnodeMaxVnodes(SDnodeObj *pDnode) {
int32_t maxVnodes = pDnode->numOfCores * tsNumOfVnodesPerCore;
maxVnodes = maxVnodes > TSDB_MAX_VNODES ? TSDB_MAX_VNODES : maxVnodes;
maxVnodes = maxVnodes < TSDB_MIN_VNODES ? TSDB_MIN_VNODES : maxVnodes;
if (pDnode->numOfTotalVnodes == 0) {
pDnode->numOfTotalVnodes = maxVnodes;
}
if (pDnode->alternativeRole == TSDB_DNODE_ROLE_MGMT) {
pDnode->numOfTotalVnodes = 0;
}
pDnode->openVnodes = 0;
pDnode->status = TSDB_DN_STATUS_OFFLINE;
mgmtUpdateDnode(pDnode);
}
bool mgmtCheckModuleInDnode(SDnodeObj *pDnode, int32_t moduleType) {
uint32_t status = pDnode->moduleStatus & (1 << moduleType);
return status > 0;
}
int32_t mgmtGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
int32_t cols = 0;
SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL);
if (pUser == NULL) return 0;
if (strcmp(pUser->user, "root") != 0) return TSDB_CODE_NO_RIGHTS;
SSchema *pSchema = pMeta->schema;
pShow->bytes[cols] = 16;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "IP");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 10;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "module type");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 10;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "module status");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pMeta->numOfColumns = htons(cols);
pShow->numOfColumns = cols;
pShow->offset[0] = 0;
for (int32_t i = 1; i < cols; ++i) {
pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1];
}
pShow->numOfRows = 0;
SDnodeObj *pDnode = NULL;
while (1) {
pShow->pNode = mgmtGetNextDnode(pShow, (SDnodeObj **)&pDnode);
if (pDnode == NULL) break;
for (int32_t moduleType = 0; moduleType < TSDB_MOD_MAX; ++moduleType) {
if (mgmtCheckModuleInDnode(pDnode, moduleType)) {
pShow->numOfRows++;
}
}
}
pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
pShow->pNode = NULL;
return 0;
}
int32_t mgmtRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pConn) {
int32_t numOfRows = 0;
SDnodeObj *pDnode = NULL;
char * pWrite;
int32_t cols = 0;
char ipstr[20];
while (numOfRows < rows) {
pShow->pNode = mgmtGetNextDnode(pShow, (SDnodeObj **)&pDnode);
if (pDnode == NULL) break;
for (int32_t moduleType = 0; moduleType < TSDB_MOD_MAX; ++moduleType) {
if (!mgmtCheckModuleInDnode(pDnode, moduleType)) {
continue;
}
cols = 0;
tinet_ntoa(ipstr, pDnode->privateIp);
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
strcpy(pWrite, ipstr);
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
strcpy(pWrite, tsModule[moduleType].name);
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
strcpy(pWrite, taosGetDnodeStatusStr(pDnode->status) );
cols++;
numOfRows++;
}
}
pShow->numOfReads += numOfRows;
return numOfRows;
}
static int32_t mgmtGetConfigMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
int32_t cols = 0;
SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL);
if (pUser == NULL) return 0;
if (strcmp(pUser->user, "root") != 0) return TSDB_CODE_NO_RIGHTS;
SSchema *pSchema = pMeta->schema;
pShow->bytes[cols] = TSDB_CFG_OPTION_LEN;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "config name");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = TSDB_CFG_VALUE_LEN;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "config value");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pMeta->numOfColumns = htons(cols);
pShow->numOfColumns = cols;
pShow->offset[0] = 0;
for (int32_t i = 1; i < cols; ++i) pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1];
pShow->numOfRows = 0;
for (int32_t i = tsGlobalConfigNum - 1; i >= 0; --i) {
SGlobalConfig *cfg = tsGlobalConfig + i;
if (!mgmtCheckConfigShow(cfg)) continue;
pShow->numOfRows++;
}
pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
pShow->pNode = NULL;
return 0;
}
static int32_t mgmtRetrieveConfigs(SShowObj *pShow, char *data, int32_t rows, void *pConn) {
int32_t numOfRows = 0;
for (int32_t i = tsGlobalConfigNum - 1; i >= 0 && numOfRows < rows; --i) {
SGlobalConfig *cfg = tsGlobalConfig + i;
if (!mgmtCheckConfigShow(cfg)) continue;
char *pWrite;
int32_t cols = 0;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
snprintf(pWrite, TSDB_CFG_OPTION_LEN, "%s", cfg->option);
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
switch (cfg->valType) {
case TSDB_CFG_VTYPE_SHORT:
snprintf(pWrite, TSDB_CFG_VALUE_LEN, "%d", *((int16_t *)cfg->ptr));
numOfRows++;
break;
case TSDB_CFG_VTYPE_INT:
snprintf(pWrite, TSDB_CFG_VALUE_LEN, "%d", *((int32_t *)cfg->ptr));
numOfRows++;
break;
case TSDB_CFG_VTYPE_UINT:
snprintf(pWrite, TSDB_CFG_VALUE_LEN, "%d", *((uint32_t *)cfg->ptr));
numOfRows++;
break;
case TSDB_CFG_VTYPE_FLOAT:
snprintf(pWrite, TSDB_CFG_VALUE_LEN, "%f", *((float *)cfg->ptr));
numOfRows++;
break;
case TSDB_CFG_VTYPE_STRING:
case TSDB_CFG_VTYPE_IPSTR:
case TSDB_CFG_VTYPE_DIRECTORY:
snprintf(pWrite, TSDB_CFG_VALUE_LEN, "%s", (char *)cfg->ptr);
numOfRows++;
break;
default:
break;
}
}
pShow->numOfReads += numOfRows;
return numOfRows;
}
static int32_t mgmtGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
int32_t cols = 0;
SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL);
if (pUser == NULL) return 0;
if (strcmp(pUser->user, "root") != 0) return TSDB_CODE_NO_RIGHTS;
SSchema *pSchema = pMeta->schema;
pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_INT;
strcpy(pSchema[cols].name, "vnode");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 12;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "status");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 12;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "sync_status");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pMeta->numOfColumns = htons(cols);
pShow->numOfColumns = cols;
pShow->offset[0] = 0;
for (int32_t i = 1; i < cols; ++i) pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1];
SDnodeObj *pDnode = NULL;
if (pShow->payloadLen > 0 ) {
uint32_t ip = ip2uint(pShow->payload);
pDnode = mgmtGetDnodeByIp(ip);
if (NULL == pDnode) {
return TSDB_CODE_NODE_OFFLINE;
}
SVnodeLoad* pVnode;
pShow->numOfRows = 0;
for (int32_t i = 0 ; i < TSDB_MAX_VNODES; i++) {
pVnode = &pDnode->vload[i];
if (0 != pVnode->vgId) {
pShow->numOfRows++;
}
}
pShow->pNode = pDnode;
} else {
while (true) {
pShow->pNode = mgmtGetNextDnode(pShow, (SDnodeObj **)&pDnode);
if (pDnode == NULL) break;
pShow->numOfRows += pDnode->openVnodes;
if (0 == pShow->numOfRows) return TSDB_CODE_NODE_OFFLINE;
}
pShow->pNode = NULL;
}
pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
return 0;
}
static int32_t mgmtRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn) {
int32_t numOfRows = 0;
SDnodeObj *pDnode = NULL;
char * pWrite;
int32_t cols = 0;
if (0 == rows) return 0;
if (pShow->payloadLen) {
// output the vnodes info of the designated dnode. And output all vnodes of this dnode, instead of rows (max 100)
pDnode = (SDnodeObj *)(pShow->pNode);
if (pDnode != NULL) {
SVnodeLoad* pVnode;
for (int32_t i = 0 ; i < TSDB_MAX_VNODES; i++) {
pVnode = &pDnode->vload[i];
if (0 == pVnode->vgId) {
continue;
}
cols = 0;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(uint32_t *)pWrite = pVnode->vgId;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
strcpy(pWrite, taosGetVnodeStatusStr(pVnode->status));
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
strcpy(pWrite, taosGetVnodeSyncStatusStr(pVnode->syncStatus));
cols++;
numOfRows++;
}
}
} else {
// TODO: output all vnodes of all dnodes
numOfRows = 0;
}
pShow->numOfReads += numOfRows;
return numOfRows;
}
extern int32_t clusterInit();
extern void clusterCleanUp();
extern int32_t clusterGetDnodesNum();
extern SDnodeObj* clusterGetDnode(int32_t dnodeId);
extern SDnodeObj* clusterGetDnodeByIp(uint32_t ip);
static SDnodeObj tsDnodeObj = {0};
int32_t mgmtInitDnodes() {
mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_MODULE, mgmtGetModuleMeta);
mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_MODULE, mgmtRetrieveModules);
mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_CONFIGS, mgmtGetConfigMeta);
mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_CONFIGS, mgmtRetrieveConfigs);
mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_VNODES, mgmtGetVnodeMeta);
mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_VNODES, mgmtRetrieveVnodes);
mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_CONFIG_DNODE, mgmtProcessCfgDnodeMsg);
mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_CONFIG_DNODE_RSP, mgmtProcessCfgDnodeMsgRsp);
mgmtAddDServerMsgHandle(TSDB_MSG_TYPE_DM_STATUS, mgmtProcessDnodeStatusMsg);
if (mgmtInitDnodesFp) {
return mgmtInitDnodesFp();
} else {
tsDnodeObj.dnodeId = 1;
tsDnodeObj.privateIp = inet_addr(tsPrivateIp);
tsDnodeObj.publicIp = inet_addr(tsPublicIp);
tsDnodeObj.createdTime = taosGetTimestampMs();
tsDnodeObj.numOfTotalVnodes = tsNumOfTotalVnodes;
tsDnodeObj.numOfCores = (uint16_t) tsNumOfCores;
tsDnodeObj.alternativeRole = TSDB_DNODE_ROLE_ANY;
tsDnodeObj.status = TSDB_DN_STATUS_OFFLINE;
tsDnodeObj.lastReboot = taosGetTimestampSec();
sprintf(tsDnodeObj.dnodeName, "%d", tsDnodeObj.dnodeId);
mgmtSetDnodeMaxVnodes(&tsDnodeObj);
#ifdef _CLUSTER
return clusterInit();
#else
tsDnodeObj.dnodeId = 1;
tsDnodeObj.privateIp = inet_addr(tsPrivateIp);
tsDnodeObj.publicIp = inet_addr(tsPublicIp);
tsDnodeObj.createdTime = taosGetTimestampMs();
tsDnodeObj.numOfTotalVnodes = tsNumOfTotalVnodes;
tsDnodeObj.status = TSDB_DN_STATUS_OFFLINE;
tsDnodeObj.lastReboot = taosGetTimestampSec();
sprintf(tsDnodeObj.dnodeName, "%d", tsDnodeObj.dnodeId);
tsDnodeObj.moduleStatus |= (1 << TSDB_MOD_MGMT);
if (tsEnableHttpModule) {
tsDnodeObj.moduleStatus |= (1 << TSDB_MOD_HTTP);
}
if (tsEnableMonitorModule) {
tsDnodeObj.moduleStatus |= (1 << TSDB_MOD_MONITOR);
}
return 0;
tsDnodeObj.moduleStatus |= (1 << TSDB_MOD_MGMT);
if (tsEnableHttpModule) {
tsDnodeObj.moduleStatus |= (1 << TSDB_MOD_HTTP);
}
if (tsEnableMonitorModule) {
tsDnodeObj.moduleStatus |= (1 << TSDB_MOD_MONITOR);
}
return 0;
#endif
}
void mgmtCleanUpDnodes() {
if (mgmtCleanUpDnodesFp) {
(*mgmtCleanUpDnodesFp)();
}
#ifdef _CLUSTER
clusterCleanUp();
#endif
}
SDnodeObj *mgmtGetDnode(int32_t dnodeId) {
if (mgmtGetDnodeFp) {
return (*mgmtGetDnodeFp)(dnodeId);
}
#ifdef _CLUSTER
return clusterGetDnode(dnodeId);
#else
if (dnodeId == 1) {
return &tsDnodeObj;
} else {
return NULL;
}
return NULL;
#endif
}
SDnodeObj *mgmtGetDnodeByIp(uint32_t ip) {
if (mgmtGetDnodeByIpFp) {
return (*mgmtGetDnodeByIpFp)(ip);
}
#ifdef _CLUSTER
return clusterGetDnodeByIp(ip);
#else
return &tsDnodeObj;
#endif
}
int32_t mgmtGetDnodesNum() {
if (mgmtGetDnodesNumFp) {
return (*mgmtGetDnodesNumFp)();
} else {
return 1;
}
}
int32_t mgmtUpdateDnode(SDnodeObj *pDnode) {
if (mgmtUpdateDnodeFp) {
return (*mgmtUpdateDnodeFp)(pDnode);
} else {
return 0;
}
}
void *mgmtGetNextDnode(SShowObj *pShow, SDnodeObj **pDnode) {
if (mgmtGetNextDnodeFp) {
return (*mgmtGetNextDnodeFp)(pShow, pDnode);
} else {
if (*pDnode == NULL) {
*pDnode = &tsDnodeObj;
} else {
*pDnode = NULL;
}
}
return *pDnode;
}
void mgmtSetDnodeUnRemove(SDnodeObj *pDnode) {
if (mgmtSetDnodeUnRemoveFp) {
(*mgmtSetDnodeUnRemoveFp)(pDnode);
}
}
bool mgmtCheckConfigShow(SGlobalConfig *cfg) {
if (!(cfg->cfgType & TSDB_CFG_CTYPE_B_SHOW))
return false;
return true;
}
bool mgmtCheckDnodeInRemoveState(SDnodeObj *pDnode) {
return pDnode->lbStatus == TSDB_DN_LB_STATUS_OFFLINE_REMOVING || pDnode->lbStatus == TSDB_DN_LB_STATE_SHELL_REMOVING;
}
bool mgmtCheckDnodeInOfflineState(SDnodeObj *pDnode) {
return pDnode->status == TSDB_DN_STATUS_OFFLINE;
#ifdef _CLUSTER
return clusterGetDnodesNum();
#else
return 1;
#endif
}
void mgmtProcessCfgDnodeMsg(SQueuedMsg *pMsg) {
@ -553,14 +175,10 @@ void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) {
pDnode->numOfCores = htons(pStatus->numOfCores);
pDnode->diskAvailable = pStatus->diskAvailable;
pDnode->alternativeRole = pStatus->alternativeRole;
if (pDnode->numOfTotalVnodes == 0) {
pDnode->numOfTotalVnodes = htons(pStatus->numOfTotalVnodes);
}
pDnode->numOfTotalVnodes = htons(pStatus->numOfTotalVnodes);
if (pStatus->dnodeId == 0) {
mTrace("dnode:%d, first access, privateIp:%s, name:%s, ", pDnode->dnodeId, taosIpStr(pDnode->privateIp), pDnode->dnodeName);
mgmtSetDnodeMaxVnodes(pDnode);
}
int32_t openVnodes = htons(pStatus->openVnodes);

64
src/mnode/src/mgmtGrant.c
View File

@ -14,58 +14,18 @@
*/
#define _DEFAULT_SOURCE
#ifndef _GRANT
#include "os.h"
#include "mgmtAcct.h"
#include "taoserror.h"
#include "tlog.h"
#include "mgmtGrant.h"
int32_t (*mgmtCheckUserGrantFp)() = NULL;
int32_t (*mgmtCheckDbGrantFp)() = NULL;
void (*mgmtAddTimeSeriesFp)(uint32_t timeSeriesNum) = NULL;
void (*mgmtRestoreTimeSeriesFp)(uint32_t timeSeriesNum) = NULL;
int32_t (*mgmtCheckTimeSeriesFp)(uint32_t timeseries) = NULL;
bool (*mgmtCheckExpiredFp)() = NULL;
int32_t grantInit() { return TSDB_CODE_SUCCESS; }
void grantCleanUp() {}
void grantParseParameter() { mError("can't parsed parameter k"); }
int32_t grantCheck(EGrantType grant) { return TSDB_CODE_SUCCESS; }
void grantReset(EGrantType grant, uint64_t value) {}
void grantAdd(EGrantType grant, uint64_t value) {}
void grantRestore(EGrantType grant, uint64_t value) {}
int32_t mgmtCheckUserGrant() {
if (mgmtCheckUserGrantFp) {
return (*mgmtCheckUserGrantFp)();
} else {
return 0;
}
}
int32_t mgmtCheckDbGrant() {
if (mgmtCheckDbGrantFp) {
return (*mgmtCheckDbGrantFp)();
} else {
return 0;
}
}
void mgmtAddTimeSeries(SAcctObj *pAcct, uint32_t timeSeriesNum) {
pAcct->acctInfo.numOfTimeSeries += timeSeriesNum;
if (mgmtAddTimeSeriesFp) {
(*mgmtAddTimeSeriesFp)(timeSeriesNum);
}
}
void mgmtRestoreTimeSeries(SAcctObj *pAcct, uint32_t timeSeriesNum) {
pAcct->acctInfo.numOfTimeSeries -= timeSeriesNum;
if (mgmtRestoreTimeSeriesFp) {
(*mgmtRestoreTimeSeriesFp)(timeSeriesNum);
}
}
int32_t mgmtCheckTimeSeries(uint32_t timeseries) {
if (mgmtCheckTimeSeriesFp) {
return (*mgmtCheckTimeSeriesFp)(timeseries);
} else {
return 0;
}
}
bool mgmtCheckExpired() {
if (mgmtCheckExpiredFp) {
return mgmtCheckExpiredFp();
} else {
return false;
}
}
#endif

11
src/mnode/src/mgmtMain.c
View File

@ -25,6 +25,7 @@
#include "mgmtDClient.h"
#include "mgmtDnode.h"
#include "mgmtDServer.h"
#include "mgmtGrant.h"
#include "mgmtMnode.h"
#include "mgmtSdb.h"
#include "mgmtVgroup.h"
@ -73,11 +74,16 @@ int32_t mgmtStartSystem() {
return -1;
}
if (mgmtInitAccts() < 0) {
if (acctInit() < 0) {
mError("failed to init accts");
return -1;
}
if (grantInit() < 0) {
mError("failed to init grants");
return -1;
}
if (mgmtInitUsers() < 0) {
mError("failed to init users");
return -1;
@ -138,6 +144,7 @@ void mgmtStopSystem() {
void mgmtCleanUpSystem() {
mPrint("starting to clean up mgmt");
grantCleanUp();
mgmtCleanupMnodes();
mgmtCleanupBalance();
mgmtCleanUpShell();
@ -148,7 +155,7 @@ void mgmtCleanUpSystem() {
mgmtCleanUpDbs();
mgmtCleanUpDnodes();
mgmtCleanUpUsers();
mgmtCleanUpAccts();
acctCleanUp();
taosTmrCleanUp(tsMgmtTmr);
mPrint("mgmt is cleaned up");
}

6
src/mnode/src/mgmtShell.c
View File

@ -137,7 +137,7 @@ static void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg) {
return;
}
if (mgmtCheckExpired()) {
if (grantCheck(TSDB_GRANT_TIME) != TSDB_CODE_SUCCESS) {
mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_GRANT_EXPIRED);
return;
}
@ -373,12 +373,12 @@ static void mgmtProcessConnectMsg(SQueuedMsg *pMsg) {
goto connect_over;
}
if (mgmtCheckExpired()) {
if (grantCheck(TSDB_GRANT_TIME) != TSDB_CODE_SUCCESS) {
code = TSDB_CODE_GRANT_EXPIRED;
goto connect_over;
}
SAcctObj *pAcct = mgmtGetAcct(pUser->acct);
SAcctObj *pAcct = acctGetAcct(pUser->acct);
if (pAcct == NULL) {
code = TSDB_CODE_INVALID_ACCT;
goto connect_over;

8
src/mnode/src/mgmtSuperTable.c
View File

@ -253,7 +253,7 @@ static int32_t mgmtAddSuperTableTag(SSuperTableObj *pStable, SSchema schema[], i
return TSDB_CODE_APP_ERROR;
}
SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct);
SAcctObj *pAcct = acctGetAcct(pDb->cfg.acct);
if (pAcct == NULL) {
mError("DB: %s not belongs to andy account", pDb->name);
return TSDB_CODE_APP_ERROR;
@ -293,7 +293,7 @@ static int32_t mgmtDropSuperTableTag(SSuperTableObj *pStable, char *tagName) {
return TSDB_CODE_APP_ERROR;
}
SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct);
SAcctObj *pAcct = acctGetAcct(pDb->cfg.acct);
if (pAcct == NULL) {
mError("DB: %s not belongs to any account", pDb->name);
return TSDB_CODE_APP_ERROR;
@ -381,7 +381,7 @@ static int32_t mgmtAddSuperTableColumn(SSuperTableObj *pStable, SSchema schema[]
return TSDB_CODE_APP_ERROR;
}
SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct);
SAcctObj *pAcct = acctGetAcct(pDb->cfg.acct);
if (pAcct == NULL) {
mError("DB: %s not belongs to andy account", pDb->name);
return TSDB_CODE_APP_ERROR;
@ -420,7 +420,7 @@ static int32_t mgmtDropSuperTableColumnByName(SSuperTableObj *pStable, char *col
return TSDB_CODE_APP_ERROR;
}
SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct);
SAcctObj *pAcct = acctGetAcct(pDb->cfg.acct);
if (pAcct == NULL) {
mError("DB: %s not belongs to any account", pDb->name);
return TSDB_CODE_APP_ERROR;

48
src/mnode/src/mgmtUser.c
View File

@ -25,10 +25,9 @@
#include "mgmtShell.h"
#include "mgmtUser.h"
static void *tsUserSdb = NULL;
void * tsUserSdb = NULL;
static int32_t tsUserUpdateSize = 0;
static int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass);
static int32_t mgmtDropUser(SAcctObj *pAcct, char *name);
static int32_t mgmtUpdateUser(SUserObj *pUser);
static int32_t mgmtGetUserMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
@ -45,10 +44,10 @@ static int32_t mgmtUserActionDestroy(SSdbOperDesc *pOper) {
static int32_t mgmtUserActionInsert(SSdbOperDesc *pOper) {
SUserObj *pUser = pOper->pObj;
SAcctObj *pAcct = mgmtGetAcct(pUser->acct);
SAcctObj *pAcct = acctGetAcct(pUser->acct);
if (pAcct != NULL) {
mgmtAddUserIntoAcct(pAcct, pUser);
acctAddUser(pAcct, pUser);
}
else {
mError("user:%s, acct:%s info not exist in sdb", pUser->user, pUser->acct);
@ -60,9 +59,9 @@ static int32_t mgmtUserActionInsert(SSdbOperDesc *pOper) {
static int32_t mgmtUserActionDelete(SSdbOperDesc *pOper) {
SUserObj *pUser = pOper->pObj;
SAcctObj *pAcct = mgmtGetAcct(pUser->acct);
SAcctObj *pAcct = acctGetAcct(pUser->acct);
mgmtRemoveUserFromAcct(pAcct, pUser);
acctRemoveUser(pAcct, pUser);
return TSDB_CODE_SUCCESS;
}
@ -115,7 +114,7 @@ int32_t mgmtInitUsers() {
return -1;
}
SAcctObj *pAcct = mgmtGetAcct("root");
SAcctObj *pAcct = acctGetAcct("root");
mgmtCreateUser(pAcct, "root", "taosdata");
mgmtCreateUser(pAcct, "monitor", tsInternalPass);
mgmtCreateUser(pAcct, "_root", tsInternalPass);
@ -155,8 +154,8 @@ static int32_t mgmtUpdateUser(SUserObj *pUser) {
return code;
}
static int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass) {
int32_t code = mgmtCheckUserLimit(pAcct);
int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass) {
int32_t code = acctCheck(pAcct, TSDB_ACCT_USER);
if (code != 0) {
return code;
}
@ -171,8 +170,8 @@ static int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass) {
return TSDB_CODE_USER_ALREADY_EXIST;
}
code = mgmtCheckUserGrant();
if (code != 0) {
code = grantCheck(TSDB_GRANT_USER);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
@ -482,3 +481,30 @@ static void mgmtProcessDropUserMsg(SQueuedMsg *pMsg) {
mgmtSendSimpleResp(pMsg->thandle, code);
}
void mgmtDropAllUsers(SAcctObj *pAcct) {
void *pNode = NULL;
void *pLastNode = NULL;
int32_t numOfUsers = 0;
int32_t acctNameLen = strlen(pAcct->user);
SUserObj *pUser = NULL;
while (1) {
pNode = sdbFetchRow(tsUserSdb, pNode, (void **)&pUser);
if (pUser == NULL) break;
if (strncmp(pUser->acct, pAcct->user, acctNameLen) == 0) {
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_LOCAL,
.table = tsUserSdb,
.pObj = pUser,
};
sdbDeleteRow(&oper);
pNode = pLastNode;
numOfUsers++;
continue;
}
}
mTrace("acct:%s, all users is dropped from sdb", pAcct->acctId, numOfUsers);
}

2
src/mnode/src/mgmtVgroup.c
View File

@ -32,7 +32,7 @@
#include "mgmtTable.h"
#include "mgmtVgroup.h"
static void *tsVgroupSdb = NULL;
void *tsVgroupSdb = NULL;
static int32_t tsVgUpdateSize = 0;
static int32_t mgmtGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);

27
src/util/CMakeLists.txt
View File

@ -9,25 +9,20 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
AUX_SOURCE_DIRECTORY(src SRC)
ADD_LIBRARY(tutil ${SRC})
TARGET_LINK_LIBRARIES(tutil thirdparty pthread os m rt)
IF (TD_CLUSTER)
FIND_PATH(ICONV_INCLUDE_EXIST iconv.h /usr/include/ /usr/local/include/)
IF (ICONV_INCLUDE_EXIST)
ADD_DEFINITIONS(-DUSE_LIBICONV)
TARGET_LINK_LIBRARIES(tutil iconv)
ELSE()
FIND_PATH(ICONV_INCLUDE_EXIST iconv.h /usr/include/ /usr/local/include/)
IF (ICONV_INCLUDE_EXIST)
ADD_DEFINITIONS(-DUSE_LIBICONV)
FIND_PATH(ICONV_LIBRARY_A_EXIST libiconv.a /usr/lib/ /usr/local/lib/ /lib64)
FIND_PATH(ICONV_LIBRARY_SO_EXIST libiconv.so /usr/lib/ /usr/local/lib/ /lib64)
IF (ICONV_LIBRARY_A_EXIST OR ICONV_LIBRARY_SO_EXIST)
MESSAGE(STATUS "Use the installed libiconv library")
TARGET_LINK_LIBRARIES(tutil iconv)
ELSE ()
# libiconv library is already included in GLIBC,
MESSAGE(STATUS "Use the iconv functions in GLIBC")
ENDIF ()
FIND_PATH(ICONV_LIBRARY_A_EXIST libiconv.a /usr/lib/ /usr/local/lib/ /lib64)
FIND_PATH(ICONV_LIBRARY_SO_EXIST libiconv.so /usr/lib/ /usr/local/lib/ /lib64)
IF (ICONV_LIBRARY_A_EXIST OR ICONV_LIBRARY_SO_EXIST)
MESSAGE(STATUS "Use the installed libiconv library")
TARGET_LINK_LIBRARIES(tutil iconv)
ELSE ()
MESSAGE(STATUS "Failed to find iconv, use default encoding method")
# libiconv library is already included in GLIBC,
MESSAGE(STATUS "Use the iconv functions in GLIBC")
ENDIF ()
ELSE ()
MESSAGE(STATUS "Failed to find iconv, use default encoding method")
ENDIF ()
ADD_SUBDIRECTORY(tests)

16
src/util/src/tglobalcfg.c
View File

@ -81,7 +81,7 @@ float tsRatioOfQueryThreads = 0.5;
char tsPublicIp[TSDB_IPv4ADDR_LEN] = {0};
char tsPrivateIp[TSDB_IPv4ADDR_LEN] = {0};
short tsNumOfVnodesPerCore = 8;
short tsNumOfTotalVnodes = 0;
short tsNumOfTotalVnodes = TSDB_INVALID_VNODE_NUM;
short tsCheckHeaderFile = 0;
#ifdef _TD_ARM_32_
@ -189,7 +189,7 @@ int tsEnableCoreFile = 0;
int tsAnyIp = 1;
uint32_t tsPublicIpInt = 0;
#ifdef CLUSTER
#ifdef _CLUSTER
int tsIsCluster = 1;
#else
int tsIsCluster = 0;
@ -946,7 +946,7 @@ bool tsReadGlobalConfig() {
if (tsSecondIp[0] == 0) {
strcpy(tsSecondIp, tsMasterIp);
}
taosGetSystemInfo();
tsSetLocale();
@ -960,6 +960,12 @@ bool tsReadGlobalConfig() {
tsNumOfCores = 1;
}
if (tsNumOfTotalVnodes == TSDB_INVALID_VNODE_NUM) {
tsNumOfTotalVnodes = tsNumOfCores * tsNumOfVnodesPerCore;
tsNumOfTotalVnodes = tsNumOfTotalVnodes > TSDB_MAX_VNODES ? TSDB_MAX_VNODES : tsNumOfTotalVnodes;
tsNumOfTotalVnodes = tsNumOfTotalVnodes < TSDB_MIN_VNODES ? TSDB_MIN_VNODES : tsNumOfTotalVnodes;
}
if (strlen(tsPrivateIp) == 0) {
pError("privateIp is null");
return false;
@ -1052,12 +1058,12 @@ void tsPrintGlobalConfig() {
if (tscEmbedded == 0 && !(cfg->cfgType & TSDB_CFG_CTYPE_B_CLIENT)) continue;
if (cfg->cfgType & TSDB_CFG_CTYPE_B_NOT_PRINT) continue;
if (cfg->cfgType & TSDB_CFG_CTYPE_B_LITE) {
#ifdef CLUSTER
#ifdef _CLUSTER
continue;
#endif
}
if (cfg->cfgType & TSDB_CFG_CTYPE_B_CLUSTER) {
#ifndef CLUSTER
#ifndef _CLUSTER
continue;
#endif
}

6
src/util/src/version.c
View File

@ -1,7 +1,7 @@
char version[64] = "1.6.5.4";
char compatible_version[64] = "1.6.1.0";
char version[64] = "2.0.0.0";
char compatible_version[64] = "2.0.0.0";
char gitinfo[128] = "3264067e97300c84caa61ac909d548c9ca56de6b";
char gitinfoOfInternal[128] = "da88f4a2474737d1f9c76adcf0ff7fd0975e7342";
char buildinfo[512] = "Built by root at 2020-02-05 14:38";
char buildinfo[512] = "Built by root at 2020-04-01 14:38";
void libtaos_1_6_5_4_Linux_x64() {};

25
src/vnode/detail/CMakeLists.txt
View File

@ -1,25 +0,0 @@
CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
PROJECT(TDengine)
IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/dnode/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/mnode/detail/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/vnode/detail/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc)
INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
INCLUDE_DIRECTORIES(inc)
AUX_SOURCE_DIRECTORY(./src SRC)
LIST(REMOVE_ITEM SRC ./src/vnodeFileUtil.c)
LIST(REMOVE_ITEM SRC ./src/taosGrant.c)
ADD_LIBRARY(vnode ${SRC})
IF (TD_CLUSTER)
TARGET_LINK_LIBRARIES(vnode vcluster)
ELSEIF (TD_LITE)
TARGET_LINK_LIBRARIES(vnode vlite)
ENDIF ()
ENDIF ()

574
src/vnode/detail/inc/vnode.h
View File

@ -1,574 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_VNODE_H
#define TDENGINE_VNODE_H
#ifdef __cplusplus
extern "C" {
#endif
#include "os.h"
#include "tglobalcfg.h"
#include "tidpool.h"
#include "tlog.h"
#include "tmempool.h"
#include "trpc.h"
#include "tsclient.h"
#include "taosdef.h"
#include "tsocket.h"
#include "ttime.h"
#include "ttimer.h"
#include "tutil.h"
#include "vnodeCache.h"
#include "vnodeFile.h"
#include "vnodePeer.h"
#include "vnodeShell.h"
#define TSDB_FILE_HEADER_LEN 512
#define TSDB_FILE_HEADER_VERSION_SIZE 32
#define TSDB_CACHE_POS_BITS 13
#define TSDB_CACHE_POS_MASK 0x1FFF
#define TSDB_ACTION_INSERT 0
#define TSDB_ACTION_IMPORT 1
#define TSDB_ACTION_DELETE 2
#define TSDB_ACTION_UPDATE 3
#define TSDB_ACTION_MAX 4
enum _data_source {
TSDB_DATA_SOURCE_METER,
TSDB_DATA_SOURCE_VNODE,
TSDB_DATA_SOURCE_SHELL,
TSDB_DATA_SOURCE_QUEUE,
TSDB_DATA_SOURCE_LOG,
};
enum _sync_cmd {
TSDB_SYNC_CMD_FILE,
TSDB_SYNC_CMD_CACHE,
TSDB_SYNC_CMD_CREATE,
TSDB_SYNC_CMD_REMOVE,
};
typedef struct {
int64_t offset : 48;
int64_t length : 16;
} SMeterObjHeader;
typedef struct {
int64_t len;
char data[];
} SData;
#pragma pack(push, 8)
typedef struct {
SVnodeStatisticInfo vnodeStatistic;
int vnode;
SVnodeCfg cfg;
// SDiskDesc tierDisk[TSDB_MAX_TIER];
SVPeerDesc vpeers[TSDB_VNODES_SUPPORT];
SVnodePeer * peerInfo[TSDB_VNODES_SUPPORT];
char selfIndex;
char vnodeStatus;
char accessState; // Vnode access state, Readable/Writable
char syncStatus;
char commitInProcess;
pthread_t commitThread;
TSKEY firstKey; // minimum key uncommitted, it may be smaller than
// commitFirstKey
TSKEY commitFirstKey; // minimum key for a commit file, it shall be
// xxxx00000, calculated from fileId
TSKEY commitLastKey; // maximum key for a commit file, it shall be xxxx99999,
// calculated fromm fileId
int commitFileId;
TSKEY lastCreate;
TSKEY lastRemove;
TSKEY lastKey; // last key for the whole vnode, updated by every insert
// operation
uint64_t version;
int streamRole;
int numOfStreams;
void *streamTimer;
TSKEY lastKeyOnFile; // maximum key on the last file, is shall be xxxx99999
int fileId;
int badFileId;
int numOfFiles;
int maxFiles;
int maxFile1;
int maxFile2;
int nfd; // temp head file FD
int hfd; // head file FD
int lfd; // last file FD
int tfd; // temp last file FD
int dfd; // data file FD
int64_t dfSize;
int64_t lfSize;
uint64_t * fmagic; // hold magic number for each file
char cfn[TSDB_FILENAME_LEN];
char nfn[TSDB_FILENAME_LEN];
char lfn[TSDB_FILENAME_LEN]; // last file name
char tfn[TSDB_FILENAME_LEN]; // temp last file name
pthread_mutex_t vmutex;
int logFd;
char * pMem;
char * pWrite;
pthread_mutex_t logMutex;
char logFn[TSDB_FILENAME_LEN];
char logOFn[TSDB_FILENAME_LEN];
int64_t mappingSize;
int64_t mappingThreshold;
void * commitTimer;
void ** meterList;
void * pCachePool;
void * pQueue;
pthread_t thread;
int peersOnline;
int shellConns;
int meterConns;
struct _qinfo *pQInfoList;
TAOS * dbConn;
SMeterObjHeader *meterIndex;
} SVnodeObj;
#pragma pack(pop)
typedef struct SColumn {
short colId;
short bytes;
char type;
} SColumn;
typedef struct _meter_obj {
uint64_t uid;
char meterId[TSDB_TABLE_ID_LEN];
int sid;
short vnode;
short numOfColumns;
short bytesPerPoint;
short maxBytes;
int32_t pointsPerBlock;
int32_t pointsPerFileBlock;
int freePoints;
TSKEY lastKey; // updated by insert operation
TSKEY lastKeyOnFile; // last key on file, updated by commit action
TSKEY timeStamp; // delete or added time
uint64_t commitCount;
int32_t sversion;
short sqlLen;
char searchAlgorithm : 4;
char compAlgorithm : 4;
char status; // 0: ok, 1: stop stream computing
char reserved[16];
int state;
int numOfQueries;
char * pSql;
void * pStream;
void * pCache;
SColumn *schema;
} SMeterObj;
typedef struct {
char type;
char pversion; // protocol version
char action; // insert, import, delete, update
int32_t sversion; // only for insert
int32_t sid;
int32_t len;
uint64_t lastVersion; // latest version
char cont[];
} SVMsgHeader;
struct tSQLBinaryExpr;
typedef struct SColumnInfoEx {
SColumnInfo data;
int16_t colIdx;
int16_t colIdxInBuf;
/*
* 0: denotes if its is required in the first round of scan of data block
* 1: denotes if its is required in the secondary scan
*/
int16_t req[2];
} SColumnInfoEx;
struct SColumnFilterElem;
typedef bool (*__filter_func_t)(struct SColumnFilterElem *pFilter, char *val1, char *val2);
typedef struct SColumnFilterElem {
int16_t bytes; // column length
__filter_func_t fp;
SColumnFilterInfo filterInfo;
} SColumnFilterElem;
typedef struct SSingleColumnFilterInfo {
SColumnInfoEx info;
int32_t numOfFilters;
SColumnFilterElem *pFilters;
char * pData;
} SSingleColumnFilterInfo;
typedef struct SQuery {
short numOfCols;
SOrderVal order;
char keyIsMet; // if key is met, it will be set
char over;
int fileId; // only for query in file
int hfd; // only for query in file, head file handle
int dfd; // only for query in file, data file handle
int lfd; // only for query in file, last file handle
SCompBlock *pBlock; // only for query in file
SField ** pFields;
int numOfBlocks; // only for query in file
int blockBufferSize; // length of pBlock buffer
int currentSlot;
int firstSlot;
/*
* the two parameters are utilized to handle the data missing situation, caused by import operation.
* When the commit slot is the first slot, and commitPoints != 0
*/
int32_t commitSlot; // which slot is committed,
int32_t commitPoint; // starting point for next commit
int slot;
int pos;
TSKEY key;
int compBlockLen; // only for import
int64_t blockId;
TSKEY skey;
TSKEY ekey;
int64_t intervalTime;
int64_t slidingTime; // sliding time for sliding window query
char intervalTimeUnit; // interval data type, used for daytime revise
int8_t precision;
int16_t numOfOutputCols;
int16_t interpoType;
int16_t checkBufferInLoop; // check if the buffer is full during scan each block
SLimitVal limit;
int32_t rowSize;
SSqlGroupbyExpr * pGroupbyExpr;
SSqlFunctionExpr * pSelectExpr;
SColumnInfoEx * colList;
int32_t numOfFilterCols;
SSingleColumnFilterInfo *pFilterInfo;
int64_t * defaultVal;
TSKEY lastKey;
// buffer info
int64_t pointsRead; // the number of points returned
int64_t pointsToRead; // maximum number of points to read
int64_t pointsOffset; // the number of points offset to save read data
SData **sdata;
SData * tsData; // timestamp column/primary key column
} SQuery;
typedef struct {
char spi;
char encrypt;
char secret[TSDB_KEY_LEN];
char cipheringKey[TSDB_KEY_LEN];
} SConnSec;
typedef struct {
char * buffer;
char * offset;
int trans;
int bufferSize;
pthread_mutex_t qmutex;
} STranQueue;
// internal globals
extern int tsMeterSizeOnFile;
extern void * tsQueryQhandle;
extern int tsVnodePeers;
extern int tsMaxVnode;
extern int tsMaxQueues;
extern int tsOpenVnodes;
extern SVnodeObj *vnodeList;
extern void * vnodeTmrCtrl;
// read API
extern int (*vnodeSearchKeyFunc[])(char *pValue, int num, TSKEY key, int order);
void *vnodeQueryOnSingleTable(SMeterObj **pMeterObj, SSqlGroupbyExpr *pGroupbyExpr, SSqlFunctionExpr *sqlExprs,
SQueryMeterMsg *pQueryMsg, int *code);
void *vnodeQueryOnMultiMeters(SMeterObj **pMeterObj, SSqlGroupbyExpr *pGroupbyExpr, SSqlFunctionExpr *pSqlExprs,
SQueryMeterMsg *pQueryMsg, int *code);
// assistant/tool functions
SSqlGroupbyExpr *vnodeCreateGroupbyExpr(SQueryMeterMsg *pQuery, int32_t *code);
SSqlFunctionExpr *vnodeCreateSqlFunctionExpr(SQueryMeterMsg *pQuery, int32_t *code);
bool vnodeValidateExprColumnInfo(SQueryMeterMsg *pQueryMsg, SSqlFuncExprMsg *pExprMsg);
bool vnodeIsValidVnodeCfg(SVnodeCfg *pCfg);
int32_t vnodeGetResultSize(void *handle, int32_t *numOfRows);
int32_t vnodeCopyQueryResultToMsg(void *handle, char *data, int32_t numOfRows);
int64_t vnodeGetOffsetVal(void *thandle);
bool vnodeHasRemainResults(void *handle);
int vnodeRetrieveQueryResult(void *handle, int *pNum, char *argv[]);
int vnodeSaveQueryResult(void *handle, char *data, int32_t* size);
int vnodeRetrieveQueryInfo(void *handle, int *numOfRows, int *rowSize, int16_t *timePrec);
void vnodeFreeQInfo(void *, bool);
void vnodeFreeQInfoInQueue(void *param);
bool vnodeIsQInfoValid(void *param);
void vnodeDecRefCount(void *param);
void vnodeAddRefCount(void *param);
int32_t vnodeConvertQueryMeterMsg(SQueryMeterMsg *pQuery);
void vnodeQueryData(SSchedMsg *pMsg);
// meter API
int vnodeOpenMetersVnode(int vnode);
void vnodeCloseMetersVnode(int vnode);
int vnodeCreateMeterObj(SMeterObj *pNew, SConnSec *pSec);
int vnodeRemoveMeterObj(int vnode, int sid);
int vnodeInsertPoints(SMeterObj *pObj, char *cont, int contLen, char source, void *, int sversion, int *numOfPoints, TSKEY now);
int vnodeImportPoints(SMeterObj *pObj, char *cont, int contLen, char source, void *, int sversion, int *numOfPoints, TSKEY now);
int vnodeInsertBufferedPoints(int vnode);
int vnodeSaveAllMeterObjToFile(int vnode);
int vnodeSaveMeterObjToFile(SMeterObj *pObj);
int vnodeSaveVnodeCfg(int vnode, SVnodeCfg *pCfg, SVPeerDesc *pDesc);
int vnodeSaveVnodeInfo(int vnode);
// cache API
void *vnodeOpenCachePool(int vnode);
void vnodeCloseCachePool(int vnode);
void *vnodeAllocateCacheInfo(SMeterObj *pObj);
void vnodeFreeCacheInfo(SMeterObj *pObj);
void vnodeSetCommitQuery(SMeterObj *pObj, SQuery *pQuery);
int vnodeInsertPointToCache(SMeterObj *pObj, char *pData);
int vnodeQueryFromCache(SMeterObj *pObj, SQuery *pQuery);
uint64_t vnodeGetPoolCount(SVnodeObj *pVnode);
void vnodeUpdateCommitInfo(SMeterObj *pObj, int slot, int pos, uint64_t count);
void vnodeCommitOver(SVnodeObj *pVnode);
TSKEY vnodeGetFirstKey(int vnode);
int vnodeSyncRetrieveCache(int vnode, int fd);
int vnodeSyncRestoreCache(int vnode, int fd);
pthread_t vnodeCreateCommitThread(SVnodeObj *pVnode);
void vnodeCancelCommit(SVnodeObj *pVnode);
void vnodeCloseStream(SVnodeObj *pVnode);
void vnodeProcessCommitTimer(void *param, void *tmrId);
void vnodeSearchPointInCache(SMeterObj *pObj, SQuery *pQuery);
int vnodeAllocateCacheBlock(SMeterObj *pObj);
int vnodeFreeCacheBlock(SCacheBlock *pCacheBlock);
int vnodeIsCacheCommitted(SMeterObj *pObj);
// file API
int vnodeInitFile(int vnode);
int vnodeQueryFromFile(SMeterObj *pObj, SQuery *pQuery);
void *vnodeCommitToFile(void *param);
void *vnodeCommitMultiToFile(SVnodeObj *pVnode, int ssid, int esid);
int vnodeSyncRetrieveFile(int vnode, int fd, uint32_t fileId, uint64_t *fmagic);
int vnodeSyncRestoreFile(int vnode, int sfd);
int vnodeWriteBlockToFile(SMeterObj *pObj, SCompBlock *pBlock, SData *data[], SData *cdata[], int pointsRead);
int vnodeSearchPointInFile(SMeterObj *pObj, SQuery *pQuery);
int vnodeReadCompBlockToMem(SMeterObj *pObj, SQuery *pQuery, SData *sdata[]);
int vnodeOpenCommitFiles(SVnodeObj *pVnode, int noTempLast);
void vnodeCloseCommitFiles(SVnodeObj *pVnode);
int vnodeReadLastBlockToMem(SMeterObj *pObj, SCompBlock *pBlock, SData *sdata[]);
// vnode API
void vnodeUpdateStreamRole(SVnodeObj *pVnode);
int vnodeInitPeer(int numOfThreads);
void vnodeCleanUpPeer();
int vnodeOpenPeerVnode(int vnode);
void vnodeClosePeerVnode(int vnode);
void *vnodeGetMeterPeerConnection(SMeterObj *pObj, int index);
int vnodeForwardToPeer(SMeterObj *pObj, char *msg, int msgLen, char action, int sversion);
void vnodeCloseAllSyncFds(int vnode);
void vnodeConfigVPeers(int vnode, int numOfPeers, SVPeerDesc peerDesc[]);
void vnodeStartSyncProcess(SVnodeObj *pVnode);
void vnodeCancelSync(int vnode);
void vnodeListPeerStatus(char *buffer);
void vnodeCheckOwnStatus(SVnodeObj *pVnode);
int vnodeSaveMeterObjToFile(SMeterObj *pObj);
int vnodeRecoverFromPeer(SVnodeObj *pVnode, int fileId);
// vnodes API
int vnodeInitVnodes();
int vnodeInitStore();
void vnodeCleanUpVnodes();
int vnodeRemoveVnode(int vnode);
int vnodeCreateVnode(int vnode, SVnodeCfg *pCfg, SVPeerDesc *pDesc);
void vnodeOpenStreams(void *param, void *tmrId);
void vnodeCreateStream(SMeterObj *pObj);
void vnodeRemoveStream(SMeterObj *pObj);
// shell API
int vnodeInitShell();
void vnodeCleanUpShell();
int vnodeOpenShellVnode(int vnode);
void vnodeCloseShellVnode(int vnode);
// memter mgmt
int vnodeInitMeterMgmt();
void vnodeCleanUpMeterMgmt();
int vnodeOpenMeterMgmtVnode(int vnode);
int vnodeOpenMeterMgmtStoreVnode(int vnode);
void vnodeCloseMeterMgmtVnode(int vnode);
int vnodeCreateMeterMgmt(SMeterObj *pObj, SConnSec *pSec);
void vnodeRemoveMeterMgmt(SMeterObj *pObj);
SConnSec *vnodeGetMeterSec(int vnode, int sid);
int vnodeCreateMeterObjFile(int vnode);
// mgmt
void vnodeCleanUpMgmt();
int vnodeRetrieveMissedCreateMsg(int vnode, int fd, uint64_t stime);
int vnodeRestoreMissedCreateMsg(int vnode, int fd);
int vnodeRetrieveMissedRemoveMsg(int vid, int fd, uint64_t stime);
int vnodeRestoreMissedRemoveMsg(int vnode, int fd);
int vnodeProcessBufferedCreateMsgs(int vnode);
void vnodeSendVpeerCfgMsg(int vnode);
int vnodeSendMeterCfgMsg(int vnode, int sid);
int vnodeMgmtConns();
void vnodeRemoveFile(int vnode, int fileId);
// commit
int vnodeInitCommit(int vnode);
void vnodeCleanUpCommit(int vnode);
int vnodeRenewCommitLog(int vnode);
void vnodeRemoveCommitLog(int vnode);
int vnodeWriteToCommitLog(SMeterObj *pObj, char action, char *cont, int contLen, int sversion);
extern int (*vnodeProcessAction[])(SMeterObj *, char *, int, char, void *, int, int *, TSKEY);
extern int (*pCompFunc[])(const char *const input, int inputSize, const int elements, char *const output,
int outputSize, char algorithm, char *const buffer, int bufferSize);
extern int (*pDecompFunc[])(const char *const input, int compressedSize, const int elements, char *const output,
int outputSize, char algorithm, char *const buffer, int bufferSize);
// global variable and APIs provided by mgmt
extern char mgmtStatus;
extern char tsMgmtDirectory[];
extern const int16_t vnodeFileVersion;
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_VNODE_H

61
src/vnode/detail/inc/vnodeCache.h
View File

@ -1,61 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_VNODECACHE_H
#define TDENGINE_VNODECACHE_H
#ifdef __cplusplus
extern "C" {
#endif
typedef struct {
short notFree;
short numOfPoints;
int slot;
int index;
int64_t blockId;
struct _meter_obj *pMeterObj;
char * offset[];
} SCacheBlock;
typedef struct {
int64_t blocks;
int maxBlocks;
int numOfBlocks;
int unCommittedBlocks;
int32_t currentSlot;
int32_t commitSlot; // which slot is committed
int32_t commitPoint; // starting point for next commit
SCacheBlock **cacheBlocks; // cache block list, circular list
} SCacheInfo;
typedef struct {
int vnode;
char ** pMem;
int64_t freeSlot;
pthread_mutex_t vmutex;
uint64_t count; // kind of transcation ID
int64_t notFreeSlots;
int64_t threshold;
char commitInProcess;
int cacheBlockSize;
int cacheNumOfBlocks;
} SCachePool;
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_VNODECACHE_H

35
src/vnode/detail/inc/vnodeDataFilterFunc.h
View File

@ -1,35 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_VNODEDATAFILTERFUNC_H
#define TDENGINE_VNODEDATAFILTERFUNC_H
#ifdef __cplusplus
extern "C" {
#endif
#include "vnode.h"
__filter_func_t *vnodeGetRangeFilterFuncArray(int32_t type);
__filter_func_t *vnodeGetValueFilterFuncArray(int32_t type);
bool vnodeSupportPrefilter(int32_t type);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_VNODEDATAFILTERFUNC_H

99
src/vnode/detail/inc/vnodeFile.h
View File

@ -1,99 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_VNODEFILE_H
#define TDENGINE_VNODEFILE_H
#ifdef __cplusplus
extern "C" {
#endif
#include "tchecksum.h"
#define TSDB_VNODE_DELIMITER 0xF00AFA0F
typedef struct { int64_t compInfoOffset; } SCompHeader;
typedef struct {
short colId;
short bytes;
int32_t numOfNullPoints;
int32_t type : 8;
int32_t offset : 24;
int32_t len; // data length
int64_t sum;
int64_t max;
int64_t min;
int16_t maxIndex;
int16_t minIndex;
char reserved[20];
} SField;
typedef struct {
int64_t last : 1;
int64_t offset : 63;
int32_t algorithm : 8; // compression algorithm can be changed
int32_t numOfPoints : 24; // how many points have been written into this block
int32_t sversion;
int32_t len; // total length of this data block
uint16_t numOfCols;
char reserved[16];
TSKEY keyFirst; // time stamp for the first point
TSKEY keyLast; // time stamp for the last point
} SCompBlock;
typedef struct {
SCompBlock *compBlock;
SField * fields;
} SCompBlockFields;
typedef struct {
uint64_t uid;
int64_t last : 1;
int64_t numOfBlocks : 62;
uint32_t delimiter; // delimiter for recovery
TSCKSUM checksum;
SCompBlock compBlocks[]; // comp block list
} SCompInfo;
typedef struct {
int64_t tempHeadOffset;
int64_t compInfoOffset;
int64_t oldCompBlockOffset;
int64_t oldNumOfBlocks;
int64_t newNumOfBlocks;
int64_t finalNumOfBlocks;
int64_t oldCompBlockLen;
int64_t newCompBlockLen;
int64_t finalCompBlockLen;
int64_t committedPoints;
int commitSlot;
int32_t last : 1;
int32_t changed : 1;
int32_t commitPos : 30;
int64_t commitCount;
SCompBlock lastBlock;
} SMeterInfo;
typedef struct { int64_t totalStorage; } SVnodeHeadInfo;
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_VNODEFILE_H

293
src/vnode/detail/inc/vnodeQueryImpl.h
View File

@ -1,293 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_VNODEQUERYIMPL_H
#define TDENGINE_VNODEQUERYIMPL_H
#ifdef __cplusplus
extern "C" {
#endif
#include "os.h"
#include "hash.h"
#include "hashfunc.h"
#define GET_QINFO_ADDR(x) ((char*)(x)-offsetof(SQInfo, query))
#define Q_STATUS_EQUAL(p, s) (((p) & (s)) != 0)
/*
* set the output buffer page size is 16k
* The page size should be sufficient for at least one output result or intermediate result.
* Some intermediate results may be extremely large, such as top/bottom(100) query.
*/
#define DEFAULT_INTERN_BUF_SIZE 16384L
#define INIT_ALLOCATE_DISK_PAGES 60L
#define DEFAULT_DATA_FILE_MAPPING_PAGES 2L
#define DEFAULT_DATA_FILE_MMAP_WINDOW_SIZE (DEFAULT_DATA_FILE_MAPPING_PAGES * DEFAULT_INTERN_BUF_SIZE)
#define IO_ENGINE_MMAP 0
#define IO_ENGINE_SYNC 1
#define DEFAULT_IO_ENGINE IO_ENGINE_SYNC
/**
* check if the primary column is load by default, otherwise, the program will
* forced to load primary column explicitly.
*/
#define PRIMARY_TSCOL_LOADED(query) ((query)->colList[0].data.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX)
typedef enum {
/*
* the program will call this function again, if this status is set.
* used to transfer from QUERY_RESBUF_FULL
*/
QUERY_NOT_COMPLETED = 0x1u,
/*
* output buffer is full, so, the next query will be employed,
* in this case, we need to set the appropriated start scan point for
* the next query.
*
* this status is only exist in group-by clause and
* diff/add/division/multiply/ query.
*/
QUERY_RESBUF_FULL = 0x2u,
/*
* query is over
* 1. this status is used in one row result query process, e.g.,
* count/sum/first/last/
* avg...etc.
* 2. when the query range on timestamp is satisfied, it is also denoted as
* query_compeleted
*/
QUERY_COMPLETED = 0x4u,
/*
* all data has been scanned, so current search is stopped,
* At last, the function will transfer this status to QUERY_COMPLETED
*/
QUERY_NO_DATA_TO_CHECK = 0x8u,
} vnodeQueryStatus;
typedef struct SPointInterpoSupporter {
int32_t numOfCols;
char** pPrevPoint;
char** pNextPoint;
} SPointInterpoSupporter;
typedef struct SBlockInfo {
TSKEY keyFirst;
TSKEY keyLast;
int32_t numOfCols;
int32_t size;
} SBlockInfo;
typedef struct SMeterDataBlockInfoEx {
SCompBlockFields pBlock;
SMeterDataInfo* pMeterDataInfo;
int32_t blockIndex;
int32_t groupIdx; /* number of group is less than the total number of meters */
} SMeterDataBlockInfoEx;
typedef enum {
DISK_DATA_LOAD_FAILED = -0x1,
DISK_DATA_LOADED = 0x0,
DISK_DATA_DISCARDED = 0x01,
} vnodeDiskLoadStatus;
#define IS_MASTER_SCAN(runtime) (((runtime)->scanFlag & 1u) == MASTER_SCAN)
#define IS_SUPPLEMENT_SCAN(runtime) ((runtime)->scanFlag == SUPPLEMENTARY_SCAN)
#define SET_SUPPLEMENT_SCAN_FLAG(runtime) ((runtime)->scanFlag = SUPPLEMENTARY_SCAN)
#define SET_MASTER_SCAN_FLAG(runtime) ((runtime)->scanFlag = MASTER_SCAN)
typedef int (*__block_search_fn_t)(char* data, int num, int64_t key, int order);
static FORCE_INLINE SMeterObj* getMeterObj(void* hashHandle, int32_t sid) {
return *(SMeterObj**)taosHashGet(hashHandle, (const char*)&sid, sizeof(sid));
}
bool isQueryKilled(SQuery* pQuery);
bool isFixedOutputQuery(SQuery* pQuery);
bool isPointInterpoQuery(SQuery* pQuery);
bool isSumAvgRateQuery(SQuery *pQuery);
bool isTopBottomQuery(SQuery* pQuery);
bool isFirstLastRowQuery(SQuery* pQuery);
bool isTSCompQuery(SQuery* pQuery);
bool notHasQueryTimeRange(SQuery* pQuery);
bool needSupplementaryScan(SQuery* pQuery);
bool onDemandLoadDatablock(SQuery* pQuery, int16_t queryRangeSet);
void setQueryStatus(SQuery* pQuery, int8_t status);
bool doRevisedResultsByLimit(SQInfo* pQInfo);
void truncateResultByLimit(SQInfo* pQInfo, int64_t* final, int32_t* interpo);
void initCtxOutputBuf(SQueryRuntimeEnv* pRuntimeEnv);
void resetCtxOutputBuf(SQueryRuntimeEnv* pRuntimeEnv);
void forwardCtxOutputBuf(SQueryRuntimeEnv* pRuntimeEnv, int64_t output);
bool needPrimaryTimestampCol(SQuery* pQuery, SBlockInfo* pBlockInfo);
void vnodeScanAllData(SQueryRuntimeEnv* pRuntimeEnv);
int32_t vnodeQueryResultInterpolate(SQInfo* pQInfo, tFilePage** pDst, tFilePage** pDataSrc, int32_t numOfRows,
int32_t* numOfInterpo);
void copyResToQueryResultBuf(STableQuerySupportObj* pSupporter, SQuery* pQuery);
void doSkipResults(SQueryRuntimeEnv* pRuntimeEnv);
void doFinalizeResult(SQueryRuntimeEnv* pRuntimeEnv);
int64_t getNumOfResult(SQueryRuntimeEnv* pRuntimeEnv);
void forwardQueryStartPosition(SQueryRuntimeEnv* pRuntimeEnv);
bool normalizedFirstQueryRange(bool dataInDisk, bool dataInCache, STableQuerySupportObj* pSupporter,
SPointInterpoSupporter* pPointInterpSupporter, int64_t* key);
void pointInterpSupporterInit(SQuery* pQuery, SPointInterpoSupporter* pInterpoSupport);
void pointInterpSupporterDestroy(SPointInterpoSupporter* pPointInterpSupport);
void pointInterpSupporterSetData(SQInfo* pQInfo, SPointInterpoSupporter* pPointInterpSupport);
int64_t loadRequiredBlockIntoMem(SQueryRuntimeEnv* pRuntimeEnv, SPositionInfo* position);
void disableFunctForSuppleScan(STableQuerySupportObj* pSupporter, int32_t order);
void enableFunctForMasterScan(SQueryRuntimeEnv* pRuntimeEnv, int32_t order);
int32_t mergeMetersResultToOneGroups(STableQuerySupportObj* pSupporter);
void copyFromWindowResToSData(SQInfo* pQInfo, SWindowResult* result);
SBlockInfo getBlockInfo(SQueryRuntimeEnv *pRuntimeEnv);
SBlockInfo getBlockBasicInfo(SQueryRuntimeEnv *pRuntimeEnv, void* pBlock, int32_t type);
SCacheBlock* getCacheDataBlock(SMeterObj* pMeterObj, SQueryRuntimeEnv* pRuntimeEnv, int32_t slot);
void stableApplyFunctionsOnBlock(STableQuerySupportObj* pSupporter, SMeterDataInfo* pMeterDataInfo,
SBlockInfo* pBlockInfo, SField* pFields, __block_search_fn_t searchFn);
int32_t vnodeFilterQualifiedMeters(SQInfo* pQInfo, int32_t vid, tSidSet* pSidSet, SMeterDataInfo* pMeterDataInfo,
int32_t* numOfMeters, SMeterDataInfo*** pReqMeterDataInfo);
int32_t vnodeGetVnodeHeaderFileIndex(int32_t* fid, SQueryRuntimeEnv* pRuntimeEnv, int32_t order);
int32_t createDataBlocksInfoEx(SMeterDataInfo** pMeterDataInfo, int32_t numOfMeters,
SMeterDataBlockInfoEx** pDataBlockInfoEx, int32_t numOfCompBlocks,
int32_t* nAllocBlocksInfoSize, int64_t addr);
void freeMeterBlockInfoEx(SMeterDataBlockInfoEx* pDataBlockInfoEx, int32_t len);
void setExecutionContext(STableQuerySupportObj* pSupporter, SMeterQueryInfo* pMeterQueryInfo, int32_t meterIdx,
int32_t groupIdx, TSKEY nextKey);
int32_t setAdditionalInfo(STableQuerySupportObj *pSupporter, int32_t meterIdx, SMeterQueryInfo *pMeterQueryInfo);
void doGetAlignedIntervalQueryRangeImpl(SQuery* pQuery, int64_t pKey, int64_t keyFirst, int64_t keyLast,
int64_t* actualSkey, int64_t* actualEkey, int64_t* skey, int64_t* ekey);
int64_t getQueryStartPositionInCache(SQueryRuntimeEnv* pRuntimeEnv, int32_t* slot, int32_t* pos, bool ignoreQueryRange);
int32_t getDataBlocksForMeters(STableQuerySupportObj* pSupporter, SQuery* pQuery, int32_t numOfMeters,
const char* filePath, SMeterDataInfo** pMeterDataInfo, uint32_t* numOfBlocks);
int32_t LoadDatablockOnDemand(SCompBlock* pBlock, SField** pFields, uint8_t* blkStatus, SQueryRuntimeEnv* pRuntimeEnv,
int32_t fileIdx, int32_t slotIdx, __block_search_fn_t searchFn, bool onDemand);
int32_t vnodeGetHeaderFile(SQueryRuntimeEnv* pRuntimeEnv, int32_t fileIndex);
/**
* Create SMeterQueryInfo.
* The MeterQueryInfo is created one for each table during super table query
*
* @param skey
* @param ekey
* @return
*/
SMeterQueryInfo* createMeterQueryInfo(STableQuerySupportObj* pSupporter, int32_t sid, TSKEY skey, TSKEY ekey);
/**
* Destroy meter query info
* @param pMeterQInfo
* @param numOfCols
*/
void destroyMeterQueryInfo(SMeterQueryInfo* pMeterQueryInfo, int32_t numOfCols);
/**
* change the meter query info for supplement scan
* @param pMeterQueryInfo
* @param skey
* @param ekey
*/
void changeMeterQueryInfoForSuppleQuery(SQuery* pQuery, SMeterQueryInfo* pMeterQueryInfo,
TSKEY skey, TSKEY ekey);
/**
* add the new allocated disk page to meter query info
* the new allocated disk page is used to keep the intermediate (interval) results
* @param pQuery
* @param pMeterQueryInfo
* @param pSupporter
*/
tFilePage* addDataPageForMeterQueryInfo(SQuery* pQuery, SMeterQueryInfo* pMeterQueryInfo,
STableQuerySupportObj* pSupporter);
/**
* restore the query range data from SMeterQueryInfo to runtime environment
*
* @param pRuntimeEnv
* @param pMeterQueryInfo
*/
void restoreIntervalQueryRange(SQueryRuntimeEnv* pRuntimeEnv, SMeterQueryInfo* pMeterQueryInfo);
/**
* set the interval query range for the interval query, when handling a data(cache) block
*
* @param pMeterQueryInfo
* @param pSupporter
* @param key
*/
void setIntervalQueryRange(SMeterQueryInfo* pMeterQueryInfo, STableQuerySupportObj* pSupporter, int64_t key);
/**
* set the meter data information
* @param pMeterDataInfo
* @param pMeterObj current query meter object
* @param meterIdx meter index in the sid list
* @param groupId group index, which the meter is belonged to
*/
void setMeterDataInfo(SMeterDataInfo* pMeterDataInfo, SMeterObj* pMeterObj, int32_t meterIdx, int32_t groupId);
void vnodeSetTagValueInParam(tSidSet* pSidSet, SQueryRuntimeEnv* pRuntimeEnv, SMeterSidExtInfo* pMeterInfo);
void vnodeCheckIfDataExists(SQueryRuntimeEnv* pRuntimeEnv, SMeterObj* pMeterObj, bool* dataInDisk, bool* dataInCache);
void displayInterResult(SData** pdata, SQuery* pQuery, int32_t numOfRows);
void vnodePrintQueryStatistics(STableQuerySupportObj* pSupporter);
void clearTimeWindowResBuf(SQueryRuntimeEnv* pRuntimeEnv, SWindowResult* pOneOutputRes);
void copyTimeWindowResBuf(SQueryRuntimeEnv* pRuntimeEnv, SWindowResult* dst, const SWindowResult* src);
int32_t initWindowResInfo(SWindowResInfo* pWindowResInfo, SQueryRuntimeEnv* pRuntimeEnv, int32_t size,
int32_t threshold, int16_t type);
void cleanupTimeWindowInfo(SWindowResInfo* pWindowResInfo, SQueryRuntimeEnv* pRuntimeEnv);
void resetTimeWindowInfo(SQueryRuntimeEnv* pRuntimeEnv, SWindowResInfo* pWindowResInfo);
void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num);
void clearClosedTimeWindow(SQueryRuntimeEnv* pRuntimeEnv);
int32_t numOfClosedTimeWindow(SWindowResInfo* pWindowResInfo);
void closeTimeWindow(SWindowResInfo* pWindowResInfo, int32_t slot);
void closeAllTimeWindow(SWindowResInfo* pWindowResInfo);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_VNODEQUERYIMPL_H

300
src/vnode/detail/inc/vnodeRead.h
View File

@ -1,300 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_VNODEREAD_H
#define TDENGINE_VNODEREAD_H
#ifdef __cplusplus
extern "C" {
#endif
#include "os.h"
#include "qresultBuf.h"
#include "qinterpolation.h"
#include "vnodeTagMgmt.h"
/*
* use to keep the first point position, consisting of position in blk and block
* id, file id
*/
typedef struct {
int32_t pos;
int32_t slot;
int32_t fileId;
} SPositionInfo;
typedef struct SLoadDataBlockInfo {
int32_t fileListIndex; /* index of this file in files list of this vnode */
int32_t fileId;
int32_t slotIdx;
int32_t sid;
bool tsLoaded; // if timestamp column of current block is loaded or not
} SLoadDataBlockInfo;
typedef struct SLoadCompBlockInfo {
int32_t sid; /* meter sid */
int32_t fileId;
int32_t fileListIndex;
} SLoadCompBlockInfo;
/*
* the header file info for one vnode
*/
typedef struct SHeaderFileInfo {
int32_t fileID; // file id
} SHeaderFileInfo;
typedef struct SQueryCostSummary {
double cacheTimeUs;
double fileTimeUs;
int64_t numOfFiles; // opened files during query
int64_t numOfTables; // num of queries tables
int64_t numOfSeek; // number of seek operation
int64_t readDiskBlocks; // accessed disk block
int64_t skippedFileBlocks; // skipped blocks
int64_t blocksInCache; // accessed cache blocks
int64_t readField; // field size
int64_t totalFieldSize; // total read fields size
double loadFieldUs; // total elapsed time to read fields info
int64_t totalBlockSize; // read data blocks
double loadBlocksUs; // total elapsed time to read data blocks
int64_t totalGenData; // in-memory generated data
int64_t readCompInfo; // read compblock info
int64_t totalCompInfoSize; // total comp block size
double loadCompInfoUs; // total elapsed time to read comp block info
int64_t tmpBufferInDisk; // size of buffer for intermediate result
} SQueryCostSummary;
typedef struct SPosInfo {
int16_t pageId;
int16_t rowId;
} SPosInfo;
typedef struct STimeWindow {
TSKEY skey;
TSKEY ekey;
} STimeWindow;
typedef struct SWindowStatus {
bool closed;
} SWindowStatus;
typedef struct SWindowResult {
uint16_t numOfRows;
SPosInfo pos; // Position of current result in disk-based output buffer
SResultInfo* resultInfo; // For each result column, there is a resultInfo
STimeWindow window; // The time window that current result covers.
SWindowStatus status;
} SWindowResult;
/*
* header files info, avoid to iterate the directory, the data is acquired
* during in query preparation function
*/
typedef struct SQueryFilesInfo {
SHeaderFileInfo* pFileInfo;
uint32_t numOfFiles; // the total available number of files for this virtual node during query execution
int32_t current; // the memory mapped header file, NOTE: only one header file can be mmap.
int32_t vnodeId;
int32_t headerFd; // header file fd
int64_t headerFileSize;
int32_t dataFd;
int32_t lastFd;
char headerFilePath[PATH_MAX]; // current opened header file name
char dataFilePath[PATH_MAX]; // current opened data file name
char lastFilePath[PATH_MAX]; // current opened last file path
char dbFilePathPrefix[PATH_MAX];
} SQueryFilesInfo;
typedef struct SWindowResInfo {
SWindowResult* pResult; // reference to SQuerySupporter->pResult
void* hashList; // hash list for quick access
int16_t type; // data type for hash key
int32_t capacity; // max capacity
int32_t curIndex; // current start active index
int32_t size;
int64_t startTime; // start time of the first time window for sliding query
int64_t prevSKey; // previous (not completed) sliding window start key
int64_t threshold; // threshold for return completed results.
} SWindowResInfo;
typedef struct SQueryRuntimeEnv {
SPositionInfo startPos; /* the start position, used for secondary/third iteration */
SPositionInfo endPos; /* the last access position in query, served as the start pos of reversed order query */
SPositionInfo nextPos; /* start position of the next scan */
SData* colDataBuffer[TSDB_MAX_COLUMNS];
SResultInfo* resultInfo; // todo refactor to merge with SWindowResInfo
uint8_t blockStatus; // Indicate if data block is loaded, the block is first/last/internal block
int32_t unzipBufSize;
SData* primaryColBuffer;
char* unzipBuffer;
char* secondaryUnzipBuffer;
SQuery* pQuery;
SMeterObj* pMeterObj;
SQLFunctionCtx* pCtx;
SLoadDataBlockInfo loadBlockInfo; /* record current block load information */
SLoadCompBlockInfo loadCompBlockInfo; /* record current compblock information in SQuery */
SQueryFilesInfo vnodeFileInfo;
int16_t numOfRowsPerPage;
int16_t offset[TSDB_MAX_COLUMNS];
uint16_t scanFlag; // denotes reversed scan of data or not
SInterpolationInfo interpoInfo;
SData** pInterpoBuf;
SWindowResInfo windowResInfo;
STSBuf* pTSBuf;
STSCursor cur;
SQueryCostSummary summary;
bool stableQuery; // is super table query or not
SDiskbasedResultBuf* pResultBuf; // query result buffer based on blocked-wised disk file
/*
* Temporarily hold the in-memory cache block info during scan cache blocks
* Here we do not use the cache block info from pMeterObj, simple because it may change anytime
* during the query by the submit/insert handling threads.
* So we keep a copy of the support structure as well as the cache block data itself.
*/
SCacheBlock cacheBlock;
} SQueryRuntimeEnv;
/* intermediate pos during multimeter query involves interval */
typedef struct SMeterQueryInfo {
int64_t lastKey;
int64_t skey;
int64_t ekey;
int32_t numOfRes;
int16_t queryRangeSet; // denote if the query range is set, only available for interval query
int64_t tag;
STSCursor cur;
int32_t sid; // for retrieve the page id list
SWindowResInfo windowResInfo;
} SMeterQueryInfo;
typedef struct SMeterDataInfo {
uint64_t offsetInHeaderFile;
int32_t numOfBlocks;
int32_t start; // start block index
SCompBlock* pBlock;
int32_t meterOrderIdx;
SMeterObj* pMeterObj;
int32_t groupIdx; // group id in meter list
SMeterQueryInfo* pMeterQInfo;
} SMeterDataInfo;
typedef struct STableQuerySupportObj {
void* pMetersHashTable; // meter table hash list
SMeterSidExtInfo** pMeterSidExtInfo;
int32_t numOfMeters;
/*
* multimeter query resultset.
* In multimeter queries, the result is temporarily stored on this structure, instead of
* directly put result into output buffer, since we have no idea how many number of
* rows may be generated by a specific subgroup. When query on all subgroups is executed,
* the result is copy to output buffer. This attribution is not used during single meter query processing.
*/
SQueryRuntimeEnv runtimeEnv;
int64_t rawSKey;
int64_t rawEKey;
int32_t subgroupIdx;
int32_t offset; /* offset in group result set of subgroup */
tSidSet* pSidSet;
/*
* the query is executed position on which meter of the whole list.
* when the index reaches the last one of the list, it means the query is completed.
* We later may refactor to remove this attribution by using another flag to denote
* whether a multimeter query is completed or not.
*/
int32_t meterIdx;
int32_t numOfGroupResultPages;
int32_t groupResultSize;
SMeterDataInfo* pMeterDataInfo;
TSKEY* tsList;
} STableQuerySupportObj;
typedef struct _qinfo {
uint64_t signature;
int32_t refCount; // QInfo reference count, when the value is 0, it can be released safely
char user[TSDB_TABLE_ID_LEN + 1];
char sql[TSDB_SHOW_SQL_LEN];
uint8_t stream;
uint16_t port;
uint32_t ip;
uint64_t startTime;
int64_t useconds;
int killed;
struct _qinfo *prev, *next;
SQuery query;
int totalPoints;
int pointsRead;
int pointsReturned;
int pointsInterpo;
int code;
char bufIndex;
char changed;
char over;
SMeterObj* pObj;
sem_t dataReady;
STableQuerySupportObj* pTableQuerySupporter;
int (*fp)(SMeterObj*, SQuery*);
} SQInfo;
int32_t vnodeQueryTablePrepare(SQInfo* pQInfo, SMeterObj* pMeterObj, STableQuerySupportObj* pSMultiMeterObj,
void* param);
void vnodeQueryFreeQInfoEx(SQInfo* pQInfo);
bool vnodeParametersSafetyCheck(SQuery* pQuery);
int32_t vnodeSTableQueryPrepare(SQInfo* pQInfo, SQuery* pQuery, void* param);
/**
* decrease the numofQuery of each table that is queried, enable the
* remove/close operation can be executed
* @param pQInfo
*/
void vnodeDecMeterRefcnt(SQInfo* pQInfo);
/* sql query handle in dnode */
void vnodeSingleTableQuery(SSchedMsg* pMsg);
/*
* handle multi-meter query process
*/
void vnodeMultiMeterQuery(SSchedMsg* pMsg);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_VNODEREAD_H

29
src/vnode/detail/inc/vnodeShell.h
View File

@ -1,29 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_VNODESHELL_H
#define TDENGINE_VNODESHELL_H
#ifdef __cplusplus
extern "C" {
#endif
#include "os.h"
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_VNODESHELL_H

35
src/vnode/detail/inc/vnodeStore.h
View File

@ -1,35 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_VNODESTORE_H
#define TDENGINE_VNODESTORE_H
#ifdef __cplusplus
extern "C" {
#endif
void vnodeProcessDataFromVnode(SIntMsg *msg, void *tcpHandle);
void vnodeCalcOpenVnodes();
bool vnodeRemoveDataFileFromLinkFile(char* linkFile, char* de_name);
int vnodeInitInfo();
#ifdef __cplusplus
}
#endif
#endif // TDEGINE_VNODESTORE_H

27
src/vnode/detail/inc/vnodeSupertableQuery.h
View File

@ -1,27 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TBASE_MNODE_SUPER_TABLE_QUERY_H
#define TBASE_MNODE_SUPER_TABLE_QUERY_H
#include "os.h"
#include "mnode.h"
#include "qast.h"
int32_t mgmtDoJoin(SSuperTableMetaMsg* pSuperTableMetaMsg, tQueryResultset* pRes);
void mgmtReorganizeMetersInMetricMeta(SSuperTableMetaMsg* pInfo, int32_t index, tQueryResultset* pRes);
#endif

29
src/vnode/detail/inc/vnodeSystem.h
View File

@ -1,29 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_VNODESYSTEM_H
#define TDENGINE_VNODESYSTEM_H
#ifdef __cplusplus
extern "C" {
#endif
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_VNODESYSTEM_H

58
src/vnode/detail/inc/vnodeTagMgmt.h
View File

@ -1,58 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_VNODETAGMGMT_H
#define TDENGINE_VNODETAGMGMT_H
#ifdef __cplusplus
extern "C" {
#endif
/*
* @version 0.1
* @date 2018/01/02
* @author liaohj
* management of the tag value of tables
* in query, client need the vnode to aggregate results according to tags
* values,
* the grouping operation is done here.
* Note:
* 1. we implement a quick sort algorithm, may remove it later.
*/
typedef int32_t (*__ext_compar_fn_t)(const void *p1, const void *p2, void *param);
tSidSet *tSidSetCreate(struct SMeterSidExtInfo **pMeterSidExtInfo, int32_t numOfMeters, SSchema *pSchema,
int32_t numOfTags, SColIndexEx *colList, int32_t numOfOrderCols);
int32_t *calculateSubGroup(void **pSids, int32_t numOfMeters, int32_t *numOfSubset, tOrderDescriptor *pOrderDesc,
__ext_compar_fn_t compareFn);
void tSidSetDestroy(tSidSet **pSets);
void tSidSetSort(tSidSet *pSets);
int32_t meterSidComparator(const void *s1, const void *s2, void *param);
int32_t doCompare(char *f1, char *f2, int32_t type, int32_t size);
void tQSortEx(void **pMeterSids, size_t size, int32_t start, int32_t end, void *param, __ext_compar_fn_t compareFn);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_VNODETAGMGMT_H

94
src/vnode/detail/inc/vnodeUtil.h
View File

@ -1,94 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_VNODE_UTIL_H
#define TDENGINE_VNODE_UTIL_H
#ifdef __cplusplus
extern "C" {
#endif
/* get the qinfo struct address from the query struct address */
#define GET_COLUMN_BYTES(query, colidx) \
((query)->colList[(query)->pSelectExpr[colidx].pBase.colInfo.colIdxInBuf].data.bytes)
#define GET_COLUMN_TYPE(query, colidx) \
((query)->colList[(query)->pSelectExpr[colidx].pBase.colInfo.colIdxInBuf].data.type)
#define QUERY_IS_ASC_QUERY(q) (GET_FORWARD_DIRECTION_FACTOR((q)->order.order) == QUERY_ASC_FORWARD_STEP)
#define EXTRA_BYTES 2 // for possible compression deflation
#define GET_COL_DATA_POS(query, index, step) ((query)->pos + (index)*(step))
int vnodeGetEid(int days);
int vnodeCheckFileIntegrity(FILE *fp);
void vnodeCreateFileHeader(FILE *fp);
void vnodeCreateFileHeaderFd(int fd);
void vnodeGetHeadFileHeaderInfo(int fd, SVnodeHeadInfo *pHeadInfo);
void vnodeUpdateHeadFileHeader(int fd, SVnodeHeadInfo *pHeadInfo);
/**
* check if two schema is identical or not
* This function does not check if a schema is valid or not
*
* @param pSSchemaFirst
* @param numOfCols1
* @param pSSchemaSecond
* @param numOfCols2
* @return
*/
bool vnodeMeterSchemaIdentical(SColumn *pSchema1, int32_t numOfCols1, SColumn *pSchema2, int32_t numOfCols2);
/**
* free SFields in SQuery
* vnodeFreeFields must be called before free(pQuery->pBlock);
* @param pQuery
*/
void vnodeFreeFields(SQuery *pQuery);
void vnodeUpdateFilterColumnIndex(SQuery* pQuery);
void vnodeUpdateQueryColumnIndex(SQuery* pQuery, SMeterObj* pMeterObj);
int32_t vnodeCreateFilterInfo(void* pQInfo, SQuery *pQuery);
bool vnodeFilterData(SQuery* pQuery, int32_t* numOfActualRead, int32_t index);
bool vnodeDoFilterData(SQuery* pQuery, int32_t elemPos);
bool vnodeIsProjectionQuery(SSqlFunctionExpr *pExpr, int32_t numOfOutput);
int32_t vnodeIncQueryRefCount(SQueryMeterMsg *pQueryMsg, SMeterSidExtInfo **pSids, SMeterObj **pMeterObjList,
int32_t *numOfInc);
void vnodeDecQueryRefCount(SQueryMeterMsg *pQueryMsg, SMeterObj **pMeterObjList, int32_t numOfInc);
int32_t vnodeSetMeterState(SMeterObj* pMeterObj, int32_t state);
void vnodeClearMeterState(SMeterObj* pMeterObj, int32_t state);
bool vnodeIsMeterState(SMeterObj* pMeterObj, int32_t state);
void vnodeSetMeterDeleting(SMeterObj* pMeterObj);
int32_t vnodeSetMeterInsertImportStateEx(SMeterObj* pObj, int32_t st);
bool vnodeIsSafeToDeleteMeter(SVnodeObj* pVnode, int32_t sid);
void vnodeFreeColumnInfo(SColumnInfo* pColumnInfo);
bool isGroupbyNormalCol(SSqlGroupbyExpr* pExpr);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_VNODE_UTIL_H

1165
src/vnode/detail/src/vnodeCache.c
View File

File diff suppressed because it is too large Load Diff

292
src/vnode/detail/src/vnodeCommit.c
View File

@ -1,292 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _GNU_SOURCE /* See feature_test_macros(7) */
#include "os.h"
#include "taosdef.h"
#include "vnode.h"
#include "vnodeUtil.h"
#include "vnodeStatus.h"
typedef struct {
int sversion;
int sid;
int contLen;
int action:8;
int simpleCheck:24;
} SCommitHead;
int vnodeOpenCommitLog(int vnode, uint64_t firstV) {
SVnodeObj *pVnode = vnodeList + vnode;
char * fileName = pVnode->logFn;
pVnode->logFd = open(fileName, O_RDWR | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO);
if (pVnode->logFd < 0) {
dError("vid:%d, failed to open file:%s, reason:%s", vnode, fileName, strerror(errno));
return -1;
}
dTrace("vid:%d, logfd:%d, open file:%s success", vnode, pVnode->logFd, fileName);
if (posix_fallocate64(pVnode->logFd, 0, pVnode->mappingSize) != 0) {
dError("vid:%d, logfd:%d, failed to alloc file size:%d, reason:%s", vnode, pVnode->logFd, pVnode->mappingSize, strerror(errno));
perror("fallocate failed");
goto _err_log_open;
}
struct stat statbuf;
stat(fileName, &statbuf);
int64_t length = statbuf.st_size;
if (length != pVnode->mappingSize) {
dError("vid:%d, logfd:%d, alloc file size:%" PRId64 " not equal to mapping size:%" PRId64, vnode, pVnode->logFd, length,
pVnode->mappingSize);
goto _err_log_open;
}
pVnode->pMem = mmap(0, pVnode->mappingSize, PROT_WRITE | PROT_READ, MAP_SHARED, pVnode->logFd, 0);
if (pVnode->pMem == MAP_FAILED) {
dError("vid:%d, logfd:%d, failed to map file, reason:%s", vnode, pVnode->logFd, strerror(errno));
goto _err_log_open;
}
pVnode->pWrite = pVnode->pMem;
memcpy(pVnode->pWrite, &(firstV), sizeof(firstV));
pVnode->pWrite += sizeof(firstV);
return pVnode->logFd;
_err_log_open:
close(pVnode->logFd);
remove(fileName);
pVnode->logFd = -1;
return -1;
}
int vnodeRenewCommitLog(int vnode) {
SVnodeObj *pVnode = vnodeList + vnode;
char * fileName = pVnode->logFn;
char * oldName = pVnode->logOFn;
pthread_mutex_lock(&(pVnode->logMutex));
if (FD_VALID(pVnode->logFd)) {
munmap(pVnode->pMem, pVnode->mappingSize);
close(pVnode->logFd);
rename(fileName, oldName);
}
if (pVnode->cfg.commitLog) vnodeOpenCommitLog(vnode, vnodeList[vnode].version);
pthread_mutex_unlock(&(pVnode->logMutex));
return pVnode->logFd;
}
void vnodeRemoveCommitLog(int vnode) { remove(vnodeList[vnode].logOFn); }
size_t vnodeRestoreDataFromLog(int vnode, char *fileName, uint64_t *firstV) {
int fd, ret;
char * cont = NULL;
size_t totalLen = 0;
int actions = 0;
SVnodeObj *pVnode = vnodeList + vnode;
if (pVnode->meterList == NULL) {
dError("vid:%d, vnode is not initialized!!!", vnode);
return 0;
}
struct stat fstat;
if (stat(fileName, &fstat) < 0) {
dTrace("vid:%d, no log file:%s", vnode, fileName);
return 0;
}
dTrace("vid:%d, uncommitted data in file:%s, restore them ...", vnode, fileName);
fd = open(fileName, O_RDWR);
if (fd < 0) {
dError("vid:%d, failed to open:%s, reason:%s", vnode, fileName, strerror(errno));
goto _error;
}
ret = read(fd, firstV, sizeof(pVnode->version));
if (ret <= 0) {
dError("vid:%d, failed to read version", vnode);
goto _error;
}
pVnode->version = *firstV;
int32_t bufLen = TSDB_PAYLOAD_SIZE;
cont = calloc(1, bufLen);
if (cont == NULL) {
dError("vid:%d, out of memory", vnode);
goto _error;
}
TSKEY now = taosGetTimestamp(pVnode->cfg.precision);
SCommitHead head;
int simpleCheck = 0;
while (1) {
ret = read(fd, &head, sizeof(head));
if (ret < 0) goto _error;
if (ret == 0) break;
if (((head.sversion+head.sid+head.contLen+head.action) & 0xFFFFFF) != head.simpleCheck) break;
simpleCheck = head.simpleCheck;
// head.contLen validation is removed
if (head.sid >= pVnode->cfg.maxSessions || head.sid < 0 || head.action >= TSDB_ACTION_MAX) {
dError("vid, invalid commit head, sid:%d contLen:%d action:%d", head.sid, head.contLen, head.action);
} else {
if (head.contLen > 0) {
if (bufLen < head.contLen+sizeof(simpleCheck)) { // pre-allocated buffer is not enough
cont = realloc(cont, head.contLen+sizeof(simpleCheck));
bufLen = head.contLen+sizeof(simpleCheck);
}
if (read(fd, cont, head.contLen+sizeof(simpleCheck)) < 0) goto _error;
if (*(int *)(cont+head.contLen) != simpleCheck) break;
SMeterObj *pObj = pVnode->meterList[head.sid];
if (pObj == NULL) {
dError("vid:%d, sid:%d not exists, ignore data in commit log, contLen:%d action:%d",
vnode, head.sid, head.contLen, head.action);
continue;
}
if (vnodeIsMeterState(pObj, TSDB_METER_STATE_DROPPING)) {
dWarn("vid:%d sid:%d id:%s, meter is dropped, ignore data in commit log, contLen:%d action:%d",
vnode, head.sid, head.contLen, head.action);
continue;
}
int32_t numOfPoints = 0;
(*vnodeProcessAction[head.action])(pObj, cont, head.contLen, TSDB_DATA_SOURCE_LOG, NULL, head.sversion,
&numOfPoints, now);
actions++;
} else {
break;
}
}
totalLen += sizeof(head) + head.contLen + sizeof(simpleCheck);
}
tclose(fd);
tfree(cont);
dTrace("vid:%d, %d pieces of uncommitted data are restored", vnode, actions);
return totalLen;
_error:
tclose(fd);
tfree(cont);
dError("vid:%d, failed to restore %s, remove this node...", vnode, fileName);
// rename to error file for future process
char *f = NULL;
taosFileRename(fileName, "error", '/', &f);
free(f);
return -1;
}
int vnodeInitCommit(int vnode) {
size_t size = 0;
uint64_t firstV = 0;
SVnodeObj *pVnode = vnodeList + vnode;
pthread_mutex_init(&(pVnode->logMutex), NULL);
sprintf(pVnode->logFn, "%s/vnode%d/db/submit%d.log", tsDirectory, vnode, vnode);
sprintf(pVnode->logOFn, "%s/vnode%d/db/submit%d.olog", tsDirectory, vnode, vnode);
pVnode->mappingSize = ((int64_t)pVnode->cfg.cacheBlockSize) * pVnode->cfg.cacheNumOfBlocks.totalBlocks * 1.5;
pVnode->mappingThreshold = pVnode->mappingSize * 0.7;
// restore from .olog file and commit to file
size = vnodeRestoreDataFromLog(vnode, pVnode->logOFn, &firstV);
if (size < 0) return -1;
if (size > 0) {
if (pVnode->commitInProcess == 0) vnodeCommitToFile(pVnode);
remove(pVnode->logOFn);
}
// restore from .log file to cache
size = vnodeRestoreDataFromLog(vnode, pVnode->logFn, &firstV);
if (size < 0) return -1;
if (pVnode->cfg.commitLog == 0) return 0;
if (size == 0) firstV = pVnode->version;
if (vnodeOpenCommitLog(vnode, firstV) < 0) {
dError("vid:%d, commit log init failed", vnode);
return -1;
}
pVnode->pWrite += size;
dPrint("vid:%d, commit log is initialized", vnode);
return 0;
}
void vnodeCleanUpCommit(int vnode) {
SVnodeObj *pVnode = vnodeList + vnode;
if (FD_VALID(pVnode->logFd)) close(pVnode->logFd);
if (pVnode->cfg.commitLog && (pVnode->logFd > 0 && remove(pVnode->logFn) < 0)) {
dError("vid:%d, failed to remove:%s", vnode, pVnode->logFn);
taosLogError("vid:%d, failed to remove:%s", vnode, pVnode->logFn);
}
pthread_mutex_destroy(&(pVnode->logMutex));
}
int vnodeWriteToCommitLog(SMeterObj *pObj, char action, char *cont, int contLen, int sverion) {
SVnodeObj *pVnode = vnodeList + pObj->vnode;
if (pVnode->pWrite == NULL) return 0;
SCommitHead head;
head.sid = pObj->sid;
head.action = action;
head.sversion = pObj->sversion;
head.contLen = contLen;
head.simpleCheck = (head.sversion+head.sid+head.contLen+head.action) & 0xFFFFFF;
int simpleCheck = head.simpleCheck;
pthread_mutex_lock(&(pVnode->logMutex));
// 100 bytes redundant mem space
if (pVnode->mappingSize - (pVnode->pWrite - pVnode->pMem) < contLen + sizeof(SCommitHead) + sizeof(simpleCheck) + 100) {
pthread_mutex_unlock(&(pVnode->logMutex));
dTrace("vid:%d, mem mapping space is not enough, wait for commit", pObj->vnode);
vnodeProcessCommitTimer(pVnode, NULL);
return TSDB_CODE_ACTION_IN_PROGRESS;
}
char *pWrite = pVnode->pWrite;
pVnode->pWrite += sizeof(head) + contLen + sizeof(simpleCheck);
memcpy(pWrite, (char *)&head, sizeof(head));
memcpy(pWrite + sizeof(head), cont, contLen);
memcpy(pWrite + sizeof(head) + contLen, &simpleCheck, sizeof(simpleCheck));
pthread_mutex_unlock(&(pVnode->logMutex));
if (pVnode->pWrite - pVnode->pMem > pVnode->mappingThreshold) {
dTrace("vid:%d, mem mapping is close to limit, commit", pObj->vnode);
vnodeProcessCommitTimer(pVnode, NULL);
}
dTrace("vid:%d sid:%d, data is written to commit log", pObj->vnode, pObj->sid);
return 0;
}

1880
src/vnode/detail/src/vnodeFile.c
View File

File diff suppressed because it is too large Load Diff

110
src/vnode/detail/src/vnodeFile.spec.c
View File

@ -1,110 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "vnode.h"
#include "vnodeFile.h"
char* vnodeGetDiskFromHeadFile(char *headName) { return tsDirectory; }
char* vnodeGetDataDir(int vnode, int fileId) { return dataDir; }
void vnodeAdustVnodeFile(SVnodeObj *pVnode) {
// Retention policy here
int fileId = pVnode->fileId - pVnode->numOfFiles + 1;
int cfile = taosGetTimestamp(pVnode->cfg.precision)/pVnode->cfg.daysPerFile/tsMsPerDay[(uint8_t)pVnode->cfg.precision];
while (fileId <= cfile - pVnode->maxFiles) {
vnodeRemoveFile(pVnode->vnode, fileId);
pVnode->numOfFiles--;
fileId++;
}
}
int vnodeCheckNewHeaderFile(int fd, SVnodeObj *pVnode) {
SCompHeader *pHeader = NULL;
SCompBlock *pBlocks = NULL;
int blockSize = 0;
SCompInfo compInfo;
int tmsize = 0;
tmsize = sizeof(SCompHeader) * pVnode->cfg.maxSessions + sizeof(TSCKSUM);
pHeader = (SCompHeader *)malloc(tmsize);
if (pHeader == NULL) return 0;
lseek(fd, TSDB_FILE_HEADER_LEN, SEEK_SET);
if (read(fd, (void *)pHeader, tmsize) != tmsize) {
goto _broken_exit;
}
if (!taosCheckChecksumWhole((uint8_t *)pHeader, tmsize)) {
goto _broken_exit;
}
for (int sid = 0; sid < pVnode->cfg.maxSessions; sid++) {
if (pVnode->meterList == NULL) goto _correct_exit;
if (pVnode->meterList[sid] == NULL || pHeader[sid].compInfoOffset == 0) continue;
lseek(fd, pHeader[sid].compInfoOffset, SEEK_SET);
if (read(fd, (void *)(&compInfo), sizeof(SCompInfo)) != sizeof(SCompInfo)) {
goto _broken_exit;
}
if (!taosCheckChecksumWhole((uint8_t *)(&compInfo), sizeof(SCompInfo))) {
goto _broken_exit;
}
if (compInfo.uid != ((SMeterObj *)pVnode->meterList[sid])->uid) continue;
int expectedSize = sizeof(SCompBlock) * compInfo.numOfBlocks + sizeof(TSCKSUM);
if (blockSize < expectedSize) {
pBlocks = (SCompBlock *)realloc(pBlocks, expectedSize);
if (pBlocks == NULL) {
tfree(pHeader);
return 0;
}
blockSize = expectedSize;
}
if (read(fd, (void *)pBlocks, expectedSize) != expectedSize) {
dError("failed to read block part");
goto _broken_exit;
}
if (!taosCheckChecksumWhole((uint8_t *)pBlocks, expectedSize)) {
dError("block part is broken");
goto _broken_exit;
}
for (int i = 0; i < compInfo.numOfBlocks; i++) {
if (pBlocks[i].last && i != compInfo.numOfBlocks-1) {
dError("last block in middle, block:%d", i);
goto _broken_exit;
}
}
}
_correct_exit:
dPrint("vid: %d new header file %s is correct", pVnode->vnode, pVnode->nfn);
tfree(pBlocks);
tfree(pHeader);
return 0;
_broken_exit:
dError("vid: %d new header file %s is broken", pVnode->vnode, pVnode->nfn);
tfree(pBlocks);
tfree(pHeader);
return -1;
}

237
src/vnode/detail/src/vnodeFileUtil.c
View File

@ -1,237 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "vnode.h"
int vnodeCheckHeaderFile(int fd, int dfd, SVnodeCfg cfg, int mode) {
SCompHeader *pHeaders = NULL;
SVnodeCfg *pCfg = &cfg;
SCompInfo compInfo;
SCompBlock *pBlocks = NULL;
int blockSize = 0;
SField *pFields = NULL;
char *pBuf = NULL;
int size = 0;
int ret = 0;
if (fd < 0 || dfd < 0) return -1;
lseek(fd, TSDB_FILE_HEADER_LEN, SEEK_SET);
size = pCfg->maxSessions*sizeof(SCompHeader)+sizeof(TSCKSUM);
pHeaders = calloc(1, size);
if (pHeaders == NULL) {
return -1;
}
read(fd, pHeaders, size);
if (!taosCheckChecksumWhole((uint8_t *)pHeaders, size)) {
return -1;
}
for (int i = 0; i < pCfg->maxSessions; i++) {
if (pHeaders[i].compInfoOffset == 0) continue;
if (pHeaders[i].compInfoOffset < 0) {
// TODO : report error here
ret = -1;
continue;
}
lseek(fd, pHeaders[i].compInfoOffset, SEEK_SET);
read(fd, &compInfo, sizeof(SCompInfo));
if (!taosCheckChecksumWhole((uint8_t *)&compInfo, sizeof(SCompInfo))) {
// TODO : report error
ret = -1;
continue;
}
int tsize = sizeof(SCompBlock) * compInfo.numOfBlocks + sizeof(TSCKSUM);
if (tsize > blockSize) {
if (pBlocks == NULL) {
pBlocks = calloc(1, tsize);
} else {
pBlocks = realloc(pBlocks, tsize);
}
blockSize = tsize;
}
read(fd, tsize);
if (!taosCheckChecksumWhole(pBlocks, tsize)) {
// TODO: Report error
ret = -1;
continue;
}
TSKEY keyLast = 0;
for (int j = 0; j < compInfo.numOfBlocks; j++) {
SCompBlock *pBlock = pBlocks + j;
if (pBlock->last != 0 && j < compInfo.numOfBlocks-1) {
// TODO: report error
ret = -1;
break;
}
if (pBlock->offset < TSDB_FILE_HEADER_LEN) {
// TODO : report erro
ret = -1;
break;
}
if (pBlock->keyLast < pBlock->keyFirst) {
// TODO : report error
ret = -1;
break;
}
if (pBlock->keyFirst <= keyLast) {
// TODO : report error
ret = -1;
break;
}
keyLast = pBlock->keyLast;
// Check block in data
lseek(dfd, pBlock->offset, SEEK_SET);
tsize = sizeof(SField) * pBlock->numOfCols + sizeof(TSCKSUM);
pFields = realloc(pFields, tsize);
read(dfd, pFields, tsize);
if (!taosCheckChecksumWhole((uint8_t*)pFields, tsize)) {
// TODO : report error
ret = -1;
continue;
}
for (int k = 0; k < pBlock->numOfCols; k++) {
// TODO: Check pFields[k] content
pBuf = realloc(pBuf, pFields[k].len);
if (!taosCheckChecksumWhole((uint8_t *)pBuf, pFields[k].len)) {
// TODO : report error;
ret = -1;
continue;
}
}
}
}
tfree(pBuf);
tfree(pFields);
tfree(pBlocks);
tfree(pHeaders);
return ret;
}
int vnodePackDataFile(int vnode, int fileId) {
// TODO: check if it is able to pack current file
// TODO: assign value to headerFile and dataFile
char *headerFile = NULL;
char *dataFile = NULL;
char *lastFile = NULL;
SVnodeObj *pVnode = vnodeList+vnode;
SCompHeader *pHeaders = NULL;
SCompBlock *pBlocks = NULL;
int blockSize = 0;
char *pBuff = 0;
int buffSize = 0;
SCompInfo compInfo;
int size = 0;
int hfd = open(headerFile, O_RDONLY);
if (hfd < 0) {
dError("vid: %d, failed to open header file:%s\n", vnode, headerFile);
return -1;
}
int dfd = open(dataFile, O_RDONLY);
if (dfd < 0) {
dError("vid: %d, failed to open data file:%s\n", vnode, dataFile);
return -1;
}
int lfd = open(lastFile, O_RDONLY);
if (lfd < 0) {
dError("vid: %d, failed to open data file:%s\n", vnode, lastFile);
return -1;
}
lseek(hfd, TSDB_FILE_HEADER_LEN, SEEK_SET);
size = sizeof(SCompHeader)*pVnode->cfg.maxSessions+sizeof(TSCKSUM);
pHeaders = malloc(size);
if (pHeaders == NULL) goto _exit_failure;
read(hfd, pHeaders, size);
if (!taosCheckChecksumWhole((uint8_t *)pHeaders, size)) {
dError("vid: %d, header file %s is broken", vnode, headerFile);
goto _exit_failure;
}
for (size_t i = 0; i < pVnode->cfg.maxSessions; i++)
{
if (pHeaders[i].compInfoOffset <= 0) continue;
SMeterObj *pObj = (SMeterObj *)pVnode->meterList[i];
// read compInfo part
lseek(hfd, pHeaders[i].compInfoOffset, SEEK_SET);
read(hfd, &compInfo, sizeof(SCompInfo));
if (!taosCheckChecksumWhole((uint8_t *)&compInfo, sizeof(SCompInfo))) {
dError("vid: %d sid:%d fileId:%d compInfo is broken", vnode, i, fileId);
goto _exit_failure;
}
// read compBlock part
int tsize = compInfo.numOfBlocks * sizeof(SCompBlock) + sizeof(TSCKSUM);
if (tsize > blockSize) {
if (blockSize == 0) {
pBlocks = malloc(tsize);
} else {
pBlocks = realloc(pBlocks, tsize);
}
blockSize = tsize;
}
read(hfd, pBlocks, tsize);
if (!taosCheckChecksumWhole((uint8_t *)pBlocks, tsize)) {
dError("vid:%d sid:%d fileId:%d block part is broken", vnode, i, fileId);
goto _exit_failure;
}
assert(compInfo.numOfBlocks > 0);
// Loop to scan the blocks and merge block when neccessary.
tsize = sizeof(SCompInfo) + compInfo.numOfBlocks *sizeof(SCompBlock) + sizeof(TSCKSUM);
pBuff = realloc(pBuff, tsize);
SCompInfo *pInfo = (SCompInfo *)pBuff;
SCompBlock *pNBlocks = pBuff + sizeof(SCompInfo);
int nCounter = 0;
for (int j; j < compInfo.numOfBlocks; j++) {
// TODO : Check if it is the last block
// if (j == compInfo.numOfBlocks - 1) {}
if (pBlocks[j].numOfPoints + pNBlocks[nCounter].numOfPoints <= pObj->pointsPerFileBlock) {
// Merge current block to current new block
} else {
// Write new block to new data file
// pNBlocks[nCounter].
nCounter++;
}
}
}
return 0;
_exit_failure:
tfree(pHeaders);
if (hfd > 0) close(hfd);
if (dfd > 0) close(dfd);
if (lfd > 0) close(lfd);
return -1;
}

558
src/vnode/detail/src/vnodeFilterFunc.c
View File

@ -1,558 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "taosmsg.h"
#include "tsqlfunction.h"
#include "vnode.h"
#include "vnodeDataFilterFunc.h"
bool less_i8(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int8_t *)minval < pFilter->filterInfo.upperBndi);
}
bool less_i16(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int16_t *)minval < pFilter->filterInfo.upperBndi);
}
bool less_i32(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int32_t *)minval < pFilter->filterInfo.upperBndi);
}
bool less_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int64_t *)minval < pFilter->filterInfo.upperBndi);
}
bool less_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(float *)minval < pFilter->filterInfo.upperBndd);
}
bool less_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(double *)minval < pFilter->filterInfo.upperBndd);
}
//////////////////////////////////////////////////////////////////
bool large_i8(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int8_t *)maxval > pFilter->filterInfo.lowerBndi);
}
bool large_i16(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int16_t *)maxval > pFilter->filterInfo.lowerBndi);
}
bool large_i32(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int32_t *)maxval > pFilter->filterInfo.lowerBndi);
}
bool large_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int64_t *)maxval > pFilter->filterInfo.lowerBndi);
}
bool large_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(float *)maxval > pFilter->filterInfo.lowerBndd);
}
bool large_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(double *)maxval > pFilter->filterInfo.lowerBndd);
}
/////////////////////////////////////////////////////////////////////
bool lessEqual_i8(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int8_t *)minval <= pFilter->filterInfo.upperBndi);
}
bool lessEqual_i16(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int16_t *)minval <= pFilter->filterInfo.upperBndi);
}
bool lessEqual_i32(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int32_t *)minval <= pFilter->filterInfo.upperBndi);
}
bool lessEqual_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int64_t *)minval <= pFilter->filterInfo.upperBndi);
}
bool lessEqual_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(float *)minval <= pFilter->filterInfo.upperBndd);
}
bool lessEqual_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(double *)minval <= pFilter->filterInfo.upperBndd);
}
//////////////////////////////////////////////////////////////////////////
bool largeEqual_i8(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int8_t *)maxval >= pFilter->filterInfo.lowerBndi);
}
bool largeEqual_i16(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int16_t *)maxval >= pFilter->filterInfo.lowerBndi);
}
bool largeEqual_i32(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int32_t *)maxval >= pFilter->filterInfo.lowerBndi);
}
bool largeEqual_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int64_t *)maxval >= pFilter->filterInfo.lowerBndi);
}
bool largeEqual_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(float *)maxval >= pFilter->filterInfo.lowerBndd);
}
bool largeEqual_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(double *)maxval >= pFilter->filterInfo.lowerBndd);
}
////////////////////////////////////////////////////////////////////////
bool equal_i8(SColumnFilterElem *pFilter, char *minval, char *maxval) {
if (*(int8_t *)minval == *(int8_t *)maxval) {
return (*(int8_t *)minval == pFilter->filterInfo.lowerBndi);
} else { /* range filter */
assert(*(int8_t *)minval < *(int8_t *)maxval);
return *(int8_t *)minval <= pFilter->filterInfo.lowerBndi && *(int8_t *)maxval >= pFilter->filterInfo.lowerBndi;
}
}
bool equal_i16(SColumnFilterElem *pFilter, char *minval, char *maxval) {
if (*(int16_t *)minval == *(int16_t *)maxval) {
return (*(int16_t *)minval == pFilter->filterInfo.lowerBndi);
} else { /* range filter */
assert(*(int16_t *)minval < *(int16_t *)maxval);
return *(int16_t *)minval <= pFilter->filterInfo.lowerBndi && *(int16_t *)maxval >= pFilter->filterInfo.lowerBndi;
}
}
bool equal_i32(SColumnFilterElem *pFilter, char *minval, char *maxval) {
if (*(int32_t *)minval == *(int32_t *)maxval) {
return (*(int32_t *)minval == pFilter->filterInfo.lowerBndi);
} else { /* range filter */
assert(*(int32_t *)minval < *(int32_t *)maxval);
return *(int32_t *)minval <= pFilter->filterInfo.lowerBndi && *(int32_t *)maxval >= pFilter->filterInfo.lowerBndi;
}
}
bool equal_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) {
if (*(int64_t *)minval == *(int64_t *)maxval) {
return (*(int64_t *)minval == pFilter->filterInfo.lowerBndi);
} else { /* range filter */
assert(*(int64_t *)minval < *(int64_t *)maxval);
return *(int64_t *)minval <= pFilter->filterInfo.lowerBndi && *(int64_t *)maxval >= pFilter->filterInfo.lowerBndi;
}
}
bool equal_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) {
if (*(float *)minval == *(float *)maxval) {
return (fabs(*(float *)minval - pFilter->filterInfo.lowerBndd) <= FLT_EPSILON);
} else { /* range filter */
assert(*(float *)minval < *(float *)maxval);
return *(float *)minval <= pFilter->filterInfo.lowerBndd && *(float *)maxval >= pFilter->filterInfo.lowerBndd;
}
}
bool equal_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) {
if (*(double *)minval == *(double *)maxval) {
return (*(double *)minval == pFilter->filterInfo.lowerBndd);
} else { /* range filter */
assert(*(double *)minval < *(double *)maxval);
return *(double *)minval <= pFilter->filterInfo.lowerBndi && *(double *)maxval >= pFilter->filterInfo.lowerBndi;
}
}
bool equal_str(SColumnFilterElem *pFilter, char *minval, char *maxval) {
// query condition string is greater than the max length of string, not qualified data
if (pFilter->filterInfo.len > pFilter->bytes) {
return false;
}
return strncmp((char *)pFilter->filterInfo.pz, minval, pFilter->bytes) == 0;
}
bool equal_nchar(SColumnFilterElem *pFilter, char *minval, char *maxval) {
// query condition string is greater than the max length of string, not qualified data
if (pFilter->filterInfo.len > pFilter->bytes) {
return false;
}
return wcsncmp((wchar_t *)pFilter->filterInfo.pz, (wchar_t*) minval, pFilter->bytes/TSDB_NCHAR_SIZE) == 0;
}
////////////////////////////////////////////////////////////////
bool like_str(SColumnFilterElem *pFilter, char *minval, char *maxval) {
SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER;
return patternMatch((char *)pFilter->filterInfo.pz, minval, pFilter->bytes, &info) == TSDB_PATTERN_MATCH;
}
bool like_nchar(SColumnFilterElem* pFilter, char* minval, char *maxval) {
SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER;
return WCSPatternMatch((wchar_t*) pFilter->filterInfo.pz, (wchar_t*) minval, pFilter->bytes/TSDB_NCHAR_SIZE, &info) == TSDB_PATTERN_MATCH;
}
////////////////////////////////////////////////////////////////
/**
* If minval equals to maxval, it may serve as the one element filter,
* or all elements of an array are identical during pref-filter stage.
* Otherwise, it must be pre-filter of array list of elements.
*
* During pre-filter stage, if there is one element that locates in [minval, maxval],
* the filter function will return true.
*/
bool nequal_i8(SColumnFilterElem *pFilter, char *minval, char *maxval) {
if (*(int8_t *)minval == *(int8_t *)maxval) {
return (*(int8_t *)minval != pFilter->filterInfo.lowerBndi);
}
return true;
}
bool nequal_i16(SColumnFilterElem *pFilter, char *minval, char *maxval) {
if (*(int16_t *)minval == *(int16_t *)maxval) {
return (*(int16_t *)minval != pFilter->filterInfo.lowerBndi);
}
return true;
}
bool nequal_i32(SColumnFilterElem *pFilter, char *minval, char *maxval) {
if (*(int32_t *)minval == *(int32_t *)maxval) {
return (*(int32_t *)minval != pFilter->filterInfo.lowerBndi);
}
return true;
}
bool nequal_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) {
if (*(int64_t *)minval == *(int64_t *)maxval) {
return (*(int64_t *)minval != pFilter->filterInfo.lowerBndi);
}
return true;
}
bool nequal_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) {
if (*(float *)minval == *(float *)maxval) {
return (*(float *)minval != pFilter->filterInfo.lowerBndd);
}
return true;
}
bool nequal_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) {
if (*(double *)minval == *(double *)maxval) {
return (*(double *)minval != pFilter->filterInfo.lowerBndd);
}
return true;
}
bool nequal_str(SColumnFilterElem *pFilter, char *minval, char *maxval) {
if (pFilter->filterInfo.len > pFilter->bytes) {
return true;
}
return strncmp((char *)pFilter->filterInfo.pz, minval, pFilter->bytes) != 0;
}
bool nequal_nchar(SColumnFilterElem *pFilter, char* minval, char *maxval) {
if (pFilter->filterInfo.len > pFilter->bytes) {
return true;
}
return wcsncmp((wchar_t *)pFilter->filterInfo.pz, (wchar_t*)minval, pFilter->bytes/TSDB_NCHAR_SIZE) != 0;
}
////////////////////////////////////////////////////////////////
bool rangeFilter_i32_ii(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int32_t *)minval <= pFilter->filterInfo.upperBndi && *(int32_t *)maxval >= pFilter->filterInfo.lowerBndi);
}
bool rangeFilter_i32_ee(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int32_t *)minval<pFilter->filterInfo.upperBndi &&*(int32_t *)maxval> pFilter->filterInfo.lowerBndi);
}
bool rangeFilter_i32_ie(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int32_t *)minval < pFilter->filterInfo.upperBndi && *(int32_t *)maxval >= pFilter->filterInfo.lowerBndi);
}
bool rangeFilter_i32_ei(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int32_t *)minval <= pFilter->filterInfo.upperBndi && *(int32_t *)maxval > pFilter->filterInfo.lowerBndi);
}
///////////////////////////////////////////////////////////////////////////////
bool rangeFilter_i8_ii(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int8_t *)minval <= pFilter->filterInfo.upperBndi && *(int8_t *)maxval >= pFilter->filterInfo.lowerBndi);
}
bool rangeFilter_i8_ee(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int8_t *)minval<pFilter->filterInfo.upperBndi &&*(int8_t *)maxval> pFilter->filterInfo.lowerBndi);
}
bool rangeFilter_i8_ie(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int8_t *)minval < pFilter->filterInfo.upperBndi && *(int8_t *)maxval >= pFilter->filterInfo.lowerBndi);
}
bool rangeFilter_i8_ei(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int8_t *)minval <= pFilter->filterInfo.upperBndi && *(int8_t *)maxval > pFilter->filterInfo.lowerBndi);
}
/////////////////////////////////////////////////////////////////////////////////////
bool rangeFilter_i16_ii(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int16_t *)minval <= pFilter->filterInfo.upperBndi && *(int16_t *)maxval >= pFilter->filterInfo.lowerBndi);
}
bool rangeFilter_i16_ee(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int16_t *)minval<pFilter->filterInfo.upperBndi &&*(int16_t *)maxval> pFilter->filterInfo.lowerBndi);
}
bool rangeFilter_i16_ie(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int16_t *)minval < pFilter->filterInfo.upperBndi && *(int16_t *)maxval >= pFilter->filterInfo.lowerBndi);
}
bool rangeFilter_i16_ei(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int16_t *)minval <= pFilter->filterInfo.upperBndi && *(int16_t *)maxval > pFilter->filterInfo.lowerBndi);
}
////////////////////////////////////////////////////////////////////////
bool rangeFilter_i64_ii(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int64_t *)minval <= pFilter->filterInfo.upperBndi && *(int64_t *)maxval >= pFilter->filterInfo.lowerBndi);
}
bool rangeFilter_i64_ee(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int64_t *)minval<pFilter->filterInfo.upperBndi &&*(int64_t *)maxval> pFilter->filterInfo.lowerBndi);
}
bool rangeFilter_i64_ie(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int64_t *)minval < pFilter->filterInfo.upperBndi && *(int64_t *)maxval >= pFilter->filterInfo.lowerBndi);
}
bool rangeFilter_i64_ei(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int64_t *)minval <= pFilter->filterInfo.upperBndi && *(int64_t *)maxval > pFilter->filterInfo.lowerBndi);
}
////////////////////////////////////////////////////////////////////////
bool rangeFilter_ds_ii(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(float *)minval <= pFilter->filterInfo.upperBndd && *(float *)maxval >= pFilter->filterInfo.lowerBndd);
}
bool rangeFilter_ds_ee(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(float *)minval<pFilter->filterInfo.upperBndd &&*(float *)maxval> pFilter->filterInfo.lowerBndd);
}
bool rangeFilter_ds_ie(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(float *)minval < pFilter->filterInfo.upperBndd && *(float *)maxval >= pFilter->filterInfo.lowerBndd);
}
bool rangeFilter_ds_ei(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(float *)minval <= pFilter->filterInfo.upperBndd && *(float *)maxval > pFilter->filterInfo.lowerBndd);
}
//////////////////////////////////////////////////////////////////////////
bool rangeFilter_dd_ii(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(double *)minval <= pFilter->filterInfo.upperBndd && *(double *)maxval >= pFilter->filterInfo.lowerBndd);
}
bool rangeFilter_dd_ee(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(double *)minval<pFilter->filterInfo.upperBndd &&*(double *)maxval> pFilter->filterInfo.lowerBndd);
}
bool rangeFilter_dd_ie(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(double *)minval < pFilter->filterInfo.upperBndd && *(double *)maxval >= pFilter->filterInfo.lowerBndd);
}
bool rangeFilter_dd_ei(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(double *)minval <= pFilter->filterInfo.upperBndd && *(double *)maxval > pFilter->filterInfo.lowerBndd);
}
////////////////////////////////////////////////////////////////////////////
bool (*filterFunc_i8[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = {
NULL,
less_i8,
large_i8,
equal_i8,
lessEqual_i8,
largeEqual_i8,
nequal_i8,
NULL,
};
bool (*filterFunc_i16[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = {
NULL,
less_i16,
large_i16,
equal_i16,
lessEqual_i16,
largeEqual_i16,
nequal_i16,
NULL,
};
bool (*filterFunc_i32[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = {
NULL,
less_i32,
large_i32,
equal_i32,
lessEqual_i32,
largeEqual_i32,
nequal_i32,
NULL,
};
bool (*filterFunc_i64[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = {
NULL,
less_i64,
large_i64,
equal_i64,
lessEqual_i64,
largeEqual_i64,
nequal_i64,
NULL,
};
bool (*filterFunc_ds[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = {
NULL,
less_ds,
large_ds,
equal_ds,
lessEqual_ds,
largeEqual_ds,
nequal_ds,
NULL,
};
bool (*filterFunc_dd[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = {
NULL,
less_dd,
large_dd,
equal_dd,
lessEqual_dd,
largeEqual_dd,
nequal_dd,
NULL,
};
bool (*filterFunc_str[])(SColumnFilterElem* pFilter, char* minval, char *maxval) = {
NULL,
NULL,
NULL,
equal_str,
NULL,
NULL,
nequal_str,
like_str,
};
bool (*filterFunc_nchar[])(SColumnFilterElem* pFitler, char* minval, char* maxval) = {
NULL,
NULL,
NULL,
equal_nchar,
NULL,
NULL,
nequal_nchar,
like_nchar,
};
bool (*rangeFilterFunc_i8[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = {
NULL,
rangeFilter_i8_ee,
rangeFilter_i8_ie,
rangeFilter_i8_ei,
rangeFilter_i8_ii,
};
bool (*rangeFilterFunc_i16[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = {
NULL,
rangeFilter_i16_ee,
rangeFilter_i16_ie,
rangeFilter_i16_ei,
rangeFilter_i16_ii,
};
bool (*rangeFilterFunc_i32[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = {
NULL,
rangeFilter_i32_ee,
rangeFilter_i32_ie,
rangeFilter_i32_ei,
rangeFilter_i32_ii,
};
bool (*rangeFilterFunc_i64[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = {
NULL,
rangeFilter_i64_ee,
rangeFilter_i64_ie,
rangeFilter_i64_ei,
rangeFilter_i64_ii,
};
bool (*rangeFilterFunc_ds[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = {
NULL,
rangeFilter_ds_ee,
rangeFilter_ds_ie,
rangeFilter_ds_ei,
rangeFilter_ds_ii,
};
bool (*rangeFilterFunc_dd[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = {
NULL,
rangeFilter_dd_ee,
rangeFilter_dd_ie,
rangeFilter_dd_ei,
rangeFilter_dd_ii,
};
__filter_func_t* vnodeGetRangeFilterFuncArray(int32_t type) {
switch(type) {
case TSDB_DATA_TYPE_BOOL: return rangeFilterFunc_i8;
case TSDB_DATA_TYPE_TINYINT: return rangeFilterFunc_i8;
case TSDB_DATA_TYPE_SMALLINT: return rangeFilterFunc_i16;
case TSDB_DATA_TYPE_INT: return rangeFilterFunc_i32;
case TSDB_DATA_TYPE_TIMESTAMP: //timestamp uses bigint filter
case TSDB_DATA_TYPE_BIGINT: return rangeFilterFunc_i64;
case TSDB_DATA_TYPE_FLOAT: return rangeFilterFunc_ds;
case TSDB_DATA_TYPE_DOUBLE: return rangeFilterFunc_dd;
default:return NULL;
}
}
__filter_func_t* vnodeGetValueFilterFuncArray(int32_t type) {
switch(type) {
case TSDB_DATA_TYPE_BOOL: return filterFunc_i8;
case TSDB_DATA_TYPE_TINYINT: return filterFunc_i8;
case TSDB_DATA_TYPE_SMALLINT: return filterFunc_i16;
case TSDB_DATA_TYPE_INT: return filterFunc_i32;
case TSDB_DATA_TYPE_TIMESTAMP: //timestamp uses bigint filter
case TSDB_DATA_TYPE_BIGINT: return filterFunc_i64;
case TSDB_DATA_TYPE_FLOAT: return filterFunc_ds;
case TSDB_DATA_TYPE_DOUBLE: return filterFunc_dd;
case TSDB_DATA_TYPE_BINARY: return filterFunc_str;
case TSDB_DATA_TYPE_NCHAR: return filterFunc_nchar;
default: return NULL;
}
}
bool vnodeSupportPrefilter(int32_t type) { return type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR; }

1573
src/vnode/detail/src/vnodeImport.c
View File

File diff suppressed because it is too large Load Diff

825
src/vnode/detail/src/vnodeMeter.c
View File

@ -1,825 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "trpc.h"
#include "tschemautil.h"
#include "ttime.h"
#include "tutil.h"
#include "vnode.h"
#include "vnodeShell.h"
#include "vnodeUtil.h"
#include "vnodeStatus.h"
#define VALID_TIMESTAMP(key, curKey, prec) (((key) >= 0) && ((key) <= ((curKey) + 36500 * tsMsPerDay[prec])))
int tsMeterSizeOnFile;
void vnodeUpdateMeter(void *param, void *tmdId);
void vnodeRecoverMeterObjectFile(int vnode);
int (*vnodeProcessAction[])(SMeterObj *, char *, int, char, void *, int, int *, TSKEY) = {vnodeInsertPoints,
vnodeImportPoints};
void vnodeFreeMeterObj(SMeterObj *pObj) {
if (pObj == NULL) return;
dTrace("vid:%d sid:%d id:%s, meter is cleaned up", pObj->vnode, pObj->sid, pObj->meterId);
vnodeFreeCacheInfo(pObj);
if (vnodeList[pObj->vnode].meterList != NULL) {
vnodeList[pObj->vnode].meterList[pObj->sid] = NULL;
}
memset(pObj->meterId, 0, tListLen(pObj->meterId));
tfree(pObj);
}
int vnodeUpdateVnodeStatistic(FILE *fp, SVnodeObj *pVnode) {
fseek(fp, TSDB_FILE_HEADER_VERSION_SIZE, SEEK_SET);
fwrite(&(pVnode->vnodeStatistic), sizeof(SVnodeStatisticInfo), 1, fp);
return 0;
}
void vnodeUpdateVnodeFileHeader(FILE *fp, SVnodeObj *pVnode) {
fseek(fp, TSDB_FILE_HEADER_LEN * 1 / 4, SEEK_SET);
#ifdef _TD_ARM_32_
fprintf(fp, "%lld %lld %lld ", pVnode->lastCreate, pVnode->lastRemove, pVnode->version);
fprintf(fp, "%lld %d %d ", pVnode->lastKeyOnFile, pVnode->fileId, pVnode->numOfFiles);
#else
fprintf(fp, "%ld %ld %ld ", pVnode->lastCreate, pVnode->lastRemove, pVnode->version);
fprintf(fp, "%ld %d %d ", pVnode->lastKeyOnFile, pVnode->fileId, pVnode->numOfFiles);
#endif
}
int vnodeCreateMeterObjFile(int vnode) {
FILE * fp;
char fileName[TSDB_FILENAME_LEN];
int32_t size;
// SMeterObj *pObj;
sprintf(fileName, "%s/vnode%d/meterObj.v%d", tsDirectory, vnode, vnode);
fp = fopen(fileName, "w+");
if (fp == NULL) {
dError("failed to create vnode:%d file:%s, errno:%d, reason:%s", vnode, fileName, errno, strerror(errno));
if (errno == EACCES) {
return TSDB_CODE_NO_DISK_PERMISSIONS;
} else if (errno == ENOSPC) {
return TSDB_CODE_SERV_NO_DISKSPACE;
} else {
return TSDB_CODE_VG_INIT_FAILED;
}
} else {
vnodeCreateFileHeader(fp);
vnodeUpdateVnodeFileHeader(fp, vnodeList + vnode);
fseek(fp, TSDB_FILE_HEADER_LEN, SEEK_SET);
size = sizeof(SMeterObjHeader) * vnodeList[vnode].cfg.maxSessions + sizeof(TSCKSUM);
tfree(vnodeList[vnode].meterIndex);
vnodeList[vnode].meterIndex = calloc(1, size);
taosCalcChecksumAppend(0, (uint8_t *)(vnodeList[vnode].meterIndex), size);
fwrite(vnodeList[vnode].meterIndex, size, 1, fp);
fclose(fp);
}
return TSDB_CODE_SUCCESS;
}
FILE *vnodeOpenMeterObjFile(int vnode) {
FILE * fp;
char fileName[TSDB_FILENAME_LEN];
struct stat fstat;
// check if directory exists
sprintf(fileName, "%s/vnode%d", tsDirectory, vnode);
if (stat(fileName, &fstat) < 0) return NULL;
sprintf(fileName, "%s/vnode%d/meterObj.v%d", tsDirectory, vnode, vnode);
if (stat(fileName, &fstat) < 0) return NULL;
fp = fopen(fileName, "r+");
if (fp != NULL) {
if (vnodeCheckFileIntegrity(fp) < 0) {
dError("file:%s is corrupted, need to restore it first, exit program", fileName);
fclose(fp);
// todo: how to recover
exit(1);
}
} else {
dError("failed to open %s, reason:%s", fileName, strerror(errno));
}
return fp;
}
int vnodeSaveMeterObjToFile(SMeterObj *pObj) {
int64_t offset, length, new_length, new_offset;
FILE * fp;
SVnodeObj *pVnode = &vnodeList[pObj->vnode];
char * buffer = NULL;
fp = vnodeOpenMeterObjFile(pObj->vnode);
if (fp == NULL) return -1;
buffer = (char *)malloc(tsMeterSizeOnFile);
if (buffer == NULL) {
dError("Failed to allocate memory while saving meter object to file, meterId", pObj->meterId);
fclose(fp);
return -1;
}
offset = pVnode->meterIndex[pObj->sid].offset;
length = pVnode->meterIndex[pObj->sid].length;
new_length = offsetof(SMeterObj, reserved) + pObj->numOfColumns * sizeof(SColumn) + pObj->sqlLen + sizeof(TSCKSUM);
memcpy(buffer, pObj, offsetof(SMeterObj, reserved));
memcpy(buffer + offsetof(SMeterObj, reserved), pObj->schema, pObj->numOfColumns * sizeof(SColumn));
memcpy(buffer + offsetof(SMeterObj, reserved) + pObj->numOfColumns * sizeof(SColumn), pObj->pSql, pObj->sqlLen);
taosCalcChecksumAppend(0, (uint8_t *)buffer, new_length);
if (offset == 0 || length < new_length) { // New, append to file end
fseek(fp, 0, SEEK_END);
new_offset = ftell(fp);
fwrite(buffer, new_length, 1, fp);
pVnode->meterIndex[pObj->sid].offset = new_offset;
pVnode->meterIndex[pObj->sid].length = new_length;
} else if (offset < 0) { // deleted meter, append to end of file
fseek(fp, -offset, SEEK_SET);
fwrite(buffer, new_length, 1, fp);
pVnode->meterIndex[pObj->sid].offset = -offset;
pVnode->meterIndex[pObj->sid].length = new_length;
} else { // meter exists, overwrite it, offset > 0
fseek(fp, offset, SEEK_SET);
fwrite(buffer, new_length, 1, fp);
pVnode->meterIndex[pObj->sid].offset = (pObj->meterId[0] == 0) ? -offset : offset;
pVnode->meterIndex[pObj->sid].length = new_length;
}
// taosCalcChecksumAppend(0, pVnode->meterIndex, sizeof(SMeterObjHeader)*pVnode->cfg.maxSessions+sizeof(TSCKSUM));
// NOTE: no checksum, since it makes creating table slow
fseek(fp, TSDB_FILE_HEADER_LEN + sizeof(SMeterObjHeader) * pObj->sid, SEEK_SET);
fwrite(&(pVnode->meterIndex[pObj->sid]), sizeof(SMeterObjHeader), 1, fp);
// update checksum
// fseek(fp, TSDB_FILE_HEADER_LEN+sizeof(SMeterObjHeader)*(pVnode->cfg.maxSessions), SEEK_SET);
// fwrite(((char *)(pVnode->meterIndex) + sizeof(SMeterObjHeader)*(pVnode->cfg.maxSessions)), sizeof(TSCKSUM), 1, fp);
tfree(buffer);
vnodeUpdateVnodeStatistic(fp, pVnode);
vnodeUpdateVnodeFileHeader(fp, pVnode);
/* vnodeUpdateFileCheckSum(fp); */
fclose(fp);
return 0;
}
int vnodeSaveAllMeterObjToFile(int vnode) {
int64_t offset, length, new_length, new_offset;
FILE * fp;
SMeterObj *pObj;
SVnodeObj *pVnode = &vnodeList[vnode];
char * buffer = NULL;
fp = vnodeOpenMeterObjFile(vnode);
if (fp == NULL) return -1;
buffer = (char *)malloc(tsMeterSizeOnFile);
if (buffer == NULL) {
dError("Failed to allocate memory while saving all meter objects to file");
return -1;
}
for (int sid = 0; sid < pVnode->cfg.maxSessions; ++sid) {
pObj = pVnode->meterList[sid];
if (pObj == NULL) continue;
offset = pVnode->meterIndex[sid].offset;
length = pVnode->meterIndex[sid].length;
new_length = offsetof(SMeterObj, reserved) + pObj->numOfColumns * sizeof(SColumn) + pObj->sqlLen + sizeof(TSCKSUM);
memcpy(buffer, pObj, offsetof(SMeterObj, reserved));
memcpy(buffer + offsetof(SMeterObj, reserved), pObj->schema, pObj->numOfColumns * sizeof(SColumn));
memcpy(buffer + offsetof(SMeterObj, reserved) + pObj->numOfColumns * sizeof(SColumn), pObj->pSql, pObj->sqlLen);
taosCalcChecksumAppend(0, (uint8_t *)buffer, new_length);
if (offset == 0 || length > new_length) { // New, append to file end
new_offset = fseek(fp, 0, SEEK_END);
fwrite(buffer, new_length, 1, fp);
pVnode->meterIndex[sid].offset = new_offset;
pVnode->meterIndex[sid].length = new_length;
} else if (offset < 0) { // deleted meter, append to end of file
fseek(fp, -offset, SEEK_SET);
fwrite(buffer, new_length, 1, fp);
pVnode->meterIndex[sid].offset = -offset;
pVnode->meterIndex[sid].length = new_length;
} else { // meter exists, overwrite it, offset > 0
fseek(fp, offset, SEEK_SET);
fwrite(buffer, new_length, 1, fp);
pVnode->meterIndex[sid].offset = offset;
pVnode->meterIndex[sid].length = new_length;
}
}
// taosCalcChecksumAppend(0, pVnode->meterIndex, sizeof(SMeterObjHeader)*pVnode->cfg.maxSessions+sizeof(TSCKSUM));
fseek(fp, TSDB_FILE_HEADER_LEN, SEEK_SET);
fwrite(pVnode->meterIndex, sizeof(SMeterObjHeader) * pVnode->cfg.maxSessions + sizeof(TSCKSUM), 1, fp);
tfree(buffer);
vnodeUpdateVnodeStatistic(fp, pVnode);
vnodeUpdateVnodeFileHeader(fp, pVnode);
/* vnodeUpdateFileCheckSum(fp); */
fclose(fp);
return 0;
}
int vnodeSaveVnodeCfg(int vnode, SVnodeCfg *pCfg, SVPeerDesc *pDesc) {
FILE *fp;
fp = vnodeOpenMeterObjFile(vnode);
if (fp == NULL) {
dError("failed to open vnode:%d file", vnode);
return -1;
}
fseek(fp, TSDB_FILE_HEADER_LEN * 2 / 4, SEEK_SET);
fwrite(pCfg, sizeof(SVnodeCfg), 1, fp);
char temp[TSDB_FILE_HEADER_LEN / 4];
memset(temp, 0, sizeof(temp));
fseek(fp, TSDB_FILE_HEADER_LEN * 3 / 4, SEEK_SET);
fwrite(temp, sizeof(temp), 1, fp);
if (pCfg->replications >= 1) {
fseek(fp, TSDB_FILE_HEADER_LEN * 3 / 4, SEEK_SET);
fwrite(pDesc, sizeof(SVPeerDesc), pCfg->replications, fp);
}
/* vnodeUpdateFileCheckSum(fp); */
fclose(fp);
return TSDB_CODE_SUCCESS;
}
int vnodeSaveVnodeInfo(int vnode) {
FILE * fp;
SVnodeObj *pVnode = &vnodeList[vnode];
fp = vnodeOpenMeterObjFile(vnode);
if (fp == NULL) return -1;
vnodeUpdateVnodeFileHeader(fp, pVnode);
/* vnodeUpdateFileCheckSum(fp); */
fclose(fp);
return 0;
}
int vnodeRestoreMeterObj(char *buffer, int64_t length) {
SMeterObj *pSavedObj, *pObj;
int size;
pSavedObj = (SMeterObj *)buffer;
if (pSavedObj->vnode < 0 || pSavedObj->vnode >= TSDB_MAX_VNODES) {
dTrace("vid:%d is out of range, corrupted meter obj file", pSavedObj->vnode);
return -1;
}
SVnodeCfg *pCfg = &vnodeList[pSavedObj->vnode].cfg;
if (pSavedObj->sid < 0 || pSavedObj->sid >= pCfg->maxSessions) {
dTrace("vid:%d, sid:%d is larger than max:%d", pSavedObj->vnode, pSavedObj->sid, pCfg->maxSessions);
return -1;
}
if (pSavedObj->meterId[0] == 0) return TSDB_CODE_SUCCESS;
size = sizeof(SMeterObj) + pSavedObj->sqlLen + 1;
pObj = (SMeterObj *)malloc(size);
if (pObj == NULL) {
dError("vid:%d sid:%d, no memory to allocate", pSavedObj->vnode, pSavedObj->sid);
return TSDB_CODE_SERV_OUT_OF_MEMORY;
}
pObj->schema = (SColumn *)malloc(pSavedObj->numOfColumns * sizeof(SColumn));
if (NULL == pObj->schema){
dError("vid:%d sid:%d, no memory to allocate for schema", pSavedObj->vnode, pSavedObj->sid);
free(pObj);
return TSDB_CODE_SERV_OUT_OF_MEMORY;
}
memcpy(pObj, pSavedObj, offsetof(SMeterObj, reserved));
pObj->numOfQueries = 0;
pObj->pCache = vnodeAllocateCacheInfo(pObj);
if (NULL == pObj->pCache){
dError("vid:%d sid:%d, no memory to allocate for cache", pSavedObj->vnode, pSavedObj->sid);
tfree(pObj->schema);
tfree(pObj);
return TSDB_CODE_SERV_OUT_OF_MEMORY;
}
vnodeList[pSavedObj->vnode].meterList[pSavedObj->sid] = pObj;
pObj->pStream = NULL;
memcpy(pObj->schema, buffer + offsetof(SMeterObj, reserved), pSavedObj->numOfColumns * sizeof(SColumn));
pObj->state = TSDB_METER_STATE_READY;
if (pObj->sqlLen > 0)
memcpy((char *)pObj + sizeof(SMeterObj),
((char *)pSavedObj) + offsetof(SMeterObj, reserved) + sizeof(SColumn) * pSavedObj->numOfColumns,
pSavedObj->sqlLen);
pObj->pSql = (char *)pObj + sizeof(SMeterObj);
pObj->lastKey = pObj->lastKeyOnFile;
if (pObj->lastKey > vnodeList[pObj->vnode].lastKey) vnodeList[pObj->vnode].lastKey = pObj->lastKey;
// taosSetSecurityInfo(pObj->vnode, pObj->sid, pObj->meterId, pObj->spi, pObj->encrypt, pObj->secret, pObj->cipheringKey);
dTrace("vid:%d sid:%d id:%s, meter is restored, uid:%" PRIu64 "", pObj->vnode, pObj->sid, pObj->meterId, pObj->uid);
return TSDB_CODE_SUCCESS;
}
int vnodeOpenMetersVnode(int vnode) {
FILE * fp;
char * buffer;
int64_t sid;
int64_t offset, length;
SVnodeObj *pVnode = &vnodeList[vnode];
fp = vnodeOpenMeterObjFile(vnode);
if (fp == NULL) return 0;
fseek(fp, TSDB_FILE_HEADER_VERSION_SIZE, SEEK_SET);
fread(&(pVnode->vnodeStatistic), sizeof(SVnodeStatisticInfo), 1, fp);
fseek(fp, TSDB_FILE_HEADER_LEN * 1 / 4, SEEK_SET);
#ifdef _TD_ARM_32_
fscanf(fp, "%lld %lld %lld ", &(pVnode->lastCreate), &(pVnode->lastRemove), &(pVnode->version));
fscanf(fp, "%lld %d %d ", &(pVnode->lastKeyOnFile), &(pVnode->fileId), &(pVnode->numOfFiles));
#else
fscanf(fp, "%ld %ld %ld ", &(pVnode->lastCreate), &(pVnode->lastRemove), &(pVnode->version));
fscanf(fp, "%ld %d %d ", &(pVnode->lastKeyOnFile), &(pVnode->fileId), &(pVnode->numOfFiles));
#endif
fseek(fp, TSDB_FILE_HEADER_LEN * 2 / 4, SEEK_SET);
fread(&pVnode->cfg, sizeof(SVnodeCfg), 1, fp);
if (vnodeIsValidVnodeCfg(&pVnode->cfg) == false) {
dError("vid:%d, maxSessions:%d cacheBlockSize:%d replications:%d daysPerFile:%d daysToKeep:%d invalid, clear it",
vnode, pVnode->cfg.maxSessions, pVnode->cfg.cacheBlockSize, pVnode->cfg.replications,
pVnode->cfg.daysPerFile, pVnode->cfg.daysToKeep);
pVnode->cfg.maxSessions = 0; // error in vnode file
return 0;
}
fseek(fp, TSDB_FILE_HEADER_LEN * 3 / 4, SEEK_SET);
fread(&pVnode->vpeers, sizeof(SVPeerDesc), TSDB_VNODES_SUPPORT, fp);
fseek(fp, TSDB_FILE_HEADER_LEN, SEEK_SET);
tsMeterSizeOnFile = sizeof(SMeterObj) + TSDB_MAX_COLUMNS * sizeof(SColumn) + TSDB_MAX_SAVED_SQL_LEN + sizeof(TSCKSUM);
int size = sizeof(SMeterObj *) * pVnode->cfg.maxSessions;
pVnode->meterList = (void *)malloc(size);
if (pVnode->meterList == NULL) return -1;
memset(pVnode->meterList, 0, size);
size = sizeof(SMeterObjHeader) * pVnode->cfg.maxSessions + sizeof(TSCKSUM);
pVnode->meterIndex = (SMeterObjHeader *)calloc(1, size);
if (pVnode->meterIndex == NULL) {
tfree(pVnode->meterList);
return -1;
}
// Read SMeterObjHeader list from file
if (fread(pVnode->meterIndex, size, 1, fp) < 0) return -1;
// if (!taosCheckChecksumWhole(pVnode->meterIndex, size)) {
// dError("vid: %d meter obj file header is broken since checksum mismatch", vnode);
// return -1;
// }
// Read the meter object from file and recover the structure
buffer = malloc(tsMeterSizeOnFile);
memset(buffer, 0, tsMeterSizeOnFile);
for (sid = 0; sid < pVnode->cfg.maxSessions; ++sid) {
offset = pVnode->meterIndex[sid].offset;
length = pVnode->meterIndex[sid].length;
if (offset <= 0 || length <= 0) continue;
fseek(fp, offset, SEEK_SET);
if (fread(buffer, length, 1, fp) <= 0) break;
if (taosCheckChecksumWhole((uint8_t *)buffer, length)) {
vnodeRestoreMeterObj(buffer, length - sizeof(TSCKSUM));
} else {
dError("meter object file is broken since checksum mismatch, vnode: %d sid: %d, try to recover", vnode, sid);
continue;
/* vnodeRecoverMeterObjectFile(vnode); */
}
}
tfree(buffer);
fclose(fp);
return 0;
}
void vnodeCloseMetersVnode(int vnode) {
SVnodeObj *pVnode = vnodeList + vnode;
SMeterObj *pObj;
if (pVnode->meterList) {
for (int sid = 0; sid < pVnode->cfg.maxSessions; ++sid) {
pObj = pVnode->meterList[sid];
if (pObj == NULL) continue;
vnodeFreeCacheInfo(pObj);
tfree(pObj->schema);
tfree(pObj);
}
tfree(pVnode->meterList);
}
pVnode->meterList = NULL;
}
int vnodeCreateMeterObj(SMeterObj *pNew, SConnSec *pSec) {
SMeterObj *pObj;
int code;
pObj = vnodeList[pNew->vnode].meterList[pNew->sid];
code = TSDB_CODE_SUCCESS;
if (pObj && pObj->uid == pNew->uid) {
if (pObj->sversion == pNew->sversion) {
dTrace("vid:%d sid:%d id:%s sversion:%d, identical meterObj, ignore create", pNew->vnode, pNew->sid,
pNew->meterId, pNew->sversion);
return -1;
}
dTrace("vid:%d sid:%d id:%s, update schema", pNew->vnode, pNew->sid, pNew->meterId);
if (!vnodeIsMeterState(pObj, TSDB_METER_STATE_UPDATING)) vnodeUpdateMeter(pNew, NULL);
return TSDB_CODE_SUCCESS;
}
if (pObj) {
dWarn("vid:%d sid:%d id:%s, old meter is there, remove it", pNew->vnode, pNew->sid, pNew->meterId);
vnodeRemoveMeterObj(pNew->vnode, pNew->sid);
}
pNew->pCache = vnodeAllocateCacheInfo(pNew);
if (pNew->pCache == NULL) {
code = TSDB_CODE_NO_RESOURCE;
} else {
vnodeList[pNew->vnode].meterList[pNew->sid] = pNew;
pNew->state = TSDB_METER_STATE_READY;
if (pNew->timeStamp > vnodeList[pNew->vnode].lastCreate) vnodeList[pNew->vnode].lastCreate = pNew->timeStamp;
vnodeSaveMeterObjToFile(pNew);
// vnodeCreateMeterMgmt(pNew, pSec);
vnodeCreateStream(pNew);
dTrace("vid:%d, sid:%d id:%s, meterObj is created, uid:%" PRIu64 "", pNew->vnode, pNew->sid, pNew->meterId, pNew->uid);
}
return code;
}
int vnodeRemoveMeterObj(int vnode, int sid) {
SMeterObj *pObj;
if (vnode < 0 || vnode >= TSDB_MAX_VNODES) {
dError("vid:%d is out of range", vnode);
return 0;
}
SVnodeCfg *pCfg = &vnodeList[vnode].cfg;
if (sid < 0 || sid >= pCfg->maxSessions) {
dError("vid:%d, sid:%d is larger than max:%d or less than 0", vnode, sid, pCfg->maxSessions);
return 0;
}
// vnode has been closed, no meters in this vnode
if (vnodeList[vnode].meterList == NULL) return 0;
pObj = vnodeList[vnode].meterList[sid];
if (pObj == NULL) {
return TSDB_CODE_SUCCESS;
}
if (!vnodeIsSafeToDeleteMeter(&vnodeList[vnode], sid)) {
return TSDB_CODE_ACTION_IN_PROGRESS;
}
// after remove this meter, change its state to DELETED
pObj->state = TSDB_METER_STATE_DROPPED;
pObj->timeStamp = taosGetTimestampMs();
vnodeList[vnode].lastRemove = pObj->timeStamp;
vnodeRemoveStream(pObj);
vnodeSaveMeterObjToFile(pObj);
vnodeFreeMeterObj(pObj);
return 0;
}
int vnodeInsertPoints(SMeterObj *pObj, char *cont, int contLen, char source, void *param, int sversion,
int *numOfInsertPoints, TSKEY now) {
int expectedLen, i;
short numOfPoints;
SSubmitMsg *pSubmit = (SSubmitMsg *)cont;
char * pData;
TSKEY tsKey;
int points = 0;
int code = TSDB_CODE_SUCCESS;
SVnodeObj * pVnode = vnodeList + pObj->vnode;
numOfPoints = htons(pSubmit->numOfRows);
expectedLen = numOfPoints * pObj->bytesPerPoint + sizeof(pSubmit->numOfRows);
if (expectedLen != contLen) {
dError("vid:%d sid:%d id:%s, invalid submit msg length:%d, expected:%d, bytesPerPoint: %d",
pObj->vnode, pObj->sid, pObj->meterId, contLen, expectedLen, pObj->bytesPerPoint);
code = TSDB_CODE_WRONG_MSG_SIZE;
goto _over;
}
// to guarantee time stamp is the same for all vnodes
pData = pSubmit->payLoad;
tsKey = now;
if (*((TSKEY *)pData) == 0) {
for (i = 0; i < numOfPoints; ++i) {
*((TSKEY *)pData) = tsKey++;
pData += pObj->bytesPerPoint;
}
}
if (numOfPoints >= (pVnode->cfg.blocksPerMeter - 2) * pObj->pointsPerBlock) {
code = TSDB_CODE_BATCH_SIZE_TOO_BIG;
dError("vid:%d sid:%d id:%s, batch size too big, insert points:%d, it shall be smaller than:%d", pObj->vnode, pObj->sid,
pObj->meterId, numOfPoints, (pVnode->cfg.blocksPerMeter - 2) * pObj->pointsPerBlock);
return code;
}
/*
* please refer to TBASE-926, data may be lost when the cache is full
*/
if (source == TSDB_DATA_SOURCE_SHELL && pVnode->cfg.replications > 1) {
code = vnodeForwardToPeer(pObj, cont, contLen, TSDB_ACTION_INSERT, sversion);
if (code != TSDB_CODE_SUCCESS) return code;
}
SCachePool *pPool = (SCachePool *)pVnode->pCachePool;
if (pObj->freePoints < numOfPoints || pObj->freePoints < (pObj->pointsPerBlock << 1) ||
pPool->notFreeSlots > pVnode->cfg.cacheNumOfBlocks.totalBlocks - 2) {
code = TSDB_CODE_ACTION_IN_PROGRESS;
dTrace("vid:%d sid:%d id:%s, cache is full, freePoints:%d, notFreeSlots:%d", pObj->vnode, pObj->sid, pObj->meterId,
pObj->freePoints, pPool->notFreeSlots);
vnodeProcessCommitTimer(pVnode, NULL);
return code;
}
// FIXME: Here should be after the comparison of sversions.
if (pVnode->cfg.commitLog && source != TSDB_DATA_SOURCE_LOG) {
if (pVnode->logFd < 0) return TSDB_CODE_INVALID_COMMIT_LOG;
code = vnodeWriteToCommitLog(pObj, TSDB_ACTION_INSERT, cont, contLen, sversion);
if (code != TSDB_CODE_SUCCESS) return code;
}
if (pObj->sversion < sversion) {
dTrace("vid:%d sid:%d id:%s, schema is changed, new:%d old:%d", pObj->vnode, pObj->sid, pObj->meterId, sversion,
pObj->sversion);
vnodeSendMeterCfgMsg(pObj->vnode, pObj->sid);
code = TSDB_CODE_ACTION_IN_PROGRESS;
return code;
} else if (pObj->sversion > sversion) {
dTrace("vid:%d sid:%d id:%s, client schema out of date, sql is invalid. client sversion:%d vnode sversion:%d",
pObj->vnode, pObj->sid, pObj->meterId, pObj->sversion, sversion);
code = TSDB_CODE_INVALID_SQL;
return code;
}
pData = pSubmit->payLoad;
TSKEY firstKey = *((TSKEY *)pData);
TSKEY lastKey = *((TSKEY *)(pData + pObj->bytesPerPoint * (numOfPoints - 1)));
int cfid = now/pVnode->cfg.daysPerFile/tsMsPerDay[(uint8_t)pVnode->cfg.precision];
TSKEY minAllowedKey = (cfid - pVnode->maxFiles + 1)*pVnode->cfg.daysPerFile*tsMsPerDay[(uint8_t)pVnode->cfg.precision];
TSKEY maxAllowedKey = (cfid + 2)*pVnode->cfg.daysPerFile*tsMsPerDay[(uint8_t)pVnode->cfg.precision] - 2;
if (firstKey < minAllowedKey || firstKey > maxAllowedKey || lastKey < minAllowedKey || lastKey > maxAllowedKey) {
dError("vid:%d sid:%d id:%s, vnode lastKeyOnFile:%" PRId64 ", data is out of range, numOfPoints:%d firstKey:%" PRId64 " lastKey:%" PRId64 " minAllowedKey:%" PRId64 " maxAllowedKey:%" PRId64,
pObj->vnode, pObj->sid, pObj->meterId, pVnode->lastKeyOnFile, numOfPoints,firstKey, lastKey, minAllowedKey, maxAllowedKey);
return TSDB_CODE_TIMESTAMP_OUT_OF_RANGE;
}
if ((code = vnodeSetMeterInsertImportStateEx(pObj, TSDB_METER_STATE_INSERTING)) != TSDB_CODE_SUCCESS) {
goto _over;
}
for (i = 0; i < numOfPoints; ++i) { // meter will be dropped, abort current insertion
if (vnodeIsMeterState(pObj, TSDB_METER_STATE_DROPPING)) {
dWarn("vid:%d sid:%d id:%s, meter is dropped, abort insert, state:%d", pObj->vnode, pObj->sid, pObj->meterId,
pObj->state);
code = TSDB_CODE_NOT_ACTIVE_TABLE;
break;
}
if (*((TSKEY *)pData) <= pObj->lastKey) {
dWarn("vid:%d sid:%d id:%s, received key:%" PRId64 " not larger than lastKey:%" PRId64, pObj->vnode, pObj->sid, pObj->meterId,
*((TSKEY *)pData), pObj->lastKey);
pData += pObj->bytesPerPoint;
continue;
}
if (!VALID_TIMESTAMP(*((TSKEY *)pData), tsKey, (uint8_t)pVnode->cfg.precision)) {
code = TSDB_CODE_TIMESTAMP_OUT_OF_RANGE;
break;
}
if (vnodeInsertPointToCache(pObj, pData) < 0) {
code = TSDB_CODE_ACTION_IN_PROGRESS;
break;
}
pObj->lastKey = *((TSKEY *)pData);
pData += pObj->bytesPerPoint;
points++;
}
atomic_fetch_add_64(&(pVnode->vnodeStatistic.pointsWritten), points * (pObj->numOfColumns - 1));
atomic_fetch_add_64(&(pVnode->vnodeStatistic.totalStorage), points * pObj->bytesPerPoint);
pthread_mutex_lock(&(pVnode->vmutex));
if (pObj->lastKey > pVnode->lastKey) pVnode->lastKey = pObj->lastKey;
if (firstKey < pVnode->firstKey) pVnode->firstKey = firstKey;
assert(pVnode->firstKey > 0);
pVnode->version++;
pthread_mutex_unlock(&(pVnode->vmutex));
vnodeClearMeterState(pObj, TSDB_METER_STATE_INSERTING);
_over:
dTrace("vid:%d sid:%d id:%s, %d out of %d points are inserted, lastKey:%" PRId64 " source:%d, vnode total storage: %" PRId64 "",
pObj->vnode, pObj->sid, pObj->meterId, points, numOfPoints, pObj->lastKey, source,
pVnode->vnodeStatistic.totalStorage);
*numOfInsertPoints = points;
return code;
}
/**
* continue running of the function may cause the free vnode crash with high probability
* todo fix it by set flag to disable commit in any cases
*
* @param param
* @param tmrId
*/
void vnodeProcessUpdateSchemaTimer(void *param, void *tmrId) {
SMeterObj * pObj = (SMeterObj *)param;
SVnodeObj * pVnode = vnodeList + pObj->vnode;
/*
* vnode may have been dropped, check it in the first place
* if the vnode is freed, the pObj is not valid any more, the pObj->vnode is meanless
* so may be the vid should be passed into this function as a parameter?
*/
if (pVnode->meterList == NULL) {
dTrace("vnode is deleted, abort update schema");
return;
}
SCachePool *pPool = (SCachePool *)pVnode->pCachePool;
pthread_mutex_lock(&pPool->vmutex);
if (pPool->commitInProcess) {
dTrace("vid:%d sid:%d mid:%s, committing in process, commit later", pObj->vnode, pObj->sid, pObj->meterId);
if (taosTmrStart(vnodeProcessUpdateSchemaTimer, 10, pObj, vnodeTmrCtrl) == NULL) {
vnodeClearMeterState(pObj, TSDB_METER_STATE_UPDATING);
}
pthread_mutex_unlock(&pPool->vmutex);
return;
}
pPool->commitInProcess = 1;
pthread_mutex_unlock(&pPool->vmutex);
vnodeCommitMultiToFile(pVnode, pObj->sid, pObj->sid);
}
void vnodeUpdateMeter(void *param, void *tmrId) {
SMeterObj *pNew = (SMeterObj *)param;
if (pNew == NULL || pNew->vnode < 0 || pNew->sid < 0) return;
SVnodeObj* pVnode = &vnodeList[pNew->vnode];
if (pVnode->meterList == NULL) {
dTrace("vid:%d sid:%d id:%s, vnode is deleted, status:%s, abort update schema",
pNew->vnode, pNew->sid, pNew->meterId, taosGetVnodeStatusStr(vnodeList[pNew->vnode].vnodeStatus));
free(pNew->schema);
free(pNew);
return;
}
SMeterObj *pObj = pVnode->meterList[pNew->sid];
if (pObj == NULL || vnodeIsMeterState(pObj, TSDB_METER_STATE_DROPPING)) {
dTrace("vid:%d sid:%d id:%s, meter is deleted, abort update schema", pNew->vnode, pNew->sid, pNew->meterId);
free(pNew->schema);
free(pNew);
return;
}
int32_t state = vnodeSetMeterState(pObj, TSDB_METER_STATE_UPDATING);
if (state >= TSDB_METER_STATE_DROPPING) {
dError("vid:%d sid:%d id:%s, meter is deleted, failed to update, state:%d",
pObj->vnode, pObj->sid, pObj->meterId, state);
return;
}
int32_t num = 0;
pthread_mutex_lock(&pVnode->vmutex);
num = pObj->numOfQueries;
pthread_mutex_unlock(&pVnode->vmutex);
if (num > 0 || state != TSDB_METER_STATE_READY) {
// the state may have been changed by vnodeSetMeterState, recover it in the first place
vnodeClearMeterState(pObj, TSDB_METER_STATE_UPDATING);
dTrace("vid:%d sid:%d id:%s, update failed, retry later, numOfQueries:%d, state:%d",
pNew->vnode, pNew->sid, pNew->meterId, num, state);
// retry update meter in 50ms
if (taosTmrStart(vnodeUpdateMeter, 50, pNew, vnodeTmrCtrl) == NULL) {
dError("vid:%d sid:%d id:%s, failed to start update timer, no retry", pNew->vnode, pNew->sid, pNew->meterId);
free(pNew->schema);
free(pNew);
}
return;
}
// commit first
if (!vnodeIsCacheCommitted(pObj)) {
// commit data first
if (taosTmrStart(vnodeProcessUpdateSchemaTimer, 0, pObj, vnodeTmrCtrl) == NULL) {
dError("vid:%d sid:%d id:%s, failed to start commit timer", pObj->vnode, pObj->sid, pObj->meterId);
vnodeClearMeterState(pObj, TSDB_METER_STATE_UPDATING);
free(pNew->schema);
free(pNew);
return;
}
if (taosTmrStart(vnodeUpdateMeter, 50, pNew, vnodeTmrCtrl) == NULL) {
dError("vid:%d sid:%d id:%s, failed to start update timer", pNew->vnode, pNew->sid, pNew->meterId);
vnodeClearMeterState(pObj, TSDB_METER_STATE_UPDATING);
free(pNew->schema);
free(pNew);
}
dTrace("vid:%d sid:%d meterId:%s, there are data in cache, commit first, update later",
pNew->vnode, pNew->sid, pNew->meterId);
vnodeClearMeterState(pObj, TSDB_METER_STATE_UPDATING);
return;
}
strcpy(pObj->meterId, pNew->meterId);
pObj->numOfColumns = pNew->numOfColumns;
pObj->timeStamp = pNew->timeStamp;
pObj->bytesPerPoint = pNew->bytesPerPoint;
pObj->maxBytes = pNew->maxBytes;
if (pObj->timeStamp > vnodeList[pObj->vnode].lastCreate) vnodeList[pObj->vnode].lastCreate = pObj->timeStamp;
tfree(pObj->schema);
pObj->schema = pNew->schema;
vnodeFreeCacheInfo(pObj);
pObj->pCache = vnodeAllocateCacheInfo(pObj);
pObj->sversion = pNew->sversion;
vnodeSaveMeterObjToFile(pObj);
vnodeClearMeterState(pObj, TSDB_METER_STATE_UPDATING);
dTrace("vid:%d sid:%d id:%s, schema is updated, state:%d", pObj->vnode, pObj->sid, pObj->meterId, pObj->state);
free(pNew);
}
void vnodeRecoverMeterObjectFile(int vnode) {
// TODO: start the recovery process
assert(0);
}

7714
src/vnode/detail/src/vnodeQueryImpl.c
View File

File diff suppressed because it is too large Load Diff

1364
src/vnode/detail/src/vnodeQueryProcess.c
View File

File diff suppressed because it is too large Load Diff

1153
src/vnode/detail/src/vnodeRead.c
View File

File diff suppressed because it is too large Load Diff

409
src/vnode/detail/src/vnodeStore.c
View File

@ -1,409 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "dnodeSystem.h"
#include "trpc.h"
#include "ttime.h"
#include "vnode.h"
#include "vnodeStore.h"
#include "vnodeUtil.h"
#include "vnodeStatus.h"
int tsMaxVnode = -1;
int tsOpenVnodes = 0;
SVnodeObj *vnodeList = NULL;
static int vnodeInitStoreVnode(int vnode) {
SVnodeObj *pVnode = vnodeList + vnode;
pVnode->vnode = vnode;
vnodeOpenMetersVnode(vnode);
if (pVnode->cfg.maxSessions <= 0) {
return TSDB_CODE_SUCCESS;
}
pVnode->firstKey = taosGetTimestamp(pVnode->cfg.precision);
pVnode->pCachePool = vnodeOpenCachePool(vnode);
if (pVnode->pCachePool == NULL) {
dError("vid:%d, cache pool init failed.", pVnode->vnode);
return TSDB_CODE_SERV_OUT_OF_MEMORY;
}
if (vnodeInitFile(vnode) != TSDB_CODE_SUCCESS) {
dError("vid:%d, files init failed.", pVnode->vnode);
return TSDB_CODE_VG_INIT_FAILED;
}
if (vnodeInitCommit(vnode) != TSDB_CODE_SUCCESS) {
dError("vid:%d, commit init failed.", pVnode->vnode);
return TSDB_CODE_VG_INIT_FAILED;
}
pthread_mutex_init(&(pVnode->vmutex), NULL);
dPrint("vid:%d, storage initialized, version:%" PRIu64 " fileId:%d numOfFiles:%d", vnode, pVnode->version, pVnode->fileId,
pVnode->numOfFiles);
return TSDB_CODE_SUCCESS;
}
int vnodeOpenVnode(int vnode) {
int32_t code = TSDB_CODE_SUCCESS;
SVnodeObj *pVnode = vnodeList + vnode;
pVnode->vnode = vnode;
pVnode->accessState = TSDB_VN_ALL_ACCCESS;
// vnode is empty
if (pVnode->cfg.maxSessions <= 0) {
return TSDB_CODE_SUCCESS;
}
if (!(pVnode->vnodeStatus == TSDB_VN_STATUS_OFFLINE || pVnode->vnodeStatus == TSDB_VN_STATUS_CREATING)) {
dError("vid:%d, status:%s, cannot enter open operation", vnode, taosGetVnodeStatusStr(pVnode->vnodeStatus));
return TSDB_CODE_INVALID_VNODE_STATUS;
}
dPrint("vid:%d, status:%s, start to open", vnode, taosGetVnodeStatusStr(pVnode->vnodeStatus));
pthread_mutex_lock(&dmutex);
// not enough memory, abort
if ((code = vnodeOpenShellVnode(vnode)) != TSDB_CODE_SUCCESS) {
pthread_mutex_unlock(&dmutex);
return code;
}
vnodeOpenPeerVnode(vnode);
if (vnode > tsMaxVnode) tsMaxVnode = vnode;
vnodeCalcOpenVnodes();
pthread_mutex_unlock(&dmutex);
#ifndef CLUSTER
vnodeOpenStreams(pVnode, NULL);
#endif
dPrint("vid:%d, vnode is opened, openVnodes:%d, status:%s", vnode, tsOpenVnodes, taosGetVnodeStatusStr(pVnode->vnodeStatus));
return TSDB_CODE_SUCCESS;
}
static int32_t vnodeMarkAllMetersDropped(SVnodeObj* pVnode) {
if (pVnode->meterList == NULL) {
return TSDB_CODE_SUCCESS;
}
bool ready = true;
for (int sid = 0; sid < pVnode->cfg.maxSessions; ++sid) {
if (!vnodeIsSafeToDeleteMeter(pVnode, sid)) {
ready = false;
} else { // set the meter is to be deleted
SMeterObj* pObj = pVnode->meterList[sid];
if (pObj != NULL) {
pObj->state = TSDB_METER_STATE_DROPPED;
}
}
}
return ready? TSDB_CODE_SUCCESS:TSDB_CODE_ACTION_IN_PROGRESS;
}
static int vnodeCloseVnode(int vnode) {
if (vnodeList == NULL) return TSDB_CODE_SUCCESS;
SVnodeObj* pVnode = &vnodeList[vnode];
pthread_mutex_lock(&dmutex);
if (pVnode->cfg.maxSessions == 0) {
pthread_mutex_unlock(&dmutex);
return TSDB_CODE_SUCCESS;
}
if (pVnode->vnodeStatus == TSDB_VN_STATUS_DELETING) {
dPrint("vid:%d, status:%s, another thread performed delete operation", vnode, taosGetVnodeStatusStr(pVnode->vnodeStatus));
return TSDB_CODE_SUCCESS;
} else {
dPrint("vid:%d, status:%s, enter close operation", vnode, taosGetVnodeStatusStr(pVnode->vnodeStatus));
pVnode->vnodeStatus = TSDB_VN_STATUS_CLOSING;
}
// set the meter is dropped flag
if (vnodeMarkAllMetersDropped(pVnode) != TSDB_CODE_SUCCESS) {
pthread_mutex_unlock(&dmutex);
return TSDB_CODE_ACTION_IN_PROGRESS;
}
dPrint("vid:%d, status:%s, enter delete operation", vnode, taosGetVnodeStatusStr(pVnode->vnodeStatus));
pVnode->vnodeStatus = TSDB_VN_STATUS_DELETING;
vnodeCloseStream(vnodeList + vnode);
vnodeCancelCommit(vnodeList + vnode);
vnodeClosePeerVnode(vnode);
vnodeCloseMetersVnode(vnode);
vnodeCloseShellVnode(vnode);
vnodeCloseCachePool(vnode);
vnodeCleanUpCommit(vnode);
pthread_mutex_destroy(&(vnodeList[vnode].vmutex));
if (tsMaxVnode == vnode) tsMaxVnode = vnode - 1;
tfree(vnodeList[vnode].meterIndex);
pthread_mutex_unlock(&dmutex);
return TSDB_CODE_SUCCESS;
}
int vnodeCreateVnode(int vnode, SVnodeCfg *pCfg, SVPeerDesc *pDesc) {
char fileName[128];
if (vnodeList[vnode].vnodeStatus != TSDB_VN_STATUS_OFFLINE) {
dError("vid:%d, status:%s, cannot enter create operation", vnode, taosGetVnodeStatusStr(vnodeList[vnode].vnodeStatus));
return TSDB_CODE_INVALID_VNODE_STATUS;
}
vnodeList[vnode].vnodeStatus = TSDB_VN_STATUS_CREATING;
sprintf(fileName, "%s/vnode%d", tsDirectory, vnode);
if (mkdir(fileName, 0755) != 0) {
dError("failed to create vnode:%d directory:%s, errno:%d, reason:%s", vnode, fileName, errno, strerror(errno));
if (errno == EACCES) {
return TSDB_CODE_NO_DISK_PERMISSIONS;
} else if (errno == ENOSPC) {
return TSDB_CODE_SERV_NO_DISKSPACE;
} else if (errno == EEXIST) {
} else {
return TSDB_CODE_VG_INIT_FAILED;
}
}
sprintf(fileName, "%s/vnode%d/db", tsDirectory, vnode);
if (mkdir(fileName, 0755) != 0) {
dError("failed to create vnode:%d directory:%s, errno:%d, reason:%s", vnode, fileName, errno, strerror(errno));
if (errno == EACCES) {
return TSDB_CODE_NO_DISK_PERMISSIONS;
} else if (errno == ENOSPC) {
return TSDB_CODE_SERV_NO_DISKSPACE;
} else if (errno == EEXIST) {
} else {
return TSDB_CODE_VG_INIT_FAILED;
}
}
vnodeList[vnode].cfg = *pCfg;
int code = vnodeCreateMeterObjFile(vnode);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
code = vnodeSaveVnodeCfg(vnode, pCfg, pDesc);
if (code != TSDB_CODE_SUCCESS) {
return TSDB_CODE_VG_INIT_FAILED;
}
code = vnodeInitStoreVnode(vnode);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
return vnodeOpenVnode(vnode);
}
static void vnodeRemoveDataFiles(int vnode) {
char vnodeDir[TSDB_FILENAME_LEN];
char dfilePath[TSDB_FILENAME_LEN];
char linkFile[TSDB_FILENAME_LEN];
struct dirent *de = NULL;
DIR * dir = NULL;
sprintf(vnodeDir, "%s/vnode%d/db", tsDirectory, vnode);
dir = opendir(vnodeDir);
if (dir == NULL) return;
while ((de = readdir(dir)) != NULL) {
if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue;
if ((strcmp(de->d_name + strlen(de->d_name) - strlen(".head"), ".head") == 0 ||
strcmp(de->d_name + strlen(de->d_name) - strlen(".data"), ".data") == 0 ||
strcmp(de->d_name + strlen(de->d_name) - strlen(".last"), ".last") == 0) &&
(de->d_type & DT_LNK)) {
sprintf(linkFile, "%s/%s", vnodeDir, de->d_name);
if (!vnodeRemoveDataFileFromLinkFile(linkFile, de->d_name)) {
continue;
}
memset(dfilePath, 0, TSDB_FILENAME_LEN);
int tcode = readlink(linkFile, dfilePath, TSDB_FILENAME_LEN);
remove(linkFile);
if (tcode >= 0) {
remove(dfilePath);
dPrint("Data file %s is removed, link file %s", dfilePath, linkFile);
}
} else {
remove(de->d_name);
}
}
closedir(dir);
rmdir(vnodeDir);
sprintf(vnodeDir, "%s/vnode%d/meterObj.v%d", tsDirectory, vnode, vnode);
remove(vnodeDir);
sprintf(vnodeDir, "%s/vnode%d", tsDirectory, vnode);
rmdir(vnodeDir);
dPrint("vid:%d, vnode is removed, status:%s", vnode, taosGetVnodeStatusStr(vnodeList[vnode].vnodeStatus));
}
int vnodeRemoveVnode(int vnode) {
if (vnodeList == NULL) return TSDB_CODE_SUCCESS;
if (vnodeList[vnode].cfg.maxSessions > 0) {
SVnodeObj* pVnode = &vnodeList[vnode];
if (pVnode->vnodeStatus == TSDB_VN_STATUS_CREATING
|| pVnode->vnodeStatus == TSDB_VN_STATUS_OFFLINE
|| pVnode->vnodeStatus == TSDB_VN_STATUS_DELETING) {
dTrace("vid:%d, status:%s, cannot enter close/delete operation", vnode, taosGetVnodeStatusStr(pVnode->vnodeStatus));
return TSDB_CODE_ACTION_IN_PROGRESS;
} else {
int32_t ret = vnodeCloseVnode(vnode);
if (ret != TSDB_CODE_SUCCESS) {
return ret;
}
dTrace("vid:%d, status:%s, do delete operation", vnode, taosGetVnodeStatusStr(pVnode->vnodeStatus));
vnodeRemoveDataFiles(vnode);
}
} else {
dPrint("vid:%d, max sessions:%d, this vnode already dropped!!!", vnode, vnodeList[vnode].cfg.maxSessions);
vnodeList[vnode].cfg.maxSessions = 0; //reset value
vnodeCalcOpenVnodes();
}
return TSDB_CODE_SUCCESS;
}
int vnodeInitStore() {
int vnode;
int size;
size = sizeof(SVnodeObj) * TSDB_MAX_VNODES;
vnodeList = (SVnodeObj *)malloc(size);
if (vnodeList == NULL) return -1;
memset(vnodeList, 0, size);
if (vnodeInitInfo() < 0) return -1;
for (vnode = 0; vnode < TSDB_MAX_VNODES; ++vnode) {
int code = vnodeInitStoreVnode(vnode);
if (code != TSDB_CODE_SUCCESS) {
// one vnode is failed to recover from commit log, continue for remain
return -1;
}
}
return 0;
}
int vnodeInitVnodes() {
int vnode;
for (vnode = 0; vnode < TSDB_MAX_VNODES; ++vnode) {
if (vnodeOpenVnode(vnode) < 0) return -1;
}
return 0;
}
void vnodeCleanUpOneVnode(int vnode) {
static int again = 0;
if (vnodeList == NULL) return;
pthread_mutex_lock(&dmutex);
if (again) {
pthread_mutex_unlock(&dmutex);
return;
}
again = 1;
if (vnodeList[vnode].pCachePool) {
vnodeList[vnode].vnodeStatus = TSDB_VN_STATUS_OFFLINE;
vnodeClosePeerVnode(vnode);
}
pthread_mutex_unlock(&dmutex);
if (vnodeList[vnode].pCachePool) {
vnodeProcessCommitTimer(vnodeList + vnode, NULL);
while (vnodeList[vnode].commitThread != 0) {
taosMsleep(10);
}
vnodeCleanUpCommit(vnode);
}
}
void vnodeCleanUpVnodes() {
static int again = 0;
if (vnodeList == NULL) return;
pthread_mutex_lock(&dmutex);
if (again) {
pthread_mutex_unlock(&dmutex);
return;
}
again = 1;
for (int vnode = 0; vnode < TSDB_MAX_VNODES; ++vnode) {
if (vnodeList[vnode].pCachePool) {
vnodeList[vnode].vnodeStatus = TSDB_VN_STATUS_OFFLINE;
vnodeClosePeerVnode(vnode);
}
}
pthread_mutex_unlock(&dmutex);
for (int vnode = 0; vnode < TSDB_MAX_VNODES; ++vnode) {
if (vnodeList[vnode].pCachePool) {
vnodeProcessCommitTimer(vnodeList + vnode, NULL);
while (vnodeList[vnode].commitThread != 0) {
taosMsleep(10);
}
vnodeCleanUpCommit(vnode);
}
}
}
void vnodeCalcOpenVnodes() {
int openVnodes = 0;
for (int vnode = 0; vnode <= tsMaxVnode; ++vnode) {
if (vnodeList[vnode].cfg.maxSessions <= 0) continue;
openVnodes++;
}
atomic_store_32(&tsOpenVnodes, openVnodes);
}
void vnodeUpdateHeadFile(int vnode, int oldTables, int newTables) {
//todo rewrite the head file with newTables
}

22
src/vnode/detail/src/vnodeStore.spec.c
View File

@ -1,22 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include <stdbool.h>
int vnodeInitInfo() { return 0; }
bool vnodeRemoveDataFileFromLinkFile(char* linkFile, char* de_name) { return true; }

207
src/vnode/detail/src/vnodeStream.c
View File

@ -1,207 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "taosmsg.h"
#include "vnode.h"
#include "vnodeUtil.h"
#include "vnodeStatus.h"
/* static TAOS *dbConn = NULL; */
void vnodeCloseStreamCallback(void *param);
void vnodeProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) {
SMeterObj *pObj = (SMeterObj *)param;
dTrace("vid:%d sid:%d id:%s, stream result is ready", pObj->vnode, pObj->sid, pObj->meterId);
// construct data
int32_t contLen = pObj->bytesPerPoint;
char * pTemp = calloc(1, sizeof(SSubmitMsg) + pObj->bytesPerPoint + sizeof(SVMsgHeader));
SSubmitMsg *pMsg = (SSubmitMsg *)(pTemp + sizeof(SVMsgHeader));
pMsg->numOfRows = htons(1);
char ncharBuf[TSDB_MAX_BYTES_PER_ROW] = {0};
int32_t offset = 0;
for (int32_t i = 0; i < pObj->numOfColumns; ++i) {
char *dst = row[i];
if (dst == NULL) {
setNull(pMsg->payLoad + offset, pObj->schema[i].type, pObj->schema[i].bytes);
} else {
// here, we need to transfer nchar(utf8) to unicode(ucs-4)
if (pObj->schema[i].type == TSDB_DATA_TYPE_NCHAR) {
taosMbsToUcs4(row[i], pObj->schema[i].bytes, ncharBuf, TSDB_MAX_BYTES_PER_ROW);
dst = ncharBuf;
}
memcpy(pMsg->payLoad + offset, dst, pObj->schema[i].bytes);
}
offset += pObj->schema[i].bytes;
}
contLen += sizeof(SSubmitMsg);
int32_t numOfPoints = 0;
int32_t code = vnodeInsertPoints(pObj, (char *)pMsg, contLen, TSDB_DATA_SOURCE_SHELL, NULL, pObj->sversion,
&numOfPoints, taosGetTimestamp(vnodeList[pObj->vnode].cfg.precision));
if (code != TSDB_CODE_SUCCESS) {
dError("vid:%d sid:%d id:%s, failed to insert continuous query results", pObj->vnode, pObj->sid, pObj->meterId);
}
assert(numOfPoints >= 0 && numOfPoints <= 1);
tfree(pTemp);
}
static void vnodeGetDBFromMeterId(SMeterObj *pObj, char *db) {
char *st = strstr(pObj->meterId, ".");
char *end = strstr(st + 1, ".");
memcpy(db, st + 1, end - (st + 1));
}
void vnodeOpenStreams(void *param, void *tmrId) {
SVnodeObj *pVnode = (SVnodeObj *)param;
SMeterObj *pObj;
if (pVnode->streamRole == TSDB_VN_STREAM_STATUS_STOP) return;
if (pVnode->meterList == NULL) return;
taosTmrStopA(&pVnode->streamTimer);
pVnode->streamTimer = NULL;
for (int sid = 0; sid < pVnode->cfg.maxSessions; ++sid) {
pObj = pVnode->meterList[sid];
if (pObj == NULL || pObj->sqlLen == 0 || vnodeIsMeterState(pObj, TSDB_METER_STATE_DROPPING)) continue;
dTrace("vid:%d sid:%d id:%s, open stream:%s", pObj->vnode, sid, pObj->meterId, pObj->pSql);
if (pVnode->dbConn == NULL) {
char db[64] = {0};
char user[64] = {0};
vnodeGetDBFromMeterId(pObj, db);
sprintf(user, "_%s", pVnode->cfg.acct);
pVnode->dbConn = taos_connect(NULL, user, tsInternalPass, db, 0);
}
if (pVnode->dbConn == NULL) {
dError("vid:%d, failed to connect to mgmt node", pVnode->vnode);
taosTmrReset(vnodeOpenStreams, 1000, param, vnodeTmrCtrl, &pVnode->streamTimer);
return;
}
if (pObj->pStream == NULL) {
pObj->pStream = taos_open_stream(pVnode->dbConn, pObj->pSql, vnodeProcessStreamRes, pObj->lastKey, pObj,
vnodeCloseStreamCallback);
if (pObj->pStream) pVnode->numOfStreams++;
}
}
}
void vnodeCreateStream(SMeterObj *pObj) {
if (pObj->sqlLen <= 0) return;
SVnodeObj *pVnode = vnodeList + pObj->vnode;
if (pVnode->streamRole == TSDB_VN_STREAM_STATUS_STOP) return;
if (pObj->pStream) return;
dTrace("vid:%d sid:%d id:%s stream:%s is created", pObj->vnode, pObj->sid, pObj->meterId, pObj->pSql);
if (pVnode->dbConn == NULL) {
if (pVnode->streamTimer == NULL) taosTmrReset(vnodeOpenStreams, 1000, pVnode, vnodeTmrCtrl, &pVnode->streamTimer);
} else {
pObj->pStream = taos_open_stream(pVnode->dbConn, pObj->pSql, vnodeProcessStreamRes, pObj->lastKey, pObj,
vnodeCloseStreamCallback);
if (pObj->pStream) pVnode->numOfStreams++;
}
}
// Close only one stream
void vnodeRemoveStream(SMeterObj *pObj) {
SVnodeObj *pVnode = vnodeList + pObj->vnode;
if (pObj->sqlLen <= 0) return;
if (pObj->pStream) {
taos_close_stream(pObj->pStream);
pVnode->numOfStreams--;
}
pObj->pStream = NULL;
if (pVnode->numOfStreams == 0) {
taos_close(pVnode->dbConn);
pVnode->dbConn = NULL;
}
dTrace("vid:%d sid:%d id:%d stream is removed", pObj->vnode, pObj->sid, pObj->meterId);
}
// Close all streams in a vnode
void vnodeCloseStream(SVnodeObj *pVnode) {
SMeterObj *pObj;
dPrint("vid:%d, stream is closed, old role %s", pVnode->vnode, taosGetVnodeStreamStatusStr(pVnode->streamRole));
// stop stream computing
for (int sid = 0; sid < pVnode->cfg.maxSessions; ++sid) {
pObj = pVnode->meterList[sid];
if (pObj == NULL) continue;
if (pObj->sqlLen > 0 && pObj->pStream) {
taos_close_stream(pObj->pStream);
pVnode->numOfStreams--;
}
pObj->pStream = NULL;
}
}
void vnodeUpdateStreamRole(SVnodeObj *pVnode) {
/* SMeterObj *pObj; */
int newRole = (pVnode->vnodeStatus == TSDB_VN_STATUS_MASTER) ? TSDB_VN_STREAM_STATUS_START : TSDB_VN_STREAM_STATUS_STOP;
if (newRole != pVnode->streamRole) {
dPrint("vid:%d, stream role is changed from %s to %s",
pVnode->vnode, taosGetVnodeStreamStatusStr(pVnode->streamRole), taosGetVnodeStreamStatusStr(newRole));
pVnode->streamRole = newRole;
if (newRole == TSDB_VN_STREAM_STATUS_START) {
vnodeOpenStreams(pVnode, NULL);
} else {
vnodeCloseStream(pVnode);
}
} else {
dPrint("vid:%d, stream role is keep to %s", pVnode->vnode, taosGetVnodeStreamStatusStr(pVnode->streamRole));
}
}
// Callback function called from client
void vnodeCloseStreamCallback(void *param) {
SMeterObj *pTable = (SMeterObj *)param;
SVnodeObj *pVnode = NULL;
if (pTable == NULL || pTable->sqlLen == 0) return;
pVnode = vnodeList + pTable->vnode;
pTable->sqlLen = 0;
pTable->pSql = NULL;
pTable->pStream = NULL;
pVnode->numOfStreams--;
if (pVnode->numOfStreams == 0) {
taos_close(pVnode->dbConn);
pVnode->dbConn = NULL;
}
vnodeSaveMeterObjToFile(pTable);
}

874
src/vnode/detail/src/vnodeSupertableQuery.c
View File

@ -1,874 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "mnode.h"
#include "os.h"
#include "qast.h"
#include "qextbuffer.h"
#include "tschemautil.h"
#include "tsqlfunction.h"
typedef struct SSyntaxTreeFilterSupporter {
SSchema* pTagSchema;
int32_t numOfTags;
int32_t optr;
} SSyntaxTreeFilterSupporter;
typedef struct SJoinSupporter {
void** val;
void** pTabObjs;
int32_t size;
int16_t type;
int16_t colIndex;
void** qualMeterObj;
int32_t qualSize;
} SJoinSupporter;
typedef struct SMeterNameFilterSupporter {
SPatternCompareInfo info;
char* pattern;
} SMeterNameFilterSupporter;
static void tansformQueryResult(tQueryResultset* pRes);
static bool tSkipListNodeFilterCallback(const void *pNode, void *param);
static int32_t tabObjVGIDComparator(const void* pLeft, const void* pRight) {
STabObj* p1 = *(STabObj**)pLeft;
STabObj* p2 = *(STabObj**)pRight;
int32_t ret = p1->gid.vgId - p2->gid.vgId;
if (ret == 0) {
return ret;
} else {
return ret > 0 ? 1 : -1;
}
}
// monotonic inc in memory address
static int32_t tabObjPointerComparator(const void* pLeft, const void* pRight) {
int64_t ret = (*(STabObj**)(pLeft))->uid - (*(STabObj**)(pRight))->uid;
if (ret == 0) {
return 0;
} else {
return ret > 0 ? 1 : -1;
}
}
static int32_t tabObjResultComparator(const void* p1, const void* p2, void* param) {
tOrderDescriptor* pOrderDesc = (tOrderDescriptor*)param;
STabObj* pNode1 = (STabObj*)p1;
STabObj* pNode2 = (STabObj*)p2;
for (int32_t i = 0; i < pOrderDesc->orderIdx.numOfCols; ++i) {
int32_t colIdx = pOrderDesc->orderIdx.pData[i];
char* f1 = NULL;
char* f2 = NULL;
SSchema schema = {0};
if (colIdx == -1) {
f1 = pNode1->meterId;
f2 = pNode2->meterId;
schema.type = TSDB_DATA_TYPE_BINARY;
schema.bytes = TSDB_TABLE_ID_LEN;
} else {
f1 = mgmtMeterGetTag(pNode1, colIdx, NULL);
f2 = mgmtMeterGetTag(pNode2, colIdx, &schema);
SSchema* pSchema = getColumnModelSchema(pOrderDesc->pColumnModel, colIdx);
assert(schema.type == pSchema->type);
}
int32_t ret = doCompare(f1, f2, schema.type, schema.bytes);
if (ret == 0) {
continue;
} else {
return ret;
}
}
return 0;
}
/**
* update the tag order index according to the tags column index. The tags column index needs to be checked one-by-one,
* since the normal columns may be passed to server for handling the group by on status column.
*
* @param pSuperTableMetaMsg
* @param tableIndex
* @param pOrderIndexInfo
* @param numOfTags
*/
static void mgmtUpdateOrderTagColIndex(SMetricMetaMsg* pMetricMetaMsg, int32_t tableIndex, SColumnOrderInfo* pOrderIndexInfo,
int32_t numOfTags) {
SMetricMetaElemMsg* pElem = (SMetricMetaElemMsg*)((char*)pSuperTableMetaMsg + pSuperTableMetaMsg->metaElem[tableIndex]);
SColIndexEx* groupColumnList = (SColIndexEx*)((char*)pSuperTableMetaMsg + pElem->groupbyTagColumnList);
int32_t numOfGroupbyTags = 0;
for (int32_t i = 0; i < pElem->numOfGroupCols; ++i) {
if (groupColumnList[i].flag == TSDB_COL_TAG) { // ignore this column if it is not a tag column.
pOrderIndexInfo->pData[numOfGroupbyTags++] = groupColumnList[i].colIdx;
assert(groupColumnList[i].colIdx < numOfTags);
}
}
pOrderIndexInfo->numOfCols = numOfGroupbyTags;
}
// todo merge sort function with losertree used
void mgmtReorganizeMetersInMetricMeta(SSuperTableMetaMsg* pSuperTableMetaMsg, int32_t tableIndex, tQueryResultset* pRes) {
if (pRes->num <= 0) { // no result, no need to pagination
return;
}
SMetricMetaElemMsg* pElem = (SMetricMetaElemMsg*)((char*)pSuperTableMetaMsg + pSuperTableMetaMsg->metaElem[tableIndex]);
STabObj* pMetric = mgmtGetTable(pElem->meterId);
SSchema* pTagSchema = (SSchema*)(pMetric->schema + pMetric->numOfColumns * sizeof(SSchema));
/*
* To apply the group limitation and group offset, we should sort the result
* list according to the order condition
*/
tOrderDescriptor* descriptor =
(tOrderDescriptor*)calloc(1, sizeof(tOrderDescriptor) + sizeof(int32_t) * pElem->numOfGroupCols);
descriptor->pColumnModel = createColumnModel(pTagSchema, pMetric->numOfTags, 1);
descriptor->orderIdx.numOfCols = pElem->numOfGroupCols;
int32_t* startPos = NULL;
int32_t numOfSubset = 1;
mgmtUpdateOrderTagColIndex(pMetricMetaMsg, tableIndex, &descriptor->orderIdx, pMetric->numOfTags);
if (descriptor->orderIdx.numOfCols > 0) {
tQSortEx(pRes->pRes, POINTER_BYTES, 0, pRes->num - 1, descriptor, tabObjResultComparator);
startPos = calculateSubGroup(pRes->pRes, pRes->num, &numOfSubset, descriptor, tabObjResultComparator);
} else {
startPos = malloc(2 * sizeof(int32_t));
startPos[0] = 0;
startPos[1] = (int32_t)pRes->num;
}
/*
* sort the result according to vgid to ensure meters with the same vgid is
* continuous in the result list
*/
qsort(pRes->pRes, (size_t)pRes->num, POINTER_BYTES, tabObjVGIDComparator);
free(descriptor->pColumnModel);
free(descriptor);
free(startPos);
}
static void mgmtRetrieveByMeterName(tQueryResultset* pRes, char* str, STabObj* pMetric) {
const char* sep = ",";
char* pToken = NULL;
int32_t s = 4; // initial size
pRes->pRes = malloc(sizeof(char*) * s);
pRes->num = 0;
for (pToken = strsep(&str, sep); pToken != NULL; pToken = strsep(&str, sep)) {
STabObj* pMeterObj = mgmtGetTable(pToken);
if (pMeterObj == NULL) {
mWarn("metric:%s error in metric query expression, invalid meter id:%s", pMetric->meterId, pToken);
continue;
}
if (pRes->num >= s) {
s += (s >> 1); // increase 50% size
pRes->pRes = realloc(pRes->pRes, sizeof(char*) * s);
}
/* not a table created from metric, ignore */
if (pMeterObj->tableType != TSDB_TABLE_TYPE_CHILD_TABLE) {
continue;
}
/*
* queried meter not belongs to this metric, ignore, metric does not have
* uid, so compare according to meterid
*/
STabObj* parentMetric = mgmtGetTable(pMeterObj->pTagData);
if (strncasecmp(parentMetric->meterId, pMetric->meterId, TSDB_TABLE_ID_LEN) != 0 ||
(parentMetric->uid != pMetric->uid)) {
continue;
}
pRes->pRes[pRes->num++] = pMeterObj;
}
}
static bool mgmtTablenameFilterCallback(tSkipListNode* pNode, void* param) {
SMeterNameFilterSupporter* pSupporter = (SMeterNameFilterSupporter*)param;
char name[TSDB_TABLE_ID_LEN] = {0};
// pattern compare for meter name
STabObj* pMeterObj = (STabObj*)pNode->pData;
extractTableName(pMeterObj->meterId, name);
return patternMatch(pSupporter->pattern, name, TSDB_TABLE_ID_LEN, &pSupporter->info) == TSDB_PATTERN_MATCH;
}
static void mgmtRetrieveFromLikeOptr(tQueryResultset* pRes, const char* str, STabObj* pMetric) {
SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER;
SMeterNameFilterSupporter supporter = {info, (char*) str};
pRes->num =
tSkipListIterateList(pMetric->pSkipList, (tSkipListNode***)&pRes->pRes, mgmtTablenameFilterCallback, &supporter);
}
static void mgmtFilterByTableNameCond(tQueryResultset* pRes, char* condStr, int32_t len, STabObj* pMetric) {
pRes->num = 0;
if (len <= 0) {
return;
}
char* str = calloc(1, (size_t)len + 1);
memcpy(str, condStr, len);
if (strncasecmp(condStr, QUERY_COND_REL_PREFIX_IN, QUERY_COND_REL_PREFIX_IN_LEN) == 0) { // handle in expression
mgmtRetrieveByMeterName(pRes, str + QUERY_COND_REL_PREFIX_IN_LEN, pMetric);
} else { // handle like expression
assert(strncasecmp(str, QUERY_COND_REL_PREFIX_LIKE, QUERY_COND_REL_PREFIX_LIKE_LEN) == 0);
mgmtRetrieveFromLikeOptr(pRes, str + QUERY_COND_REL_PREFIX_LIKE_LEN, pMetric);
tansformQueryResult(pRes);
}
free(str);
}
UNUSED_FUNC static bool mgmtJoinFilterCallback(tSkipListNode* pNode, void* param) {
SJoinSupporter* pSupporter = (SJoinSupporter*)param;
SSchema s = {0};
char* v = mgmtTableGetTag((STabObj*)pNode->pData, pSupporter->colIndex, &s);
for (int32_t i = 0; i < pSupporter->size; ++i) {
int32_t ret = doCompare(v, pSupporter->val[i], pSupporter->type, s.bytes);
if (ret == 0) {
pSupporter->qualMeterObj[pSupporter->qualSize++] = pSupporter->pTabObjs[i];
/*
* Once a value is qualified according to the join condition, it is remove from the
* candidate list, as well as its corresponding meter object.
*
* last one does not need to move forward
*/
if (i < pSupporter->size - 1) {
memmove(pSupporter->val[i], pSupporter->val[i + 1], pSupporter->size - (i + 1));
}
pSupporter->size -= 1;
return true;
}
}
return false;
}
static void orderResult(SSuperTableMetaMsg* pSuperTableMetaMsg, tQueryResultset* pRes, int16_t colIndex, int32_t tableIndex) {
SMetricMetaElemMsg* pElem = (SMetricMetaElemMsg*)((char*)pSuperTableMetaMsg + pSuperTableMetaMsg->metaElem[tableIndex]);
tOrderDescriptor* descriptor =
(tOrderDescriptor*)calloc(1, sizeof(tOrderDescriptor) + sizeof(int32_t) * 1); // only one column for join
STabObj* pMetric = mgmtGetTable(pElem->meterId);
SSchema* pTagSchema = (SSchema*)(pMetric->schema + pMetric->numOfColumns * sizeof(SSchema));
descriptor->pColumnModel = createColumnModel(pTagSchema, pMetric->numOfTags, 1);
descriptor->orderIdx.pData[0] = colIndex;
descriptor->orderIdx.numOfCols = 1;
// sort results list
tQSortEx(pRes->pRes, POINTER_BYTES, 0, pRes->num - 1, descriptor, tabObjResultComparator);
free(descriptor->pColumnModel);
free(descriptor);
}
// check for duplicate join tags
static int32_t mgmtCheckForDuplicateTagValue(tQueryResultset* pRes, int32_t index, int32_t tagCol) {
SSchema s = {0};
for (int32_t k = 1; k < pRes[index].num; ++k) {
STabObj* pObj1 = pRes[index].pRes[k - 1];
STabObj* pObj2 = pRes[index].pRes[k];
char* val1 = mgmtTableGetTag(pObj1, tagCol, &s);
char* val2 = mgmtTableGetTag(pObj2, tagCol, NULL);
if (doCompare(val1, val2, s.type, s.bytes) == 0) {
return TSDB_CODE_DUPLICATE_TAGS;
}
}
return TSDB_CODE_SUCCESS;
}
int32_t mgmtDoJoin(SSuperTableMetaMsg* pSuperTableMetaMsg, tQueryResultset* pRes) {
if (pSuperTableMetaMsg->numOfMeters == 1) {
return TSDB_CODE_SUCCESS;
}
bool allEmpty = false;
for (int32_t i = 0; i < pSuperTableMetaMsg->numOfMeters; ++i) {
if (pRes[i].num == 0) { // all results are empty if one of them is empty
allEmpty = true;
break;
}
}
if (allEmpty) {
for (int32_t i = 0; i < pSuperTableMetaMsg->numOfMeters; ++i) {
pRes[i].num = 0;
tfree(pRes[i].pRes);
}
return TSDB_CODE_SUCCESS;
}
char* cond = (char*)pSuperTableMetaMsg + pSuperTableMetaMsg->join;
char left[TSDB_TABLE_ID_LEN + 1] = {0};
strcpy(left, cond);
int16_t leftTagColIndex = *(int16_t*)(cond + TSDB_TABLE_ID_LEN);
char right[TSDB_TABLE_ID_LEN + 1] = {0};
strcpy(right, cond + TSDB_TABLE_ID_LEN + sizeof(int16_t));
int16_t rightTagColIndex = *(int16_t*)(cond + TSDB_TABLE_ID_LEN * 2 + sizeof(int16_t));
STabObj* pLeftMetric = mgmtGetTable(left);
STabObj* pRightMetric = mgmtGetTable(right);
// decide the pRes belongs to
int32_t leftIndex = 0;
int32_t rightIndex = 0;
for (int32_t i = 0; i < pSuperTableMetaMsg->numOfMeters; ++i) {
STabObj* pObj = (STabObj*)pRes[i].pRes[0];
STabObj* pMetric1 = mgmtGetTable(pObj->pTagData);
if (pMetric1 == pLeftMetric) {
leftIndex = i;
} else if (pMetric1 == pRightMetric) {
rightIndex = i;
}
}
orderResult(pSuperTableMetaMsg, &pRes[leftIndex], leftTagColIndex, leftIndex);
orderResult(pSuperTableMetaMsg, &pRes[rightIndex], rightTagColIndex, rightIndex);
int32_t i = 0;
int32_t j = 0;
SSchema s = {0};
int32_t res = 0;
// check for duplicated tag values
int32_t ret1 = mgmtCheckForDuplicateTagValue(pRes, leftIndex, leftTagColIndex);
int32_t ret2 = mgmtCheckForDuplicateTagValue(pRes, rightIndex, rightTagColIndex);
if (ret1 != TSDB_CODE_SUCCESS || ret2 != TSDB_CODE_SUCCESS) {
return ret1;
}
while (i < pRes[leftIndex].num && j < pRes[rightIndex].num) {
STabObj* pLeftObj = pRes[leftIndex].pRes[i];
STabObj* pRightObj = pRes[rightIndex].pRes[j];
char* v1 = mgmtTableGetTag(pLeftObj, leftTagColIndex, &s);
char* v2 = mgmtTableGetTag(pRightObj, rightTagColIndex, NULL);
int32_t ret = doCompare(v1, v2, s.type, s.bytes);
if (ret == 0) { // qualified
pRes[leftIndex].pRes[res] = pRes[leftIndex].pRes[i++];
pRes[rightIndex].pRes[res] = pRes[rightIndex].pRes[j++];
res++;
} else if (ret < 0) {
i++;
} else {
j++;
}
}
pRes[leftIndex].num = res;
pRes[rightIndex].num = res;
return TSDB_CODE_SUCCESS;
}
/**
* convert the result pointer to STabObj instead of tSkipListNode
* @param pRes
*/
static void tansformQueryResult(tQueryResultset* pRes) {
if (pRes == NULL || pRes->num == 0) {
return;
}
for (int32_t i = 0; i < pRes->num; ++i) {
pRes->pRes[i] = ((tSkipListNode*)(pRes->pRes[i]))->pData;
}
}
static tQueryResultset* doNestedLoopIntersect(tQueryResultset* pRes1, tQueryResultset* pRes2) {
int32_t num = 0;
void** pResult = pRes1->pRes;
for (int32_t i = 0; i < pRes1->num; ++i) {
for (int32_t j = 0; j < pRes2->num; ++j) {
if (pRes1->pRes[i] == pRes2->pRes[j]) {
pResult[num++] = pRes1->pRes[i];
break;
}
}
}
tQueryResultClean(pRes2);
memset(pRes1->pRes + num, 0, sizeof(void*) * (pRes1->num - num));
pRes1->num = num;
return pRes1;
}
static tQueryResultset* doSortIntersect(tQueryResultset* pRes1, tQueryResultset* pRes2) {
size_t sizePtr = sizeof(void *);
qsort(pRes1->pRes, pRes1->num, sizePtr, tabObjPointerComparator);
qsort(pRes2->pRes, pRes2->num, sizePtr, tabObjPointerComparator);
int32_t i = 0;
int32_t j = 0;
int32_t num = 0;
while (i < pRes1->num && j < pRes2->num) {
if (pRes1->pRes[i] == pRes2->pRes[j]) {
j++;
pRes1->pRes[num++] = pRes1->pRes[i++];
} else if (pRes1->pRes[i] < pRes2->pRes[j]) {
i++;
} else {
j++;
}
}
tQueryResultClean(pRes2);
memset(pRes1->pRes + num, 0, sizeof(void*) * (pRes1->num - num));
pRes1->num = num;
return pRes1;
}
static void queryResultIntersect(tQueryResultset* pFinalRes, tQueryResultset* pRes) {
const int32_t NUM_OF_RES_THRESHOLD = 20;
// for small result, use nested loop join
if (pFinalRes->num <= NUM_OF_RES_THRESHOLD && pRes->num <= NUM_OF_RES_THRESHOLD) {
doNestedLoopIntersect(pFinalRes, pRes);
} else { // for larger result, sort merge is employed
doSortIntersect(pFinalRes, pRes);
}
}
static void queryResultUnion(tQueryResultset* pFinalRes, tQueryResultset* pRes) {
if (pRes->num == 0) {
tQueryResultClean(pRes);
return;
}
int32_t total = pFinalRes->num + pRes->num;
void* tmp = realloc(pFinalRes->pRes, total * POINTER_BYTES);
if (tmp == NULL) {
return;
}
pFinalRes->pRes = tmp;
memcpy(&pFinalRes->pRes[pFinalRes->num], pRes->pRes, POINTER_BYTES * pRes->num);
qsort(pFinalRes->pRes, total, POINTER_BYTES, tabObjPointerComparator);
int32_t num = 1;
for (int32_t i = 1; i < total; ++i) {
if (pFinalRes->pRes[i] != pFinalRes->pRes[i - 1]) {
pFinalRes->pRes[num++] = pFinalRes->pRes[i];
}
}
if (num < total) {
memset(&pFinalRes->pRes[num], 0, POINTER_BYTES * (total - num));
}
pFinalRes->num = num;
tQueryResultClean(pRes);
}
static int32_t compareIntVal(const void* pLeft, const void* pRight) {
DEFAULT_COMP(GET_INT64_VAL(pLeft), GET_INT64_VAL(pRight));
}
static int32_t compareIntDoubleVal(const void* pLeft, const void* pRight) {
DEFAULT_COMP(GET_INT64_VAL(pLeft), GET_DOUBLE_VAL(pRight));
}
static int32_t compareDoubleVal(const void* pLeft, const void* pRight) {
DEFAULT_COMP(GET_DOUBLE_VAL(pLeft), GET_DOUBLE_VAL(pRight));
}
static int32_t compareDoubleIntVal(const void* pLeft, const void* pRight) {
double ret = (*(double*)pLeft) - (*(int64_t*)pRight);
if (fabs(ret) < DBL_EPSILON) {
return 0;
} else {
return ret > 0 ? 1 : -1;
}
}
static int32_t compareStrVal(const void* pLeft, const void* pRight) {
int32_t ret = strcmp(pLeft, pRight);
if (ret == 0) {
return 0;
} else {
return ret > 0 ? 1 : -1;
}
}
static int32_t compareWStrVal(const void* pLeft, const void* pRight) {
int32_t ret = wcscmp(pLeft, pRight);
if (ret == 0) {
return 0;
} else {
return ret > 0 ? 1 : -1;
}
}
static int32_t compareStrPatternComp(const void* pLeft, const void* pRight) {
SPatternCompareInfo pInfo = {'%', '_'};
const char* pattern = pRight;
const char* str = pLeft;
int32_t ret = patternMatch(pattern, str, strlen(str), &pInfo);
return (ret == TSDB_PATTERN_MATCH) ? 0 : 1;
}
static int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) {
SPatternCompareInfo pInfo = {'%', '_'};
const wchar_t* pattern = pRight;
const wchar_t* str = pLeft;
int32_t ret = WCSPatternMatch(pattern, str, wcslen(str), &pInfo);
return (ret == TSDB_PATTERN_MATCH) ? 0 : 1;
}
static __compar_fn_t getFilterComparator(int32_t type, int32_t filterType, int32_t optr) {
__compar_fn_t comparator = NULL;
switch (type) {
case TSDB_DATA_TYPE_TINYINT:
case TSDB_DATA_TYPE_SMALLINT:
case TSDB_DATA_TYPE_INT:
case TSDB_DATA_TYPE_BIGINT:
case TSDB_DATA_TYPE_BOOL: {
if (filterType >= TSDB_DATA_TYPE_BOOL && filterType <= TSDB_DATA_TYPE_BIGINT) {
comparator = compareIntVal;
} else if (filterType >= TSDB_DATA_TYPE_FLOAT && filterType <= TSDB_DATA_TYPE_DOUBLE) {
comparator = compareIntDoubleVal;
}
break;
}
case TSDB_DATA_TYPE_FLOAT:
case TSDB_DATA_TYPE_DOUBLE: {
if (filterType >= TSDB_DATA_TYPE_BOOL && filterType <= TSDB_DATA_TYPE_BIGINT) {
comparator = compareDoubleIntVal;
} else if (filterType >= TSDB_DATA_TYPE_FLOAT && filterType <= TSDB_DATA_TYPE_DOUBLE) {
comparator = compareDoubleVal;
}
break;
}
case TSDB_DATA_TYPE_BINARY: {
assert(filterType == TSDB_DATA_TYPE_BINARY);
if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */
comparator = compareStrPatternComp;
} else { /* normal relational comparator */
comparator = compareStrVal;
}
break;
}
case TSDB_DATA_TYPE_NCHAR: {
assert(filterType == TSDB_DATA_TYPE_NCHAR);
if (optr == TSDB_RELATION_LIKE) {
comparator = compareWStrPatternComp;
} else {
comparator = compareWStrVal;
}
break;
}
default:
comparator = compareIntVal;
break;
}
return comparator;
}
static void getTagColumnInfo(SSyntaxTreeFilterSupporter* pSupporter, SSchema* pSchema, int32_t* index,
int32_t* offset) {
*index = 0;
*offset = 0;
// filter on table name(TBNAME)
if (strcasecmp(pSchema->name, TSQL_TBNAME_L) == 0) {
*index = TSDB_TBNAME_COLUMN_INDEX;
*offset = TSDB_TBNAME_COLUMN_INDEX;
return;
}
while ((*index) < pSupporter->numOfTags) {
if (pSupporter->pTagSchema[*index].bytes == pSchema->bytes &&
pSupporter->pTagSchema[*index].type == pSchema->type &&
strcmp(pSupporter->pTagSchema[*index].name, pSchema->name) == 0) {
break;
} else {
(*offset) += pSupporter->pTagSchema[(*index)++].bytes;
}
}
}
void filterPrepare(void* expr, void* param) {
tSQLBinaryExpr *pExpr = (tSQLBinaryExpr*) expr;
if (pExpr->info != NULL) {
return;
}
int32_t i = 0, offset = 0;
pExpr->info = calloc(1, sizeof(tQueryInfo));
tQueryInfo* pInfo = pExpr->info;
SSyntaxTreeFilterSupporter* pSupporter = (SSyntaxTreeFilterSupporter*)param;
tVariant* pCond = pExpr->pRight->pVal;
SSchema* pSchema = pExpr->pLeft->pSchema;
getTagColumnInfo(pSupporter, pSchema, &i, &offset);
assert((i >= 0 && i < TSDB_MAX_TAGS) || (i == TSDB_TBNAME_COLUMN_INDEX));
assert((offset >= 0 && offset < TSDB_MAX_TAGS_LEN) || (offset == TSDB_TBNAME_COLUMN_INDEX));
pInfo->sch = *pSchema;
pInfo->colIdx = i;
pInfo->optr = pExpr->nSQLBinaryOptr;
pInfo->offset = offset;
pInfo->compare = getFilterComparator(pSchema->type, pCond->nType, pInfo->optr);
tVariantAssign(&pInfo->q, pCond);
tVariantTypeSetType(&pInfo->q, pInfo->sch.type);
}
void tSQLListTraverseDestroyInfo(void* param) {
if (param == NULL) {
return;
}
tQueryInfo* pInfo = (tQueryInfo*)param;
tVariantDestroy(&(pInfo->q));
free(param);
}
static int32_t mgmtFilterMeterByIndex(STabObj* pMetric, tQueryResultset* pRes, char* pCond, int32_t condLen) {
SSchema* pTagSchema = (SSchema*)(pMetric->schema + pMetric->numOfColumns * sizeof(SSchema));
tSQLBinaryExpr* pExpr = NULL;
tSQLBinaryExprFromString(&pExpr, pTagSchema, pMetric->numOfTags, pCond, condLen);
// failed to build expression, no result, return immediately
if (pExpr == NULL) {
mError("metric:%s, no result returned, error in super table query expression:%s", pMetric->meterId, pCond);
tfree(pCond);
return TSDB_CODE_OPS_NOT_SUPPORT;
} else { // query according to the binary expression
SSyntaxTreeFilterSupporter s = {.pTagSchema = pTagSchema, .numOfTags = pMetric->numOfTags};
SBinaryFilterSupp supp = {.fp = (__result_filter_fn_t)tSkipListNodeFilterCallback,
.setupInfoFn = (__do_filter_suppl_fn_t)filterPrepare,
.pExtInfo = &s};
// tSQLBinaryExprTraverse(pExpr, pMetric->pSkipList, pRes, &supp);
tSQLBinaryExprDestroy(&pExpr, tSQLListTraverseDestroyInfo);
}
tansformQueryResult(pRes);
return TSDB_CODE_SUCCESS;
}
int32_t mgmtRetrieveMetersFromSuperTable(SSuperTableMetaMsg* pMsg, int32_t tableIndex, tQueryResultset* pRes) {
SMetricMetaElemMsg* pElem = (SMetricMetaElemMsg*)((char*)pMsg + pMsg->metaElem[tableIndex]);
STabObj* pMetric = mgmtGetTable(pElem->meterId);
char* pCond = NULL;
char* tmpTableNameCond = NULL;
// no table created in accordance with this metric.
if (pMetric->pSkipList == NULL || pMetric->pSkipList->nSize == 0) {
assert(pMetric->numOfMeters == 0);
return TSDB_CODE_SUCCESS;
}
char* pQueryCond = (char*)pMsg + pElem->cond;
int32_t condLen = pElem->condLen;
// transfer the unicode string to mbs binary expression
if (condLen > 0) {
pCond = calloc(1, (condLen + 1) * TSDB_NCHAR_SIZE);
taosUcs4ToMbs(pQueryCond, condLen * TSDB_NCHAR_SIZE, pCond);
condLen = strlen(pCond) + 1;
mTrace("metric:%s len:%d, metric query condition:%s", pMetric->meterId, condLen, pCond);
}
char* tablenameCond = (char*)pMsg + pElem->tableCond;
if (pElem->tableCondLen > 0) {
tmpTableNameCond = calloc(1, pElem->tableCondLen + 1);
strncpy(tmpTableNameCond, tablenameCond, pElem->tableCondLen);
mTrace("metric:%s rel:%d, len:%d, table name cond:%s", pMetric->meterId, pElem->rel, pElem->tableCondLen,
tmpTableNameCond);
}
if (pElem->tableCondLen > 0 || condLen > 0) {
mgmtFilterByTableNameCond(pRes, tmpTableNameCond, pElem->tableCondLen, pMetric);
bool noNextCal = (pRes->num == 0 && pElem->rel == TSDB_RELATION_AND); // no need to calculate next result
if (!noNextCal && condLen > 0) {
tQueryResultset filterRes = {0};
int32_t ret = mgmtFilterMeterByIndex(pMetric, &filterRes, pCond, condLen);
if (ret != TSDB_CODE_SUCCESS) {
tfree(pCond);
tfree(tmpTableNameCond);
return ret;
}
// union or intersect of two results
assert(pElem->rel == TSDB_RELATION_AND || pElem->rel == TSDB_RELATION_OR);
if (pElem->rel == TSDB_RELATION_AND) {
if (filterRes.num == 0 || pRes->num == 0) { // intersect two sets
tQueryResultClean(pRes);
} else {
queryResultIntersect(pRes, &filterRes);
}
} else { // union two sets
queryResultUnion(pRes, &filterRes);
}
tQueryResultClean(&filterRes);
}
} else {
mTrace("metric:%s retrieve all meter, no query condition", pMetric->meterId);
pRes->num = tSkipListIterateList(pMetric->pSkipList, (tSkipListNode***)&pRes->pRes, NULL, NULL);
tansformQueryResult(pRes);
}
tfree(pCond);
tfree(tmpTableNameCond);
mTrace("metric:%s numOfRes:%d", pMetric->meterId, pRes->num);
return TSDB_CODE_SUCCESS;
}
// todo refactor!!!!!
static char* getTagValueFromMeter(STabObj* pTable, int32_t offset, int32_t len, char* param) {
if (offset == TSDB_TBNAME_COLUMN_INDEX) {
extractTableName(pTable->meterId, param);
} else {
char* tags = pTable->pTagData + offset + TSDB_TABLE_ID_LEN; // tag start position
memcpy(param, tags, len); // make sure the value is null-terminated string
}
return param;
}
bool tSkipListNodeFilterCallback(const void* pNode, void* param) {
tQueryInfo* pInfo = (tQueryInfo*)param;
STabObj* pTable = (STabObj*)(((tSkipListNode*)pNode)->pData);
char buf[TSDB_MAX_TAGS_LEN] = {0};
char* val = getTagValueFromMeter(pTable, pInfo->offset, pInfo->sch.bytes, buf);
int8_t type = pInfo->sch.type;
int32_t ret = 0;
if (pInfo->q.nType == TSDB_DATA_TYPE_BINARY || pInfo->q.nType == TSDB_DATA_TYPE_NCHAR) {
ret = pInfo->compare(val, pInfo->q.pz);
} else {
tVariant t = {0};
tVariantCreateFromBinary(&t, val, (uint32_t) pInfo->sch.bytes, type);
ret = pInfo->compare(&t.i64Key, &pInfo->q.i64Key);
}
switch (pInfo->optr) {
case TSDB_RELATION_EQUAL: {
return ret == 0;
}
case TSDB_RELATION_NOT_EQUAL: {
return ret != 0;
}
case TSDB_RELATION_LARGE_EQUAL: {
return ret >= 0;
}
case TSDB_RELATION_LARGE: {
return ret > 0;
}
case TSDB_RELATION_LESS_EQUAL: {
return ret <= 0;
}
case TSDB_RELATION_LESS: {
return ret < 0;
}
case TSDB_RELATION_LIKE: {
return ret == 0;
}
default:
assert(false);
}
return true;
}

391
src/vnode/detail/src/vnodeTagMgmt.c
View File

@ -1,391 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "qast.h"
#include "qextbuffer.h"
#include "taosdef.h"
#include "taosmsg.h"
#include "tlog.h"
#include "tutil.h"
#include "vnodeTagMgmt.h"
#define GET_TAG_VAL_POINTER(s, col, sc, t) ((t *)(&((s)->tags[getColumnModelOffset(sc, col)])))
#define GET_TAG_VAL(s, col, sc, t) (*GET_TAG_VAL_POINTER(s, col, sc, t))
static void tTagsPrints(SMeterSidExtInfo *pMeterInfo, SColumnModel *pSchema, SColumnOrderInfo *pOrder);
static void tSidSetDisplay(tSidSet *pSets);
//todo merge with losertree_compar/ext_comp
int32_t doCompare(char* f1, char* f2, int32_t type, int32_t size) {
switch (type) {
case TSDB_DATA_TYPE_INT: DEFAULT_COMP(GET_INT32_VAL(f1), GET_INT32_VAL(f2));
case TSDB_DATA_TYPE_DOUBLE: DEFAULT_COMP(GET_DOUBLE_VAL(f1), GET_DOUBLE_VAL(f2));
case TSDB_DATA_TYPE_FLOAT: DEFAULT_COMP(GET_FLOAT_VAL(f1), GET_FLOAT_VAL(f2));
case TSDB_DATA_TYPE_BIGINT: DEFAULT_COMP(GET_INT64_VAL(f1), GET_INT64_VAL(f2));
case TSDB_DATA_TYPE_SMALLINT: DEFAULT_COMP(GET_INT16_VAL(f1), GET_INT16_VAL(f2));
case TSDB_DATA_TYPE_TINYINT:
case TSDB_DATA_TYPE_BOOL: DEFAULT_COMP(GET_INT8_VAL(f1), GET_INT8_VAL(f2));
case TSDB_DATA_TYPE_NCHAR: {
int32_t ret = wcsncmp((wchar_t*) f1, (wchar_t*) f2, size/TSDB_NCHAR_SIZE);
if (ret == 0) {
return ret;
}
return (ret < 0) ? -1 : 1;
}
default: {
int32_t ret = strncmp(f1, f2, (size_t)size);
if (ret == 0) {
return ret;
}
return (ret < 0) ? -1 : 1;
}
}
}
int32_t meterSidComparator(const void *p1, const void *p2, void *param) {
tOrderDescriptor *pOrderDesc = (tOrderDescriptor *)param;
SMeterSidExtInfo *s1 = (SMeterSidExtInfo *)p1;
SMeterSidExtInfo *s2 = (SMeterSidExtInfo *)p2;
for (int32_t i = 0; i < pOrderDesc->orderIdx.numOfCols; ++i) {
int32_t colIdx = pOrderDesc->orderIdx.pData[i];
char * f1 = NULL;
char * f2 = NULL;
int32_t type = 0;
int32_t bytes = 0;
if (colIdx == -1) {
f1 = s1->tags;
f2 = s2->tags;
type = TSDB_DATA_TYPE_BINARY;
bytes = TSDB_METER_NAME_LEN;
} else {
f1 = GET_TAG_VAL_POINTER(s1, colIdx, pOrderDesc->pColumnModel, char);
f2 = GET_TAG_VAL_POINTER(s2, colIdx, pOrderDesc->pColumnModel, char);
SSchema *pSchema = getColumnModelSchema(pOrderDesc->pColumnModel, colIdx);
type = pSchema->type;
bytes = pSchema->bytes;
}
int32_t ret = doCompare(f1, f2, type, bytes);
if (ret == 0) {
continue;
} else {
return ret;
}
}
return 0;
}
static void median(void **pMeterSids, size_t size, int32_t s1, int32_t s2, tOrderDescriptor *pOrderDesc,
__ext_compar_fn_t compareFn) {
int32_t midIdx = ((s2 - s1) >> 1) + s1;
if (compareFn(pMeterSids[midIdx], pMeterSids[s1], pOrderDesc) == 1) {
tsDataSwap(&pMeterSids[midIdx], &pMeterSids[s1], TSDB_DATA_TYPE_BINARY, size);
}
if (compareFn(pMeterSids[midIdx], pMeterSids[s2], pOrderDesc) == 1) {
tsDataSwap(&pMeterSids[midIdx], &pMeterSids[s1], TSDB_DATA_TYPE_BINARY, size);
tsDataSwap(&pMeterSids[midIdx], &pMeterSids[s2], TSDB_DATA_TYPE_BINARY, size);
} else if (compareFn(pMeterSids[s1], pMeterSids[s2], pOrderDesc) == 1) {
tsDataSwap(&pMeterSids[s1], &pMeterSids[s2], TSDB_DATA_TYPE_BINARY, size);
}
assert(compareFn(pMeterSids[midIdx], pMeterSids[s1], pOrderDesc) <= 0 &&
compareFn(pMeterSids[s1], pMeterSids[s2], pOrderDesc) <= 0);
#ifdef _DEBUG_VIEW
tTagsPrints(pMeterSids[s1], pOrderDesc->pColumnModel, &pOrderDesc->orderIdx);
tTagsPrints(pMeterSids[midIdx], pOrderDesc->pColumnModel, &pOrderDesc->orderIdx);
tTagsPrints(pMeterSids[s2], pOrderDesc->pColumnModel, &pOrderDesc->orderIdx);
#endif
}
static void tInsertSort(void **pMeterSids, size_t size, int32_t startPos, int32_t endPos, void *param,
__ext_compar_fn_t compareFn) {
for (int32_t i = startPos + 1; i <= endPos; ++i) {
for (int32_t j = i; j > startPos; --j) {
if (compareFn(pMeterSids[j], pMeterSids[j - 1], param) == -1) {
tsDataSwap(&pMeterSids[j], &pMeterSids[j - 1], TSDB_DATA_TYPE_BINARY, size);
} else {
break;
}
}
}
}
void tQSortEx(void **pMeterSids, size_t size, int32_t start, int32_t end, void *param, __ext_compar_fn_t compareFn) {
tOrderDescriptor *pOrderDesc = (tOrderDescriptor *)param;
// short array sort, incur another sort procedure instead of quick sort process
if (end - start + 1 <= 8) {
tInsertSort(pMeterSids, size, start, end, pOrderDesc, compareFn);
return;
}
median(pMeterSids, size, start, end, pOrderDesc, compareFn);
int32_t s = start, e = end;
int32_t endRightS = end, startLeftS = start;
while (s < e) {
while (e > s) {
int32_t ret = compareFn(pMeterSids[e], pMeterSids[s], pOrderDesc);
if (ret < 0) {
break;
}
/*
* move the data that equals to pivotal value to the right end of the list
*/
if (ret == 0 && e != endRightS) {
tsDataSwap(&pMeterSids[e], &pMeterSids[endRightS--], TSDB_DATA_TYPE_BINARY, size);
}
e--;
}
if (e != s) {
tsDataSwap(&pMeterSids[e], &pMeterSids[s], TSDB_DATA_TYPE_BINARY, size);
}
while (s < e) {
int32_t ret = compareFn(pMeterSids[s], pMeterSids[e], pOrderDesc);
if (ret > 0) {
break;
}
if (ret == 0 && s != startLeftS) {
tsDataSwap(&pMeterSids[s], &pMeterSids[startLeftS++], TSDB_DATA_TYPE_BINARY, size);
}
s++;
}
if (e != s) {
tsDataSwap(&pMeterSids[s], &pMeterSids[e], TSDB_DATA_TYPE_BINARY, size);
}
}
int32_t rightPartStart = e + 1;
if (endRightS != end && e < end) {
int32_t left = rightPartStart;
int32_t right = end;
while (right > endRightS && left <= endRightS) {
tsDataSwap(&pMeterSids[left++], &pMeterSids[right--], TSDB_DATA_TYPE_BINARY, size);
}
rightPartStart += (end - endRightS);
}
int32_t leftPartEnd = e - 1;
if (startLeftS != end && s > start) {
int32_t left = start;
int32_t right = leftPartEnd;
while (left < startLeftS && right >= startLeftS) {
tsDataSwap(&pMeterSids[left++], &pMeterSids[right--], TSDB_DATA_TYPE_BINARY, size);
}
leftPartEnd -= (startLeftS - start);
}
if (leftPartEnd > start) {
tQSortEx(pMeterSids, size, start, leftPartEnd, pOrderDesc, compareFn);
}
if (rightPartStart < end) {
tQSortEx(pMeterSids, size, rightPartStart, end, pOrderDesc, compareFn);
}
}
int32_t *calculateSubGroup(void **pSids, int32_t numOfMeters, int32_t *numOfSubset, tOrderDescriptor *pOrderDesc,
__ext_compar_fn_t compareFn) {
int32_t *starterPos = (int32_t *)malloc((numOfMeters + 1) * sizeof(int32_t)); // add additional buffer
starterPos[0] = 0;
*numOfSubset = 1;
for (int32_t i = 1; i < numOfMeters; ++i) {
int32_t ret = compareFn(pSids[i - 1], pSids[i], pOrderDesc);
if (ret != 0) {
assert(ret == -1);
starterPos[(*numOfSubset)++] = i;
}
}
starterPos[*numOfSubset] = numOfMeters;
assert(*numOfSubset <= numOfMeters);
return starterPos;
}
tSidSet *tSidSetCreate(struct SMeterSidExtInfo **pMeterSidExtInfo, int32_t numOfMeters, SSchema *pSchema,
int32_t numOfTags, SColIndexEx *colList, int32_t numOfCols) {
tSidSet *pSidSet = (tSidSet *)calloc(1, sizeof(tSidSet) + numOfCols * sizeof(int16_t));
if (pSidSet == NULL) {
return NULL;
}
pSidSet->numOfSids = numOfMeters;
pSidSet->pSids = pMeterSidExtInfo;
pSidSet->pColumnModel = createColumnModel(pSchema, numOfTags, 1);
pSidSet->orderIdx.numOfCols = numOfCols;
/*
* in case of "group by tbname,normal_col", the normal_col is ignored
*/
int32_t numOfTagCols = 0;
for(int32_t i = 0; i < numOfCols; ++i) {
if (colList[i].flag == TSDB_COL_TAG) {
pSidSet->orderIdx.pData[numOfTagCols++] = colList[i].colIdx;
}
}
pSidSet->orderIdx.numOfCols = numOfTagCols;
pSidSet->starterPos = NULL;
return pSidSet;
}
void tSidSetDestroy(tSidSet **pSets) {
if ((*pSets) != NULL) {
tfree((*pSets)->starterPos);
tfree((*pSets)->pColumnModel)(*pSets)->pSids = NULL;
tfree(*pSets);
}
}
void tTagsPrints(SMeterSidExtInfo *pMeterInfo, SColumnModel *pSchema, SColumnOrderInfo *pOrder) {
if (pSchema == NULL) {
return;
}
printf("sid: %-5d tags(", pMeterInfo->sid);
for (int32_t i = 0; i < pOrder->numOfCols; ++i) {
int32_t colIndex = pOrder->pData[i];
// it is the tbname column
if (colIndex == -1) {
printf("%s, ", pMeterInfo->tags);
continue;
}
SSchema* s = getColumnModelSchema(pSchema, colIndex);
switch (s->type) {
case TSDB_DATA_TYPE_INT:
printf("%d, ", GET_TAG_VAL(pMeterInfo, colIndex, pSchema, int32_t));
break;
case TSDB_DATA_TYPE_DOUBLE:
printf("%lf, ", GET_TAG_VAL(pMeterInfo, colIndex, pSchema, double));
break;
case TSDB_DATA_TYPE_FLOAT:
printf("%f, ", GET_TAG_VAL(pMeterInfo, colIndex, pSchema, float));
break;
case TSDB_DATA_TYPE_BIGINT:
printf("%" PRId64 ", ", GET_TAG_VAL(pMeterInfo, colIndex, pSchema, int64_t));
break;
case TSDB_DATA_TYPE_SMALLINT:
printf("%d, ", GET_TAG_VAL(pMeterInfo, colIndex, pSchema, int16_t));
break;
case TSDB_DATA_TYPE_TINYINT:
printf("%d, ", GET_TAG_VAL(pMeterInfo, colIndex, pSchema, int8_t));
break;
case TSDB_DATA_TYPE_BINARY:
printf("%s, ", GET_TAG_VAL_POINTER(pMeterInfo, colIndex, pSchema, char));
break;
case TSDB_DATA_TYPE_NCHAR: {
char *data = GET_TAG_VAL_POINTER(pMeterInfo, colIndex, pSchema, char);
char buffer[512] = {0};
taosUcs4ToMbs(data, s->bytes, buffer);
printf("%s, ", buffer);
break;
}
case TSDB_DATA_TYPE_BOOL:
printf("%d, ", GET_TAG_VAL(pMeterInfo, colIndex, pSchema, int8_t));
break;
default:
assert(false);
}
}
printf(")\n");
}
/*
* display all the subset groups for debug purpose only
*/
static void UNUSED_FUNC tSidSetDisplay(tSidSet *pSets) {
printf("%d meters.\n", pSets->numOfSids);
for (int32_t i = 0; i < pSets->numOfSids; ++i) {
printf("%d\t", pSets->pSids[i]->sid);
}
printf("\n");
printf("total number of subset group is: %d\n", pSets->numOfSubSet);
for (int32_t i = 0; i < pSets->numOfSubSet; ++i) {
int32_t s = pSets->starterPos[i];
int32_t e = pSets->starterPos[i + 1];
printf("the %d-th subgroup: \n", i + 1);
for (int32_t j = s; j < e; ++j) {
tTagsPrints(pSets->pSids[j], pSets->pColumnModel, &pSets->orderIdx);
}
}
}
void tSidSetSort(tSidSet *pSets) {
pTrace("number of meters in sort: %d", pSets->numOfSids);
SColumnOrderInfo *pOrderIdx = &pSets->orderIdx;
if (pOrderIdx->numOfCols == 0 || pSets->numOfSids <= 1 || pSets->pColumnModel == NULL) { // no group by tags clause
pSets->numOfSubSet = 1;
pSets->starterPos = (int32_t *)malloc(sizeof(int32_t) * (pSets->numOfSubSet + 1));
pSets->starterPos[0] = 0;
pSets->starterPos[1] = pSets->numOfSids;
pTrace("all meters belong to one subgroup, no need to subgrouping ops");
#ifdef _DEBUG_VIEW
tSidSetDisplay(pSets);
#endif
} else {
tOrderDescriptor *descriptor =
(tOrderDescriptor *)calloc(1, sizeof(tOrderDescriptor) + sizeof(int16_t) * pSets->orderIdx.numOfCols);
descriptor->pColumnModel = pSets->pColumnModel;
descriptor->orderIdx = pSets->orderIdx;
memcpy(descriptor->orderIdx.pData, pOrderIdx->pData, sizeof(int16_t) * pSets->orderIdx.numOfCols);
tQSortEx((void **)pSets->pSids, POINTER_BYTES, 0, pSets->numOfSids - 1, descriptor, meterSidComparator);
pSets->starterPos =
calculateSubGroup((void **)pSets->pSids, pSets->numOfSids, &pSets->numOfSubSet, descriptor, meterSidComparator);
#ifdef _DEBUG_VIEW
tSidSetDisplay(pSets);
#endif
tfree(descriptor);
}
}

766
src/vnode/detail/src/vnodeUtil.c
View File

@ -1,766 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "qast.h"
#include "tscUtil.h"
#include "tschemautil.h"
#include "vnode.h"
#include "vnodeDataFilterFunc.h"
#include "vnodeStatus.h"
#include "vnodeUtil.h"
int vnodeCheckFileIntegrity(FILE* fp) {
/*
int savedSessions, savedMeterSize;
fseek(fp, TSDB_FILE_HEADER_LEN/3, SEEK_SET);
fscanf(fp, "%d %d", &savedSessions, &savedMeterSize);
if ( (savedSessions != tsSessionsPerVnode) || (savedMeterSize != tsMeterSizeOnFile) ) {
dError("file structure is changed");
return -1;
}
uint64_t checkSum = 0, savedCheckSum=0;
checkSum = taosGetCheckSum(fp, TSDB_FILE_HEADER_LEN);
fseek(fp, TSDB_FILE_HEADER_LEN - cksumsize, SEEK_SET);
fread(&savedCheckSum, cksumsize, 1, fp);
if ( savedCheckSum != checkSum ) {
dError("check sum is not matched:0x%x 0x%x", checkSum, savedCheckSum);
return -1;
}
*/
return 0;
}
void vnodeCreateFileHeaderFd(int fd) {
char temp[TSDB_FILE_HEADER_LEN / 4];
int lineLen;
lineLen = sizeof(temp);
// write the first line`
memset(temp, 0, lineLen);
*(int16_t*)temp = vnodeFileVersion;
sprintf(temp + sizeof(int16_t), "tsdb version: %s\n", version);
/* *((int16_t *)(temp + TSDB_FILE_HEADER_LEN/8)) = vnodeFileVersion; */
lseek(fd, 0, SEEK_SET);
twrite(fd, temp, lineLen);
// second line
memset(temp, 0, lineLen);
twrite(fd, temp, lineLen);
// the third/forth line is the dynamic info
memset(temp, 0, lineLen);
twrite(fd, temp, lineLen);
twrite(fd, temp, lineLen);
}
void vnodeGetHeadFileHeaderInfo(int fd, SVnodeHeadInfo* pHeadInfo) {
lseek(fd, TSDB_FILE_HEADER_LEN / 4, SEEK_SET);
read(fd, pHeadInfo, sizeof(SVnodeHeadInfo));
}
void vnodeUpdateHeadFileHeader(int fd, SVnodeHeadInfo* pHeadInfo) {
lseek(fd, TSDB_FILE_HEADER_LEN / 4, SEEK_SET);
twrite(fd, pHeadInfo, sizeof(SVnodeHeadInfo));
}
void vnodeCreateFileHeader(FILE* fp) {
char temp[TSDB_FILE_HEADER_LEN / 4];
int lineLen;
lineLen = sizeof(temp);
// write the first line
memset(temp, 0, lineLen);
*(int16_t*)temp = vnodeFileVersion;
sprintf(temp + sizeof(int16_t), "tsdb version: %s\n", version);
/* *((int16_t *)(temp + TSDB_FILE_HEADER_LEN/8)) = vnodeFileVersion; */
fseek(fp, 0, SEEK_SET);
fwrite(temp, lineLen, 1, fp);
// second line
memset(temp, 0, lineLen);
fwrite(temp, lineLen, 1, fp);
// the third line is the dynamic info
memset(temp, 0, lineLen);
fwrite(temp, lineLen, 1, fp);
fwrite(temp, lineLen, 1, fp);
}
SSqlGroupbyExpr* vnodeCreateGroupbyExpr(SQueryMeterMsg* pQueryMsg, int32_t* code) {
if (pQueryMsg->numOfGroupCols == 0) {
return NULL;
}
// using group by tag columns
SSqlGroupbyExpr* pGroupbyExpr =
(SSqlGroupbyExpr*)malloc(sizeof(SSqlGroupbyExpr) + pQueryMsg->numOfGroupCols * sizeof(SColIndexEx));
if (pGroupbyExpr == NULL) {
*code = TSDB_CODE_SERV_OUT_OF_MEMORY;
return NULL;
}
SColIndexEx* pGroupbyColInfo = (SColIndexEx*)pQueryMsg->groupbyTagIds;
pGroupbyExpr->numOfGroupCols = pQueryMsg->numOfGroupCols;
pGroupbyExpr->orderType = pQueryMsg->orderType;
pGroupbyExpr->orderIndex = pQueryMsg->orderByIdx;
memcpy(pGroupbyExpr->columnInfo, pGroupbyColInfo, sizeof(SColIndexEx) * pGroupbyExpr->numOfGroupCols);
// TODO: update the colIndexInBuf for each column in group by clause
return pGroupbyExpr;
}
static SSchema* toSchema(SQueryMeterMsg* pQuery, SColumnInfo* pCols, int32_t numOfCols) {
char* start = (char*)pQuery->colNameList;
char* end = start;
SSchema* pSchema = calloc(1, sizeof(SSchema) * numOfCols);
for (int32_t i = 0; i < numOfCols; ++i) {
pSchema[i].type = pCols[i].type;
pSchema[i].bytes = pCols[i].bytes;
pSchema[i].colId = pCols[i].colId;
end = strstr(start, ",");
memcpy(pSchema[i].name, start, end - start);
start = end + 1;
}
return pSchema;
}
static int32_t id_compar(const void* left, const void* right) {
DEFAULT_COMP(GET_INT16_VAL(left), GET_INT16_VAL(right));
}
static int32_t vnodeBuildExprFromArithmeticStr(SSqlFunctionExpr* pExpr, SQueryMeterMsg* pQueryMsg) {
SSqlBinaryExprInfo* pBinaryExprInfo = &pExpr->pBinExprInfo;
SColumnInfo* pColMsg = pQueryMsg->colList;
tSQLBinaryExpr* pBinExpr = NULL;
SSchema* pSchema = toSchema(pQueryMsg, pColMsg, pQueryMsg->numOfCols);
dTrace("qmsg:%p create binary expr from string:%s", pQueryMsg, pExpr->pBase.arg[0].argValue.pz);
tSQLBinaryExprFromString(&pBinExpr, pSchema, pQueryMsg->numOfCols, pExpr->pBase.arg[0].argValue.pz);
if (pBinExpr == NULL) {
dError("qmsg:%p failed to create arithmetic expression string from:%s", pQueryMsg, pExpr->pBase.arg[0].argValue.pz);
return TSDB_CODE_APP_ERROR;
}
pBinaryExprInfo->pBinExpr = pBinExpr;
int32_t num = 0;
int16_t ids[TSDB_MAX_COLUMNS] = {0};
tSQLBinaryExprTrv(pBinExpr, &num, ids);
qsort(ids, num, sizeof(int16_t), id_compar);
int32_t i = 0, j = 0;
while (i < num && j < num) {
if (ids[i] == ids[j]) {
j++;
} else {
ids[++i] = ids[j++];
}
}
assert(i <= num);
// there may be duplicated referenced columns.
num = i + 1;
pBinaryExprInfo->pReqColumns = malloc(sizeof(SColIndexEx) * num);
for (int32_t k = 0; k < num; ++k) {
SColIndexEx* pColIndex = &pBinaryExprInfo->pReqColumns[k];
pColIndex->colId = ids[k];
}
pBinaryExprInfo->numOfCols = num;
free(pSchema);
return TSDB_CODE_SUCCESS;
}
static int32_t getColumnIndexInSource(SQueryMeterMsg* pQueryMsg, SSqlFuncExprMsg* pExprMsg) {
int32_t j = 0;
while(j < pQueryMsg->numOfCols) {
if (pExprMsg->colInfo.colId == pQueryMsg->colList[j].colId) {
break;
}
j += 1;
}
return j;
}
bool vnodeValidateExprColumnInfo(SQueryMeterMsg* pQueryMsg, SSqlFuncExprMsg* pExprMsg) {
int32_t j = getColumnIndexInSource(pQueryMsg, pExprMsg);
return j < pQueryMsg->numOfCols;
}
SSqlFunctionExpr* vnodeCreateSqlFunctionExpr(SQueryMeterMsg* pQueryMsg, int32_t* code) {
SSqlFunctionExpr* pExprs = (SSqlFunctionExpr*)calloc(1, sizeof(SSqlFunctionExpr) * pQueryMsg->numOfOutputCols);
if (pExprs == NULL) {
tfree(pQueryMsg->pSqlFuncExprs);
*code = TSDB_CODE_SERV_OUT_OF_MEMORY;
return NULL;
}
bool isSuperTable = QUERY_IS_STABLE_QUERY(pQueryMsg->queryType);
int16_t tagLen = 0;
SSchema* pTagSchema = (SSchema*)pQueryMsg->pTagSchema;
for (int32_t i = 0; i < pQueryMsg->numOfOutputCols; ++i) {
pExprs[i].pBase = *((SSqlFuncExprMsg**)pQueryMsg->pSqlFuncExprs)[i];
pExprs[i].resBytes = 0;
int16_t type = 0;
int16_t bytes = 0;
SColIndexEx* pColumnIndexExInfo = &pExprs[i].pBase.colInfo;
// tag column schema is kept in pQueryMsg->pColumnModel
if (TSDB_COL_IS_TAG(pColumnIndexExInfo->flag)) {
if (pColumnIndexExInfo->colIdx >= pQueryMsg->numOfTagsCols) {
*code = TSDB_CODE_INVALID_QUERY_MSG;
tfree(pExprs);
return NULL;
}
type = pTagSchema[pColumnIndexExInfo->colIdx].type;
bytes = pTagSchema[pColumnIndexExInfo->colIdx].bytes;
} else { // parse the arithmetic expression
if (pExprs[i].pBase.functionId == TSDB_FUNC_ARITHM) {
*code = vnodeBuildExprFromArithmeticStr(&pExprs[i], pQueryMsg);
if (*code != TSDB_CODE_SUCCESS) {
tfree(pExprs);
return NULL;
}
type = TSDB_DATA_TYPE_DOUBLE;
bytes = tDataTypeDesc[type].nSize;
} else { // parse the normal column
int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].pBase);
assert(j < pQueryMsg->numOfCols);
SColumnInfo* pCol = &pQueryMsg->colList[j];
type = pCol->type;
bytes = pCol->bytes;
}
}
int32_t param = pExprs[i].pBase.arg[0].argValue.i64;
if (getResultDataInfo(type, bytes, pExprs[i].pBase.functionId, param, &pExprs[i].resType, &pExprs[i].resBytes,
&pExprs[i].interResBytes, 0, isSuperTable) != TSDB_CODE_SUCCESS) {
*code = TSDB_CODE_INVALID_QUERY_MSG;
return NULL;
}
if (pExprs[i].pBase.functionId == TSDB_FUNC_TAG_DUMMY || pExprs[i].pBase.functionId == TSDB_FUNC_TS_DUMMY) {
tagLen += pExprs[i].resBytes;
}
assert(isValidDataType(pExprs[i].resType, pExprs[i].resBytes));
}
//get the correct result size for top/bottom query, according to the number of tags columns in selection clause
// TODO refactor
for(int32_t i = 0; i < pQueryMsg->numOfOutputCols; ++i) {
pExprs[i].pBase = *((SSqlFuncExprMsg**)pQueryMsg->pSqlFuncExprs)[i];
int16_t functId = pExprs[i].pBase.functionId;
if (functId == TSDB_FUNC_TOP || functId == TSDB_FUNC_BOTTOM) {
int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].pBase);
assert(j < pQueryMsg->numOfCols);
SColumnInfo* pCol = &pQueryMsg->colList[j];
int16_t type = pCol->type;
int16_t bytes = pCol->bytes;
int32_t ret = getResultDataInfo(type, bytes, pExprs[i].pBase.functionId, pExprs[i].pBase.arg[0].argValue.i64,
&pExprs[i].resType, &pExprs[i].resBytes, &pExprs[i].interResBytes, tagLen, isSuperTable);
assert(ret == TSDB_CODE_SUCCESS);
}
}
tfree(pQueryMsg->pSqlFuncExprs);
return pExprs;
}
bool vnodeIsValidVnodeCfg(SVnodeCfg* pCfg) {
if (pCfg == NULL) return false;
if (pCfg->maxSessions <= 0 || pCfg->cacheBlockSize <= 0 || pCfg->replications <= 0 || pCfg->replications > 20 ||
pCfg->daysPerFile <= 0 || pCfg->daysToKeep <= 0) {
return false;
}
return true;
}
/**
* compare if schema of two tables are identical.
* when multi-table query is issued, the schemas of all requested tables
* should be identical. Otherwise,query process will abort.
*/
bool vnodeMeterSchemaIdentical(SColumn* pSchema1, int32_t numOfCols1, SColumn* pSchema2, int32_t numOfCols2) {
if (!VALIDNUMOFCOLS(numOfCols1) || !VALIDNUMOFCOLS(numOfCols2) || numOfCols1 != numOfCols2) {
return false;
}
return memcmp((char*)pSchema1, (char*)pSchema2, sizeof(SColumn) * numOfCols1) == 0;
}
void vnodeFreeFields(SQuery* pQuery) {
if (pQuery == NULL || pQuery->pFields == NULL) {
return;
}
for (int32_t i = 0; i < pQuery->numOfBlocks; ++i) {
tfree(pQuery->pFields[i]);
}
/*
* pQuery->pFields does not need to be released, it is allocated at the last part of pBlock
* so free(pBlock) can release this memory at the same time.
*/
pQuery->pFields = NULL;
pQuery->numOfBlocks = 0;
}
void vnodeUpdateFilterColumnIndex(SQuery* pQuery) {
for (int32_t i = 0; i < pQuery->numOfFilterCols; ++i) {
for (int16_t j = 0; j < pQuery->numOfCols; ++j) {
if (pQuery->pFilterInfo[i].info.data.colId == pQuery->colList[j].data.colId) {
pQuery->pFilterInfo[i].info.colIdx = pQuery->colList[j].colIdx;
pQuery->pFilterInfo[i].info.colIdxInBuf = pQuery->colList[j].colIdxInBuf;
// supplementary scan is also require this column
pQuery->colList[j].req[1] = 1;
break;
}
}
}
// set the column index in buffer for arithmetic operation
if (pQuery->pSelectExpr == NULL) {
return;
}
for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) {
SSqlBinaryExprInfo* pBinExprInfo = &pQuery->pSelectExpr[i].pBinExprInfo;
if (pBinExprInfo->pBinExpr == NULL) {
continue;
}
for (int16_t j = 0; j < pBinExprInfo->numOfCols; ++j) {
for (int32_t k = 0; k < pQuery->numOfCols; ++k) {
if (pBinExprInfo->pReqColumns[j].colId == pQuery->colList[k].data.colId) {
pBinExprInfo->pReqColumns[j].colIdxInBuf = pQuery->colList[k].colIdxInBuf;
assert(pQuery->colList[k].colIdxInBuf == k);
break;
}
}
}
}
}
// TODO support k<12 and k<>9
int32_t vnodeCreateFilterInfo(void* pQInfo, SQuery* pQuery) {
for (int32_t i = 0; i < pQuery->numOfCols; ++i) {
if (pQuery->colList[i].data.numOfFilters > 0) {
pQuery->numOfFilterCols++;
}
}
if (pQuery->numOfFilterCols == 0) {
return TSDB_CODE_SUCCESS;
}
pQuery->pFilterInfo = calloc(1, sizeof(SSingleColumnFilterInfo) * pQuery->numOfFilterCols);
for (int32_t i = 0, j = 0; i < pQuery->numOfCols; ++i) {
if (pQuery->colList[i].data.numOfFilters > 0) {
SSingleColumnFilterInfo* pFilterInfo = &pQuery->pFilterInfo[j];
memcpy(&pFilterInfo->info, &pQuery->colList[i], sizeof(SColumnInfoEx));
pFilterInfo->info.data.filters = NULL;
pFilterInfo->numOfFilters = pQuery->colList[i].data.numOfFilters;
pFilterInfo->pFilters = calloc(pFilterInfo->numOfFilters, sizeof(SColumnFilterElem));
for(int32_t f = 0; f < pFilterInfo->numOfFilters; ++f) {
SColumnFilterElem *pSingleColFilter = &pFilterInfo->pFilters[f];
pSingleColFilter->filterInfo = pQuery->colList[i].data.filters[f];
int32_t lower = pSingleColFilter->filterInfo.lowerRelOptr;
int32_t upper = pSingleColFilter->filterInfo.upperRelOptr;
if (lower == TSDB_RELATION_INVALID && upper == TSDB_RELATION_INVALID) {
dError("QInfo:%p invalid filter info", pQInfo);
return TSDB_CODE_INVALID_QUERY_MSG;
}
int16_t type = pQuery->colList[i].data.type;
int16_t bytes = pQuery->colList[i].data.bytes;
__filter_func_t *rangeFilterArray = vnodeGetRangeFilterFuncArray(type);
__filter_func_t *filterArray = vnodeGetValueFilterFuncArray(type);
if (rangeFilterArray == NULL && filterArray == NULL) {
dError("QInfo:%p failed to get filter function, invalid data type:%d", pQInfo, type);
return TSDB_CODE_INVALID_QUERY_MSG;
}
if ((lower == TSDB_RELATION_LARGE_EQUAL || lower == TSDB_RELATION_LARGE) &&
(upper == TSDB_RELATION_LESS_EQUAL || upper == TSDB_RELATION_LESS)) {
if (lower == TSDB_RELATION_LARGE_EQUAL) {
if (upper == TSDB_RELATION_LESS_EQUAL) {
pSingleColFilter->fp = rangeFilterArray[4];
} else {
pSingleColFilter->fp = rangeFilterArray[2];
}
} else {
if (upper == TSDB_RELATION_LESS_EQUAL) {
pSingleColFilter->fp = rangeFilterArray[3];
} else {
pSingleColFilter->fp = rangeFilterArray[1];
}
}
} else { // set callback filter function
if (lower != TSDB_RELATION_INVALID) {
pSingleColFilter->fp = filterArray[lower];
if (upper != TSDB_RELATION_INVALID) {
dError("pQInfo:%p failed to get filter function, invalid filter condition", pQInfo, type);
return TSDB_CODE_INVALID_QUERY_MSG;
}
} else {
pSingleColFilter->fp = filterArray[upper];
}
}
assert (pSingleColFilter->fp != NULL);
pSingleColFilter->bytes = bytes;
}
j++;
}
}
return TSDB_CODE_SUCCESS;
}
bool vnodeDoFilterData(SQuery* pQuery, int32_t elemPos) {
for (int32_t k = 0; k < pQuery->numOfFilterCols; ++k) {
SSingleColumnFilterInfo *pFilterInfo = &pQuery->pFilterInfo[k];
char* pElem = pFilterInfo->pData + pFilterInfo->info.data.bytes * elemPos;
if(isNull(pElem, pFilterInfo->info.data.type)) {
return false;
}
int32_t num = pFilterInfo->numOfFilters;
bool qualified = false;
for(int32_t j = 0; j < num; ++j) {
SColumnFilterElem* pFilterElem = &pFilterInfo->pFilters[j];
if (pFilterElem->fp(pFilterElem, pElem, pElem)) {
qualified = true;
break;
}
}
if (!qualified) {
return false;
}
}
return true;
}
bool vnodeFilterData(SQuery* pQuery, int32_t* numOfActualRead, int32_t index) {
(*numOfActualRead)++;
if (!vnodeDoFilterData(pQuery, index)) {
return false;
}
if (pQuery->limit.offset > 0) {
pQuery->limit.offset--; // ignore this qualified row
return false;
}
return true;
}
bool vnodeIsProjectionQuery(SSqlFunctionExpr* pExpr, int32_t numOfOutput) {
for (int32_t i = 0; i < numOfOutput; ++i) {
if (pExpr[i].pBase.functionId != TSDB_FUNC_PRJ) {
return false;
}
}
return true;
}
/*
* the pTable->state may be changed by vnodeIsSafeToDeleteMeter and import/update processor, the check of
* the state will not always be correct.
*
* The import/update/deleting is actually blocked by current query processing if the check of meter state is
* passed, but later queries are denied.
*
* 1. vnodeIsSafeToDelete will wait for this complete, since it also use the vmutex to check the numOfQueries
* 2. import will check the numOfQueries again after setting state to be TSDB_METER_STATE_IMPORTING, while the
* vmutex is also used.
* 3. insert has nothing to do with the query processing.
*/
int32_t vnodeIncQueryRefCount(SQueryMeterMsg* pQueryMsg, SMeterSidExtInfo** pSids, SMeterObj** pMeterObjList,
int32_t* numOfIncTables) {
SVnodeObj* pVnode = &vnodeList[pQueryMsg->vnode];
int32_t num = 0;
int32_t index = 0;
int32_t code = TSDB_CODE_SUCCESS;
for (int32_t i = 0; i < pQueryMsg->numOfSids; ++i) {
SMeterObj* pTable = pVnode->meterList[pSids[i]->sid];
/*
* If table is missing or is in dropping status, config it from management node, and ignore it
* during query processing. The error code of TSDB_CODE_NOT_ACTIVE_TABLE will never return to client.
* The missing table needs to be removed from pSids list
*/
if (pTable == NULL || vnodeIsMeterState(pTable, TSDB_METER_STATE_DROPPING)) {
dWarn("qmsg:%p, vid:%d sid:%d, not there or will be dropped, ignore this table in query", pQueryMsg,
pQueryMsg->vnode, pSids[i]->sid);
vnodeSendMeterCfgMsg(pQueryMsg->vnode, pSids[i]->sid);
continue;
} else if (pTable->uid != pSids[i]->uid || pTable->sid != pSids[i]->sid) {
code = TSDB_CODE_TABLE_ID_MISMATCH;
dError("qmsg:%p, vid:%d sid:%d id:%s uid:%" PRIu64 ", id mismatch. sid:%d uid:%" PRId64 " in msg", pQueryMsg,
pQueryMsg->vnode, pTable->sid, pTable->meterId, pTable->uid, pSids[i]->sid, pSids[i]->uid);
vnodeSendMeterCfgMsg(pQueryMsg->vnode, pSids[i]->sid);
continue;
} else if (pTable->state > TSDB_METER_STATE_INSERTING) { //update or import
code = TSDB_CODE_ACTION_IN_PROGRESS;
dTrace("qmsg:%p, vid:%d sid:%d id:%s, it is in state:%s, wait!", pQueryMsg, pQueryMsg->vnode, pSids[i]->sid,
pTable->meterId, taosGetTableStatusStr(pTable->state));
continue;
}
/*
* vnodeIsSafeToDeleteMeter will wait for this function complete, and then it can
* check if the numOfQueries is 0 or not.
*/
pMeterObjList[(*numOfIncTables)++] = pTable;
atomic_fetch_add_32(&pTable->numOfQueries, 1);
pSids[index++] = pSids[i];
// output for meter more than one query executed
if (pTable->numOfQueries > 1) {
dTrace("qmsg:%p, vid:%d sid:%d id:%s, inc query ref, numOfQueries:%d", pQueryMsg, pTable->vnode, pTable->sid,
pTable->meterId, pTable->numOfQueries);
num++;
}
}
dTrace("qmsg:%p, query meters: %d, inc query ref %d, numOfQueries on %d meters are 1, queried meters:%d after "
"filter missing meters", pQueryMsg, pQueryMsg->numOfSids, *numOfIncTables, (*numOfIncTables) - num, index);
assert(pQueryMsg->numOfSids >= (*numOfIncTables) && pQueryMsg->numOfSids >= index);
pQueryMsg->numOfSids = index;
return code;
}
void vnodeDecQueryRefCount(SQueryMeterMsg* pQueryMsg, SMeterObj** pMeterObjList, int32_t numOfIncTables) {
int32_t num = 0;
for (int32_t i = 0; i < numOfIncTables; ++i) {
SMeterObj* pTable = pMeterObjList[i];
if (pTable != NULL) { // here, do not need to lock to perform operations
atomic_fetch_sub_32(&pTable->numOfQueries, 1);
if (pTable->numOfQueries > 0) {
dTrace("qmsg:%p, vid:%d sid:%d id:%s dec query ref, numOfQueries:%d", pQueryMsg, pTable->vnode, pTable->sid,
pTable->meterId, pTable->numOfQueries);
num++;
}
}
}
dTrace("qmsg:%p, dec query ref for %d meters, numOfQueries on %d meters are 0", pQueryMsg, numOfIncTables, numOfIncTables - num);
}
void vnodeUpdateQueryColumnIndex(SQuery* pQuery, SMeterObj* pMeterObj) {
if (pQuery == NULL || pMeterObj == NULL) {
return;
}
int32_t i = 0, j = 0;
while (i < pQuery->numOfCols && j < pMeterObj->numOfColumns) {
if (pQuery->colList[i].data.colId == pMeterObj->schema[j].colId) {
pQuery->colList[i++].colIdx = (int16_t)j++;
} else if (pQuery->colList[i].data.colId < pMeterObj->schema[j].colId) {
pQuery->colList[i++].colIdx = -1;
} else if (pQuery->colList[i].data.colId > pMeterObj->schema[j].colId) {
j++;
}
}
while (i < pQuery->numOfCols) {
pQuery->colList[i++].colIdx = -1; // not such column in current meter
}
// sql expression has not been created yet
if (pQuery->pSelectExpr == NULL) {
return;
}
for(int32_t k = 0; k < pQuery->numOfOutputCols; ++k) {
SSqlFuncExprMsg* pSqlExprMsg = &pQuery->pSelectExpr[k].pBase;
if (pSqlExprMsg->functionId == TSDB_FUNC_ARITHM || pSqlExprMsg->colInfo.flag == TSDB_COL_TAG) {
continue;
}
SColIndexEx* pColIndexEx = &pSqlExprMsg->colInfo;
for(int32_t f = 0; f < pQuery->numOfCols; ++f) {
if (pColIndexEx->colId == pQuery->colList[f].data.colId) {
pColIndexEx->colIdx = pQuery->colList[f].colIdx;
break;
}
}
}
}
int32_t vnodeSetMeterState(SMeterObj* pMeterObj, int32_t state) {
return atomic_val_compare_exchange_32(&pMeterObj->state, TSDB_METER_STATE_READY, state);
}
void vnodeClearMeterState(SMeterObj* pMeterObj, int32_t state) {
pMeterObj->state &= (~state);
}
bool vnodeIsMeterState(SMeterObj* pMeterObj, int32_t state) {
if (state == TSDB_METER_STATE_READY) {
return pMeterObj->state == TSDB_METER_STATE_READY;
} else if (state == TSDB_METER_STATE_DROPPING) {
return pMeterObj->state >= state;
} else {
return (((pMeterObj->state) & state) == state);
}
}
void vnodeSetMeterDeleting(SMeterObj* pMeterObj) {
if (pMeterObj == NULL) {
return;
}
pMeterObj->state |= TSDB_METER_STATE_DROPPING;
}
int32_t vnodeSetMeterInsertImportStateEx(SMeterObj* pObj, int32_t st) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t state = vnodeSetMeterState(pObj, st);
if (state != TSDB_METER_STATE_READY) {//return to denote import is not performed
if (vnodeIsMeterState(pObj, TSDB_METER_STATE_DROPPING)) {
dTrace("vid:%d sid:%d id:%s, meter is deleted, state:%d", pObj->vnode, pObj->sid, pObj->meterId,
pObj->state);
code = TSDB_CODE_NOT_ACTIVE_TABLE;
} else {// waiting for 300ms by default and try again
dTrace("vid:%d sid:%d id:%s, try submit again since in state:%d", pObj->vnode, pObj->sid,
pObj->meterId, pObj->state);
code = TSDB_CODE_ACTION_IN_PROGRESS;
}
}
return code;
}
bool vnodeIsSafeToDeleteMeter(SVnodeObj* pVnode, int32_t sid) {
SMeterObj* pObj = pVnode->meterList[sid];
if (pObj == NULL || vnodeIsMeterState(pObj, TSDB_METER_STATE_DROPPED)) {
return true;
}
int32_t prev = vnodeSetMeterState(pObj, TSDB_METER_STATE_DROPPING);
/*
* if the meter is not in ready/deleting state, it must be in insert/import/update,
* set the deleting state and wait the procedure to be completed
*/
if (prev != TSDB_METER_STATE_READY && prev < TSDB_METER_STATE_DROPPING) {
vnodeSetMeterDeleting(pObj);
dWarn("vid:%d sid:%d id:%s, can not be deleted, state:%d, wait", pObj->vnode, pObj->sid, pObj->meterId, prev);
return false;
}
bool ready = true;
/*
* the query will be stopped ASAP, since the state of meter is set to TSDB_METER_STATE_DROPPING,
* and new query will abort since the meter is deleted.
*/
pthread_mutex_lock(&pVnode->vmutex);
if (pObj->numOfQueries > 0) {
dWarn("vid:%d sid:%d id:%s %d queries executing on it, wait query to be finished",
pObj->vnode, pObj->sid, pObj->meterId, pObj->numOfQueries);
ready = false;
}
pthread_mutex_unlock(&pVnode->vmutex);
return ready;
}
void vnodeFreeColumnInfo(SColumnInfo* pColumnInfo) {
if (pColumnInfo == NULL) {
return;
}
if (pColumnInfo->numOfFilters > 0) {
if (pColumnInfo->type == TSDB_DATA_TYPE_BINARY) {
for (int32_t i = 0; i < pColumnInfo->numOfFilters; ++i) {
tfree(pColumnInfo->filters[i].pz);
pColumnInfo->filters[i].len = 0;
}
}
tfree(pColumnInfo->filters);
}
}