xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' into fix/TD-25717-3.0

This commit is contained in:
kailixu 2023-08-16 10:19:34 +08:00
parent 3b69736b29 63bc50a723
commit 6ee95dca30
67 changed files with 4873 additions and 411 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
cmake/apr-util_CMakeLists.txt.in Normal file
View File

@ -0,0 +1,19 @@
# apr-util
ExternalProject_Add(aprutil-1
URL https://dlcdn.apache.org//apr/apr-util-1.6.3.tar.gz
URL_HASH SHA256=2b74d8932703826862ca305b094eef2983c27b39d5c9414442e9976a9acf1983
DOWNLOAD_NO_PROGRESS 1
DOWNLOAD_DIR "${TD_CONTRIB_DIR}/deps-download"
#GIT_REPOSITORY https://github.com/apache/apr-util.git
#GIT_TAG 1.5.4
SOURCE_DIR "${TD_CONTRIB_DIR}/apr-util"
#BINARY_DIR ""
BUILD_IN_SOURCE TRUE
BUILD_ALWAYS 1
#UPDATE_COMMAND ""
CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1/ --with-apr=$ENV{HOME}/.cos-local.1
#CONFIGURE_COMMAND ./configure --with-apr=/usr/local/apr
BUILD_COMMAND make
INSTALL_COMMAND make install
TEST_COMMAND ""
)

19
cmake/apr_CMakeLists.txt.in Normal file
View File

@ -0,0 +1,19 @@
# apr
ExternalProject_Add(apr-1
URL https://dlcdn.apache.org//apr/apr-1.7.4.tar.gz
URL_HASH SHA256=a4137dd82a185076fa50ba54232d920a17c6469c30b0876569e1c2a05ff311d9
DOWNLOAD_NO_PROGRESS 1
DOWNLOAD_DIR "${TD_CONTRIB_DIR}/deps-download"
#GIT_REPOSITORY https://github.com/apache/apr.git
#GIT_TAG 1.5.2
SOURCE_DIR "${TD_CONTRIB_DIR}/apr"
BUILD_IN_SOURCE TRUE
UPDATE_DISCONNECTED TRUE
BUILD_ALWAYS 1
#UPDATE_COMMAND ""
CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1/ --enable-shared=no
#CONFIGURE_COMMAND ./configure
BUILD_COMMAND make
INSTALL_COMMAND make install
TEST_COMMAND ""
)

10
cmake/cmake.options
View File

@ -125,6 +125,16 @@ option(
ON ON
) )
IF(${TD_LINUX})
option(
BUILD_WITH_COS
"If build with cos"
ON
)
ENDIF ()
option( option(
BUILD_WITH_SQLITE BUILD_WITH_SQLITE
"If build with sqlite" "If build with sqlite"

12
cmake/cos_CMakeLists.txt.in Normal file
View File

@ -0,0 +1,12 @@
# cos
ExternalProject_Add(cos
GIT_REPOSITORY https://github.com/tencentyun/cos-c-sdk-v5.git
GIT_TAG v5.0.16
SOURCE_DIR "${TD_CONTRIB_DIR}/cos-c-sdk-v5"
BINARY_DIR ""
#BUILD_IN_SOURCE TRUE
CONFIGURE_COMMAND ""
BUILD_COMMAND ""
INSTALL_COMMAND ""
TEST_COMMAND ""
)

17
cmake/curl_CMakeLists.txt.in Normal file
View File

@ -0,0 +1,17 @@
# curl
ExternalProject_Add(curl
URL https://curl.se/download/curl-8.2.1.tar.gz
DOWNLOAD_NO_PROGRESS 1
DOWNLOAD_DIR "${TD_CONTRIB_DIR}/deps-download"
#GIT_REPOSITORY https://github.com/curl/curl.git
#GIT_TAG curl-7_88_1
SOURCE_DIR "${TD_CONTRIB_DIR}/curl"
BUILD_IN_SOURCE TRUE
BUILD_ALWAYS 1
#UPDATE_COMMAND ""
CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --without-ssl --enable-shared=no --disable-ldap --disable-ldaps --without-brotli
#CONFIGURE_COMMAND ./configure --without-ssl
BUILD_COMMAND make
INSTALL_COMMAND make install
TEST_COMMAND ""
)

14
cmake/mxml_CMakeLists.txt.in Normal file
View File

@ -0,0 +1,14 @@
# cos
ExternalProject_Add(mxml
GIT_REPOSITORY https://github.com/michaelrsweet/mxml.git
GIT_TAG release-2.10
SOURCE_DIR "${TD_CONTRIB_DIR}/mxml"
#BINARY_DIR ""
BUILD_IN_SOURCE TRUE
#UPDATE_COMMAND ""
CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --enable-shared=no
#CONFIGURE_COMMAND ./configure
BUILD_COMMAND make
INSTALL_COMMAND make install
TEST_COMMAND ""
)

68
contrib/CMakeLists.txt
View File

@ -6,6 +6,39 @@ function(cat IN_FILE OUT_FILE)
file(APPEND ${OUT_FILE} "${CONTENTS}") file(APPEND ${OUT_FILE} "${CONTENTS}")
endfunction(cat IN_FILE OUT_FILE) endfunction(cat IN_FILE OUT_FILE)
if(${TD_LINUX})
set(CONTRIB_TMP_FILE3 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in3")
configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3})
if(${BUILD_WITH_COS})
file(MAKE_DIRECTORY $ENV{HOME}/.cos-local.1/)
cat("${TD_SUPPORT_DIR}/mxml_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3})
cat("${TD_SUPPORT_DIR}/apr_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3})
cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3})
endif(${BUILD_WITH_COS})
configure_file(${CONTRIB_TMP_FILE3} "${TD_CONTRIB_DIR}/deps-download/CMakeLists.txt")
execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" .
WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download")
execute_process(COMMAND "${CMAKE_COMMAND}" --build .
WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download")
set(CONTRIB_TMP_FILE2 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in2")
configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE2})
if(${BUILD_WITH_COS})
cat("${TD_SUPPORT_DIR}/apr-util_CMakeLists.txt.in" ${CONTRIB_TMP_FILE2})
endif(${BUILD_WITH_COS})
configure_file(${CONTRIB_TMP_FILE2} "${TD_CONTRIB_DIR}/deps-download/CMakeLists.txt")
execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" .
WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download")
execute_process(COMMAND "${CMAKE_COMMAND}" --build .
WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download")
endif(${TD_LINUX})
set(CONTRIB_TMP_FILE "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in") set(CONTRIB_TMP_FILE "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in")
configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
@ -122,6 +155,16 @@ if(${BUILD_WITH_SQLITE})
cat("${TD_SUPPORT_DIR}/sqlite_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) cat("${TD_SUPPORT_DIR}/sqlite_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
endif(${BUILD_WITH_SQLITE}) endif(${BUILD_WITH_SQLITE})
# cos
if(${BUILD_WITH_COS})
#cat("${TD_SUPPORT_DIR}/mxml_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
#cat("${TD_SUPPORT_DIR}/apr_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
#cat("${TD_SUPPORT_DIR}/apr-util_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
#cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
cat("${TD_SUPPORT_DIR}/cos_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
add_definitions(-DUSE_COS)
endif(${BUILD_WITH_COS})
# lucene # lucene
if(${BUILD_WITH_LUCENE}) if(${BUILD_WITH_LUCENE})
cat("${TD_SUPPORT_DIR}/lucene_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) cat("${TD_SUPPORT_DIR}/lucene_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
@ -347,6 +390,31 @@ if (${BUILD_WITH_ROCKSDB})
endif() endif()
endif() endif()
# cos
if(${BUILD_WITH_COS})
if(${TD_LINUX})
set(CMAKE_PREFIX_PATH $ENV{HOME}/.cos-local.1)
#ADD_DEFINITIONS(-DMINIXML_LIBRARY=${CMAKE_BINARY_DIR}/build/lib/libxml.a)
option(ENABLE_TEST "Enable the tests" OFF)
INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.1/include)
MESSAGE("$ENV{HOME}/.cos-local.1/include")
set(CMAKE_BUILD_TYPE debug)
set(ORIG_CMAKE_PROJECT_NAME ${CMAKE_PROJECT_NAME})
set(CMAKE_PROJECT_NAME cos_c_sdk)
add_subdirectory(cos-c-sdk-v5 EXCLUDE_FROM_ALL)
target_include_directories(
cos_c_sdk
PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cos-c-sdk-v5/cos_c_sdk>
)
set(CMAKE_PROJECT_NAME ${ORIG_CMAKE_PROJECT_NAME})
else()
endif(${TD_LINUX})
endif(${BUILD_WITH_COS})
# lucene # lucene
# To support build on ubuntu: sudo apt-get install libboost-all-dev # To support build on ubuntu: sudo apt-get install libboost-all-dev
if(${BUILD_WITH_LUCENE}) if(${BUILD_WITH_LUCENE})

5
contrib/test/CMakeLists.txt
View File

@ -3,6 +3,11 @@ if(${BUILD_WITH_ROCKSDB})
add_subdirectory(rocksdb) add_subdirectory(rocksdb)
endif(${BUILD_WITH_ROCKSDB}) endif(${BUILD_WITH_ROCKSDB})
# cos
if(${BUILD_WITH_COS})
add_subdirectory(cos)
endif(${BUILD_WITH_COS})
if(${BUILD_WITH_LUCENE}) if(${BUILD_WITH_LUCENE})
add_subdirectory(lucene) add_subdirectory(lucene)
endif(${BUILD_WITH_LUCENE}) endif(${BUILD_WITH_LUCENE})

49
contrib/test/cos/CMakeLists.txt Normal file
View File

@ -0,0 +1,49 @@
add_executable(cosTest "")
target_sources(cosTest
PRIVATE
"${CMAKE_CURRENT_SOURCE_DIR}/main.c"
)
#find_path(APR_INCLUDE_DIR apr-1/apr_time.h)
#find_path(APR_UTIL_INCLUDE_DIR apr/include/apr-1/apr_md5.h)
#find_path(MINIXML_INCLUDE_DIR mxml.h)
#find_path(CURL_INCLUDE_DIR curl/curl.h)
#include_directories (${MINIXML_INCLUDE_DIR})
#include_directories (${CURL_INCLUDE_DIR})
FIND_PROGRAM(APR_CONFIG_BIN NAMES apr-config apr-1-config PATHS /usr/bin /usr/local/bin /usr/local/apr/bin/)
#FIND_PROGRAM(APU_CONFIG_BIN NAMES apu-config apu-1-config PATHS /usr/bin /usr/local/bin /usr/local/apr/bin/)
IF (APR_CONFIG_BIN)
EXECUTE_PROCESS(
COMMAND ${APR_CONFIG_BIN} --includedir
OUTPUT_VARIABLE APR_INCLUDE_DIR
OUTPUT_STRIP_TRAILING_WHITESPACE
)
ENDIF()
#IF (APU_CONFIG_BIN)
# EXECUTE_PROCESS(
# COMMAND ${APU_CONFIG_BIN} --includedir
# OUTPUT_VARIABLE APR_UTIL_INCLUDE_DIR
# OUTPUT_STRIP_TRAILING_WHITESPACE
# )
#ENDIF()
include_directories (${APR_INCLUDE_DIR})
#include_directories (${APR_UTIL_INCLUDE_DIR})
target_include_directories(
cosTest
PUBLIC "${TD_SOURCE_DIR}/contrib/cos-c-sdk-v5/cos_c_sdk"
)
#find_library(APR_LIBRARY apr-1 PATHS /usr/local/apr/lib/)
#find_library(APR_UTIL_LIBRARY aprutil-1 PATHS /usr/local/apr/lib/)
#find_library(MINIXML_LIBRARY mxml)
#find_library(CURL_LIBRARY curl)
target_link_libraries(cosTest cos_c_sdk)
target_link_libraries(cosTest apr-1})
target_link_libraries(cosTest aprutil-1})
target_link_libraries(cosTest mxml)
target_link_libraries(cosTest curl)

3095
contrib/test/cos/main.c Normal file
View File

File diff suppressed because it is too large Load Diff

14
include/libs/executor/storageapi.h
View File

@ -98,6 +98,16 @@ typedef struct SMTbCursor {
int8_t paused; int8_t paused;
} SMTbCursor; } SMTbCursor;
typedef struct SMCtbCursor {
SMeta *pMeta;
void *pCur;
tb_uid_t suid;
void *pKey;
void *pVal;
int kLen;
int vLen;
} SMCtbCursor;
typedef struct SRowBuffPos { typedef struct SRowBuffPos {
void* pRowBuff; void* pRowBuff;
void* pKey; void* pKey;
@ -278,13 +288,15 @@ typedef struct SStoreMeta {
void (*getBasicInfo)(void* pVnode, const char** dbname, int32_t* vgId, int64_t* numOfTables, void (*getBasicInfo)(void* pVnode, const char** dbname, int32_t* vgId, int64_t* numOfTables,
int64_t* numOfNormalTables); // vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId) & int64_t* numOfNormalTables); // vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId) &
// metaGetTbNum(SMeta *pMeta) & metaGetNtbNum(SMeta *pMeta); // metaGetTbNum(SMeta *pMeta) & metaGetNtbNum(SMeta *pMeta);
int64_t (*getNumOfRowsInMem)(void* pVnode); int64_t (*getNumOfRowsInMem)(void* pVnode);
/** /**
int32_t vnodeGetCtbIdList(void *pVnode, int64_t suid, SArray *list); int32_t vnodeGetCtbIdList(void *pVnode, int64_t suid, SArray *list);
int32_t vnodeGetCtbIdListByFilter(void *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg), void *arg); int32_t vnodeGetCtbIdListByFilter(void *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg), void *arg);
int32_t vnodeGetStbIdList(void *pVnode, int64_t suid, SArray *list); int32_t vnodeGetStbIdList(void *pVnode, int64_t suid, SArray *list);
*/ */
SMCtbCursor* (*openCtbCursor)(void *pVnode, tb_uid_t uid, int lock);
void (*closeCtbCursor)(SMCtbCursor *pCtbCur, int lock);
tb_uid_t (*ctbCursorNext)(SMCtbCursor* pCur);
} SStoreMeta; } SStoreMeta;
typedef struct SStoreMetaReader { typedef struct SStoreMetaReader {

7
include/libs/nodes/plannodes.h
View File

@ -107,6 +107,7 @@ typedef struct SScanLogicNode {
bool sortPrimaryKey; bool sortPrimaryKey;
bool igLastNull; bool igLastNull;
bool groupOrderScan; bool groupOrderScan;
bool onlyMetaCtbIdx; // for tag scan with no tbname
} SScanLogicNode; } SScanLogicNode;
typedef struct SJoinLogicNode { typedef struct SJoinLogicNode {
@ -334,7 +335,11 @@ typedef struct SScanPhysiNode {
bool groupOrderScan; bool groupOrderScan;
} SScanPhysiNode; } SScanPhysiNode;
typedef SScanPhysiNode STagScanPhysiNode; typedef struct STagScanPhysiNode {
SScanPhysiNode scan;
bool onlyMetaCtbIdx; //no tbname, tag index not used.
} STagScanPhysiNode;
typedef SScanPhysiNode SBlockDistScanPhysiNode; typedef SScanPhysiNode SBlockDistScanPhysiNode;
typedef struct SLastRowScanPhysiNode { typedef struct SLastRowScanPhysiNode {

2
include/os/osFile.h
View File

@ -76,7 +76,7 @@ int32_t taosUnLockFile(TdFilePtr pFile);
int32_t taosUmaskFile(int32_t maskVal); int32_t taosUmaskFile(int32_t maskVal);
int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime); int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime, int32_t *atime);
int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno); int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno);
int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime); int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime);
bool taosCheckExistFile(const char *pathname); bool taosCheckExistFile(const char *pathname);

2
include/os/osRand.h
View File

@ -32,6 +32,8 @@ void taosSeedRand(uint32_t seed);
uint32_t taosRand(void); uint32_t taosRand(void);
uint32_t taosRandR(uint32_t* pSeed); uint32_t taosRandR(uint32_t* pSeed);
void taosRandStr(char* str, int32_t size); void taosRandStr(char* str, int32_t size);
void taosRandStr2(char* str, int32_t size);
uint32_t taosSafeRand(void); uint32_t taosSafeRand(void);
#ifdef __cplusplus #ifdef __cplusplus

8
source/common/CMakeLists.txt
View File

@ -16,6 +16,14 @@ ENDIF ()
IF (TD_STORAGE) IF (TD_STORAGE)
ADD_DEFINITIONS(-D_STORAGE) ADD_DEFINITIONS(-D_STORAGE)
TARGET_LINK_LIBRARIES(common PRIVATE storage) TARGET_LINK_LIBRARIES(common PRIVATE storage)
IF(${TD_LINUX})
IF(${BUILD_WITH_COS})
add_definitions(-DUSE_COS)
ENDIF(${BUILD_WITH_COS})
ENDIF(${TD_LINUX})
ENDIF () ENDIF ()
target_include_directories( target_include_directories(

26
source/common/src/tdatablock.c
View File

@ -142,7 +142,8 @@ int32_t colDataSetVal(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const
return 0; return 0;
} }
int32_t colDataReassignVal(SColumnInfoData* pColumnInfoData, uint32_t dstRowIdx, uint32_t srcRowIdx, const char* pData) { int32_t colDataReassignVal(SColumnInfoData* pColumnInfoData, uint32_t dstRowIdx, uint32_t srcRowIdx,
const char* pData) {
int32_t type = pColumnInfoData->info.type; int32_t type = pColumnInfoData->info.type;
if (IS_VAR_DATA_TYPE(type)) { if (IS_VAR_DATA_TYPE(type)) {
int32_t dataLen = 0; int32_t dataLen = 0;
@ -164,7 +165,6 @@ int32_t colDataReassignVal(SColumnInfoData* pColumnInfoData, uint32_t dstRowIdx,
return 0; return 0;
} }
static int32_t colDataReserve(SColumnInfoData* pColumnInfoData, size_t newSize) { static int32_t colDataReserve(SColumnInfoData* pColumnInfoData, size_t newSize) {
if (!IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { if (!IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -190,7 +190,8 @@ static int32_t colDataReserve(SColumnInfoData* pColumnInfoData, size_t newSize)
static int32_t doCopyNItems(struct SColumnInfoData* pColumnInfoData, int32_t currentRow, const char* pData, static int32_t doCopyNItems(struct SColumnInfoData* pColumnInfoData, int32_t currentRow, const char* pData,
int32_t itemLen, int32_t numOfRows, bool trimValue) { int32_t itemLen, int32_t numOfRows, bool trimValue) {
if (pColumnInfoData->info.bytes < itemLen) { if (pColumnInfoData->info.bytes < itemLen) {
uWarn("column/tag actual data len %d is bigger than schema len %d, trim it:%d", itemLen, pColumnInfoData->info.bytes, trimValue); uWarn("column/tag actual data len %d is bigger than schema len %d, trim it:%d", itemLen,
pColumnInfoData->info.bytes, trimValue);
if (trimValue) { if (trimValue) {
itemLen = pColumnInfoData->info.bytes; itemLen = pColumnInfoData->info.bytes;
} else { } else {
@ -230,8 +231,8 @@ static int32_t doCopyNItems(struct SColumnInfoData* pColumnInfoData, int32_t cur
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t colDataSetNItems(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, int32_t colDataSetNItems(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, uint32_t numOfRows,
uint32_t numOfRows, bool trimValue) { bool trimValue) {
int32_t len = pColumnInfoData->info.bytes; int32_t len = pColumnInfoData->info.bytes;
if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
len = varDataTLen(pData); len = varDataTLen(pData);
@ -698,8 +699,7 @@ int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static bool colDataIsNNull(const SColumnInfoData* pColumnInfoData, int32_t startIndex, static bool colDataIsNNull(const SColumnInfoData* pColumnInfoData, int32_t startIndex, uint32_t nRows) {
uint32_t nRows) {
if (!pColumnInfoData->hasNull) { if (!pColumnInfoData->hasNull) {
return false; return false;
} }
@ -880,7 +880,6 @@ int32_t dataBlockCompar(const void* p1, const void* p2, const void* param) {
} }
static int32_t blockDataAssign(SColumnInfoData* pCols, const SSDataBlock* pDataBlock, const int32_t* index) { static int32_t blockDataAssign(SColumnInfoData* pCols, const SSDataBlock* pDataBlock, const int32_t* index) {
size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pDst = &pCols[i]; SColumnInfoData* pDst = &pCols[i];
@ -1131,6 +1130,7 @@ static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo*
if (tmp == NULL) { if (tmp == NULL) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
// memset(tmp, 0, numOfRows * pColumn->info.bytes);
// copy back the existed data // copy back the existed data
if (pColumn->pData != NULL) { if (pColumn->pData != NULL) {
@ -1780,8 +1780,8 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf)
int32_t rows = pDataBlock->info.rows; int32_t rows = pDataBlock->info.rows;
int32_t len = 0; int32_t len = 0;
len += snprintf(dumpBuf + len, size - len, len += snprintf(dumpBuf + len, size - len,
"===stream===%s|block type %d|child id %d|group id:%" PRIu64 "|uid:%" PRId64 "===stream===%s|block type %d|child id %d|group id:%" PRIu64 "|uid:%" PRId64 "|rows:%" PRId64
"|rows:%" PRId64 "|version:%" PRIu64 "|cal start:%" PRIu64 "|cal end:%" PRIu64 "|tbl:%s\n", "|version:%" PRIu64 "|cal start:%" PRIu64 "|cal end:%" PRIu64 "|tbl:%s\n",
flag, (int32_t)pDataBlock->info.type, pDataBlock->info.childId, pDataBlock->info.id.groupId, flag, (int32_t)pDataBlock->info.type, pDataBlock->info.childId, pDataBlock->info.id.groupId,
pDataBlock->info.id.uid, pDataBlock->info.rows, pDataBlock->info.version, pDataBlock->info.id.uid, pDataBlock->info.rows, pDataBlock->info.version,
pDataBlock->info.calWin.skey, pDataBlock->info.calWin.ekey, pDataBlock->info.parTbName); pDataBlock->info.calWin.skey, pDataBlock->info.calWin.ekey, pDataBlock->info.parTbName);
@ -2181,7 +2181,8 @@ int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols) {
} }
colSizes[col] = htonl(colSizes[col]); colSizes[col] = htonl(colSizes[col]);
// uError("blockEncode col bytes:%d, type:%d, size:%d, htonl size:%d", pColRes->info.bytes, pColRes->info.type, htonl(colSizes[col]), colSizes[col]); // uError("blockEncode col bytes:%d, type:%d, size:%d, htonl size:%d", pColRes->info.bytes, pColRes->info.type,
// htonl(colSizes[col]), colSizes[col]);
} }
*actualLen = dataLen; *actualLen = dataLen;
@ -2310,7 +2311,8 @@ void trimDataBlock(SSDataBlock* pBlock, int32_t totalRows, const bool* pBoolList
if (colDataIsNull_var(pDst, j)) { if (colDataIsNull_var(pDst, j)) {
colDataSetNull_var(pDst, numOfRows); colDataSetNull_var(pDst, numOfRows);
} else { } else {
// fix address sanitizer error. p1 may point to memory that will change during realloc of colDataSetVal, first copy it to p2 // fix address sanitizer error. p1 may point to memory that will change during realloc of colDataSetVal, first
// copy it to p2
char* p1 = colDataGetVarData(pDst, j); char* p1 = colDataGetVarData(pDst, j);
int32_t len = 0; int32_t len = 0;
if (pDst->info.type == TSDB_DATA_TYPE_JSON) { if (pDst->info.type == TSDB_DATA_TYPE_JSON) {

51
source/common/src/tglobal.c
View File

@ -237,6 +237,14 @@ int64_t tsCheckpointInterval = 3 * 60 * 60 * 1000;
bool tsFilterScalarMode = false; bool tsFilterScalarMode = false;
int32_t tsKeepTimeOffset = 0; // latency of data migration int32_t tsKeepTimeOffset = 0; // latency of data migration
char tsS3Endpoint[TSDB_FQDN_LEN] = "<endpoint>";
char tsS3AccessKey[TSDB_FQDN_LEN] = "<accesskey>";
char tsS3AccessKeyId[TSDB_FQDN_LEN] = "<accesskeyid>";
char tsS3AccessKeySecret[TSDB_FQDN_LEN] = "<accesskeysecrect>";
char tsS3BucketName[TSDB_FQDN_LEN] = "<bucketname>";
char tsS3AppId[TSDB_FQDN_LEN] = "<appid>";
int8_t tsS3Enabled = false;
#ifndef _STORAGE #ifndef _STORAGE
int32_t taosSetTfsCfg(SConfig *pCfg) { int32_t taosSetTfsCfg(SConfig *pCfg) {
SConfigItem *pItem = cfgGetItem(pCfg, "dataDir"); SConfigItem *pItem = cfgGetItem(pCfg, "dataDir");
@ -258,7 +266,43 @@ int32_t taosSetTfsCfg(SConfig *pCfg) {
int32_t taosSetTfsCfg(SConfig *pCfg); int32_t taosSetTfsCfg(SConfig *pCfg);
#endif #endif
struct SConfig *taosGetCfg() { return tsCfg; } int32_t taosSetS3Cfg(SConfig *pCfg) {
tstrncpy(tsS3AccessKey, cfgGetItem(pCfg, "s3Accesskey")->str, TSDB_FQDN_LEN);
if (tsS3AccessKey[0] == '<') {
return 0;
}
char *colon = strchr(tsS3AccessKey, ':');
if (!colon) {
uError("invalid access key:%s", tsS3AccessKey);
return -1;
}
*colon = '\0';
tstrncpy(tsS3AccessKeyId, tsS3AccessKey, TSDB_FQDN_LEN);
tstrncpy(tsS3AccessKeySecret, colon + 1, TSDB_FQDN_LEN);
tstrncpy(tsS3Endpoint, cfgGetItem(pCfg, "s3Endpoint")->str, TSDB_FQDN_LEN);
tstrncpy(tsS3BucketName, cfgGetItem(pCfg, "s3BucketName")->str, TSDB_FQDN_LEN);
char *cos = strstr(tsS3Endpoint, "cos.");
if (cos) {
char *appid = strrchr(tsS3BucketName, '-');
if (!appid) {
uError("failed to locate appid in bucket:%s", tsS3BucketName);
return -1;
} else {
tstrncpy(tsS3AppId, appid + 1, TSDB_FQDN_LEN);
}
}
if (tsS3BucketName[0] != '<' && tsDiskCfgNum > 1) {
#ifdef USE_COS
tsS3Enabled = true;
#endif
}
return 0;
}
struct SConfig *taosGetCfg() {
return tsCfg;
}
static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *inputCfgDir, const char *envFile, static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *inputCfgDir, const char *envFile,
char *apolloUrl) { char *apolloUrl) {
@ -581,6 +625,10 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddInt32(pCfg, "maxStreamBackendCache", tsMaxStreamBackendCache, 16, 1024, CFG_SCOPE_SERVER) != 0) return -1; if (cfgAddInt32(pCfg, "maxStreamBackendCache", tsMaxStreamBackendCache, 16, 1024, CFG_SCOPE_SERVER) != 0) return -1;
if (cfgAddInt32(pCfg, "pqSortMemThreshold", tsPQSortMemThreshold, 1, 10240, CFG_SCOPE_SERVER) != 0) return -1; if (cfgAddInt32(pCfg, "pqSortMemThreshold", tsPQSortMemThreshold, 1, 10240, CFG_SCOPE_SERVER) != 0) return -1;
if (cfgAddString(pCfg, "s3Accesskey", tsS3AccessKey, CFG_SCOPE_SERVER) != 0) return -1;
if (cfgAddString(pCfg, "s3Endpoint", tsS3Endpoint, CFG_SCOPE_SERVER) != 0) return -1;
if (cfgAddString(pCfg, "s3BucketName", tsS3BucketName, CFG_SCOPE_SERVER) != 0) return -1;
GRANT_CFG_ADD; GRANT_CFG_ADD;
return 0; return 0;
} }
@ -1502,6 +1550,7 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile
if (taosSetServerCfg(tsCfg)) return -1; if (taosSetServerCfg(tsCfg)) return -1;
if (taosSetReleaseCfg(tsCfg)) return -1; if (taosSetReleaseCfg(tsCfg)) return -1;
if (taosSetTfsCfg(tsCfg) != 0) return -1; if (taosSetTfsCfg(tsCfg) != 0) return -1;
if (taosSetS3Cfg(tsCfg) != 0) return -1;
} }
taosSetSystemCfg(tsCfg); taosSetSystemCfg(tsCfg);

2
source/dnode/mgmt/mgmt_mnode/src/mmFile.c
View File

@ -65,7 +65,7 @@ int32_t mmReadFile(const char *path, SMnodeOpt *pOption) {
char file[PATH_MAX] = {0}; char file[PATH_MAX] = {0};
snprintf(file, sizeof(file), "%s%smnode.json", path, TD_DIRSEP); snprintf(file, sizeof(file), "%s%smnode.json", path, TD_DIRSEP);
if (taosStatFile(file, NULL, NULL) < 0) { if (taosStatFile(file, NULL, NULL, NULL) < 0) {
dInfo("mnode file:%s not exist", file); dInfo("mnode file:%s not exist", file);
return 0; return 0;
} }

2
source/dnode/mgmt/mgmt_vnode/src/vmFile.c
View File

@ -97,7 +97,7 @@ int32_t vmGetVnodeListFromFile(SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t
SWrapperCfg *pCfgs = NULL; SWrapperCfg *pCfgs = NULL;
snprintf(file, sizeof(file), "%s%svnodes.json", pMgmt->path, TD_DIRSEP); snprintf(file, sizeof(file), "%s%svnodes.json", pMgmt->path, TD_DIRSEP);
if (taosStatFile(file, NULL, NULL) < 0) { if (taosStatFile(file, NULL, NULL, NULL) < 0) {
dInfo("vnode file:%s not exist", file); dInfo("vnode file:%s not exist", file);
return 0; return 0;
} }

4
source/dnode/mgmt/node_util/src/dmEps.c
View File

@ -100,7 +100,7 @@ int32_t dmReadEps(SDnodeData *pData) {
goto _OVER; goto _OVER;
} }
if (taosStatFile(file, NULL, NULL) < 0) { if (taosStatFile(file, NULL, NULL, NULL) < 0) {
dInfo("dnode file:%s not exist", file); dInfo("dnode file:%s not exist", file);
code = 0; code = 0;
goto _OVER; goto _OVER;
@ -469,7 +469,7 @@ static int32_t dmReadDnodePairs(SDnodeData *pData) {
char file[PATH_MAX] = {0}; char file[PATH_MAX] = {0};
snprintf(file, sizeof(file), "%s%sdnode%sep.json", tsDataDir, TD_DIRSEP, TD_DIRSEP); snprintf(file, sizeof(file), "%s%sdnode%sep.json", tsDataDir, TD_DIRSEP, TD_DIRSEP);
if (taosStatFile(file, NULL, NULL) < 0) { if (taosStatFile(file, NULL, NULL, NULL) < 0) {
dDebug("dnode file:%s not exist", file); dDebug("dnode file:%s not exist", file);
code = 0; code = 0;
goto _OVER; goto _OVER;

2
source/dnode/mgmt/node_util/src/dmFile.c
View File

@ -38,7 +38,7 @@ int32_t dmReadFile(const char *path, const char *name, bool *pDeployed) {
char file[PATH_MAX] = {0}; char file[PATH_MAX] = {0};
snprintf(file, sizeof(file), "%s%s%s.json", path, TD_DIRSEP, name); snprintf(file, sizeof(file), "%s%s%s.json", path, TD_DIRSEP, name);
if (taosStatFile(file, NULL, NULL) < 0) { if (taosStatFile(file, NULL, NULL, NULL) < 0) {
dInfo("file:%s not exist", file); dInfo("file:%s not exist", file);
code = 0; code = 0;
goto _OVER; goto _OVER;

72
source/dnode/mnode/impl/src/mndIndex.c
View File

@ -79,9 +79,12 @@ int32_t mndInitIdx(SMnode *pMnode) {
return sdbSetTable(pMnode->pSdb, table); return sdbSetTable(pMnode->pSdb, table);
} }
static int32_t mndFindSuperTableTagId(const SStbObj *pStb, const char *tagName) { static int32_t mndFindSuperTableTagId(const SStbObj *pStb, const char *tagName, int8_t *hasIdx) {
for (int32_t tag = 0; tag < pStb->numOfTags; tag++) { for (int32_t tag = 0; tag < pStb->numOfTags; tag++) {
if (strcasecmp(pStb->pTags[tag].name, tagName) == 0) { if (strcasecmp(pStb->pTags[tag].name, tagName) == 0) {
if (IS_IDX_ON(&pStb->pTags[tag])) {
*hasIdx = 1;
}
return tag; return tag;
} }
} }
@ -597,7 +600,8 @@ static int32_t mndSetUpdateIdxStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStb
pNew->updateTime = taosGetTimestampMs(); pNew->updateTime = taosGetTimestampMs();
pNew->lock = 0; pNew->lock = 0;
int32_t tag = mndFindSuperTableTagId(pOld, tagName); int8_t hasIdx = 0;
int32_t tag = mndFindSuperTableTagId(pOld, tagName, &hasIdx);
if (tag < 0) { if (tag < 0) {
terrno = TSDB_CODE_MND_TAG_NOT_EXIST; terrno = TSDB_CODE_MND_TAG_NOT_EXIST;
return -1; return -1;
@ -612,14 +616,14 @@ static int32_t mndSetUpdateIdxStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStb
SSchema *pTag = pNew->pTags + tag; SSchema *pTag = pNew->pTags + tag;
if (on == 1) { if (on == 1) {
if (IS_IDX_ON(pTag)) { if (hasIdx && tag != 0) {
terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST; terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST;
return -1; return -1;
} else { } else {
SSCHMEA_SET_IDX_ON(pTag); SSCHMEA_SET_IDX_ON(pTag);
} }
} else { } else {
if (!IS_IDX_ON(pTag)) { if (hasIdx == 0) {
terrno = TSDB_CODE_MND_SMA_NOT_EXIST; terrno = TSDB_CODE_MND_SMA_NOT_EXIST;
} else { } else {
SSCHMEA_SET_IDX_OFF(pTag); SSCHMEA_SET_IDX_OFF(pTag);
@ -667,7 +671,42 @@ _OVER:
mndTransDrop(pTrans); mndTransDrop(pTrans);
return code; return code;
} }
int8_t mndCheckIndexNameByTagName(SMnode *pMnode, SIdxObj *pIdxObj) {
// build index on first tag, and no index name;
int8_t exist = 0;
SDbObj *pDb = NULL;
if (strlen(pIdxObj->db) > 0) {
pDb = mndAcquireDb(pMnode, pIdxObj->db);
if (pDb == NULL) return 0;
}
SSmaAndTagIter *pIter = NULL;
SIdxObj *pIdx = NULL;
SSdb *pSdb = pMnode->pSdb;
while (1) {
pIter = sdbFetch(pSdb, SDB_IDX, pIter, (void **)&pIdx);
if (pIter == NULL) break;
if (NULL != pDb && pIdx->dbUid != pDb->uid) {
sdbRelease(pSdb, pIdx);
continue;
}
if (pIdxObj->stbUid != pIdx->stbUid) {
sdbRelease(pSdb, pIdx);
continue;
}
if (strncmp(pIdxObj->colName, pIdx->colName, TSDB_COL_NAME_LEN) == 0) {
sdbCancelFetch(pSdb, pIter);
sdbRelease(pSdb, pIdx);
exist = 1;
break;
}
sdbRelease(pSdb, pIdx);
}
mndReleaseDb(pMnode, pDb);
return exist;
}
static int32_t mndAddIndex(SMnode *pMnode, SRpcMsg *pReq, SCreateTagIndexReq *req, SDbObj *pDb, SStbObj *pStb) { static int32_t mndAddIndex(SMnode *pMnode, SRpcMsg *pReq, SCreateTagIndexReq *req, SDbObj *pDb, SStbObj *pStb) {
int32_t code = -1; int32_t code = -1;
SIdxObj idxObj = {0}; SIdxObj idxObj = {0};
@ -681,11 +720,20 @@ static int32_t mndAddIndex(SMnode *pMnode, SRpcMsg *pReq, SCreateTagIndexReq *re
idxObj.stbUid = pStb->uid; idxObj.stbUid = pStb->uid;
idxObj.dbUid = pStb->dbUid; idxObj.dbUid = pStb->dbUid;
int32_t tag = mndFindSuperTableTagId(pStb, req->colName); int8_t hasIdx = 0;
int32_t tag = mndFindSuperTableTagId(pStb, req->colName, &hasIdx);
if (tag < 0) { if (tag < 0) {
terrno = TSDB_CODE_MND_TAG_NOT_EXIST; terrno = TSDB_CODE_MND_TAG_NOT_EXIST;
return -1; return -1;
} else if (tag == 0) { }
int8_t exist = 0;
if (tag == 0 && hasIdx == 1) {
exist = mndCheckIndexNameByTagName(pMnode, &idxObj);
if (exist) {
terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST;
return -1;
}
} else if (hasIdx == 1) {
terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST; terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST;
return -1; return -1;
} }
@ -695,11 +743,11 @@ static int32_t mndAddIndex(SMnode *pMnode, SRpcMsg *pReq, SCreateTagIndexReq *re
return -1; return -1;
} }
SSchema *pTag = pStb->pTags + tag; // SSchema *pTag = pStb->pTags + tag;
if (IS_IDX_ON(pTag)) { // if (IS_IDX_ON(pTag)) {
terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST; // terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST;
return -1; // return -1;
} // }
code = mndAddIndexImpl(pMnode, pReq, pDb, pStb, &idxObj); code = mndAddIndexImpl(pMnode, pReq, pDb, pStb, &idxObj);
return code; return code;
@ -806,8 +854,8 @@ int32_t mndDropIdxsByStb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *p
if (pIdx->stbUid == pStb->uid) { if (pIdx->stbUid == pStb->uid) {
if (mndSetDropIdxCommitLogs(pMnode, pTrans, pIdx) != 0) { if (mndSetDropIdxCommitLogs(pMnode, pTrans, pIdx) != 0) {
sdbCancelFetch(pSdb, pIter);
sdbRelease(pSdb, pIdx); sdbRelease(pSdb, pIdx);
sdbCancelFetch(pSdb, pIdx);
return -1; return -1;
} }
} }

36
source/dnode/mnode/impl/src/mndStb.c
View File

@ -18,6 +18,7 @@
#include "mndDb.h" #include "mndDb.h"
#include "mndDnode.h" #include "mndDnode.h"
#include "mndIndex.h" #include "mndIndex.h"
#include "mndIndexComm.h"
#include "mndInfoSchema.h" #include "mndInfoSchema.h"
#include "mndMnode.h" #include "mndMnode.h"
#include "mndPerfSchema.h" #include "mndPerfSchema.h"
@ -857,11 +858,39 @@ static int32_t mndCreateStb(SMnode *pMnode, SRpcMsg *pReq, SMCreateStbReq *pCrea
SStbObj stbObj = {0}; SStbObj stbObj = {0};
int32_t code = -1; int32_t code = -1;
char fullIdxName[TSDB_INDEX_FNAME_LEN * 2] = {0};
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pReq, "create-stb"); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pReq, "create-stb");
if (pTrans == NULL) goto _OVER; if (pTrans == NULL) goto _OVER;
mInfo("trans:%d, used to create stb:%s", pTrans->id, pCreate->name); mInfo("trans:%d, used to create stb:%s", pTrans->id, pCreate->name);
if (mndBuildStbFromReq(pMnode, &stbObj, pCreate, pDb) != 0) goto _OVER; if (mndBuildStbFromReq(pMnode, &stbObj, pCreate, pDb) != 0) goto _OVER;
char randStr[24] = {0};
taosRandStr2(randStr, tListLen(randStr) - 1);
SSchema *pSchema = &(stbObj.pTags[0]);
sprintf(fullIdxName, "%s.%s_%s", pDb->name, pSchema->name, randStr);
SSIdx idx = {0};
if (mndAcquireGlobalIdx(pMnode, fullIdxName, SDB_IDX, &idx) == 0 && idx.pIdx != NULL) {
terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST;
mndReleaseIdx(pMnode, idx.pIdx);
goto _OVER;
}
SIdxObj idxObj = {0};
memcpy(idxObj.name, fullIdxName, TSDB_INDEX_FNAME_LEN);
memcpy(idxObj.stb, stbObj.name, TSDB_TABLE_FNAME_LEN);
memcpy(idxObj.db, stbObj.db, TSDB_DB_FNAME_LEN);
memcpy(idxObj.colName, pSchema->name, TSDB_COL_NAME_LEN);
idxObj.createdTime = taosGetTimestampMs();
idxObj.uid = mndGenerateUid(fullIdxName, strlen(fullIdxName));
idxObj.stbUid = stbObj.uid;
idxObj.dbUid = stbObj.dbUid;
if (mndSetCreateIdxCommitLogs(pMnode, pTrans, &idxObj) < 0) goto _OVER;
if (mndAddStbToTrans(pMnode, pTrans, pDb, &stbObj) < 0) goto _OVER; if (mndAddStbToTrans(pMnode, pTrans, pDb, &stbObj) < 0) goto _OVER;
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
code = 0; code = 0;
@ -1478,7 +1507,8 @@ static int32_t mndAlterStbTagBytes(SMnode *pMnode, const SStbObj *pOld, SStbObj
SSchema *pTag = pNew->pTags + tag; SSchema *pTag = pNew->pTags + tag;
if (!(pTag->type == TSDB_DATA_TYPE_BINARY || pTag->type == TSDB_DATA_TYPE_NCHAR || pTag->type == TSDB_DATA_TYPE_GEOMETRY)) { if (!(pTag->type == TSDB_DATA_TYPE_BINARY || pTag->type == TSDB_DATA_TYPE_NCHAR ||
pTag->type == TSDB_DATA_TYPE_GEOMETRY)) {
terrno = TSDB_CODE_MND_INVALID_STB_OPTION; terrno = TSDB_CODE_MND_INVALID_STB_OPTION;
return -1; return -1;
} }
@ -1598,7 +1628,8 @@ static int32_t mndAlterStbColumnBytes(SMnode *pMnode, const SStbObj *pOld, SStbO
} }
SSchema *pCol = pNew->pColumns + col; SSchema *pCol = pNew->pColumns + col;
if (!(pCol->type == TSDB_DATA_TYPE_BINARY || pCol->type == TSDB_DATA_TYPE_NCHAR || pCol->type == TSDB_DATA_TYPE_GEOMETRY)) { if (!(pCol->type == TSDB_DATA_TYPE_BINARY || pCol->type == TSDB_DATA_TYPE_NCHAR ||
pCol->type == TSDB_DATA_TYPE_GEOMETRY)) {
terrno = TSDB_CODE_MND_INVALID_STB_OPTION; terrno = TSDB_CODE_MND_INVALID_STB_OPTION;
return -1; return -1;
} }
@ -3182,7 +3213,6 @@ static int32_t mndRetrieveStbCol(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
SStbObj *pStb = NULL; SStbObj *pStb = NULL;
int32_t numOfRows = 0; int32_t numOfRows = 0;
if (!pShow->sysDbRsp) { if (!pShow->sysDbRsp) {
numOfRows = buildSysDbColsInfo(pBlock, pShow->db, pShow->filterTb); numOfRows = buildSysDbColsInfo(pBlock, pShow->db, pShow->filterTb);

42
source/dnode/vnode/CMakeLists.txt
View File

@ -8,6 +8,7 @@ set(
"src/vnd/vnodeCommit.c" "src/vnd/vnodeCommit.c"
"src/vnd/vnodeQuery.c" "src/vnd/vnodeQuery.c"
"src/vnd/vnodeModule.c" "src/vnd/vnodeModule.c"
"src/vnd/vnodeCos.c"
"src/vnd/vnodeSvr.c" "src/vnd/vnodeSvr.c"
"src/vnd/vnodeSync.c" "src/vnd/vnodeSync.c"
"src/vnd/vnodeSnapshot.c" "src/vnd/vnodeSnapshot.c"
@ -155,6 +156,45 @@ target_link_libraries(
PUBLIC index PUBLIC index
) )
if(${TD_LINUX})
set(CMAKE_FIND_LIBRARY_SUFFIXES ".a")
find_library(APR_LIBRARY apr-1 PATHS /usr/local/apr/lib/)
find_library(APR_UTIL_LIBRARY aprutil-1 PATHS /usr/local/apr/lib/)
find_library(MINIXML_LIBRARY mxml)
find_library(CURL_LIBRARY curl)
target_link_libraries(
vnode
# s3
PUBLIC cos_c_sdk_static
PUBLIC ${APR_UTIL_LIBRARY}
PUBLIC ${APR_LIBRARY}
PUBLIC ${MINIXML_LIBRARY}
PUBLIC ${CURL_LIBRARY}
)
# s3
FIND_PROGRAM(APR_CONFIG_BIN NAMES apr-config apr-1-config PATHS /usr/bin /usr/local/bin /usr/local/apr/bin/)
IF (APR_CONFIG_BIN)
EXECUTE_PROCESS(
COMMAND ${APR_CONFIG_BIN} --includedir
OUTPUT_VARIABLE APR_INCLUDE_DIR
OUTPUT_STRIP_TRAILING_WHITESPACE
)
ENDIF()
include_directories (${APR_INCLUDE_DIR})
target_include_directories(
vnode
PUBLIC "${TD_SOURCE_DIR}/contrib/cos-c-sdk-v5/cos_c_sdk"
PUBLIC "$ENV{HOME}/.cos-local.1/include"
)
if(${BUILD_WITH_COS})
add_definitions(-DUSE_COS)
endif(${BUILD_WITH_COS})
endif(${TD_LINUX})
IF (TD_GRANT) IF (TD_GRANT)
TARGET_LINK_LIBRARIES(vnode PUBLIC grant) TARGET_LINK_LIBRARIES(vnode PUBLIC grant)
ENDIF () ENDIF ()
@ -169,8 +209,6 @@ if(${BUILD_WITH_ROCKSDB})
add_definitions(-DUSE_ROCKSDB) add_definitions(-DUSE_ROCKSDB)
endif(${BUILD_WITH_ROCKSDB}) endif(${BUILD_WITH_ROCKSDB})
if(${BUILD_TEST}) if(${BUILD_TEST})
add_subdirectory(test) add_subdirectory(test)
endif(${BUILD_TEST}) endif(${BUILD_TEST})

40
source/dnode/vnode/src/inc/vndCos.h Normal file
View File

@ -0,0 +1,40 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_VND_COS_H_
#define _TD_VND_COS_H_
#include "vnd.h"
#ifdef __cplusplus
extern "C" {
#endif
extern int8_t tsS3Enabled;
int32_t s3Init();
void s3CleanUp();
int32_t s3PutObjectFromFile(const char *file, const char *object);
void s3DeleteObjects(const char *object_name[], int nobject);
bool s3Exists(const char *object_name);
bool s3Get(const char *object_name, const char *path);
void s3EvictCache(const char *path, long object_size);
long s3Size(const char *object_name);
#ifdef __cplusplus
}
#endif
#endif /*_TD_VND_COS_H_*/

2
source/dnode/vnode/src/inc/vnodeInt.h
View File

@ -167,7 +167,7 @@ int metaAddIndexToSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq);
int metaDropIndexFromSTable(SMeta* pMeta, int64_t version, SDropIndexReq* pReq); int metaDropIndexFromSTable(SMeta* pMeta, int64_t version, SDropIndexReq* pReq);
int64_t metaGetTimeSeriesNum(SMeta* pMeta); int64_t metaGetTimeSeriesNum(SMeta* pMeta);
SMCtbCursor* metaOpenCtbCursor(SMeta* pMeta, tb_uid_t uid, int lock); SMCtbCursor* metaOpenCtbCursor(void* pVnode, tb_uid_t uid, int lock);
void metaCloseCtbCursor(SMCtbCursor* pCtbCur, int lock); void metaCloseCtbCursor(SMCtbCursor* pCtbCur, int lock);
tb_uid_t metaCtbCursorNext(SMCtbCursor* pCtbCur); tb_uid_t metaCtbCursorNext(SMCtbCursor* pCtbCur);
SMStbCursor* metaOpenStbCursor(SMeta* pMeta, tb_uid_t uid); SMStbCursor* metaOpenStbCursor(SMeta* pMeta, tb_uid_t uid);

18
source/dnode/vnode/src/meta/metaQuery.c
View File

@ -408,17 +408,9 @@ _err:
return NULL; return NULL;
} }
struct SMCtbCursor {
SMeta *pMeta;
TBC *pCur;
tb_uid_t suid;
void *pKey;
void *pVal;
int kLen;
int vLen;
};
SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid, int lock) { SMCtbCursor *metaOpenCtbCursor(void* pVnode, tb_uid_t uid, int lock) {
SMeta* pMeta = ((SVnode*)pVnode)->pMeta;
SMCtbCursor *pCtbCur = NULL; SMCtbCursor *pCtbCur = NULL;
SCtbIdxKey ctbIdxKey; SCtbIdxKey ctbIdxKey;
int ret = 0; int ret = 0;
@ -435,7 +427,7 @@ SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid, int lock) {
metaRLock(pMeta); metaRLock(pMeta);
} }
ret = tdbTbcOpen(pMeta->pCtbIdx, &pCtbCur->pCur, NULL); ret = tdbTbcOpen(pMeta->pCtbIdx, (TBC**)&pCtbCur->pCur, NULL);
if (ret < 0) { if (ret < 0) {
metaULock(pMeta); metaULock(pMeta);
taosMemoryFree(pCtbCur); taosMemoryFree(pCtbCur);
@ -1194,7 +1186,7 @@ int32_t metaFilterTableIds(void *pVnode, SMetaFltParam *arg, SArray *pUids) {
ret = -1; ret = -1;
for (int i = 0; i < oStbEntry.stbEntry.schemaTag.nCols; i++) { for (int i = 0; i < oStbEntry.stbEntry.schemaTag.nCols; i++) {
SSchema *schema = oStbEntry.stbEntry.schemaTag.pSchema + i; SSchema *schema = oStbEntry.stbEntry.schemaTag.pSchema + i;
if (schema->colId == param->cid && param->type == schema->type && (IS_IDX_ON(schema) || i == 0)) { if (schema->colId == param->cid && param->type == schema->type && (IS_IDX_ON(schema))) {
ret = 0; ret = 0;
} }
} }
@ -1373,7 +1365,7 @@ int32_t metaGetTableTagsByUids(void *pVnode, int64_t suid, SArray *uidList) {
} }
int32_t metaGetTableTags(void *pVnode, uint64_t suid, SArray *pUidTagInfo) { int32_t metaGetTableTags(void *pVnode, uint64_t suid, SArray *pUidTagInfo) {
SMCtbCursor *pCur = metaOpenCtbCursor(((SVnode *)pVnode)->pMeta, suid, 1); SMCtbCursor *pCur = metaOpenCtbCursor(pVnode, suid, 1);
// If len > 0 means there already have uids, and we only want the // If len > 0 means there already have uids, and we only want the
// tags of the specified tables, of which uid in the uid list. Otherwise, all table tags are retrieved and kept // tags of the specified tables, of which uid in the uid list. Otherwise, all table tags are retrieved and kept

9
source/dnode/vnode/src/meta/metaTable.c
View File

@ -450,12 +450,13 @@ int metaAddIndexToSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
goto _err; goto _err;
} }
if (IS_IDX_ON(pNew) && !IS_IDX_ON(pOld)) { if (IS_IDX_ON(pNew) && !IS_IDX_ON(pOld)) {
if (diffIdx != -1) goto _err; // if (diffIdx != -1) goto _err;
diffIdx = i; diffIdx = i;
break;
} }
} }
if (diffIdx == -1 || diffIdx == 0) { if (diffIdx == -1) {
goto _err; goto _err;
} }
@ -586,7 +587,7 @@ int metaDropIndexFromSTable(SMeta *pMeta, int64_t version, SDropIndexReq *pReq)
for (int i = 0; i < oStbEntry.stbEntry.schemaTag.nCols; i++) { for (int i = 0; i < oStbEntry.stbEntry.schemaTag.nCols; i++) {
SSchema *schema = oStbEntry.stbEntry.schemaTag.pSchema + i; SSchema *schema = oStbEntry.stbEntry.schemaTag.pSchema + i;
if (0 == strncmp(schema->name, pReq->colName, sizeof(pReq->colName))) { if (0 == strncmp(schema->name, pReq->colName, sizeof(pReq->colName))) {
if (i != 0 || IS_IDX_ON(schema)) { if (IS_IDX_ON(schema)) {
pCol = schema; pCol = schema;
} }
break; break;
@ -2094,7 +2095,7 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) {
} else { } else {
for (int i = 0; i < pTagSchema->nCols; i++) { for (int i = 0; i < pTagSchema->nCols; i++) {
pTagColumn = &pTagSchema->pSchema[i]; pTagColumn = &pTagSchema->pSchema[i];
if (i != 0 && !IS_IDX_ON(pTagColumn)) continue; if (!IS_IDX_ON(pTagColumn)) continue;
STagVal tagVal = {.cid = pTagColumn->colId}; STagVal tagVal = {.cid = pTagColumn->colId};
tTagGet((const STag *)pCtbEntry->ctbEntry.pTags, &tagVal); tTagGet((const STag *)pCtbEntry->ctbEntry.pTags, &tagVal);

2
source/dnode/vnode/src/tq/tqOffsetSnapshot.c
View File

@ -60,7 +60,7 @@ int32_t tqOffsetSnapRead(STqOffsetReader* pReader, uint8_t** ppData) {
} }
int64_t sz = 0; int64_t sz = 0;
if (taosStatFile(fname, &sz, NULL) < 0) { if (taosStatFile(fname, &sz, NULL, NULL) < 0) {
taosCloseFile(&pFile); taosCloseFile(&pFile);
taosMemoryFree(fname); taosMemoryFree(fname);
return -1; return -1;

10
source/dnode/vnode/src/tsdb/tsdbFS.c
View File

@ -176,7 +176,7 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) {
// SDelFile // SDelFile
if (pTsdb->fs.pDelFile) { if (pTsdb->fs.pDelFile) {
tsdbDelFileName(pTsdb, pTsdb->fs.pDelFile, fname); tsdbDelFileName(pTsdb, pTsdb->fs.pDelFile, fname);
if (taosStatFile(fname, &size, NULL)) { if (taosStatFile(fname, &size, NULL, NULL)) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }
@ -195,7 +195,7 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) {
// head ========= // head =========
tsdbHeadFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pHeadF, fname); tsdbHeadFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pHeadF, fname);
if (taosStatFile(fname, &size, NULL)) { if (taosStatFile(fname, &size, NULL, NULL)) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }
@ -206,7 +206,7 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) {
// data ========= // data =========
tsdbDataFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pDataF, fname); tsdbDataFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pDataF, fname);
if (taosStatFile(fname, &size, NULL)) { if (taosStatFile(fname, &size, NULL, NULL)) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }
@ -221,7 +221,7 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) {
// sma ============= // sma =============
tsdbSmaFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pSmaF, fname); tsdbSmaFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pSmaF, fname);
if (taosStatFile(fname, &size, NULL)) { if (taosStatFile(fname, &size, NULL, NULL)) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }
@ -237,7 +237,7 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) {
// stt =========== // stt ===========
for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) { for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) {
tsdbSttFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSttF[iStt], fname); tsdbSttFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSttF[iStt], fname);
if (taosStatFile(fname, &size, NULL)) { if (taosStatFile(fname, &size, NULL, NULL)) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }

18
source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
View File

@ -14,6 +14,7 @@
*/ */
#include "tsdb.h" #include "tsdb.h"
#include "vndCos.h"
// =============== PAGE-WISE FILE =============== // =============== PAGE-WISE FILE ===============
int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t flag, STsdbFD **ppFD) { int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t flag, STsdbFD **ppFD) {
@ -33,11 +34,26 @@ int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t flag, STsdbFD **p
pFD->szPage = szPage; pFD->szPage = szPage;
pFD->flag = flag; pFD->flag = flag;
pFD->pFD = taosOpenFile(path, flag); pFD->pFD = taosOpenFile(path, flag);
if (pFD->pFD == NULL) {
const char *object_name = taosDirEntryBaseName((char *)path);
long s3_size = s3Size(object_name);
if (!strncmp(path + strlen(path) - 5, ".data", 5) && s3_size > 0) {
s3EvictCache(path, s3_size);
s3Get(object_name, path);
pFD->pFD = taosOpenFile(path, flag);
if (pFD->pFD == NULL) { if (pFD->pFD == NULL) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
taosMemoryFree(pFD); taosMemoryFree(pFD);
goto _exit; goto _exit;
} }
} else {
code = TAOS_SYSTEM_ERROR(errno);
taosMemoryFree(pFD);
goto _exit;
}
}
pFD->szPage = szPage; pFD->szPage = szPage;
pFD->pgno = 0; pFD->pgno = 0;
pFD->pBuf = taosMemoryCalloc(1, szPage); pFD->pBuf = taosMemoryCalloc(1, szPage);
@ -50,7 +66,7 @@ int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t flag, STsdbFD **p
// not check file size when reading data files. // not check file size when reading data files.
if (flag != TD_FILE_READ) { if (flag != TD_FILE_READ) {
if (taosStatFile(path, &pFD->szFile, NULL) < 0) { if (taosStatFile(path, &pFD->szFile, NULL, NULL) < 0) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
taosMemoryFree(pFD->pBuf); taosMemoryFree(pFD->pBuf);
taosCloseFile(&pFD->pFD); taosCloseFile(&pFD->pFD);

111
source/dnode/vnode/src/tsdb/tsdbRetention.c
View File

@ -15,6 +15,7 @@
#include "tsdb.h" #include "tsdb.h"
#include "tsdbFS2.h" #include "tsdbFS2.h"
#include "vndCos.h"
typedef struct { typedef struct {
STsdb *tsdb; STsdb *tsdb;
@ -41,6 +42,28 @@ static int32_t tsdbDoRemoveFileObject(SRTNer *rtner, const STFileObj *fobj) {
return TARRAY2_APPEND(rtner->fopArr, op); return TARRAY2_APPEND(rtner->fopArr, op);
} }
static int32_t tsdbRemoveFileObjectS3(SRTNer *rtner, const STFileObj *fobj) {
int32_t code = 0, lino = 0;
STFileOp op = {
.optype = TSDB_FOP_REMOVE,
.fid = fobj->f->fid,
.of = fobj->f[0],
};
code = TARRAY2_APPEND(rtner->fopArr, op);
TSDB_CHECK_CODE(code, lino, _exit);
const char *object_name = taosDirEntryBaseName((char *)fobj->fname);
s3DeleteObjects(&object_name, 1);
_exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
}
return code;
}
static int32_t tsdbDoCopyFile(SRTNer *rtner, const STFileObj *from, const STFile *to) { static int32_t tsdbDoCopyFile(SRTNer *rtner, const STFileObj *from, const STFile *to) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
@ -76,6 +99,34 @@ _exit:
return code; return code;
} }
static int32_t tsdbCopyFileS3(SRTNer *rtner, const STFileObj *from, const STFile *to) {
int32_t code = 0;
int32_t lino = 0;
char fname[TSDB_FILENAME_LEN];
TdFilePtr fdFrom = NULL;
TdFilePtr fdTo = NULL;
tsdbTFileName(rtner->tsdb, to, fname);
fdFrom = taosOpenFile(from->fname, TD_FILE_READ);
if (fdFrom == NULL) code = terrno;
TSDB_CHECK_CODE(code, lino, _exit);
char *object_name = taosDirEntryBaseName(fname);
code = s3PutObjectFromFile(from->fname, object_name);
TSDB_CHECK_CODE(code, lino, _exit);
taosCloseFile(&fdFrom);
_exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
taosCloseFile(&fdFrom);
}
return code;
}
static int32_t tsdbDoMigrateFileObj(SRTNer *rtner, const STFileObj *fobj, const SDiskID *did) { static int32_t tsdbDoMigrateFileObj(SRTNer *rtner, const STFileObj *fobj, const SDiskID *did) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
@ -123,6 +174,53 @@ _exit:
return code; return code;
} }
static int32_t tsdbMigrateDataFileS3(SRTNer *rtner, const STFileObj *fobj, const SDiskID *did) {
int32_t code = 0;
int32_t lino = 0;
STFileOp op = {0};
// remove old
op = (STFileOp){
.optype = TSDB_FOP_REMOVE,
.fid = fobj->f->fid,
.of = fobj->f[0],
};
code = TARRAY2_APPEND(rtner->fopArr, op);
TSDB_CHECK_CODE(code, lino, _exit);
// create new
op = (STFileOp){
.optype = TSDB_FOP_CREATE,
.fid = fobj->f->fid,
.nf =
{
.type = fobj->f->type,
.did = did[0],
.fid = fobj->f->fid,
.cid = fobj->f->cid,
.size = fobj->f->size,
.stt[0] =
{
.level = fobj->f->stt[0].level,
},
},
};
code = TARRAY2_APPEND(rtner->fopArr, op);
TSDB_CHECK_CODE(code, lino, _exit);
// do copy the file
code = tsdbCopyFileS3(rtner, fobj, &op.nf);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
}
return code;
}
typedef struct { typedef struct {
STsdb *tsdb; STsdb *tsdb;
int32_t sync; int32_t sync;
@ -201,9 +299,15 @@ static int32_t tsdbDoRetention2(void *arg) {
for (int32_t ftype = 0; (ftype < TSDB_FTYPE_MAX) && (fobj = rtner->ctx->fset->farr[ftype], 1); ++ftype) { for (int32_t ftype = 0; (ftype < TSDB_FTYPE_MAX) && (fobj = rtner->ctx->fset->farr[ftype], 1); ++ftype) {
if (fobj == NULL) continue; if (fobj == NULL) continue;
int32_t nlevel = tfsGetLevel(rtner->tsdb->pVnode->pTfs);
if (tsS3Enabled && nlevel > 1 && TSDB_FTYPE_DATA == ftype && fobj->f->did.level == nlevel - 1) {
code = tsdbRemoveFileObjectS3(rtner, fobj);
TSDB_CHECK_CODE(code, lino, _exit);
} else {
code = tsdbDoRemoveFileObject(rtner, fobj); code = tsdbDoRemoveFileObject(rtner, fobj);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }
}
SSttLvl *lvl; SSttLvl *lvl;
TARRAY2_FOREACH(rtner->ctx->fset->lvlArr, lvl) { TARRAY2_FOREACH(rtner->ctx->fset->lvlArr, lvl) {
@ -228,9 +332,16 @@ static int32_t tsdbDoRetention2(void *arg) {
if (fobj == NULL) continue; if (fobj == NULL) continue;
if (fobj->f->did.level == did.level) continue; if (fobj->f->did.level == did.level) continue;
int32_t nlevel = tfsGetLevel(rtner->tsdb->pVnode->pTfs);
if (tsS3Enabled && nlevel > 1 && TSDB_FTYPE_DATA == ftype && did.level == nlevel - 1) {
code = tsdbMigrateDataFileS3(rtner, fobj, &did);
TSDB_CHECK_CODE(code, lino, _exit);
} else {
code = tsdbDoMigrateFileObj(rtner, fobj, &did); code = tsdbDoMigrateFileObj(rtner, fobj, &did);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }
}
// stt // stt
SSttLvl *lvl; SSttLvl *lvl;

2
source/dnode/vnode/src/tsdb/tsdbWrite.c
View File

@ -76,7 +76,7 @@ int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq2 *pMsg) {
int32_t code = 0; int32_t code = 0;
STsdbKeepCfg *pCfg = &pTsdb->keepCfg; STsdbKeepCfg *pCfg = &pTsdb->keepCfg;
TSKEY now = taosGetTimestamp(pCfg->precision); TSKEY now = taosGetTimestamp(pCfg->precision);
TSKEY minKey = now - tsTickPerMin[pCfg->precision] * pCfg->keep2; TSKEY minKey = now - tsTickPerMin[pCfg->precision] * pCfg->keep1;
TSKEY maxKey = tsMaxKeyByPrecision[pCfg->precision]; TSKEY maxKey = tsMaxKeyByPrecision[pCfg->precision];
int32_t size = taosArrayGetSize(pMsg->aSubmitTbData); int32_t size = taosArrayGetSize(pMsg->aSubmitTbData);

323
source/dnode/vnode/src/vnd/vnodeCos.c Normal file
View File

@ -0,0 +1,323 @@
#define ALLOW_FORBID_FUNC
#include "vndCos.h"
extern char tsS3Endpoint[];
extern char tsS3AccessKeyId[];
extern char tsS3AccessKeySecret[];
extern char tsS3BucketName[];
extern char tsS3AppId[];
#ifdef USE_COS
#include "cos_api.h"
#include "cos_http_io.h"
#include "cos_log.h"
int32_t s3Init() {
if (cos_http_io_initialize(NULL, 0) != COSE_OK) {
return -1;
}
// set log level, default COS_LOG_WARN
cos_log_set_level(COS_LOG_WARN);
// set log output, default stderr
cos_log_set_output(NULL);
return 0;
}
void s3CleanUp() { cos_http_io_deinitialize(); }
static void log_status(cos_status_t *s) {
cos_warn_log("status->code: %d", s->code);
if (s->error_code) cos_warn_log("status->error_code: %s", s->error_code);
if (s->error_msg) cos_warn_log("status->error_msg: %s", s->error_msg);
if (s->req_id) cos_warn_log("status->req_id: %s", s->req_id);
}
static void s3InitRequestOptions(cos_request_options_t *options, int is_cname) {
options->config = cos_config_create(options->pool);
cos_config_t *config = options->config;
cos_str_set(&config->endpoint, tsS3Endpoint);
cos_str_set(&config->access_key_id, tsS3AccessKeyId);
cos_str_set(&config->access_key_secret, tsS3AccessKeySecret);
cos_str_set(&config->appid, tsS3AppId);
config->is_cname = is_cname;
options->ctl = cos_http_controller_create(options->pool, 0);
}
int32_t s3PutObjectFromFile(const char *file_str, const char *object_str) {
int32_t code = 0;
cos_pool_t *p = NULL;
int is_cname = 0;
cos_status_t *s = NULL;
cos_request_options_t *options = NULL;
cos_string_t bucket, object, file;
cos_table_t *resp_headers;
int traffic_limit = 0;
cos_pool_create(&p, NULL);
options = cos_request_options_create(p);
s3InitRequestOptions(options, is_cname);
cos_table_t *headers = NULL;
if (traffic_limit) {
// 限速值设置范围为819200 - 838860800，即100KB/s - 100MB/s，如果超出该范围将返回400错误
headers = cos_table_make(p, 1);
cos_table_add_int(headers, "x-cos-traffic-limit", 819200);
}
cos_str_set(&bucket, tsS3BucketName);
cos_str_set(&file, file_str);
cos_str_set(&object, object_str);
s = cos_put_object_from_file(options, &bucket, &object, &file, headers, &resp_headers);
log_status(s);
cos_pool_destroy(p);
if (s->code != 200) {
return code = s->code;
}
return code;
}
void s3DeleteObjects(const char *object_name[], int nobject) {
cos_pool_t *p = NULL;
int is_cname = 0;
cos_string_t bucket;
cos_table_t *resp_headers = NULL;
cos_request_options_t *options = NULL;
cos_list_t object_list;
cos_list_t deleted_object_list;
int is_quiet = COS_TRUE;
cos_pool_create(&p, NULL);
options = cos_request_options_create(p);
s3InitRequestOptions(options, is_cname);
cos_str_set(&bucket, tsS3BucketName);
cos_list_init(&object_list);
cos_list_init(&deleted_object_list);
for (int i = 0; i < nobject; ++i) {
cos_object_key_t *content = cos_create_cos_object_key(p);
cos_str_set(&content->key, object_name[i]);
cos_list_add_tail(&content->node, &object_list);
}
cos_status_t *s = cos_delete_objects(options, &bucket, &object_list, is_quiet, &resp_headers, &deleted_object_list);
log_status(s);
cos_pool_destroy(p);
if (cos_status_is_ok(s)) {
cos_warn_log("delete objects succeeded\n");
} else {
cos_warn_log("delete objects failed\n");
}
}
bool s3Exists(const char *object_name) {
bool ret = false;
cos_pool_t *p = NULL;
int is_cname = 0;
cos_status_t *s = NULL;
cos_request_options_t *options = NULL;
cos_string_t bucket;
cos_string_t object;
cos_table_t *resp_headers;
cos_table_t *headers = NULL;
cos_object_exist_status_e object_exist;
cos_pool_create(&p, NULL);
options = cos_request_options_create(p);
s3InitRequestOptions(options, is_cname);
cos_str_set(&bucket, tsS3BucketName);
cos_str_set(&object, object_name);
s = cos_check_object_exist(options, &bucket, &object, headers, &object_exist, &resp_headers);
if (object_exist == COS_OBJECT_NON_EXIST) {
cos_warn_log("object: %.*s non exist.\n", object.len, object.data);
} else if (object_exist == COS_OBJECT_EXIST) {
ret = true;
cos_warn_log("object: %.*s exist.\n", object.len, object.data);
} else {
cos_warn_log("object: %.*s unknown status.\n", object.len, object.data);
log_status(s);
}
cos_pool_destroy(p);
return ret;
}
bool s3Get(const char *object_name, const char *path) {
bool ret = false;
cos_pool_t *p = NULL;
int is_cname = 0;
cos_status_t *s = NULL;
cos_request_options_t *options = NULL;
cos_string_t bucket;
cos_string_t object;
cos_string_t file;
cos_table_t *resp_headers = NULL;
cos_table_t *headers = NULL;
int traffic_limit = 0;
//创建内存池
cos_pool_create(&p, NULL);
//初始化请求选项
options = cos_request_options_create(p);
s3InitRequestOptions(options, is_cname);
cos_str_set(&bucket, tsS3BucketName);
if (traffic_limit) {
//限速值设置范围为819200 - 838860800，即100KB/s - 100MB/s，如果超出该范围将返回400错误
headers = cos_table_make(p, 1);
cos_table_add_int(headers, "x-cos-traffic-limit", 819200);
}
//下载对象
cos_str_set(&file, path);
cos_str_set(&object, object_name);
s = cos_get_object_to_file(options, &bucket, &object, headers, NULL, &file, &resp_headers);
if (cos_status_is_ok(s)) {
ret = true;
cos_warn_log("get object succeeded\n");
} else {
cos_warn_log("get object failed\n");
}
//销毁内存池
cos_pool_destroy(p);
return ret;
}
typedef struct {
int64_t size;
int32_t atime;
char name[TSDB_FILENAME_LEN];
} SEvictFile;
static int32_t evictFileCompareAsce(const void *pLeft, const void *pRight) {
SEvictFile *lhs = (SEvictFile *)pLeft;
SEvictFile *rhs = (SEvictFile *)pRight;
return lhs->atime < rhs->atime ? -1 : 1;
}
void s3EvictCache(const char *path, long object_size) {
SDiskSize disk_size = {0};
char dir_name[TSDB_FILENAME_LEN] = "\0";
tstrncpy(dir_name, path, TSDB_FILENAME_LEN);
taosDirName(dir_name);
if (taosGetDiskSize((char *)dir_name, &disk_size) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
vError("failed to get disk:%s size since %s", path, terrstr());
return;
}
if (object_size >= disk_size.avail - (1 << 30)) {
// evict too old files
// 1, list data files' atime under dir(path)
tdbDirPtr pDir = taosOpenDir(dir_name);
if (pDir == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
vError("failed to open %s since %s", dir_name, terrstr());
}
SArray *evict_files = taosArrayInit(16, sizeof(SEvictFile));
tdbDirEntryPtr pDirEntry;
while ((pDirEntry = taosReadDir(pDir)) != NULL) {
char *name = taosGetDirEntryName(pDirEntry);
if (!strncmp(name + strlen(name) - 5, ".data", 5)) {
SEvictFile e_file = {0};
char entry_name[TSDB_FILENAME_LEN] = "\0";
int dir_len = strlen(dir_name);
memcpy(e_file.name, dir_name, dir_len);
e_file.name[dir_len] = '/';
memcpy(e_file.name + dir_len + 1, name, strlen(name));
taosStatFile(e_file.name, &e_file.size, NULL, &e_file.atime);
taosArrayPush(evict_files, &e_file);
}
}
taosCloseDir(&pDir);
// 2, sort by atime
taosArraySort(evict_files, evictFileCompareAsce);
// 3, remove files ascendingly until we get enough object_size space
long evict_size = 0;
size_t ef_size = TARRAY_SIZE(evict_files);
for (size_t i = 0; i < ef_size; ++i) {
SEvictFile *evict_file = taosArrayGet(evict_files, i);
taosRemoveFile(evict_file->name);
evict_size += evict_file->size;
if (evict_size >= object_size) {
break;
}
}
taosArrayDestroy(evict_files);
}
}
long s3Size(const char *object_name) {
long size = 0;
cos_pool_t *p = NULL;
int is_cname = 0;
cos_status_t *s = NULL;
cos_request_options_t *options = NULL;
cos_string_t bucket;
cos_string_t object;
cos_table_t *resp_headers = NULL;
//创建内存池
cos_pool_create(&p, NULL);
//初始化请求选项
options = cos_request_options_create(p);
s3InitRequestOptions(options, is_cname);
cos_str_set(&bucket, tsS3BucketName);
//获取对象元数据
cos_str_set(&object, object_name);
s = cos_head_object(options, &bucket, &object, NULL, &resp_headers);
// print_headers(resp_headers);
if (cos_status_is_ok(s)) {
char *content_length_str = (char *)apr_table_get(resp_headers, COS_CONTENT_LENGTH);
if (content_length_str != NULL) {
size = atol(content_length_str);
}
cos_warn_log("head object succeeded: %ld\n", size);
} else {
cos_warn_log("head object failed\n");
}
//销毁内存池
cos_pool_destroy(p);
return size;
}
#else
int32_t s3Init() { return 0; }
void s3CleanUp() {}
int32_t s3PutObjectFromFile(const char *file, const char *object) { return 0; }
void s3DeleteObjects(const char *object_name[], int nobject) {}
bool s3Exists(const char *object_name) { return false; }
bool s3Get(const char *object_name, const char *path) { return false; }
void s3EvictCache(const char *path, long object_size) {}
long s3Size(const char *object_name) { return 0; }
#endif

4
source/dnode/vnode/src/vnd/vnodeInitApi.c
View File

@ -96,6 +96,10 @@ void initMetadataAPI(SStoreMeta* pMeta) {
pMeta->metaGetCachedTbGroup = metaGetCachedTbGroup; pMeta->metaGetCachedTbGroup = metaGetCachedTbGroup;
pMeta->metaPutTbGroupToCache = metaPutTbGroupToCache; pMeta->metaPutTbGroupToCache = metaPutTbGroupToCache;
pMeta->openCtbCursor = metaOpenCtbCursor;
pMeta->closeCtbCursor = metaCloseCtbCursor;
pMeta->ctbCursorNext = metaCtbCursorNext;
} }
void initTqAPI(SStoreTqReader* pTq) { void initTqAPI(SStoreTqReader* pTq) {

5
source/dnode/vnode/src/vnd/vnodeModule.c
View File

@ -14,6 +14,7 @@
*/ */
#include "vnd.h" #include "vnd.h"
#include "vndCos.h"
typedef struct SVnodeTask SVnodeTask; typedef struct SVnodeTask SVnodeTask;
struct SVnodeTask { struct SVnodeTask {
@ -81,6 +82,9 @@ int vnodeInit(int nthreads) {
if (tqInit() < 0) { if (tqInit() < 0) {
return -1; return -1;
} }
if (s3Init() < 0) {
return -1;
}
return 0; return 0;
} }
@ -112,6 +116,7 @@ void vnodeCleanup() {
walCleanUp(); walCleanUp();
tqCleanUp(); tqCleanUp();
smaCleanUp(); smaCleanUp();
s3CleanUp();
} }
int vnodeScheduleTaskEx(int tpid, int (*execute)(void*), void* arg) { int vnodeScheduleTaskEx(int tpid, int (*execute)(void*), void* arg) {

6
source/dnode/vnode/src/vnd/vnodeQuery.c
View File

@ -440,7 +440,7 @@ int32_t vnodeGetTableList(void* pVnode, int8_t type, SArray* pList) {
} }
int32_t vnodeGetAllTableList(SVnode *pVnode, uint64_t uid, SArray *list) { int32_t vnodeGetAllTableList(SVnode *pVnode, uint64_t uid, SArray *list) {
SMCtbCursor *pCur = metaOpenCtbCursor(pVnode->pMeta, uid, 1); SMCtbCursor *pCur = metaOpenCtbCursor(pVnode, uid, 1);
while (1) { while (1) {
tb_uid_t id = metaCtbCursorNext(pCur); tb_uid_t id = metaCtbCursorNext(pCur);
@ -462,7 +462,7 @@ int32_t vnodeGetCtbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bo
int32_t vnodeGetCtbIdList(void *pVnode, int64_t suid, SArray *list) { int32_t vnodeGetCtbIdList(void *pVnode, int64_t suid, SArray *list) {
SVnode *pVnodeObj = pVnode; SVnode *pVnodeObj = pVnode;
SMCtbCursor *pCur = metaOpenCtbCursor(pVnodeObj->pMeta, suid, 1); SMCtbCursor *pCur = metaOpenCtbCursor(pVnodeObj, suid, 1);
while (1) { while (1) {
tb_uid_t id = metaCtbCursorNext(pCur); tb_uid_t id = metaCtbCursorNext(pCur);
@ -521,7 +521,7 @@ int32_t vnodeGetStbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bo
} }
int32_t vnodeGetCtbNum(SVnode *pVnode, int64_t suid, int64_t *num) { int32_t vnodeGetCtbNum(SVnode *pVnode, int64_t suid, int64_t *num) {
SMCtbCursor *pCur = metaOpenCtbCursor(pVnode->pMeta, suid, 0); SMCtbCursor *pCur = metaOpenCtbCursor(pVnode, suid, 0);
if (!pCur) { if (!pCur) {
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }

16
source/libs/command/src/explain.c
View File

@ -291,17 +291,17 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
switch (pNode->type) { switch (pNode->type) {
case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: { case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: {
STagScanPhysiNode *pTagScanNode = (STagScanPhysiNode *)pNode; STagScanPhysiNode *pTagScanNode = (STagScanPhysiNode *)pNode;
EXPLAIN_ROW_NEW(level, EXPLAIN_TAG_SCAN_FORMAT, pTagScanNode->tableName.tname); EXPLAIN_ROW_NEW(level, EXPLAIN_TAG_SCAN_FORMAT, pTagScanNode->scan.tableName.tname);
EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT);
if (pResNode->pExecInfo) { if (pResNode->pExecInfo) {
QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen));
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
} }
if (pTagScanNode->pScanPseudoCols) { if (pTagScanNode->scan.pScanPseudoCols) {
EXPLAIN_ROW_APPEND(EXPLAIN_PSEUDO_COLUMNS_FORMAT, pTagScanNode->pScanPseudoCols->length); EXPLAIN_ROW_APPEND(EXPLAIN_PSEUDO_COLUMNS_FORMAT, pTagScanNode->scan.pScanPseudoCols->length);
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
} }
EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pTagScanNode->node.pOutputDataBlockDesc->totalRowSize); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pTagScanNode->scan.node.pOutputDataBlockDesc->totalRowSize);
EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT);
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
@ -309,11 +309,11 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
if (verbose) { if (verbose) {
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT); EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT);
EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT,
nodesGetOutputNumFromSlotList(pTagScanNode->node.pOutputDataBlockDesc->pSlots)); nodesGetOutputNumFromSlotList(pTagScanNode->scan.node.pOutputDataBlockDesc->pSlots));
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pTagScanNode->node.pOutputDataBlockDesc->outputRowSize); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pTagScanNode->scan.node.pOutputDataBlockDesc->outputRowSize);
EXPLAIN_ROW_APPEND_LIMIT(pTagScanNode->node.pLimit); EXPLAIN_ROW_APPEND_LIMIT(pTagScanNode->scan.node.pLimit);
EXPLAIN_ROW_APPEND_SLIMIT(pTagScanNode->node.pSlimit); EXPLAIN_ROW_APPEND_SLIMIT(pTagScanNode->scan.node.pSlimit);
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));

2
source/libs/executor/inc/executil.h
View File

@ -190,4 +190,6 @@ void printDataBlock(SSDataBlock* pBlock, const char* flag);
void getNextTimeWindow(const SInterval* pInterval, STimeWindow* tw, int32_t order); void getNextTimeWindow(const SInterval* pInterval, STimeWindow* tw, int32_t order);
void getInitialStartTimeWindow(SInterval* pInterval, TSKEY ts, STimeWindow* w, bool ascQuery); void getInitialStartTimeWindow(SInterval* pInterval, TSKEY ts, STimeWindow* w, bool ascQuery);
SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, SArray* pUidTagList, void* pVnode,
SStorageAPI* pStorageAPI);
#endif // TDENGINE_EXECUTIL_H #endif // TDENGINE_EXECUTIL_H

14
source/libs/executor/inc/executorInt.h
View File

@ -251,6 +251,12 @@ typedef struct STableMergeScanInfo {
SSortExecInfo sortExecInfo; SSortExecInfo sortExecInfo;
} STableMergeScanInfo; } STableMergeScanInfo;
typedef struct STagScanFilterContext {
SHashObj* colHash;
int32_t index;
SArray* cInfoList;
} STagScanFilterContext;
typedef struct STagScanInfo { typedef struct STagScanInfo {
SColumnInfo* pCols; SColumnInfo* pCols;
SSDataBlock* pRes; SSDataBlock* pRes;
@ -259,6 +265,14 @@ typedef struct STagScanInfo {
SLimitNode* pSlimit; SLimitNode* pSlimit;
SReadHandle readHandle; SReadHandle readHandle;
STableListInfo* pTableListInfo; STableListInfo* pTableListInfo;
uint64_t suid;
void* pCtbCursor;
SNode* pTagCond;
SNode* pTagIndexCond;
STagScanFilterContext filterCtx;
SArray* aUidTags; // SArray<STUidTagInfo>
SArray* aFilterIdxs; // SArray<int32_t>
SStorageAPI* pStorageAPI;
} STagScanInfo; } STagScanInfo;
typedef enum EStreamScanMode { typedef enum EStreamScanMode {

2
source/libs/executor/inc/operator.h
View File

@ -81,7 +81,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode,
SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* readHandle, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* readHandle, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode, STableListInfo* pTableListInfo, SNode* pTagCond, SNode*pTagIndexCond, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode* pScanPhyNode, const char* pUser, SExecTaskInfo* pTaskInfo); SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode* pScanPhyNode, const char* pUser, SExecTaskInfo* pTaskInfo);

4
source/libs/executor/src/executil.c
View File

@ -47,8 +47,6 @@ static int32_t optimizeTbnameInCondImpl(void* metaHandle, SArray* list, SNode* p
static int32_t getTableList(void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, SNode* pTagIndexCond, static int32_t getTableList(void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, SNode* pTagIndexCond,
STableListInfo* pListInfo, uint8_t* digest, const char* idstr, SStorageAPI* pStorageAPI); STableListInfo* pListInfo, uint8_t* digest, const char* idstr, SStorageAPI* pStorageAPI);
static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, SArray* pUidTagList, void* pVnode,
SStorageAPI* pStorageAPI);
static int64_t getLimit(const SNode* pLimit) { return NULL == pLimit ? -1 : ((SLimitNode*)pLimit)->limit; } static int64_t getLimit(const SNode* pLimit) { return NULL == pLimit ? -1 : ((SLimitNode*)pLimit)->limit; }
static int64_t getOffset(const SNode* pLimit) { return NULL == pLimit ? -1 : ((SLimitNode*)pLimit)->offset; } static int64_t getOffset(const SNode* pLimit) { return NULL == pLimit ? -1 : ((SLimitNode*)pLimit)->offset; }
@ -846,7 +844,7 @@ static int32_t optimizeTbnameInCondImpl(void* pVnode, SArray* pExistedUidList, S
return -1; return -1;
} }
static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, SArray* pUidTagList, void* pVnode, SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, SArray* pUidTagList, void* pVnode,
SStorageAPI* pStorageAPI) { SStorageAPI* pStorageAPI) {
SSDataBlock* pResBlock = createDataBlock(); SSDataBlock* pResBlock = createDataBlock();
if (pResBlock == NULL) { if (pResBlock == NULL) {

9
source/libs/executor/src/operator.c
View File

@ -370,17 +370,18 @@ SOperatorInfo* createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SR
STableCountScanPhysiNode* pTblCountScanNode = (STableCountScanPhysiNode*)pPhyNode; STableCountScanPhysiNode* pTblCountScanNode = (STableCountScanPhysiNode*)pPhyNode;
pOperator = createTableCountScanOperatorInfo(pHandle, pTblCountScanNode, pTaskInfo); pOperator = createTableCountScanOperatorInfo(pHandle, pTblCountScanNode, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN == type) { } else if (QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN == type) {
STagScanPhysiNode* pScanPhyNode = (STagScanPhysiNode*)pPhyNode; STagScanPhysiNode* pTagScanPhyNode = (STagScanPhysiNode*)pPhyNode;
STableListInfo* pTableListInfo = tableListCreate(); STableListInfo* pTableListInfo = tableListCreate();
int32_t code = createScanTableListInfo(pScanPhyNode, NULL, false, pHandle, pTableListInfo, pTagCond, if (!pTagScanPhyNode->onlyMetaCtbIdx) {
int32_t code = createScanTableListInfo((SScanPhysiNode*)pTagScanPhyNode, NULL, false, pHandle, pTableListInfo, pTagCond,
pTagIndexCond, pTaskInfo); pTagIndexCond, pTaskInfo);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
pTaskInfo->code = code; pTaskInfo->code = code;
qError("failed to getTableList, code: %s", tstrerror(code)); qError("failed to getTableList, code: %s", tstrerror(code));
return NULL; return NULL;
} }
}
pOperator = createTagScanOperatorInfo(pHandle, pScanPhyNode, pTableListInfo, pTaskInfo); pOperator = createTagScanOperatorInfo(pHandle, pTagScanPhyNode, pTableListInfo, pTagCond, pTagIndexCond, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN == type) { } else if (QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN == type) {
SBlockDistScanPhysiNode* pBlockNode = (SBlockDistScanPhysiNode*)pPhyNode; SBlockDistScanPhysiNode* pBlockNode = (SBlockDistScanPhysiNode*)pPhyNode;
STableListInfo* pTableListInfo = tableListCreate(); STableListInfo* pTableListInfo = tableListCreate();

261
source/libs/executor/src/scanoperator.c
View File

@ -2688,7 +2688,236 @@ static void doTagScanOneTable(SOperatorInfo* pOperator, const SSDataBlock* pRes,
} }
} }
static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { static void tagScanFreeUidTag(void* p) {
STUidTagInfo* pInfo = p;
if (pInfo->pTagVal != NULL) {
taosMemoryFree(pInfo->pTagVal);
}
}
static int32_t tagScanCreateResultData(SDataType* pType, int32_t numOfRows, SScalarParam* pParam) {
SColumnInfoData* pColumnData = taosMemoryCalloc(1, sizeof(SColumnInfoData));
if (pColumnData == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
pColumnData->info.type = pType->type;
pColumnData->info.bytes = pType->bytes;
pColumnData->info.scale = pType->scale;
pColumnData->info.precision = pType->precision;
int32_t code = colInfoDataEnsureCapacity(pColumnData, numOfRows, true);
if (code != TSDB_CODE_SUCCESS) {
terrno = code;
taosMemoryFree(pColumnData);
return terrno;
}
pParam->columnData = pColumnData;
pParam->colAlloced = true;
return TSDB_CODE_SUCCESS;
}
static EDealRes tagScanRewriteTagColumn(SNode** pNode, void* pContext) {
SColumnNode* pSColumnNode = NULL;
if (QUERY_NODE_COLUMN == nodeType((*pNode))) {
pSColumnNode = *(SColumnNode**)pNode;
} else if (QUERY_NODE_FUNCTION == nodeType((*pNode))) {
SFunctionNode* pFuncNode = *(SFunctionNode**)(pNode);
if (pFuncNode->funcType == FUNCTION_TYPE_TBNAME) {
pSColumnNode = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN);
if (NULL == pSColumnNode) {
return DEAL_RES_ERROR;
}
pSColumnNode->colId = -1;
pSColumnNode->colType = COLUMN_TYPE_TBNAME;
pSColumnNode->node.resType.type = TSDB_DATA_TYPE_VARCHAR;
pSColumnNode->node.resType.bytes = TSDB_TABLE_FNAME_LEN - 1 + VARSTR_HEADER_SIZE;
nodesDestroyNode(*pNode);
*pNode = (SNode*)pSColumnNode;
} else {
return DEAL_RES_CONTINUE;
}
} else {
return DEAL_RES_CONTINUE;
}
STagScanFilterContext* pCtx = (STagScanFilterContext*)pContext;
void* data = taosHashGet(pCtx->colHash, &pSColumnNode->colId, sizeof(pSColumnNode->colId));
if (!data) {
taosHashPut(pCtx->colHash, &pSColumnNode->colId, sizeof(pSColumnNode->colId), pNode, sizeof((*pNode)));
pSColumnNode->slotId = pCtx->index++;
SColumnInfo cInfo = {.colId = pSColumnNode->colId,
.type = pSColumnNode->node.resType.type,
.bytes = pSColumnNode->node.resType.bytes};
taosArrayPush(pCtx->cInfoList, &cInfo);
} else {
SColumnNode* col = *(SColumnNode**)data;
pSColumnNode->slotId = col->slotId;
}
return DEAL_RES_CONTINUE;
}
static void tagScanFilterByTagCond(SArray* aUidTags, SNode* pTagCond, SArray* aFilterIdxs, void* pVnode, SStorageAPI* pAPI, STagScanInfo* pInfo) {
int32_t code = 0;
int32_t numOfTables = taosArrayGetSize(aUidTags);
SSDataBlock* pResBlock = createTagValBlockForFilter(pInfo->filterCtx.cInfoList, numOfTables, aUidTags, pVnode, pAPI);
SArray* pBlockList = taosArrayInit(1, POINTER_BYTES);
taosArrayPush(pBlockList, &pResBlock);
SDataType type = {.type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)};
SScalarParam output = {0};
tagScanCreateResultData(&type, numOfTables, &output);
scalarCalculate(pTagCond, pBlockList, &output);
bool* result = (bool*)output.columnData->pData;
for (int32_t i = 0 ; i < numOfTables; ++i) {
if (result[i]) {
taosArrayPush(aFilterIdxs, &i);
}
}
colDataDestroy(output.columnData);
taosMemoryFreeClear(output.columnData);
blockDataDestroy(pResBlock);
taosArrayDestroy(pBlockList);
}
static void tagScanFillOneCellWithTag(const STUidTagInfo* pUidTagInfo, SExprInfo* pExprInfo, SColumnInfoData* pColInfo, int rowIndex, const SStorageAPI* pAPI, void* pVnode) {
if (fmIsScanPseudoColumnFunc(pExprInfo->pExpr->_function.functionId)) { // tbname
char str[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
// if (pUidTagInfo->name != NULL) {
// STR_TO_VARSTR(str, pUidTagInfo->name);
// } else { // name is not retrieved during filter
// pAPI->metaFn.getTableNameByUid(pVnode, pUidTagInfo->uid, str);
// }
STR_TO_VARSTR(str, "ctbidx");
colDataSetVal(pColInfo, rowIndex, str, false);
} else {
STagVal tagVal = {0};
tagVal.cid = pExprInfo->base.pParam[0].pCol->colId;
if (pUidTagInfo->pTagVal == NULL) {
colDataSetNULL(pColInfo, rowIndex);
} else {
const char* p = pAPI->metaFn.extractTagVal(pUidTagInfo->pTagVal, pColInfo->info.type, &tagVal);
if (p == NULL || (pColInfo->info.type == TSDB_DATA_TYPE_JSON && ((STag*)p)->nTag == 0)) {
colDataSetNULL(pColInfo, rowIndex);
} else if (pColInfo->info.type == TSDB_DATA_TYPE_JSON) {
colDataSetVal(pColInfo, rowIndex, p, false);
} else if (IS_VAR_DATA_TYPE(pColInfo->info.type)) {
char* tmp = taosMemoryMalloc(tagVal.nData + VARSTR_HEADER_SIZE + 1);
varDataSetLen(tmp, tagVal.nData);
memcpy(tmp + VARSTR_HEADER_SIZE, tagVal.pData, tagVal.nData);
colDataSetVal(pColInfo, rowIndex, tmp, false);
taosMemoryFree(tmp);
} else {
colDataSetVal(pColInfo, rowIndex, (const char*)&tagVal.i64, false);
}
}
}
}
static int32_t tagScanFillResultBlock(SOperatorInfo* pOperator, SSDataBlock* pRes, SArray* aUidTags, SArray* aFilterIdxs, bool ignoreFilterIdx,
SStorageAPI* pAPI) {
STagScanInfo* pInfo = pOperator->info;
SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[0];
if (!ignoreFilterIdx) {
size_t szTables = taosArrayGetSize(aFilterIdxs);
for (int i = 0; i < szTables; ++i) {
int32_t idx = *(int32_t*)taosArrayGet(aFilterIdxs, i);
STUidTagInfo* pUidTagInfo = taosArrayGet(aUidTags, idx);
for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) {
SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, pExprInfo[j].base.resSchema.slotId);
tagScanFillOneCellWithTag(pUidTagInfo, &pExprInfo[j], pDst, i, pAPI, pInfo->readHandle.vnode);
}
}
} else {
size_t szTables = taosArrayGetSize(aUidTags);
for (int i = 0; i < szTables; ++i) {
STUidTagInfo* pUidTagInfo = taosArrayGet(aUidTags, i);
for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) {
SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, pExprInfo[j].base.resSchema.slotId);
tagScanFillOneCellWithTag(pUidTagInfo, &pExprInfo[j], pDst, i, pAPI, pInfo->readHandle.vnode);
}
}
}
return 0;
}
static SSDataBlock* doTagScanFromCtbIdx(SOperatorInfo* pOperator) {
if (pOperator->status == OP_EXEC_DONE) {
return NULL;
}
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStorageAPI* pAPI = &pTaskInfo->storageAPI;
STagScanInfo* pInfo = pOperator->info;
SSDataBlock* pRes = pInfo->pRes;
blockDataCleanup(pRes);
if (pInfo->pCtbCursor == NULL) {
pInfo->pCtbCursor = pAPI->metaFn.openCtbCursor(pInfo->readHandle.vnode, pInfo->suid, 1);
}
SArray* aUidTags = pInfo->aUidTags;
SArray* aFilterIdxs = pInfo->aFilterIdxs;
int32_t count = 0;
while (1) {
taosArrayClearEx(aUidTags, tagScanFreeUidTag);
taosArrayClear(aFilterIdxs);
int32_t numTables = 0;
while (numTables < pOperator->resultInfo.capacity) {
SMCtbCursor* pCur = pInfo->pCtbCursor;
tb_uid_t uid = pAPI->metaFn.ctbCursorNext(pInfo->pCtbCursor);
if (uid == 0) {
break;
}
STUidTagInfo info = {.uid = uid, .pTagVal = pCur->pVal};
info.pTagVal = taosMemoryMalloc(pCur->vLen);
memcpy(info.pTagVal, pCur->pVal, pCur->vLen);
taosArrayPush(aUidTags, &info);
++numTables;
}
if (numTables == 0) {
break;
}
bool ignoreFilterIdx = true;
if (pInfo->pTagCond != NULL) {
ignoreFilterIdx = false;
tagScanFilterByTagCond(aUidTags, pInfo->pTagCond, aFilterIdxs, pInfo->readHandle.vnode, pAPI, pInfo);
} else {
ignoreFilterIdx = true;
}
tagScanFillResultBlock(pOperator, pRes, aUidTags, aFilterIdxs, ignoreFilterIdx, pAPI);
count = ignoreFilterIdx ? taosArrayGetSize(aUidTags): taosArrayGetSize(aFilterIdxs);
if (count != 0) {
break;
}
}
pRes->info.rows = count;
pOperator->resultInfo.totalRows += count;
return (pRes->info.rows == 0) ? NULL : pInfo->pRes;
}
static SSDataBlock* doTagScanFromMetaEntry(SOperatorInfo* pOperator) {
if (pOperator->status == OP_EXEC_DONE) { if (pOperator->status == OP_EXEC_DONE) {
return NULL; return NULL;
} }
@ -2746,14 +2975,23 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) {
static void destroyTagScanOperatorInfo(void* param) { static void destroyTagScanOperatorInfo(void* param) {
STagScanInfo* pInfo = (STagScanInfo*)param; STagScanInfo* pInfo = (STagScanInfo*)param;
if (pInfo->pCtbCursor != NULL) {
pInfo->pStorageAPI->metaFn.closeCtbCursor(pInfo->pCtbCursor, 1);
}
taosHashCleanup(pInfo->filterCtx.colHash);
taosArrayDestroy(pInfo->filterCtx.cInfoList);
taosArrayDestroy(pInfo->aFilterIdxs);
taosArrayDestroyEx(pInfo->aUidTags, tagScanFreeUidTag);
pInfo->pRes = blockDataDestroy(pInfo->pRes); pInfo->pRes = blockDataDestroy(pInfo->pRes);
taosArrayDestroy(pInfo->matchInfo.pList); taosArrayDestroy(pInfo->matchInfo.pList);
pInfo->pTableListInfo = tableListDestroy(pInfo->pTableListInfo); pInfo->pTableListInfo = tableListDestroy(pInfo->pTableListInfo);
taosMemoryFreeClear(param); taosMemoryFreeClear(param);
} }
SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode, SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pTagScanNode,
STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo) { STableListInfo* pTableListInfo, SNode* pTagCond, SNode* pTagIndexCond, SExecTaskInfo* pTaskInfo) {
SScanPhysiNode* pPhyNode = (SScanPhysiNode*)pTagScanNode;
STagScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STagScanInfo)); STagScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STagScanInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) { if (pInfo == NULL || pOperator == NULL) {
@ -2775,6 +3013,11 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi
goto _error; goto _error;
} }
pInfo->pTagCond = pTagCond;
pInfo->pTagIndexCond = pTagIndexCond;
pInfo->suid = pPhyNode->suid;
pInfo->pStorageAPI = &pTaskInfo->storageAPI;
pInfo->pTableListInfo = pTableListInfo; pInfo->pTableListInfo = pTableListInfo;
pInfo->pRes = createDataBlockFromDescNode(pDescNode); pInfo->pRes = createDataBlockFromDescNode(pDescNode);
pInfo->readHandle = *pReadHandle; pInfo->readHandle = *pReadHandle;
@ -2786,8 +3029,18 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi
initResultSizeInfo(&pOperator->resultInfo, 4096); initResultSizeInfo(&pOperator->resultInfo, 4096);
blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);
if (pTagScanNode->onlyMetaCtbIdx) {
pInfo->aUidTags = taosArrayInit(pOperator->resultInfo.capacity, sizeof(STUidTagInfo));
pInfo->aFilterIdxs = taosArrayInit(pOperator->resultInfo.capacity, sizeof(int32_t));
pInfo->filterCtx.colHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_SMALLINT), false, HASH_NO_LOCK);
pInfo->filterCtx.cInfoList = taosArrayInit(4, sizeof(SColumnInfo));
if (pInfo->pTagCond != NULL) {
nodesRewriteExprPostOrder(&pTagCond, tagScanRewriteTagColumn, (void*)&pInfo->filterCtx);
}
}
__optr_fn_t tagScanNextFn = (pTagScanNode->onlyMetaCtbIdx) ? doTagScanFromCtbIdx : doTagScanFromMetaEntry;
pOperator->fpSet = pOperator->fpSet =
createOperatorFpSet(optrDummyOpenFn, doTagScan, NULL, destroyTagScanOperatorInfo, optrDefaultBufFn, NULL); createOperatorFpSet(optrDummyOpenFn, tagScanNextFn, NULL, destroyTagScanOperatorInfo, optrDefaultBufFn, NULL);
return pOperator; return pOperator;

2
source/libs/function/src/udfd.c
View File

@ -848,7 +848,7 @@ int32_t udfdSaveFuncBodyToFile(SFuncInfo *pFuncInfo, SUdf *udf) {
char path[PATH_MAX] = {0}; char path[PATH_MAX] = {0};
udfdGetFuncBodyPath(udf, path); udfdGetFuncBodyPath(udf, path);
bool fileExist = !(taosStatFile(path, NULL, NULL) < 0); bool fileExist = !(taosStatFile(path, NULL, NULL, NULL) < 0);
if (fileExist) { if (fileExist) {
strncpy(udf->path, path, PATH_MAX); strncpy(udf->path, path, PATH_MAX);
fnInfo("udfd func body file. reuse existing file %s", path); fnInfo("udfd func body file. reuse existing file %s", path);

4
source/libs/index/src/indexFstFile.c
View File

@ -162,7 +162,7 @@ static FORCE_INLINE int idxFileCtxGetSize(IFileCtx* ctx) {
return ctx->offset; return ctx->offset;
} else { } else {
int64_t file_size = 0; int64_t file_size = 0;
taosStatFile(ctx->file.buf, &file_size, NULL); taosStatFile(ctx->file.buf, &file_size, NULL, NULL);
return (int)file_size; return (int)file_size;
} }
} }
@ -199,7 +199,7 @@ IFileCtx* idxFileCtxCreate(WriterType type, const char* path, bool readOnly, int
code = taosFtruncateFile(ctx->file.pFile, 0); code = taosFtruncateFile(ctx->file.pFile, 0);
UNUSED(code); UNUSED(code);
code = taosStatFile(path, &ctx->file.size, NULL); code = taosStatFile(path, &ctx->file.size, NULL, NULL);
UNUSED(code); UNUSED(code);
ctx->file.wBufOffset = 0; ctx->file.wBufOffset = 0;

4
source/libs/nodes/src/nodesCloneFuncs.c
View File

@ -564,7 +564,9 @@ static int32_t physiScanCopy(const SScanPhysiNode* pSrc, SScanPhysiNode* pDst) {
} }
static int32_t physiTagScanCopy(const STagScanPhysiNode* pSrc, STagScanPhysiNode* pDst) { static int32_t physiTagScanCopy(const STagScanPhysiNode* pSrc, STagScanPhysiNode* pDst) {
return physiScanCopy(pSrc, pDst); COPY_BASE_OBJECT_FIELD(scan, physiScanCopy);
COPY_SCALAR_FIELD(onlyMetaCtbIdx);
return TSDB_CODE_SUCCESS;
} }
static int32_t physiTableScanCopy(const STableScanPhysiNode* pSrc, STableScanPhysiNode* pDst) { static int32_t physiTableScanCopy(const STableScanPhysiNode* pSrc, STableScanPhysiNode* pDst) {

38
source/libs/nodes/src/nodesCodeFuncs.c
View File

@ -662,6 +662,7 @@ static const char* jkScanLogicPlanDynamicScanFuncs = "DynamicScanFuncs";
static const char* jkScanLogicPlanDataRequired = "DataRequired"; static const char* jkScanLogicPlanDataRequired = "DataRequired";
static const char* jkScanLogicPlanTagCond = "TagCond"; static const char* jkScanLogicPlanTagCond = "TagCond";
static const char* jkScanLogicPlanGroupTags = "GroupTags"; static const char* jkScanLogicPlanGroupTags = "GroupTags";
static const char* jkScanLogicPlanOnlyMetaCtbIdx = "OnlyMetaCtbIdx";
static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) { static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) {
const SScanLogicNode* pNode = (const SScanLogicNode*)pObj; const SScanLogicNode* pNode = (const SScanLogicNode*)pObj;
@ -703,7 +704,9 @@ static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) {
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkScanLogicPlanGroupTags, pNode->pGroupTags); code = nodeListToJson(pJson, jkScanLogicPlanGroupTags, pNode->pGroupTags);
} }
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddBoolToObject(pJson, jkScanLogicPlanOnlyMetaCtbIdx, pNode->onlyMetaCtbIdx);
}
return code; return code;
} }
@ -748,6 +751,9 @@ static int32_t jsonToLogicScanNode(const SJson* pJson, void* pObj) {
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkScanLogicPlanGroupTags, &pNode->pGroupTags); code = jsonToNodeList(pJson, jkScanLogicPlanGroupTags, &pNode->pGroupTags);
} }
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBoolValue(pJson, jkScanLogicPlanOnlyMetaCtbIdx, &pNode->onlyMetaCtbIdx);
}
return code; return code;
} }
@ -1564,7 +1570,7 @@ static const char* jkScanPhysiPlanTableName = "TableName";
static const char* jkScanPhysiPlanGroupOrderScan = "GroupOrderScan"; static const char* jkScanPhysiPlanGroupOrderScan = "GroupOrderScan";
static int32_t physiScanNodeToJson(const void* pObj, SJson* pJson) { static int32_t physiScanNodeToJson(const void* pObj, SJson* pJson) {
const STagScanPhysiNode* pNode = (const STagScanPhysiNode*)pObj; const SScanPhysiNode* pNode = (const SScanPhysiNode*)pObj;
int32_t code = physicPlanNodeToJson(pObj, pJson); int32_t code = physicPlanNodeToJson(pObj, pJson);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
@ -1593,7 +1599,7 @@ static int32_t physiScanNodeToJson(const void* pObj, SJson* pJson) {
} }
static int32_t jsonToPhysiScanNode(const SJson* pJson, void* pObj) { static int32_t jsonToPhysiScanNode(const SJson* pJson, void* pObj) {
STagScanPhysiNode* pNode = (STagScanPhysiNode*)pObj; SScanPhysiNode* pNode = (SScanPhysiNode*)pObj;
int32_t code = jsonToPhysicPlanNode(pJson, pObj); int32_t code = jsonToPhysicPlanNode(pJson, pObj);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
@ -1621,6 +1627,30 @@ static int32_t jsonToPhysiScanNode(const SJson* pJson, void* pObj) {
return code; return code;
} }
static const char* jkTagScanPhysiOnlyMetaCtbIdx = "OnlyMetaCtbIdx";
static int32_t physiTagScanNodeToJson(const void* pObj, SJson* pJson) {
const STagScanPhysiNode* pNode = (const STagScanPhysiNode*)pObj;
int32_t code = physiScanNodeToJson(pObj, pJson);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddBoolToObject(pJson, jkTagScanPhysiOnlyMetaCtbIdx, pNode->onlyMetaCtbIdx);
}
return code;
}
static int32_t jsonToPhysiTagScanNode(const SJson* pJson, void* pObj) {
STagScanPhysiNode* pNode = (STagScanPhysiNode*)pObj;
int32_t code = jsonToPhysiScanNode(pJson, pObj);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBoolValue(pJson, jkTagScanPhysiOnlyMetaCtbIdx, &pNode->onlyMetaCtbIdx);
}
return code;
}
static const char* jkLastRowScanPhysiPlanGroupTags = "GroupTags"; static const char* jkLastRowScanPhysiPlanGroupTags = "GroupTags";
static const char* jkLastRowScanPhysiPlanGroupSort = "GroupSort"; static const char* jkLastRowScanPhysiPlanGroupSort = "GroupSort";
@ -6592,6 +6622,7 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
case QUERY_NODE_LOGIC_PLAN: case QUERY_NODE_LOGIC_PLAN:
return logicPlanToJson(pObj, pJson); return logicPlanToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN:
return physiTagScanNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN: case QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN:
return physiScanNodeToJson(pObj, pJson); return physiScanNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN: case QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN:
@ -6910,6 +6941,7 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
case QUERY_NODE_LOGIC_PLAN: case QUERY_NODE_LOGIC_PLAN:
return jsonToLogicPlan(pJson, pObj); return jsonToLogicPlan(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN:
return jsonToPhysiTagScanNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN: case QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN:
case QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN: case QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN:
return jsonToPhysiScanNode(pJson, pObj); return jsonToPhysiScanNode(pJson, pObj);

41
source/libs/nodes/src/nodesMsgFuncs.c
View File

@ -2003,6 +2003,43 @@ static int32_t msgToPhysiScanNode(STlvDecoder* pDecoder, void* pObj) {
return code; return code;
} }
enum {
PHY_TAG_SCAN_CODE_SCAN = 1,
PHY_TAG_SCAN_CODE_ONLY_META_CTB_IDX
};
static int32_t physiTagScanNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
const STagScanPhysiNode* pNode = (const STagScanPhysiNode*)pObj;
int32_t code = tlvEncodeObj(pEncoder, PHY_TAG_SCAN_CODE_SCAN, physiScanNodeToMsg, &pNode->scan);
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeBool(pEncoder, PHY_TAG_SCAN_CODE_ONLY_META_CTB_IDX, pNode->onlyMetaCtbIdx);
}
return code;
}
static int32_t msgToPhysiTagScanNode(STlvDecoder* pDecoder, void* pObj) {
STagScanPhysiNode* pNode = (STagScanPhysiNode*)pObj;
int32_t code = TSDB_CODE_SUCCESS;
STlv* pTlv = NULL;
tlvForEach(pDecoder, pTlv, code) {
switch (pTlv->type) {
case PHY_TAG_SCAN_CODE_SCAN:
code = tlvDecodeObjFromTlv(pTlv, msgToPhysiScanNode, &pNode->scan);
break;
case PHY_TAG_SCAN_CODE_ONLY_META_CTB_IDX:
code = tlvDecodeBool(pTlv, &pNode->onlyMetaCtbIdx);
break;
default:
break;
}
}
return code;
}
enum { enum {
PHY_LAST_ROW_SCAN_CODE_SCAN = 1, PHY_LAST_ROW_SCAN_CODE_SCAN = 1,
PHY_LAST_ROW_SCAN_CODE_GROUP_TAGS, PHY_LAST_ROW_SCAN_CODE_GROUP_TAGS,
@ -3726,6 +3763,8 @@ static int32_t specificNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
code = caseWhenNodeToMsg(pObj, pEncoder); code = caseWhenNodeToMsg(pObj, pEncoder);
break; break;
case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN:
code = physiTagScanNodeToMsg(pObj, pEncoder);
break;
case QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN: case QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN:
code = physiScanNodeToMsg(pObj, pEncoder); code = physiScanNodeToMsg(pObj, pEncoder);
break; break;
@ -3869,6 +3908,8 @@ static int32_t msgToSpecificNode(STlvDecoder* pDecoder, void* pObj) {
code = msgToCaseWhenNode(pDecoder, pObj); code = msgToCaseWhenNode(pDecoder, pObj);
break; break;
case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN:
code = msgToPhysiTagScanNode(pDecoder, pObj);
break;
case QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN: case QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN:
code = msgToPhysiScanNode(pDecoder, pObj); code = msgToPhysiScanNode(pDecoder, pObj);
break; break;

7
source/libs/nodes/test/nodesCloneTest.cpp
View File

@ -199,9 +199,10 @@ TEST_F(NodesCloneTest, physiScan) {
ASSERT_EQ(nodeType(pSrc), nodeType(pDst)); ASSERT_EQ(nodeType(pSrc), nodeType(pDst));
STagScanPhysiNode* pSrcNode = (STagScanPhysiNode*)pSrc; STagScanPhysiNode* pSrcNode = (STagScanPhysiNode*)pSrc;
STagScanPhysiNode* pDstNode = (STagScanPhysiNode*)pDst; STagScanPhysiNode* pDstNode = (STagScanPhysiNode*)pDst;
ASSERT_EQ(pSrcNode->uid, pDstNode->uid); ASSERT_EQ(pSrcNode->scan.uid, pDstNode->scan.uid);
ASSERT_EQ(pSrcNode->suid, pDstNode->suid); ASSERT_EQ(pSrcNode->scan.suid, pDstNode->scan.suid);
ASSERT_EQ(pSrcNode->tableType, pDstNode->tableType); ASSERT_EQ(pSrcNode->scan.tableType, pDstNode->scan.tableType);
ASSERT_EQ(pSrcNode->onlyMetaCtbIdx, pDstNode->onlyMetaCtbIdx);
}); });
std::unique_ptr<SNode, void (*)(SNode*)> srcNode(nullptr, nodesDestroyNode); std::unique_ptr<SNode, void (*)(SNode*)> srcNode(nullptr, nodesDestroyNode);

67
source/libs/parser/src/parTranslater.c
View File

@ -280,7 +280,8 @@ static const SSysTableShowAdapter sysTableShowAdapter[] = {
static int32_t translateSubquery(STranslateContext* pCxt, SNode* pNode); static int32_t translateSubquery(STranslateContext* pCxt, SNode* pNode);
static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode); static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode);
static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal); static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal);
static int32_t createSimpleSelectStmtFromProjList(const char* pDb, const char* pTable, SNodeList* pProjectionList, SSelectStmt** pStmt); static int32_t createSimpleSelectStmtFromProjList(const char* pDb, const char* pTable, SNodeList* pProjectionList,
SSelectStmt** pStmt);
static int32_t createLastTsSelectStmt(char* pDb, char* pTable, STableMeta* pMeta, SNode** pQuery); static int32_t createLastTsSelectStmt(char* pDb, char* pTable, STableMeta* pMeta, SNode** pQuery);
static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery); static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery);
static int32_t setRefreshMate(STranslateContext* pCxt, SQuery* pQuery); static int32_t setRefreshMate(STranslateContext* pCxt, SQuery* pQuery);
@ -774,7 +775,8 @@ static SNodeList* getProjectList(const SNode* pNode) {
static bool isTimeLineQuery(SNode* pStmt) { static bool isTimeLineQuery(SNode* pStmt) {
if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) {
return (TIME_LINE_MULTI == ((SSelectStmt*)pStmt)->timeLineResMode) || (TIME_LINE_GLOBAL == ((SSelectStmt*)pStmt)->timeLineResMode); return (TIME_LINE_MULTI == ((SSelectStmt*)pStmt)->timeLineResMode) ||
(TIME_LINE_GLOBAL == ((SSelectStmt*)pStmt)->timeLineResMode);
} else if (QUERY_NODE_SET_OPERATOR == nodeType(pStmt)) { } else if (QUERY_NODE_SET_OPERATOR == nodeType(pStmt)) {
return TIME_LINE_GLOBAL == ((SSetOperator*)pStmt)->timeLineResMode; return TIME_LINE_GLOBAL == ((SSetOperator*)pStmt)->timeLineResMode;
} else { } else {
@ -862,7 +864,7 @@ static void setColumnInfoBySchema(const SRealTableNode* pTable, const SSchema* p
pCol->tableType = pTable->pMeta->tableType; pCol->tableType = pTable->pMeta->tableType;
pCol->colId = pColSchema->colId; pCol->colId = pColSchema->colId;
pCol->colType = (tagFlag >= 0 ? COLUMN_TYPE_TAG : COLUMN_TYPE_COLUMN); pCol->colType = (tagFlag >= 0 ? COLUMN_TYPE_TAG : COLUMN_TYPE_COLUMN);
pCol->hasIndex = ((0 == tagFlag) || (pColSchema != NULL && IS_IDX_ON(pColSchema))); pCol->hasIndex = (pColSchema != NULL && IS_IDX_ON(pColSchema));
pCol->node.resType.type = pColSchema->type; pCol->node.resType.type = pColSchema->type;
pCol->node.resType.bytes = pColSchema->bytes; pCol->node.resType.bytes = pColSchema->bytes;
if (TSDB_DATA_TYPE_TIMESTAMP == pCol->node.resType.type) { if (TSDB_DATA_TYPE_TIMESTAMP == pCol->node.resType.type) {
@ -1610,9 +1612,11 @@ static int32_t translateInterpFunc(STranslateContext* pCxt, SFunctionNode* pFunc
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC); return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC);
} }
if (pSelect->hasInterpFunc && (FUNC_RETURN_ROWS_INDEFINITE == pSelect->returnRows || pSelect->returnRows != fmGetFuncReturnRows(pFunc))) { if (pSelect->hasInterpFunc &&
(FUNC_RETURN_ROWS_INDEFINITE == pSelect->returnRows || pSelect->returnRows != fmGetFuncReturnRows(pFunc))) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC, return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC,
"%s ignoring null value options cannot be used when applying to multiple columns", pFunc->functionName); "%s ignoring null value options cannot be used when applying to multiple columns",
pFunc->functionName);
} }
if (NULL != pSelect->pWindow || NULL != pSelect->pGroupByList) { if (NULL != pSelect->pWindow || NULL != pSelect->pGroupByList) {
@ -1650,7 +1654,8 @@ static int32_t translateTimelineFunc(STranslateContext* pCxt, SFunctionNode* pFu
} }
SSelectStmt* pSelect = (SSelectStmt*)pCxt->pCurrStmt; SSelectStmt* pSelect = (SSelectStmt*)pCxt->pCurrStmt;
if (NULL != pSelect->pFromTable && QUERY_NODE_TEMP_TABLE == nodeType(pSelect->pFromTable) && if (NULL != pSelect->pFromTable && QUERY_NODE_TEMP_TABLE == nodeType(pSelect->pFromTable) &&
!isGlobalTimeLineQuery(((STempTableNode*)pSelect->pFromTable)->pSubquery) && !isTimeLineAlignedQuery(pCxt->pCurrStmt)) { !isGlobalTimeLineQuery(((STempTableNode*)pSelect->pFromTable)->pSubquery) &&
!isTimeLineAlignedQuery(pCxt->pCurrStmt)) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC, return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC,
"%s function requires valid time series input", pFunc->functionName); "%s function requires valid time series input", pFunc->functionName);
} }
@ -2310,7 +2315,8 @@ static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) {
} }
} }
if (isScanPseudoColumnFunc(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) { if (isScanPseudoColumnFunc(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) {
if (pSelect->selectFuncNum > 1 || pSelect->hasOtherVectorFunc || !pSelect->hasSelectFunc || (isDistinctOrderBy(pCxt) && pCxt->currClause == SQL_CLAUSE_ORDER_BY)) { if (pSelect->selectFuncNum > 1 || pSelect->hasOtherVectorFunc || !pSelect->hasSelectFunc ||
(isDistinctOrderBy(pCxt) && pCxt->currClause == SQL_CLAUSE_ORDER_BY)) {
return generateDealNodeErrMsg(pCxt, getGroupByErrorCode(pCxt), ((SExprNode*)(*pNode))->userAlias); return generateDealNodeErrMsg(pCxt, getGroupByErrorCode(pCxt), ((SExprNode*)(*pNode))->userAlias);
} else { } else {
return rewriteColToSelectValFunc(pCxt, pNode); return rewriteColToSelectValFunc(pCxt, pNode);
@ -2713,7 +2719,8 @@ static int32_t replaceTbName(STranslateContext* pCxt, SSelectStmt* pSelect) {
} }
SRealTableNode* pTable = (SRealTableNode*)pSelect->pFromTable; SRealTableNode* pTable = (SRealTableNode*)pSelect->pFromTable;
if (TSDB_CHILD_TABLE != pTable->pMeta->tableType && TSDB_NORMAL_TABLE != pTable->pMeta->tableType && TSDB_SYSTEM_TABLE != pTable->pMeta->tableType) { if (TSDB_CHILD_TABLE != pTable->pMeta->tableType && TSDB_NORMAL_TABLE != pTable->pMeta->tableType &&
TSDB_SYSTEM_TABLE != pTable->pMeta->tableType) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -3124,7 +3131,8 @@ static int32_t convertFillValue(STranslateContext* pCxt, SDataType dt, SNodeList
code = scalarCalculateConstants(pCastFunc, &pCell->pNode); code = scalarCalculateConstants(pCastFunc, &pCell->pNode);
} }
if (TSDB_CODE_SUCCESS == code && QUERY_NODE_VALUE != nodeType(pCell->pNode)) { if (TSDB_CODE_SUCCESS == code && QUERY_NODE_VALUE != nodeType(pCell->pNode)) {
code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Fill value can only accept constant"); code =
generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Fill value can only accept constant");
} else if (TSDB_CODE_SUCCESS != code) { } else if (TSDB_CODE_SUCCESS != code) {
code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Filled data type mismatch"); code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Filled data type mismatch");
} }
@ -3590,7 +3598,6 @@ static int32_t createDefaultEveryNode(STranslateContext* pCxt, SNode** pOutput)
pEvery->isDuration = true; pEvery->isDuration = true;
pEvery->literal = taosStrdup("1s"); pEvery->literal = taosStrdup("1s");
*pOutput = (SNode*)pEvery; *pOutput = (SNode*)pEvery;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -3689,8 +3696,8 @@ static int32_t translatePartitionBy(STranslateContext* pCxt, SSelectStmt* pSelec
if (pSelect->pPartitionByList) { if (pSelect->pPartitionByList) {
int8_t typeType = getTableTypeFromTableNode(pSelect->pFromTable); int8_t typeType = getTableTypeFromTableNode(pSelect->pFromTable);
SNode* pPar = nodesListGetNode(pSelect->pPartitionByList, 0); SNode* pPar = nodesListGetNode(pSelect->pPartitionByList, 0);
if (!((TSDB_NORMAL_TABLE == typeType || TSDB_CHILD_TABLE == typeType) && if (!((TSDB_NORMAL_TABLE == typeType || TSDB_CHILD_TABLE == typeType) && 1 == pSelect->pPartitionByList->length &&
1 == pSelect->pPartitionByList->length && (QUERY_NODE_FUNCTION == nodeType(pPar) && FUNCTION_TYPE_TBNAME == ((SFunctionNode*)pPar)->funcType))) { (QUERY_NODE_FUNCTION == nodeType(pPar) && FUNCTION_TYPE_TBNAME == ((SFunctionNode*)pPar)->funcType))) {
pSelect->timeLineResMode = TIME_LINE_MULTI; pSelect->timeLineResMode = TIME_LINE_MULTI;
} }
@ -3957,9 +3964,9 @@ static int32_t translateSetOperProject(STranslateContext* pCxt, SSetOperator* pS
} }
snprintf(pRightExpr->aliasName, sizeof(pRightExpr->aliasName), "%s", pLeftExpr->aliasName); snprintf(pRightExpr->aliasName, sizeof(pRightExpr->aliasName), "%s", pLeftExpr->aliasName);
SNode* pProj = createSetOperProject(pSetOperator->stmtName, pLeft); SNode* pProj = createSetOperProject(pSetOperator->stmtName, pLeft);
if (QUERY_NODE_COLUMN == nodeType(pLeft) && QUERY_NODE_COLUMN == nodeType(pRight) if (QUERY_NODE_COLUMN == nodeType(pLeft) && QUERY_NODE_COLUMN == nodeType(pRight) &&
&& ((SColumnNode*)pLeft)->colId == PRIMARYKEY_TIMESTAMP_COL_ID ((SColumnNode*)pLeft)->colId == PRIMARYKEY_TIMESTAMP_COL_ID &&
&& ((SColumnNode*)pRight)->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { ((SColumnNode*)pRight)->colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
((SColumnNode*)pProj)->colId = PRIMARYKEY_TIMESTAMP_COL_ID; ((SColumnNode*)pProj)->colId = PRIMARYKEY_TIMESTAMP_COL_ID;
} }
if (TSDB_CODE_SUCCESS != nodesListMakeStrictAppend(&pSetOperator->pProjectionList, pProj)) { if (TSDB_CODE_SUCCESS != nodesListMakeStrictAppend(&pSetOperator->pProjectionList, pProj)) {
@ -5739,7 +5746,6 @@ static int32_t translateRestoreDnode(STranslateContext* pCxt, SRestoreComponentN
return buildCmdMsg(pCxt, TDMT_MND_RESTORE_DNODE, (FSerializeFunc)tSerializeSRestoreDnodeReq, &restoreReq); return buildCmdMsg(pCxt, TDMT_MND_RESTORE_DNODE, (FSerializeFunc)tSerializeSRestoreDnodeReq, &restoreReq);
} }
static int32_t getSmaIndexDstVgId(STranslateContext* pCxt, const char* pDbName, const char* pTableName, static int32_t getSmaIndexDstVgId(STranslateContext* pCxt, const char* pDbName, const char* pTableName,
int32_t* pVgId) { int32_t* pVgId) {
SVgroupInfo vg = {0}; SVgroupInfo vg = {0};
@ -5881,8 +5887,10 @@ int32_t createIntervalFromCreateSmaIndexStmt(SCreateIndexStmt* pStmt, SInterval*
pInterval->interval = ((SValueNode*)pStmt->pOptions->pInterval)->datum.i; pInterval->interval = ((SValueNode*)pStmt->pOptions->pInterval)->datum.i;
pInterval->intervalUnit = ((SValueNode*)pStmt->pOptions->pInterval)->unit; pInterval->intervalUnit = ((SValueNode*)pStmt->pOptions->pInterval)->unit;
pInterval->offset = NULL != pStmt->pOptions->pOffset ? ((SValueNode*)pStmt->pOptions->pOffset)->datum.i : 0; pInterval->offset = NULL != pStmt->pOptions->pOffset ? ((SValueNode*)pStmt->pOptions->pOffset)->datum.i : 0;
pInterval->sliding = NULL != pStmt->pOptions->pSliding ? ((SValueNode*)pStmt->pOptions->pSliding)->datum.i : pInterval->interval; pInterval->sliding =
pInterval->slidingUnit = NULL != pStmt->pOptions->pSliding ? ((SValueNode*)pStmt->pOptions->pSliding)->unit : pInterval->intervalUnit; NULL != pStmt->pOptions->pSliding ? ((SValueNode*)pStmt->pOptions->pSliding)->datum.i : pInterval->interval;
pInterval->slidingUnit =
NULL != pStmt->pOptions->pSliding ? ((SValueNode*)pStmt->pOptions->pSliding)->unit : pInterval->intervalUnit;
pInterval->precision = pStmt->pOptions->tsPrecision; pInterval->precision = pStmt->pOptions->tsPrecision;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -6114,7 +6122,8 @@ static EDealRes checkColumnTagsInCond(SNode* pNode, void* pContext) {
return DEAL_RES_CONTINUE; return DEAL_RES_CONTINUE;
} }
static int32_t checkCollectTopicTags(STranslateContext* pCxt, SCreateTopicStmt* pStmt, STableMeta* pMeta, SNodeList** ppProjection) { static int32_t checkCollectTopicTags(STranslateContext* pCxt, SCreateTopicStmt* pStmt, STableMeta* pMeta,
SNodeList** ppProjection) {
SBuildTopicContext colCxt = {.colExists = false, .colNotFound = false, .pMeta = pMeta, .pTags = NULL}; SBuildTopicContext colCxt = {.colExists = false, .colNotFound = false, .pMeta = pMeta, .pTags = NULL};
nodesWalkExprPostOrder(pStmt->pWhere, checkColumnTagsInCond, &colCxt); nodesWalkExprPostOrder(pStmt->pWhere, checkColumnTagsInCond, &colCxt);
if (colCxt.colNotFound) { if (colCxt.colNotFound) {
@ -6556,7 +6565,8 @@ static int32_t checkStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStm
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY,
"SUBTABLE expression must be of VARCHAR type"); "SUBTABLE expression must be of VARCHAR type");
} }
if (NULL != pSelect->pSubtable && 0 == LIST_LENGTH(pSelect->pPartitionByList) && subtableExprHasColumnOrPseudoColumn(pSelect->pSubtable)) { if (NULL != pSelect->pSubtable && 0 == LIST_LENGTH(pSelect->pPartitionByList) &&
subtableExprHasColumnOrPseudoColumn(pSelect->pSubtable)) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY,
"SUBTABLE expression must not has column when no partition by clause"); "SUBTABLE expression must not has column when no partition by clause");
} }
@ -7138,7 +7148,6 @@ int32_t translatePostCreateStream(SParseContext* pParseCxt, SQuery* pQuery, void
return code; return code;
} }
static int32_t translateDropStream(STranslateContext* pCxt, SDropStreamStmt* pStmt) { static int32_t translateDropStream(STranslateContext* pCxt, SDropStreamStmt* pStmt) {
SMDropStreamReq dropReq = {0}; SMDropStreamReq dropReq = {0};
SName name; SName name;
@ -7169,7 +7178,7 @@ static int32_t translateResumeStream(STranslateContext* pCxt, SResumeStreamStmt*
static int32_t readFromFile(char* pName, int32_t* len, char** buf) { static int32_t readFromFile(char* pName, int32_t* len, char** buf) {
int64_t filesize = 0; int64_t filesize = 0;
if (taosStatFile(pName, &filesize, NULL) < 0) { if (taosStatFile(pName, &filesize, NULL, NULL) < 0) {
return TAOS_SYSTEM_ERROR(errno); return TAOS_SYSTEM_ERROR(errno);
} }
@ -7823,7 +7832,8 @@ static SNodeList* createProjectCols(int32_t ncols, const char* const pCols[]) {
return pProjections; return pProjections;
} }
static int32_t createSimpleSelectStmtImpl(const char* pDb, const char* pTable, SNodeList* pProjectionList, SSelectStmt** pStmt) { static int32_t createSimpleSelectStmtImpl(const char* pDb, const char* pTable, SNodeList* pProjectionList,
SSelectStmt** pStmt) {
SSelectStmt* pSelect = (SSelectStmt*)nodesMakeNode(QUERY_NODE_SELECT_STMT); SSelectStmt* pSelect = (SSelectStmt*)nodesMakeNode(QUERY_NODE_SELECT_STMT);
if (NULL == pSelect) { if (NULL == pSelect) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
@ -7846,7 +7856,6 @@ static int32_t createSimpleSelectStmtImpl(const char* pDb, const char* pTable, S
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t createSimpleSelectStmtFromCols(const char* pDb, const char* pTable, int32_t numOfProjs, static int32_t createSimpleSelectStmtFromCols(const char* pDb, const char* pTable, int32_t numOfProjs,
const char* const pProjCol[], SSelectStmt** pStmt) { const char* const pProjCol[], SSelectStmt** pStmt) {
SNodeList* pProjectionList = NULL; SNodeList* pProjectionList = NULL;
@ -7860,13 +7869,15 @@ static int32_t createSimpleSelectStmtFromCols(const char* pDb, const char* pTabl
return createSimpleSelectStmtImpl(pDb, pTable, pProjectionList, pStmt); return createSimpleSelectStmtImpl(pDb, pTable, pProjectionList, pStmt);
} }
static int32_t createSimpleSelectStmtFromProjList(const char* pDb, const char* pTable, SNodeList* pProjectionList, SSelectStmt** pStmt) { static int32_t createSimpleSelectStmtFromProjList(const char* pDb, const char* pTable, SNodeList* pProjectionList,
SSelectStmt** pStmt) {
return createSimpleSelectStmtImpl(pDb, pTable, pProjectionList, pStmt); return createSimpleSelectStmtImpl(pDb, pTable, pProjectionList, pStmt);
} }
static int32_t createSelectStmtForShow(ENodeType showType, SSelectStmt** pStmt) { static int32_t createSelectStmtForShow(ENodeType showType, SSelectStmt** pStmt) {
const SSysTableShowAdapter* pShow = &sysTableShowAdapter[showType - SYSTABLE_SHOW_TYPE_OFFSET]; const SSysTableShowAdapter* pShow = &sysTableShowAdapter[showType - SYSTABLE_SHOW_TYPE_OFFSET];
return createSimpleSelectStmtFromCols(pShow->pDbName, pShow->pTableName, pShow->numOfShowCols, pShow->pShowCols, pStmt); return createSimpleSelectStmtFromCols(pShow->pDbName, pShow->pTableName, pShow->numOfShowCols, pShow->pShowCols,
pStmt);
} }
static int32_t createSelectStmtForShowTableDist(SShowTableDistributedStmt* pStmt, SSelectStmt** pOutput) { static int32_t createSelectStmtForShowTableDist(SShowTableDistributedStmt* pStmt, SSelectStmt** pOutput) {
@ -8004,8 +8015,8 @@ static int32_t createShowTableTagsProjections(SNodeList** pProjections, SNodeLis
static int32_t rewriteShowStableTags(STranslateContext* pCxt, SQuery* pQuery) { static int32_t rewriteShowStableTags(STranslateContext* pCxt, SQuery* pQuery) {
SShowTableTagsStmt* pShow = (SShowTableTagsStmt*)pQuery->pRoot; SShowTableTagsStmt* pShow = (SShowTableTagsStmt*)pQuery->pRoot;
SSelectStmt* pSelect = NULL; SSelectStmt* pSelect = NULL;
int32_t code = createSimpleSelectStmtFromCols(((SValueNode*)pShow->pDbName)->literal, ((SValueNode*)pShow->pTbName)->literal, int32_t code = createSimpleSelectStmtFromCols(((SValueNode*)pShow->pDbName)->literal,
-1, NULL, &pSelect); ((SValueNode*)pShow->pTbName)->literal, -1, NULL, &pSelect);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = createShowTableTagsProjections(&pSelect->pProjectionList, &pShow->pTags); code = createShowTableTagsProjections(&pSelect->pProjectionList, &pShow->pTags);
} }

3
source/libs/planner/src/planOptimizer.c
View File

@ -2679,6 +2679,9 @@ static int32_t tagScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubp
} }
nodesDestroyNode((SNode*)pAgg); nodesDestroyNode((SNode*)pAgg);
tagScanOptCloneAncestorSlimit((SLogicNode*)pScanNode); tagScanOptCloneAncestorSlimit((SLogicNode*)pScanNode);
pScanNode->onlyMetaCtbIdx = false;
pCxt->optimized = true; pCxt->optimized = true;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }

15
source/libs/planner/src/planPhysiCreater.c
View File

@ -511,6 +511,20 @@ static int32_t createSimpleScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSub
return createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, pScan, pPhyNode); return createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, pScan, pPhyNode);
} }
static int32_t createTagScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode,
SPhysiNode** pPhyNode) {
STagScanPhysiNode* pScan =
(STagScanPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pScanLogicNode, QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN);
if (NULL == pScan) {
return TSDB_CODE_OUT_OF_MEMORY;
}
vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode);
pScan->onlyMetaCtbIdx = pScanLogicNode->onlyMetaCtbIdx;
return createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pScan, pPhyNode);
}
static int32_t createLastRowScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode, static int32_t createLastRowScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode,
SPhysiNode** pPhyNode) { SPhysiNode** pPhyNode) {
SLastRowScanPhysiNode* pScan = SLastRowScanPhysiNode* pScan =
@ -646,6 +660,7 @@ static int32_t createScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan,
pCxt->hasScan = true; pCxt->hasScan = true;
switch (pScanLogicNode->scanType) { switch (pScanLogicNode->scanType) {
case SCAN_TYPE_TAG: case SCAN_TYPE_TAG:
return createTagScanPhysiNode(pCxt, pSubplan, pScanLogicNode, pPhyNode);
case SCAN_TYPE_BLOCK_INFO: case SCAN_TYPE_BLOCK_INFO:
return createSimpleScanPhysiNode(pCxt, pSubplan, pScanLogicNode, pPhyNode); return createSimpleScanPhysiNode(pCxt, pSubplan, pScanLogicNode, pPhyNode);
case SCAN_TYPE_TABLE_COUNT: case SCAN_TYPE_TABLE_COUNT:

2
source/libs/sync/src/syncRaftStore.c
View File

@ -42,7 +42,7 @@ int32_t raftStoreReadFile(SSyncNode *pNode) {
const char *file = pNode->raftStorePath; const char *file = pNode->raftStorePath;
SRaftStore *pStore = &pNode->raftStore; SRaftStore *pStore = &pNode->raftStore;
if (taosStatFile(file, NULL, NULL) < 0) { if (taosStatFile(file, NULL, NULL, NULL) < 0) {
sInfo("vgId:%d, raft store file:%s not exist, use default value", pNode->vgId, file); sInfo("vgId:%d, raft store file:%s not exist, use default value", pNode->vgId, file);
pStore->currentTerm = 0; pStore->currentTerm = 0;
pStore->voteFor.addr = 0; pStore->voteFor.addr = 0;

12
source/libs/wal/src/walMeta.c
View File

@ -53,7 +53,7 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) {
walBuildLogName(pWal, pFileInfo->firstVer, fnameStr); walBuildLogName(pWal, pFileInfo->firstVer, fnameStr);
int64_t fileSize = 0; int64_t fileSize = 0;
taosStatFile(fnameStr, &fileSize, NULL); taosStatFile(fnameStr, &fileSize, NULL, NULL);
TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ | TD_FILE_WRITE); TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ | TD_FILE_WRITE);
if (pFile == NULL) { if (pFile == NULL) {
@ -304,7 +304,7 @@ int walRepairLogFileTs(SWal* pWal, bool* updateMeta) {
walBuildLogName(pWal, pFileInfo->firstVer, fnameStr); walBuildLogName(pWal, pFileInfo->firstVer, fnameStr);
int32_t mtime = 0; int32_t mtime = 0;
if (taosStatFile(fnameStr, NULL, &mtime) < 0) { if (taosStatFile(fnameStr, NULL, &mtime, NULL) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, failed to stat file due to %s, file:%s", pWal->cfg.vgId, strerror(errno), fnameStr); wError("vgId:%d, failed to stat file due to %s, file:%s", pWal->cfg.vgId, strerror(errno), fnameStr);
return -1; return -1;
@ -353,7 +353,7 @@ int walTrimIdxFile(SWal* pWal, int32_t fileIdx) {
walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr); walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr);
int64_t fileSize = 0; int64_t fileSize = 0;
taosStatFile(fnameStr, &fileSize, NULL); taosStatFile(fnameStr, &fileSize, NULL, NULL);
int64_t records = TMAX(0, pFileInfo->lastVer - pFileInfo->firstVer + 1); int64_t records = TMAX(0, pFileInfo->lastVer - pFileInfo->firstVer + 1);
int64_t lastEndOffset = records * sizeof(SWalIdxEntry); int64_t lastEndOffset = records * sizeof(SWalIdxEntry);
@ -436,7 +436,7 @@ int walCheckAndRepairMeta(SWal* pWal) {
SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx); SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx);
walBuildLogName(pWal, pFileInfo->firstVer, fnameStr); walBuildLogName(pWal, pFileInfo->firstVer, fnameStr);
int32_t code = taosStatFile(fnameStr, &fileSize, NULL); int32_t code = taosStatFile(fnameStr, &fileSize, NULL, NULL);
if (code < 0) { if (code < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
wError("failed to stat file since %s. file:%s", terrstr(), fnameStr); wError("failed to stat file since %s. file:%s", terrstr(), fnameStr);
@ -522,7 +522,7 @@ int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) {
walBuildLogName(pWal, pFileInfo->firstVer, fLogNameStr); walBuildLogName(pWal, pFileInfo->firstVer, fLogNameStr);
int64_t fileSize = 0; int64_t fileSize = 0;
if (taosStatFile(fnameStr, &fileSize, NULL) < 0 && errno != ENOENT) { if (taosStatFile(fnameStr, &fileSize, NULL, NULL) < 0 && errno != ENOENT) {
wError("vgId:%d, failed to stat file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr); wError("vgId:%d, failed to stat file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr);
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
return -1; return -1;
@ -935,7 +935,7 @@ int walLoadMeta(SWal* pWal) {
walBuildMetaName(pWal, metaVer, fnameStr); walBuildMetaName(pWal, metaVer, fnameStr);
// read metafile // read metafile
int64_t fileSize = 0; int64_t fileSize = 0;
taosStatFile(fnameStr, &fileSize, NULL); taosStatFile(fnameStr, &fileSize, NULL, NULL);
if (fileSize == 0) { if (fileSize == 0) {
(void)taosRemoveFile(fnameStr); (void)taosRemoveFile(fnameStr);
wDebug("vgId:%d, wal find empty meta ver %d", pWal->cfg.vgId, metaVer); wDebug("vgId:%d, wal find empty meta ver %d", pWal->cfg.vgId, metaVer);

6
source/os/src/osFile.c
View File

@ -191,7 +191,7 @@ int32_t taosRenameFile(const char *oldName, const char *newName) {
#endif #endif
} }
int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime) { int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime, int32_t *atime) {
#ifdef WINDOWS #ifdef WINDOWS
struct _stati64 fileStat; struct _stati64 fileStat;
int32_t code = _stati64(path, &fileStat); int32_t code = _stati64(path, &fileStat);
@ -211,6 +211,10 @@ int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime) {
*mtime = fileStat.st_mtime; *mtime = fileStat.st_mtime;
} }
if (atime != NULL) {
*atime = fileStat.st_mtime;
}
return 0; return 0;
} }
int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno) { int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno) {

9
source/os/src/osRand.c
View File

@ -84,3 +84,12 @@ void taosRandStr(char* str, int32_t size) {
str[i] = set[taosRand() % len]; str[i] = set[taosRand() % len];
} }
} }
void taosRandStr2(char* str, int32_t size) {
const char* set = "abcdefghijklmnopqrstuvwxyz0123456789";
int32_t len = 36;
for (int32_t i = 0; i < size; ++i) {
str[i] = set[taosRand() % len];
}
}

12
source/util/src/tlog.c
View File

@ -17,8 +17,8 @@
#include "tlog.h" #include "tlog.h"
#include "os.h" #include "os.h"
#include "tconfig.h" #include "tconfig.h"
#include "tjson.h"
#include "tglobal.h" #include "tglobal.h"
#include "tjson.h"
#define LOG_MAX_LINE_SIZE (10024) #define LOG_MAX_LINE_SIZE (10024)
#define LOG_MAX_LINE_BUFFER_SIZE (LOG_MAX_LINE_SIZE + 3) #define LOG_MAX_LINE_BUFFER_SIZE (LOG_MAX_LINE_SIZE + 3)
@ -403,13 +403,13 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) {
strcpy(name, fn); strcpy(name, fn);
strcat(name, ".0"); strcat(name, ".0");
} }
bool log0Exist = taosStatFile(name, NULL, &logstat0_mtime) >= 0; bool log0Exist = taosStatFile(name, NULL, &logstat0_mtime, NULL) >= 0;
if (strlen(fn) < LOG_FILE_NAME_LEN + 50 - 2) { if (strlen(fn) < LOG_FILE_NAME_LEN + 50 - 2) {
strcpy(name, fn); strcpy(name, fn);
strcat(name, ".1"); strcat(name, ".1");
} }
bool log1Exist = taosStatFile(name, NULL, &logstat1_mtime) >= 0; bool log1Exist = taosStatFile(name, NULL, &logstat1_mtime, NULL) >= 0;
// if none of the log files exist, open 0, if both exists, open the old one // if none of the log files exist, open 0, if both exists, open the old one
if (!log0Exist && !log1Exist) { if (!log0Exist && !log1Exist) {
@ -902,7 +902,7 @@ void taosReadCrashInfo(char* filepath, char** pMsg, int64_t* pMsgLen, TdFilePtr*
if (NULL == *pFd) { if (NULL == *pFd) {
int64_t filesize = 0; int64_t filesize = 0;
if (taosStatFile(filepath, &filesize, NULL) < 0) { if (taosStatFile(filepath, &filesize, NULL, NULL) < 0) {
if (ENOENT == errno) { if (ENOENT == errno) {
return; return;
} }
@ -952,8 +952,8 @@ void taosReadCrashInfo(char* filepath, char** pMsg, int64_t* pMsgLen, TdFilePtr*
readSize = taosReadFile(pFile, buf, msgLen); readSize = taosReadFile(pFile, buf, msgLen);
if (msgLen != readSize) { if (msgLen != readSize) {
truncateFile = true; truncateFile = true;
taosPrintLog(flags, level, dflag, "failed to read file:%s,%p wlen:%" PRId64 " tlen:%" PRId64 " since %s", taosPrintLog(flags, level, dflag, "failed to read file:%s,%p wlen:%" PRId64 " tlen:%" PRId64 " since %s", filepath,
filepath, pFile, readSize, msgLen, terrstr()); pFile, readSize, msgLen, terrstr());
goto _return; goto _return;
} }

3
tests/parallel_test/cases.task
View File

@ -1204,6 +1204,9 @@
,,y,script,./test.sh -f tsim/tag/drop_tag.sim ,,y,script,./test.sh -f tsim/tag/drop_tag.sim
,,y,script,./test.sh -f tsim/tag/tbNameIn.sim ,,y,script,./test.sh -f tsim/tag/tbNameIn.sim
,,y,script,./test.sh -f tmp/monitor.sim ,,y,script,./test.sh -f tmp/monitor.sim
,,y,script,./test.sh -f tsim/tagindex/add_index.sim
,,n,script,./test.sh -f tsim/tagindex/sma_and_tag_index.sim
#develop test #develop test
,,n,develop-test,python3 ./test.py -f 2-query/table_count_scan.py ,,n,develop-test,python3 ./test.py -f 2-query/table_count_scan.py

2
tests/parallel_test/container_build.sh
View File

@ -60,6 +60,7 @@ docker run \
-v /root/.cargo/git:/root/.cargo/git \ -v /root/.cargo/git:/root/.cargo/git \
-v /root/go/pkg/mod:/root/go/pkg/mod \ -v /root/go/pkg/mod:/root/go/pkg/mod \
-v /root/.cache/go-build:/root/.cache/go-build \ -v /root/.cache/go-build:/root/.cache/go-build \
-v /root/.cos-local.1:/root/.cos-local.1 \
-v ${REP_REAL_PATH}/enterprise/src/plugins/taosx/target:${REP_DIR}/enterprise/src/plugins/taosx/target \ -v ${REP_REAL_PATH}/enterprise/src/plugins/taosx/target:${REP_DIR}/enterprise/src/plugins/taosx/target \
-v ${REP_REAL_PATH}/community/tools/taosws-rs/target:${REP_DIR}/community/tools/taosws-rs/target \ -v ${REP_REAL_PATH}/community/tools/taosws-rs/target:${REP_DIR}/community/tools/taosws-rs/target \
-v ${REP_REAL_PATH}/community/contrib/cJson/:${REP_DIR}/community/contrib/cJson \ -v ${REP_REAL_PATH}/community/contrib/cJson/:${REP_DIR}/community/contrib/cJson \
@ -88,6 +89,7 @@ docker run \
-v /root/.cargo/git:/root/.cargo/git \ -v /root/.cargo/git:/root/.cargo/git \
-v /root/go/pkg/mod:/root/go/pkg/mod \ -v /root/go/pkg/mod:/root/go/pkg/mod \
-v /root/.cache/go-build:/root/.cache/go-build \ -v /root/.cache/go-build:/root/.cache/go-build \
-v /root/.cos-local.1:/root/.cos-local.1 \
-v ${REP_REAL_PATH}/enterprise/src/plugins/taosx/target:${REP_DIR}/enterprise/src/plugins/taosx/target \ -v ${REP_REAL_PATH}/enterprise/src/plugins/taosx/target:${REP_DIR}/enterprise/src/plugins/taosx/target \
-v ${REP_REAL_PATH}/community/tools/taosws-rs/target:${REP_DIR}/community/tools/taosws-rs/target \ -v ${REP_REAL_PATH}/community/tools/taosws-rs/target:${REP_DIR}/community/tools/taosws-rs/target \
-v ${REP_REAL_PATH}/community/contrib/cJson/:${REP_DIR}/community/contrib/cJson \ -v ${REP_REAL_PATH}/community/contrib/cJson/:${REP_DIR}/community/contrib/cJson \

33
tests/script/tsim/sma/drop_sma.sim
View File

@ -52,9 +52,21 @@ sql create sma index sma_index_name1 on stb function(max(c1),max(c2),min(c1)) in
print --> show sma print --> show sma
sql show indexes from stb from d1; sql show indexes from stb from d1;
if $rows != 1 then if $rows != 2 then
return -1 return -1
endi endi
if $data[0][6] == tag_index then
if $data[1][0] != sma_index_name1 then
return -1
endi
if $data[1][1] != d1 then
return -1
endi
if $data[1][2] != stb then
return -1
endi
else
if $data[0][0] != sma_index_name1 then if $data[0][0] != sma_index_name1 then
return -1 return -1
endi endi
@ -64,6 +76,10 @@ endi
if $data[0][2] != stb then if $data[0][2] != stb then
return -1 return -1
endi endi
endi
print --> drop stb print --> drop stb
sql drop table stb; sql drop table stb;
@ -78,9 +94,21 @@ sql create sma index sma_index_name1 on stb function(max(c1),max(c2),min(c1)) in
print --> show sma print --> show sma
sql show indexes from stb from d1; sql show indexes from stb from d1;
if $rows != 1 then if $rows != 2 then
return -1 return -1
endi endi
if $data[0][6] == tag_index then
if $data[1][0] != sma_index_name1 then
return -1
endi
if $data[1][1] != d1 then
return -1
endi
if $data[1][2] != stb then
return -1
endi
else
if $data[0][0] != sma_index_name1 then if $data[0][0] != sma_index_name1 then
return -1 return -1
endi endi
@ -90,6 +118,7 @@ endi
if $data[0][2] != stb then if $data[0][2] != stb then
return -1 return -1
endi endi
endi
print --> drop stb print --> drop stb
sql drop table stb; sql drop table stb;

48
tests/script/tsim/tagindex/add_index.sim
View File

@ -7,7 +7,7 @@ print ======== step0
$dbPrefix = ta_3_db $dbPrefix = ta_3_db
$tbPrefix = ta_3_tb $tbPrefix = ta_3_tb
$mtPrefix = ta_3_mt $mtPrefix = ta_3_mt
$tbNum = 500 $tbNum = 50
$rowNum = 20 $rowNum = 20
$totalNum = 200 $totalNum = 200
@ -48,12 +48,16 @@ while $i < $tbNum
$i = $i + 1 $i = $i + 1
endw endw
sql_error create index ti1 on $mtPrefix (t1)
sql create index ti2 on $mtPrefix (t2) sql create index ti2 on $mtPrefix (t2)
sql create index ti5 on $mtPrefix (t5) sql create index ti5 on $mtPrefix (t5)
print ==== test name conflict print ==== test name conflict
# #
sql_error create index ti1 on $mtPrefix(t1)
sql_error create index ti11 on $mtPrefix(t1)
sql_error create index ti3 on $mtPrefix(t2) sql_error create index ti3 on $mtPrefix(t2)
sql_error create index ti2 on $mtPrefix(t2) sql_error create index ti2 on $mtPrefix(t2)
@ -73,6 +77,15 @@ while $i < $tbNum
$i = $i + 1 $i = $i + 1
endw endw
$i = 0
while $i < $tbNum
sql select * from $mtPrefix where t1= $i ;
if $rows != 1 then
return -1
endi
$i = $i + 1
endw
print ===== test operator great equal print ===== test operator great equal
@ -250,7 +263,7 @@ endw
print === show index print === show index
sql select * from information_schema.ins_indexes sql select * from information_schema.ins_indexes
if $rows != 1 then if $rows != 2 then
return -1 return -1
endi endi
@ -259,12 +272,41 @@ print === drop index ti2
sql drop index ti2 sql drop index ti2
print === drop not exist index print === drop not exist index
sql select * from information_schema.ins_indexes
if $rows != 1 then
return -1
endi
sql drop index $data[0][0]
if $rows != 0 then
return -1
endi
sql_error drop index t2 sql_error drop index t2
sql_error drop index t3 sql_error drop index t3
sql create index ti0 on $mtPrefix (t1)
$i = $interval
while $i < $limit
sql select * from $mtPrefix where t1 <= $i ;
$tmp = $i - $interval
$tmp = $tmp + 1
if $rows != $tmp then
return -1
endi
$i = $i + 1
endw
sql_error create index ti0 on $mtPrefix (t1) sql_error create index ti0 on $mtPrefix (t1)
sql_error create index ti2 on $mtPrefix (t1)
sql_error create index t2i on ta_3_tb17 (t2) sql_error create index t2i on ta_3_tb17 (t2)

22
tests/script/tsim/tagindex/sma_and_tag_index.sim
View File

@ -69,7 +69,7 @@ sql create sma index smat2i on $mtPrefix function(max(c1)) interval(6m,10s) slid
sql select * from information_schema.ins_indexes sql select * from information_schema.ins_indexes
if $rows != 2 then if $rows != 3 then
return -1 return -1
endi endi
@ -84,7 +84,7 @@ while $i < 5
endw endw
sql select * from information_schema.ins_indexes sql select * from information_schema.ins_indexes
if $rows != 6 then if $rows != 7 then
return -1 return -1
endi endi
@ -114,13 +114,13 @@ sql use $dbPrefix
sql create table if not exists $mtPrefix (ts timestamp, c1 int) tags (t1 int, t2 int, t3 int, t4 int, t5 int) sql create table if not exists $mtPrefix (ts timestamp, c1 int) tags (t1 int, t2 int, t3 int, t4 int, t5 int)
sql create index tagt2i on $mtPrefix (t2) sql create index tagt2i on $mtPrefix (t2)
sql select * from information_schema.ins_indexes sql select * from information_schema.ins_indexes
if $rows != 1 then if $rows != 2 then
return -1 return -1
endi endi
sql alter table $mtPrefix drop tag t2 sql alter table $mtPrefix drop tag t2
sql select * from information_schema.ins_indexes sql select * from information_schema.ins_indexes
if $rows != 0 then if $rows != 1 then
return -1 return -1
endi endi
@ -128,18 +128,22 @@ endi
print ==== rename tag name, and update index colName print ==== rename tag name, and update index colName
sql create index tagt3i on $mtPrefix (t3) sql create index tagt3i on $mtPrefix (t3)
sql select * from information_schema.ins_indexes sql select * from information_schema.ins_indexes
if $rows != 1 then if $rows != 2 then
return -1 return -1
endi endi
sql alter table $mtPrefix rename tag t3 txxx sql alter table $mtPrefix rename tag t3 txxx
sql select * from information_schema.ins_indexes sql select * from information_schema.ins_indexes
if $rows != 1 then if $rows != 2 then
return -1 return -1
endi endi
if $data05 != txxx then if $data05 == txxx then
return -1 print "manual created index"
elif $data15 == txxx then
print "auto created index at tag0"
else
return -1;
endi endi
@ -153,7 +157,7 @@ sql create table if not exists $mtPrefix (ts timestamp, c1 int) tags (t1 int, t2
sql create index tagt3i on $mtPrefix (t3) sql create index tagt3i on $mtPrefix (t3)
sql select * from information_schema.ins_indexes sql select * from information_schema.ins_indexes
if $rows != 2 then if $rows != 4 then
return -1 return -1
endi endi

18
tests/system-test/0-others/show_tag_index.py
View File

@ -59,14 +59,18 @@ class TDTestCase:
tdSql.checkData(1, 2, 2) tdSql.checkData(1, 2, 2)
def check_indexes(self): def check_indexes(self):
tdSql.checkRows(1) tdSql.checkRows(2)
for i in range(2):
col_name = tdSql.getData(i, 5)
if col_name == "t0":
continue
tdSql.checkCols(7) tdSql.checkCols(7)
tdSql.checkData(0, 0, 'idx1') tdSql.checkData(i, 0, 'idx1')
tdSql.checkData(0, 1, 'db') tdSql.checkData(i, 1, 'db')
tdSql.checkData(0, 2, 'stb') tdSql.checkData(i, 2, 'stb')
tdSql.checkData(0, 3, None) tdSql.checkData(i, 3, None)
tdSql.checkData(0, 5, 't1') tdSql.checkData(i, 5, 't1')
tdSql.checkData(0, 6, 'tag_index') tdSql.checkData(i, 6, 'tag_index')
def run(self): def run(self):
tdSql.execute(f'create database db') tdSql.execute(f'create database db')

7
tests/system-test/0-others/tag_index_basic.py
View File

@ -118,12 +118,15 @@ class TDTestCase:
def show_tagidx(self, stbname): def show_tagidx(self, stbname):
sql = f'select index_name,column_name from information_schema.ins_indexes where db_name="db"' sql = f'select index_name,column_name from information_schema.ins_indexes where db_name="db"'
tdSql.query(sql) tdSql.query(sql)
rows = len(self.tag_dict.keys())-1 rows = len(self.tag_dict.keys())
tdSql.checkRows(rows) tdSql.checkRows(rows)
for i in range(rows): for i in range(rows):
col_name = tdSql.getData(i, 1) col_name = tdSql.getData(i, 1)
idx_name = f'idx_{col_name}' idx_name = f'idx_{col_name}'
# skip first tag
if col_name == "t1":
continue
tdSql.checkData(i, 0, idx_name) tdSql.checkData(i, 0, idx_name)
tdLog.info(f' show {rows} tag indexs ok.') tdLog.info(f' show {rows} tag indexs ok.')
@ -201,7 +204,7 @@ class TDTestCase:
# check idx result is 0 # check idx result is 0
sql = f'select index_name,column_name from information_schema.ins_indexes where db_name="db"' sql = f'select index_name,column_name from information_schema.ins_indexes where db_name="db"'
tdSql.query(sql) tdSql.query(sql)
tdSql.checkRows(0) tdSql.checkRows(1)
tdLog.info(f' drop {cnt} tag indexs ok.') tdLog.info(f' drop {cnt} tag indexs ok.')
# create long name idx # create long name idx

28
tools/shell/src/shellEngine.c
View File

@ -18,9 +18,9 @@
#define _GNU_SOURCE #define _GNU_SOURCE
#define _XOPEN_SOURCE #define _XOPEN_SOURCE
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "shellInt.h"
#include "shellAuto.h"
#include "geosWrapper.h" #include "geosWrapper.h"
#include "shellAuto.h"
#include "shellInt.h"
static bool shellIsEmptyCommand(const char *cmd); static bool shellIsEmptyCommand(const char *cmd);
static int32_t shellRunSingleCommand(char *command); static int32_t shellRunSingleCommand(char *command);
@ -387,8 +387,7 @@ void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, i
break; break;
case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_JSON: case TSDB_DATA_TYPE_JSON: {
{
int32_t bufIndex = 0; int32_t bufIndex = 0;
for (int32_t i = 0; i < length; i++) { for (int32_t i = 0; i < length; i++) {
buf[bufIndex] = val[i]; buf[bufIndex] = val[i];
@ -401,8 +400,7 @@ void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, i
buf[bufIndex] = 0; buf[bufIndex] = 0;
taosFprintfFile(pFile, "%s%s%s", quotationStr, buf, quotationStr); taosFprintfFile(pFile, "%s%s%s", quotationStr, buf, quotationStr);
} } break;
break;
case TSDB_DATA_TYPE_GEOMETRY: case TSDB_DATA_TYPE_GEOMETRY:
shellDumpHexValue(buf, val, length); shellDumpHexValue(buf, val, length);
taosFprintfFile(pFile, "%s", buf); taosFprintfFile(pFile, "%s", buf);
@ -535,12 +533,10 @@ void shellPrintString(const char *str, int32_t width) {
if (width == 0) { if (width == 0) {
printf("%s", str); printf("%s", str);
} } else if (len > width) {
else if (len > width) {
if (width <= 3) { if (width <= 3) {
printf("%.*s.", width - 1, str); printf("%.*s.", width - 1, str);
} } else {
else {
printf("%.*s...", width - 3, str); printf("%.*s...", width - 3, str);
} }
} else { } else {
@ -616,7 +612,6 @@ void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t
} else { } else {
n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.7f", width, GET_FLOAT_VAL(val)); n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.7f", width, GET_FLOAT_VAL(val));
if (n > SHELL_FLOAT_WIDTH) { if (n > SHELL_FLOAT_WIDTH) {
printf("%*.7e", width, GET_FLOAT_VAL(val)); printf("%*.7e", width, GET_FLOAT_VAL(val));
} else { } else {
printf("%s", buf); printf("%s", buf);
@ -922,7 +917,7 @@ void shellReadHistory() {
taosMemoryFreeClear(line); taosMemoryFreeClear(line);
taosCloseFile(&pFile); taosCloseFile(&pFile);
int64_t file_size; int64_t file_size;
if (taosStatFile(pHistory->file, &file_size, NULL) == 0 && file_size > SHELL_MAX_COMMAND_SIZE) { if (taosStatFile(pHistory->file, &file_size, NULL, NULL) == 0 && file_size > SHELL_MAX_COMMAND_SIZE) {
TdFilePtr pFile = taosOpenFile(pHistory->file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_STREAM | TD_FILE_TRUNC); TdFilePtr pFile = taosOpenFile(pHistory->file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_STREAM | TD_FILE_TRUNC);
if (pFile == NULL) return; if (pFile == NULL) return;
int32_t endIndex = pHistory->hstart; int32_t endIndex = pHistory->hstart;
@ -1044,7 +1039,8 @@ void shellGetGrantInfo() {
int32_t code = taos_errno(tres); int32_t code = taos_errno(tres);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
if (code != TSDB_CODE_OPS_NOT_SUPPORT && code != TSDB_CODE_MND_NO_RIGHTS && code != TSDB_CODE_PAR_PERMISSION_DENIED) { if (code != TSDB_CODE_OPS_NOT_SUPPORT && code != TSDB_CODE_MND_NO_RIGHTS &&
code != TSDB_CODE_PAR_PERMISSION_DENIED) {
fprintf(stderr, "Failed to check Server Edition, Reason:0x%04x:%s\r\n\r\n", code, taos_errstr(tres)); fprintf(stderr, "Failed to check Server Edition, Reason:0x%04x:%s\r\n\r\n", code, taos_errstr(tres));
} }
return; return;
@ -1080,7 +1076,8 @@ void shellGetGrantInfo() {
} else if (strcmp(expiretime, "unlimited") == 0) { } else if (strcmp(expiretime, "unlimited") == 0) {
fprintf(stdout, "Server is Enterprise %s Edition, %s and will never expire.\r\n", serverVersion, sinfo); fprintf(stdout, "Server is Enterprise %s Edition, %s and will never expire.\r\n", serverVersion, sinfo);
} else { } else {
fprintf(stdout, "Server is Enterprise %s Edition, %s and will expire at %s.\r\n", serverVersion, sinfo, expiretime); fprintf(stdout, "Server is Enterprise %s Edition, %s and will expire at %s.\r\n", serverVersion, sinfo,
expiretime);
} }
taos_free_result(tres); taos_free_result(tres);
@ -1165,8 +1162,7 @@ void *shellThreadLoop(void *arg) {
} }
int32_t shellExecute() { int32_t shellExecute() {
printf(shell.info.clientVersion, shell.info.cusName, printf(shell.info.clientVersion, shell.info.cusName, taos_get_client_info(), shell.info.cusName);
taos_get_client_info(), shell.info.cusName);
fflush(stdout); fflush(stdout);
SShellArgs *pArgs = &shell.args; SShellArgs *pArgs = &shell.args;

3
utils/test/c/tmqDemo.c
View File

@ -221,7 +221,7 @@ int64_t getDirectorySize(char* dir) {
totalSize += subDirSize; totalSize += subDirSize;
} else if (0 == strcmp(strchr(fileName, '.'), ".log")) { // only calc .log file size, and not include .idx file } else if (0 == strcmp(strchr(fileName, '.'), ".log")) { // only calc .log file size, and not include .idx file
int64_t file_size = 0; int64_t file_size = 0;
taosStatFile(subdir, &file_size, NULL); taosStatFile(subdir, &file_size, NULL, NULL);
totalSize += file_size; totalSize += file_size;
} }
} }
@ -702,4 +702,3 @@ int main(int32_t argc, char* argv[]) {
taosCloseFile(&g_fp); taosCloseFile(&g_fp);
return 0; return 0;
} }