xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/3.0' into feat/TD-24700

This commit is contained in:
dapan1121 2023-08-16 14:50:31 +08:00
parent 91d6734b34 a6ecb7253e
commit 143c657571
86 changed files with 5064 additions and 460 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
cmake/apr-util_CMakeLists.txt.in Normal file
View File

@ -0,0 +1,19 @@
# apr-util
ExternalProject_Add(aprutil-1
URL https://dlcdn.apache.org//apr/apr-util-1.6.3.tar.gz
URL_HASH SHA256=2b74d8932703826862ca305b094eef2983c27b39d5c9414442e9976a9acf1983
DOWNLOAD_NO_PROGRESS 1
DOWNLOAD_DIR "${TD_CONTRIB_DIR}/deps-download"
#GIT_REPOSITORY https://github.com/apache/apr-util.git
#GIT_TAG 1.5.4
SOURCE_DIR "${TD_CONTRIB_DIR}/apr-util"
#BINARY_DIR ""
BUILD_IN_SOURCE TRUE
BUILD_ALWAYS 1
#UPDATE_COMMAND ""
CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1/ --with-apr=$ENV{HOME}/.cos-local.1
#CONFIGURE_COMMAND ./configure --with-apr=/usr/local/apr
BUILD_COMMAND make
INSTALL_COMMAND make install
TEST_COMMAND ""
)

19
cmake/apr_CMakeLists.txt.in Normal file
View File

@ -0,0 +1,19 @@
# apr
ExternalProject_Add(apr-1
URL https://dlcdn.apache.org//apr/apr-1.7.4.tar.gz
URL_HASH SHA256=a4137dd82a185076fa50ba54232d920a17c6469c30b0876569e1c2a05ff311d9
DOWNLOAD_NO_PROGRESS 1
DOWNLOAD_DIR "${TD_CONTRIB_DIR}/deps-download"
#GIT_REPOSITORY https://github.com/apache/apr.git
#GIT_TAG 1.5.2
SOURCE_DIR "${TD_CONTRIB_DIR}/apr"
BUILD_IN_SOURCE TRUE
UPDATE_DISCONNECTED TRUE
BUILD_ALWAYS 1
#UPDATE_COMMAND ""
CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1/ --enable-shared=no
#CONFIGURE_COMMAND ./configure
BUILD_COMMAND make
INSTALL_COMMAND make install
TEST_COMMAND ""
)

6
cmake/cmake.define
View File

@ -78,6 +78,12 @@ ELSE ()
SET(TD_TAOS_TOOLS TRUE)
ENDIF ()
IF (${TD_WINDOWS})
SET(TAOS_LIB taos_static)
ELSE ()
SET(TAOS_LIB taos)
ENDIF ()
IF (TD_WINDOWS)
MESSAGE("${Yellow} set compiler flag for Windows! ${ColourReset}")
SET(COMMON_FLAGS "/w /D_WIN32 /DWIN32 /Zi /MTd")

10
cmake/cmake.options
View File

@ -125,6 +125,16 @@ option(
ON
)
IF(${TD_LINUX})
option(
BUILD_WITH_COS
"If build with cos"
ON
)
ENDIF ()
option(
BUILD_WITH_SQLITE
"If build with sqlite"

12
cmake/cos_CMakeLists.txt.in Normal file
View File

@ -0,0 +1,12 @@
# cos
ExternalProject_Add(cos
GIT_REPOSITORY https://github.com/tencentyun/cos-c-sdk-v5.git
GIT_TAG v5.0.16
SOURCE_DIR "${TD_CONTRIB_DIR}/cos-c-sdk-v5"
BINARY_DIR ""
#BUILD_IN_SOURCE TRUE
CONFIGURE_COMMAND ""
BUILD_COMMAND ""
INSTALL_COMMAND ""
TEST_COMMAND ""
)

17
cmake/curl_CMakeLists.txt.in Normal file
View File

@ -0,0 +1,17 @@
# curl
ExternalProject_Add(curl
URL https://curl.se/download/curl-8.2.1.tar.gz
DOWNLOAD_NO_PROGRESS 1
DOWNLOAD_DIR "${TD_CONTRIB_DIR}/deps-download"
#GIT_REPOSITORY https://github.com/curl/curl.git
#GIT_TAG curl-7_88_1
SOURCE_DIR "${TD_CONTRIB_DIR}/curl"
BUILD_IN_SOURCE TRUE
BUILD_ALWAYS 1
#UPDATE_COMMAND ""
CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --without-ssl --enable-shared=no --disable-ldap --disable-ldaps --without-brotli
#CONFIGURE_COMMAND ./configure --without-ssl
BUILD_COMMAND make
INSTALL_COMMAND make install
TEST_COMMAND ""
)

14
cmake/mxml_CMakeLists.txt.in Normal file
View File

@ -0,0 +1,14 @@
# cos
ExternalProject_Add(mxml
GIT_REPOSITORY https://github.com/michaelrsweet/mxml.git
GIT_TAG release-2.10
SOURCE_DIR "${TD_CONTRIB_DIR}/mxml"
#BINARY_DIR ""
BUILD_IN_SOURCE TRUE
#UPDATE_COMMAND ""
CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --enable-shared=no
#CONFIGURE_COMMAND ./configure
BUILD_COMMAND make
INSTALL_COMMAND make install
TEST_COMMAND ""
)

68
contrib/CMakeLists.txt
View File

@ -6,6 +6,39 @@ function(cat IN_FILE OUT_FILE)
file(APPEND ${OUT_FILE} "${CONTENTS}")
endfunction(cat IN_FILE OUT_FILE)
if(${TD_LINUX})
set(CONTRIB_TMP_FILE3 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in3")
configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3})
if(${BUILD_WITH_COS})
file(MAKE_DIRECTORY $ENV{HOME}/.cos-local.1/)
cat("${TD_SUPPORT_DIR}/mxml_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3})
cat("${TD_SUPPORT_DIR}/apr_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3})
cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3})
endif(${BUILD_WITH_COS})
configure_file(${CONTRIB_TMP_FILE3} "${TD_CONTRIB_DIR}/deps-download/CMakeLists.txt")
execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" .
WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download")
execute_process(COMMAND "${CMAKE_COMMAND}" --build .
WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download")
set(CONTRIB_TMP_FILE2 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in2")
configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE2})
if(${BUILD_WITH_COS})
cat("${TD_SUPPORT_DIR}/apr-util_CMakeLists.txt.in" ${CONTRIB_TMP_FILE2})
endif(${BUILD_WITH_COS})
configure_file(${CONTRIB_TMP_FILE2} "${TD_CONTRIB_DIR}/deps-download/CMakeLists.txt")
execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" .
WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download")
execute_process(COMMAND "${CMAKE_COMMAND}" --build .
WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download")
endif(${TD_LINUX})
set(CONTRIB_TMP_FILE "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in")
configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
@ -122,6 +155,16 @@ if(${BUILD_WITH_SQLITE})
cat("${TD_SUPPORT_DIR}/sqlite_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
endif(${BUILD_WITH_SQLITE})
# cos
if(${BUILD_WITH_COS})
#cat("${TD_SUPPORT_DIR}/mxml_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
#cat("${TD_SUPPORT_DIR}/apr_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
#cat("${TD_SUPPORT_DIR}/apr-util_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
#cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
cat("${TD_SUPPORT_DIR}/cos_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
add_definitions(-DUSE_COS)
endif(${BUILD_WITH_COS})
# lucene
if(${BUILD_WITH_LUCENE})
cat("${TD_SUPPORT_DIR}/lucene_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
@ -347,6 +390,31 @@ if (${BUILD_WITH_ROCKSDB})
endif()
endif()
# cos
if(${BUILD_WITH_COS})
if(${TD_LINUX})
set(CMAKE_PREFIX_PATH $ENV{HOME}/.cos-local.1)
#ADD_DEFINITIONS(-DMINIXML_LIBRARY=${CMAKE_BINARY_DIR}/build/lib/libxml.a)
option(ENABLE_TEST "Enable the tests" OFF)
INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.1/include)
MESSAGE("$ENV{HOME}/.cos-local.1/include")
set(CMAKE_BUILD_TYPE debug)
set(ORIG_CMAKE_PROJECT_NAME ${CMAKE_PROJECT_NAME})
set(CMAKE_PROJECT_NAME cos_c_sdk)
add_subdirectory(cos-c-sdk-v5 EXCLUDE_FROM_ALL)
target_include_directories(
cos_c_sdk
PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cos-c-sdk-v5/cos_c_sdk>
)
set(CMAKE_PROJECT_NAME ${ORIG_CMAKE_PROJECT_NAME})
else()
endif(${TD_LINUX})
endif(${BUILD_WITH_COS})
# lucene
# To support build on ubuntu: sudo apt-get install libboost-all-dev
if(${BUILD_WITH_LUCENE})

5
contrib/test/CMakeLists.txt
View File

@ -3,6 +3,11 @@ if(${BUILD_WITH_ROCKSDB})
add_subdirectory(rocksdb)
endif(${BUILD_WITH_ROCKSDB})
# cos
if(${BUILD_WITH_COS})
add_subdirectory(cos)
endif(${BUILD_WITH_COS})
if(${BUILD_WITH_LUCENE})
add_subdirectory(lucene)
endif(${BUILD_WITH_LUCENE})

49
contrib/test/cos/CMakeLists.txt Normal file
View File

@ -0,0 +1,49 @@
add_executable(cosTest "")
target_sources(cosTest
PRIVATE
"${CMAKE_CURRENT_SOURCE_DIR}/main.c"
)
#find_path(APR_INCLUDE_DIR apr-1/apr_time.h)
#find_path(APR_UTIL_INCLUDE_DIR apr/include/apr-1/apr_md5.h)
#find_path(MINIXML_INCLUDE_DIR mxml.h)
#find_path(CURL_INCLUDE_DIR curl/curl.h)
#include_directories (${MINIXML_INCLUDE_DIR})
#include_directories (${CURL_INCLUDE_DIR})
FIND_PROGRAM(APR_CONFIG_BIN NAMES apr-config apr-1-config PATHS /usr/bin /usr/local/bin /usr/local/apr/bin/)
#FIND_PROGRAM(APU_CONFIG_BIN NAMES apu-config apu-1-config PATHS /usr/bin /usr/local/bin /usr/local/apr/bin/)
IF (APR_CONFIG_BIN)
EXECUTE_PROCESS(
COMMAND ${APR_CONFIG_BIN} --includedir
OUTPUT_VARIABLE APR_INCLUDE_DIR
OUTPUT_STRIP_TRAILING_WHITESPACE
)
ENDIF()
#IF (APU_CONFIG_BIN)
# EXECUTE_PROCESS(
# COMMAND ${APU_CONFIG_BIN} --includedir
# OUTPUT_VARIABLE APR_UTIL_INCLUDE_DIR
# OUTPUT_STRIP_TRAILING_WHITESPACE
# )
#ENDIF()
include_directories (${APR_INCLUDE_DIR})
#include_directories (${APR_UTIL_INCLUDE_DIR})
target_include_directories(
cosTest
PUBLIC "${TD_SOURCE_DIR}/contrib/cos-c-sdk-v5/cos_c_sdk"
)
#find_library(APR_LIBRARY apr-1 PATHS /usr/local/apr/lib/)
#find_library(APR_UTIL_LIBRARY aprutil-1 PATHS /usr/local/apr/lib/)
#find_library(MINIXML_LIBRARY mxml)
#find_library(CURL_LIBRARY curl)
target_link_libraries(cosTest cos_c_sdk)
target_link_libraries(cosTest apr-1})
target_link_libraries(cosTest aprutil-1})
target_link_libraries(cosTest mxml)
target_link_libraries(cosTest curl)

3095
contrib/test/cos/main.c Normal file
View File

File diff suppressed because it is too large Load Diff

14
include/libs/executor/storageapi.h
View File

@ -98,6 +98,16 @@ typedef struct SMTbCursor {
int8_t paused;
} SMTbCursor;
typedef struct SMCtbCursor {
SMeta *pMeta;
void *pCur;
tb_uid_t suid;
void *pKey;
void *pVal;
int kLen;
int vLen;
} SMCtbCursor;
typedef struct SRowBuffPos {
void* pRowBuff;
void* pKey;
@ -278,13 +288,15 @@ typedef struct SStoreMeta {
void (*getBasicInfo)(void* pVnode, const char** dbname, int32_t* vgId, int64_t* numOfTables,
int64_t* numOfNormalTables); // vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId) &
// metaGetTbNum(SMeta *pMeta) & metaGetNtbNum(SMeta *pMeta);
int64_t (*getNumOfRowsInMem)(void* pVnode);
/**
int32_t vnodeGetCtbIdList(void *pVnode, int64_t suid, SArray *list);
int32_t vnodeGetCtbIdListByFilter(void *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg), void *arg);
int32_t vnodeGetStbIdList(void *pVnode, int64_t suid, SArray *list);
*/
SMCtbCursor* (*openCtbCursor)(void *pVnode, tb_uid_t uid, int lock);
void (*closeCtbCursor)(SMCtbCursor *pCtbCur, int lock);
tb_uid_t (*ctbCursorNext)(SMCtbCursor* pCur);
} SStoreMeta;
typedef struct SStoreMetaReader {

9
include/libs/nodes/plannodes.h
View File

@ -110,6 +110,7 @@ typedef struct SScanLogicNode {
bool sortPrimaryKey;
bool igLastNull;
bool groupOrderScan;
bool onlyMetaCtbIdx; // for tag scan with no tbname
} SScanLogicNode;
typedef struct SJoinLogicNode {
@ -364,7 +365,11 @@ typedef struct SScanPhysiNode {
bool groupOrderScan;
} SScanPhysiNode;
typedef SScanPhysiNode STagScanPhysiNode;
typedef struct STagScanPhysiNode {
SScanPhysiNode scan;
bool onlyMetaCtbIdx; //no tbname, tag index not used.
} STagScanPhysiNode;
typedef SScanPhysiNode SBlockDistScanPhysiNode;
typedef struct SLastRowScanPhysiNode {
@ -671,6 +676,8 @@ typedef struct SSubplan {
SNode* pTagCond;
SNode* pTagIndexCond;
bool showRewrite;
int32_t rowsThreshold;
bool dynamicRowThreshold;
} SSubplan;
typedef enum EExplainMode { EXPLAIN_MODE_DISABLE = 1, EXPLAIN_MODE_STATIC, EXPLAIN_MODE_ANALYZE } EExplainMode;

2
include/os/osFile.h
View File

@ -76,7 +76,7 @@ int32_t taosUnLockFile(TdFilePtr pFile);
int32_t taosUmaskFile(int32_t maskVal);
int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime);
int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime, int32_t *atime);
int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno);
int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime);
bool taosCheckExistFile(const char *pathname);

2
include/os/osRand.h
View File

@ -32,6 +32,8 @@ void taosSeedRand(uint32_t seed);
uint32_t taosRand(void);
uint32_t taosRandR(uint32_t* pSeed);
void taosRandStr(char* str, int32_t size);
void taosRandStr2(char* str, int32_t size);
uint32_t taosSafeRand(void);
#ifdef __cplusplus

1
include/util/taoserror.h
View File

@ -191,6 +191,7 @@ int32_t* taosGetErrno();
// #define TSDB_CODE_MND_FAILED_TO_CREATE_DIR TAOS_DEF_ERROR_CODE(0, 0x0313) // 2.x
// #define TSDB_CODE_MND_FAILED_TO_INIT_STEP TAOS_DEF_ERROR_CODE(0, 0x0314) // 2.x
#define TSDB_CODE_MND_USER_DISABLED TAOS_DEF_ERROR_CODE(0, 0x0315)
#define TSDB_CODE_MND_INVALID_PLATFORM TAOS_DEF_ERROR_CODE(0, 0x0316)
// mnode-sdb
#define TSDB_CODE_SDB_OBJ_ALREADY_THERE TAOS_DEF_ERROR_CODE(0, 0x0320) // internal

8
source/common/CMakeLists.txt
View File

@ -16,6 +16,14 @@ ENDIF ()
IF (TD_STORAGE)
ADD_DEFINITIONS(-D_STORAGE)
TARGET_LINK_LIBRARIES(common PRIVATE storage)
IF(${TD_LINUX})
IF(${BUILD_WITH_COS})
add_definitions(-DUSE_COS)
ENDIF(${BUILD_WITH_COS})
ENDIF(${TD_LINUX})
ENDIF ()
target_include_directories(

78
source/common/src/tdatablock.c
View File

@ -26,7 +26,7 @@ int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRo
if (pColumnInfoData->reassigned) {
int32_t totalSize = 0;
for (int32_t row = 0; row < numOfRows; ++row) {
char* pColData = pColumnInfoData->pData + pColumnInfoData->varmeta.offset[row];
char* pColData = pColumnInfoData->pData + pColumnInfoData->varmeta.offset[row];
int32_t colSize = 0;
if (pColumnInfoData->info.type == TSDB_DATA_TYPE_JSON) {
colSize = getJsonValueLen(pColData);
@ -142,7 +142,8 @@ int32_t colDataSetVal(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const
return 0;
}
int32_t colDataReassignVal(SColumnInfoData* pColumnInfoData, uint32_t dstRowIdx, uint32_t srcRowIdx, const char* pData) {
int32_t colDataReassignVal(SColumnInfoData* pColumnInfoData, uint32_t dstRowIdx, uint32_t srcRowIdx,
const char* pData) {
int32_t type = pColumnInfoData->info.type;
if (IS_VAR_DATA_TYPE(type)) {
pColumnInfoData->varmeta.offset[dstRowIdx] = pColumnInfoData->varmeta.offset[srcRowIdx];
@ -155,7 +156,6 @@ int32_t colDataReassignVal(SColumnInfoData* pColumnInfoData, uint32_t dstRowIdx,
return 0;
}
static int32_t colDataReserve(SColumnInfoData* pColumnInfoData, size_t newSize) {
if (!IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
return TSDB_CODE_SUCCESS;
@ -179,16 +179,17 @@ static int32_t colDataReserve(SColumnInfoData* pColumnInfoData, size_t newSize)
}
static int32_t doCopyNItems(struct SColumnInfoData* pColumnInfoData, int32_t currentRow, const char* pData,
int32_t itemLen, int32_t numOfRows, bool trimValue) {
int32_t itemLen, int32_t numOfRows, bool trimValue) {
if (pColumnInfoData->info.bytes < itemLen) {
uWarn("column/tag actual data len %d is bigger than schema len %d, trim it:%d", itemLen, pColumnInfoData->info.bytes, trimValue);
uWarn("column/tag actual data len %d is bigger than schema len %d, trim it:%d", itemLen,
pColumnInfoData->info.bytes, trimValue);
if (trimValue) {
itemLen = pColumnInfoData->info.bytes;
} else {
return TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER;
}
}
size_t start = 1;
// the first item
@ -221,8 +222,8 @@ static int32_t doCopyNItems(struct SColumnInfoData* pColumnInfoData, int32_t cur
return TSDB_CODE_SUCCESS;
}
int32_t colDataSetNItems(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData,
uint32_t numOfRows, bool trimValue) {
int32_t colDataSetNItems(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, uint32_t numOfRows,
bool trimValue) {
int32_t len = pColumnInfoData->info.bytes;
if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
len = varDataTLen(pData);
@ -329,7 +330,7 @@ static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, c
uint8_t* p = (uint8_t*)pSource->nullbitmap;
pColumnInfoData->nullbitmap[BitmapLen(numOfRow1) - 1] &= (0B11111111 << shiftBits); // clear remind bits
pColumnInfoData->nullbitmap[BitmapLen(numOfRow1) - 1] |= (p[0] >> remindBits); // copy remind bits
pColumnInfoData->nullbitmap[BitmapLen(numOfRow1) - 1] |= (p[0] >> remindBits); // copy remind bits
if (BitmapLen(numOfRow1) == BitmapLen(total)) {
return;
@ -417,7 +418,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, int
pColumnInfoData->pData = tmp;
if (BitmapLen(numOfRow1) < BitmapLen(finalNumOfRows)) {
char* btmp = taosMemoryRealloc(pColumnInfoData->nullbitmap, BitmapLen(finalNumOfRows));
char* btmp = taosMemoryRealloc(pColumnInfoData->nullbitmap, BitmapLen(finalNumOfRows));
if (btmp == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
@ -689,7 +690,7 @@ int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock) {
if (pCol->reassigned && IS_VAR_DATA_TYPE(pCol->info.type)) {
for (int32_t row = 0; row < numOfRows; ++row) {
char* pColData = pCol->pData + pCol->varmeta.offset[row];
char* pColData = pCol->pData + pCol->varmeta.offset[row];
int32_t colSize = 0;
if (pCol->info.type == TSDB_DATA_TYPE_JSON) {
colSize = getJsonValueLen(pColData);
@ -765,8 +766,7 @@ int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf) {
return TSDB_CODE_SUCCESS;
}
static bool colDataIsNNull(const SColumnInfoData* pColumnInfoData, int32_t startIndex,
uint32_t nRows) {
static bool colDataIsNNull(const SColumnInfoData* pColumnInfoData, int32_t startIndex, uint32_t nRows) {
if (!pColumnInfoData->hasNull) {
return false;
}
@ -947,7 +947,6 @@ int32_t dataBlockCompar(const void* p1, const void* p2, const void* param) {
}
static int32_t blockDataAssign(SColumnInfoData* pCols, const SSDataBlock* pDataBlock, const int32_t* index) {
size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pDst = &pCols[i];
@ -1198,6 +1197,7 @@ static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo*
if (tmp == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
// memset(tmp, 0, numOfRows * pColumn->info.bytes);
// copy back the existed data
if (pColumn->pData != NULL) {
@ -1541,8 +1541,8 @@ size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize, int
int end = nRows;
while (start <= end) {
int mid = start + (end - start) / 2;
//data size + var data type columns offset + fixed data type columns bitmap len
int midSize = rowSize * mid + numVarCols * sizeof(int32_t) * mid + numFixCols * BitmapLen(mid);
// data size + var data type columns offset + fixed data type columns bitmap len
int midSize = rowSize * mid + numVarCols * sizeof(int32_t) * mid + numFixCols * BitmapLen(mid);
if (midSize > payloadSize) {
result = mid;
end = mid - 1;
@ -1736,7 +1736,7 @@ int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) {
if (pColData->reassigned && IS_VAR_DATA_TYPE(pColData->info.type)) {
for (int32_t row = 0; row < rows; ++row) {
char* pData = pColData->pData + pColData->varmeta.offset[row];
char* pData = pColData->pData + pColData->varmeta.offset[row];
int32_t colSize = 0;
if (pColData->info.type == TSDB_DATA_TYPE_JSON) {
colSize = getJsonValueLen(pData);
@ -1839,7 +1839,7 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) {
// for debug
char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) {
int32_t size = 2048*1024;
int32_t size = 2048 * 1024;
*pDataBuf = taosMemoryCalloc(size, 1);
char* dumpBuf = *pDataBuf;
char pBuf[128] = {0};
@ -1847,8 +1847,8 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf)
int32_t rows = pDataBlock->info.rows;
int32_t len = 0;
len += snprintf(dumpBuf + len, size - len,
"===stream===%s|block type %d|child id %d|group id:%" PRIu64 "|uid:%" PRId64
"|rows:%" PRId64 "|version:%" PRIu64 "|cal start:%" PRIu64 "|cal end:%" PRIu64 "|tbl:%s\n",
"===stream===%s|block type %d|child id %d|group id:%" PRIu64 "|uid:%" PRId64 "|rows:%" PRId64
"|version:%" PRIu64 "|cal start:%" PRIu64 "|cal end:%" PRIu64 "|tbl:%s\n",
flag, (int32_t)pDataBlock->info.type, pDataBlock->info.childId, pDataBlock->info.id.groupId,
pDataBlock->info.id.uid, pDataBlock->info.rows, pDataBlock->info.version,
pDataBlock->info.calWin.skey, pDataBlock->info.calWin.ekey, pDataBlock->info.parTbName);
@ -2223,21 +2223,21 @@ int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols) {
data += metaSize;
dataLen += metaSize;
if (pColRes->reassigned && IS_VAR_DATA_TYPE(pColRes->info.type)) {
colSizes[col] = 0;
for (int32_t row = 0; row < numOfRows; ++row) {
char* pColData = pColRes->pData + pColRes->varmeta.offset[row];
int32_t colSize = 0;
if (pColRes->info.type == TSDB_DATA_TYPE_JSON) {
colSize = getJsonValueLen(pColData);
} else {
colSize = varDataTLen(pColData);
}
colSizes[col] += colSize;
dataLen += colSize;
memmove(data, pColData, colSize);
data += colSize;
if (pColRes->reassigned && IS_VAR_DATA_TYPE(pColRes->info.type)) {
colSizes[col] = 0;
for (int32_t row = 0; row < numOfRows; ++row) {
char* pColData = pColRes->pData + pColRes->varmeta.offset[row];
int32_t colSize = 0;
if (pColRes->info.type == TSDB_DATA_TYPE_JSON) {
colSize = getJsonValueLen(pColData);
} else {
colSize = varDataTLen(pColData);
}
colSizes[col] += colSize;
dataLen += colSize;
memmove(data, pColData, colSize);
data += colSize;
}
} else {
colSizes[col] = colDataGetLength(pColRes, numOfRows);
dataLen += colSizes[col];
@ -2248,7 +2248,8 @@ int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols) {
}
colSizes[col] = htonl(colSizes[col]);
// uError("blockEncode col bytes:%d, type:%d, size:%d, htonl size:%d", pColRes->info.bytes, pColRes->info.type, htonl(colSizes[col]), colSizes[col]);
// uError("blockEncode col bytes:%d, type:%d, size:%d, htonl size:%d", pColRes->info.bytes, pColRes->info.type,
// htonl(colSizes[col]), colSizes[col]);
}
*actualLen = dataLen;
@ -2350,7 +2351,7 @@ const char* blockDecode(SSDataBlock* pBlock, const char* pData) {
}
void trimDataBlock(SSDataBlock* pBlock, int32_t totalRows, const bool* pBoolList) {
// int32_t totalRows = pBlock->info.rows;
// int32_t totalRows = pBlock->info.rows;
int32_t bmLen = BitmapLen(totalRows);
char* pBitmap = NULL;
int32_t maxRows = 0;
@ -2377,8 +2378,9 @@ void trimDataBlock(SSDataBlock* pBlock, int32_t totalRows, const bool* pBoolList
if (colDataIsNull_var(pDst, j)) {
colDataSetNull_var(pDst, numOfRows);
} else {
// fix address sanitizer error. p1 may point to memory that will change during realloc of colDataSetVal, first copy it to p2
char* p1 = colDataGetVarData(pDst, j);
// fix address sanitizer error. p1 may point to memory that will change during realloc of colDataSetVal, first
// copy it to p2
char* p1 = colDataGetVarData(pDst, j);
int32_t len = 0;
if (pDst->info.type == TSDB_DATA_TYPE_JSON) {
len = getJsonValueLen(p1);

57
source/common/src/tglobal.c
View File

@ -216,7 +216,11 @@ uint32_t tsCurRange = 100; // range
char tsCompressor[32] = "ZSTD_COMPRESSOR"; // ZSTD_COMPRESSOR or GZIP_COMPRESSOR
// udf
bool tsStartUdfd = true;
#ifdef WINDOWS
bool tsStartUdfd = false;
#else
bool tsStartUdfd = true;
#endif
// wal
int64_t tsWalFsyncDataSizeLimit = (100 * 1024 * 1024L);
@ -237,6 +241,14 @@ int64_t tsCheckpointInterval = 3 * 60 * 60 * 1000;
bool tsFilterScalarMode = false;
int32_t tsKeepTimeOffset = 0; // latency of data migration
char tsS3Endpoint[TSDB_FQDN_LEN] = "<endpoint>";
char tsS3AccessKey[TSDB_FQDN_LEN] = "<accesskey>";
char tsS3AccessKeyId[TSDB_FQDN_LEN] = "<accesskeyid>";
char tsS3AccessKeySecret[TSDB_FQDN_LEN] = "<accesskeysecrect>";
char tsS3BucketName[TSDB_FQDN_LEN] = "<bucketname>";
char tsS3AppId[TSDB_FQDN_LEN] = "<appid>";
int8_t tsS3Enabled = false;
#ifndef _STORAGE
int32_t taosSetTfsCfg(SConfig *pCfg) {
SConfigItem *pItem = cfgGetItem(pCfg, "dataDir");
@ -258,7 +270,43 @@ int32_t taosSetTfsCfg(SConfig *pCfg) {
int32_t taosSetTfsCfg(SConfig *pCfg);
#endif
struct SConfig *taosGetCfg() { return tsCfg; }
int32_t taosSetS3Cfg(SConfig *pCfg) {
tstrncpy(tsS3AccessKey, cfgGetItem(pCfg, "s3Accesskey")->str, TSDB_FQDN_LEN);
if (tsS3AccessKey[0] == '<') {
return 0;
}
char *colon = strchr(tsS3AccessKey, ':');
if (!colon) {
uError("invalid access key:%s", tsS3AccessKey);
return -1;
}
*colon = '\0';
tstrncpy(tsS3AccessKeyId, tsS3AccessKey, TSDB_FQDN_LEN);
tstrncpy(tsS3AccessKeySecret, colon + 1, TSDB_FQDN_LEN);
tstrncpy(tsS3Endpoint, cfgGetItem(pCfg, "s3Endpoint")->str, TSDB_FQDN_LEN);
tstrncpy(tsS3BucketName, cfgGetItem(pCfg, "s3BucketName")->str, TSDB_FQDN_LEN);
char *cos = strstr(tsS3Endpoint, "cos.");
if (cos) {
char *appid = strrchr(tsS3BucketName, '-');
if (!appid) {
uError("failed to locate appid in bucket:%s", tsS3BucketName);
return -1;
} else {
tstrncpy(tsS3AppId, appid + 1, TSDB_FQDN_LEN);
}
}
if (tsS3BucketName[0] != '<' && tsDiskCfgNum > 1) {
#ifdef USE_COS
tsS3Enabled = true;
#endif
}
return 0;
}
struct SConfig *taosGetCfg() {
return tsCfg;
}
static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *inputCfgDir, const char *envFile,
char *apolloUrl) {
@ -581,6 +629,10 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddInt32(pCfg, "maxStreamBackendCache", tsMaxStreamBackendCache, 16, 1024, CFG_SCOPE_SERVER) != 0) return -1;
if (cfgAddInt32(pCfg, "pqSortMemThreshold", tsPQSortMemThreshold, 1, 10240, CFG_SCOPE_SERVER) != 0) return -1;
if (cfgAddString(pCfg, "s3Accesskey", tsS3AccessKey, CFG_SCOPE_SERVER) != 0) return -1;
if (cfgAddString(pCfg, "s3Endpoint", tsS3Endpoint, CFG_SCOPE_SERVER) != 0) return -1;
if (cfgAddString(pCfg, "s3BucketName", tsS3BucketName, CFG_SCOPE_SERVER) != 0) return -1;
GRANT_CFG_ADD;
return 0;
}
@ -1502,6 +1554,7 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile
if (taosSetServerCfg(tsCfg)) return -1;
if (taosSetReleaseCfg(tsCfg)) return -1;
if (taosSetTfsCfg(tsCfg) != 0) return -1;
if (taosSetS3Cfg(tsCfg) != 0) return -1;
}
taosSetSystemCfg(tsCfg);

4
source/dnode/mgmt/mgmt_mnode/src/mmFile.c
View File

@ -46,7 +46,7 @@ static int32_t mmDecodeOption(SJson *pJson, SMnodeOpt *pOption) {
if (code < 0) return -1;
tjsonGetInt32ValueFromDouble(replica, "role", pOption->nodeRoles[i], code);
if (code < 0) return -1;
if(pOption->nodeRoles[i] == TAOS_SYNC_ROLE_VOTER){
if (pOption->nodeRoles[i] == TAOS_SYNC_ROLE_VOTER) {
pOption->numOfReplicas++;
}
}
@ -65,7 +65,7 @@ int32_t mmReadFile(const char *path, SMnodeOpt *pOption) {
char file[PATH_MAX] = {0};
snprintf(file, sizeof(file), "%s%smnode.json", path, TD_DIRSEP);
if (taosStatFile(file, NULL, NULL) < 0) {
if (taosStatFile(file, NULL, NULL, NULL) < 0) {
dInfo("mnode file:%s not exist", file);
return 0;
}

2
source/dnode/mgmt/mgmt_vnode/src/vmFile.c
View File

@ -97,7 +97,7 @@ int32_t vmGetVnodeListFromFile(SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t
SWrapperCfg *pCfgs = NULL;
snprintf(file, sizeof(file), "%s%svnodes.json", pMgmt->path, TD_DIRSEP);
if (taosStatFile(file, NULL, NULL) < 0) {
if (taosStatFile(file, NULL, NULL, NULL) < 0) {
dInfo("vnode file:%s not exist", file);
return 0;
}

6
source/dnode/mgmt/node_util/src/dmEps.c
View File

@ -100,7 +100,7 @@ int32_t dmReadEps(SDnodeData *pData) {
goto _OVER;
}
if (taosStatFile(file, NULL, NULL) < 0) {
if (taosStatFile(file, NULL, NULL, NULL) < 0) {
dInfo("dnode file:%s not exist", file);
code = 0;
goto _OVER;
@ -350,7 +350,7 @@ void dmRotateMnodeEpSet(SDnodeData *pData) {
}
void dmGetMnodeEpSetForRedirect(SDnodeData *pData, SRpcMsg *pMsg, SEpSet *pEpSet) {
if(!pData->validMnodeEps) return;
if (!pData->validMnodeEps) return;
dmGetMnodeEpSet(pData, pEpSet);
dTrace("msg is redirected, handle:%p num:%d use:%d", pMsg->info.handle, pEpSet->numOfEps, pEpSet->inUse);
for (int32_t i = 0; i < pEpSet->numOfEps; ++i) {
@ -469,7 +469,7 @@ static int32_t dmReadDnodePairs(SDnodeData *pData) {
char file[PATH_MAX] = {0};
snprintf(file, sizeof(file), "%s%sdnode%sep.json", tsDataDir, TD_DIRSEP, TD_DIRSEP);
if (taosStatFile(file, NULL, NULL) < 0) {
if (taosStatFile(file, NULL, NULL, NULL) < 0) {
dDebug("dnode file:%s not exist", file);
code = 0;
goto _OVER;

2
source/dnode/mgmt/node_util/src/dmFile.c
View File

@ -38,7 +38,7 @@ int32_t dmReadFile(const char *path, const char *name, bool *pDeployed) {
char file[PATH_MAX] = {0};
snprintf(file, sizeof(file), "%s%s%s.json", path, TD_DIRSEP, name);
if (taosStatFile(file, NULL, NULL) < 0) {
if (taosStatFile(file, NULL, NULL, NULL) < 0) {
dInfo("file:%s not exist", file);
code = 0;
goto _OVER;

7
source/dnode/mnode/impl/src/mndDb.c
View File

@ -666,7 +666,12 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) {
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
#ifdef WINDOWS
if (taosArrayGetSize(createReq.pRetensions) > 0) {
terrno = TSDB_CODE_MND_INVALID_PLATFORM;
goto _OVER;
}
#endif
mInfo("db:%s, start to create, vgroups:%d", createReq.db, createReq.numOfVgroups);
if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_DB, NULL) != 0) {
goto _OVER;

5
source/dnode/mnode/impl/src/mndFunc.c
View File

@ -359,7 +359,10 @@ static int32_t mndProcessCreateFuncReq(SRpcMsg *pReq) {
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
#ifdef WINDOWS
terrno = TSDB_CODE_MND_INVALID_PLATFORM;
goto _OVER;
#endif
mInfo("func:%s, start to create, size:%d", createReq.name, createReq.codeLen);
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_FUNC) != 0) {
goto _OVER;

72
source/dnode/mnode/impl/src/mndIndex.c
View File

@ -79,9 +79,12 @@ int32_t mndInitIdx(SMnode *pMnode) {
return sdbSetTable(pMnode->pSdb, table);
}
static int32_t mndFindSuperTableTagId(const SStbObj *pStb, const char *tagName) {
static int32_t mndFindSuperTableTagId(const SStbObj *pStb, const char *tagName, int8_t *hasIdx) {
for (int32_t tag = 0; tag < pStb->numOfTags; tag++) {
if (strcasecmp(pStb->pTags[tag].name, tagName) == 0) {
if (IS_IDX_ON(&pStb->pTags[tag])) {
*hasIdx = 1;
}
return tag;
}
}
@ -597,7 +600,8 @@ static int32_t mndSetUpdateIdxStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStb
pNew->updateTime = taosGetTimestampMs();
pNew->lock = 0;
int32_t tag = mndFindSuperTableTagId(pOld, tagName);
int8_t hasIdx = 0;
int32_t tag = mndFindSuperTableTagId(pOld, tagName, &hasIdx);
if (tag < 0) {
terrno = TSDB_CODE_MND_TAG_NOT_EXIST;
return -1;
@ -612,14 +616,14 @@ static int32_t mndSetUpdateIdxStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStb
SSchema *pTag = pNew->pTags + tag;
if (on == 1) {
if (IS_IDX_ON(pTag)) {
if (hasIdx && tag != 0) {
terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST;
return -1;
} else {
SSCHMEA_SET_IDX_ON(pTag);
}
} else {
if (!IS_IDX_ON(pTag)) {
if (hasIdx == 0) {
terrno = TSDB_CODE_MND_SMA_NOT_EXIST;
} else {
SSCHMEA_SET_IDX_OFF(pTag);
@ -667,7 +671,42 @@ _OVER:
mndTransDrop(pTrans);
return code;
}
int8_t mndCheckIndexNameByTagName(SMnode *pMnode, SIdxObj *pIdxObj) {
// build index on first tag, and no index name;
int8_t exist = 0;
SDbObj *pDb = NULL;
if (strlen(pIdxObj->db) > 0) {
pDb = mndAcquireDb(pMnode, pIdxObj->db);
if (pDb == NULL) return 0;
}
SSmaAndTagIter *pIter = NULL;
SIdxObj *pIdx = NULL;
SSdb *pSdb = pMnode->pSdb;
while (1) {
pIter = sdbFetch(pSdb, SDB_IDX, pIter, (void **)&pIdx);
if (pIter == NULL) break;
if (NULL != pDb && pIdx->dbUid != pDb->uid) {
sdbRelease(pSdb, pIdx);
continue;
}
if (pIdxObj->stbUid != pIdx->stbUid) {
sdbRelease(pSdb, pIdx);
continue;
}
if (strncmp(pIdxObj->colName, pIdx->colName, TSDB_COL_NAME_LEN) == 0) {
sdbCancelFetch(pSdb, pIter);
sdbRelease(pSdb, pIdx);
exist = 1;
break;
}
sdbRelease(pSdb, pIdx);
}
mndReleaseDb(pMnode, pDb);
return exist;
}
static int32_t mndAddIndex(SMnode *pMnode, SRpcMsg *pReq, SCreateTagIndexReq *req, SDbObj *pDb, SStbObj *pStb) {
int32_t code = -1;
SIdxObj idxObj = {0};
@ -681,11 +720,20 @@ static int32_t mndAddIndex(SMnode *pMnode, SRpcMsg *pReq, SCreateTagIndexReq *re
idxObj.stbUid = pStb->uid;
idxObj.dbUid = pStb->dbUid;
int32_t tag = mndFindSuperTableTagId(pStb, req->colName);
int8_t hasIdx = 0;
int32_t tag = mndFindSuperTableTagId(pStb, req->colName, &hasIdx);
if (tag < 0) {
terrno = TSDB_CODE_MND_TAG_NOT_EXIST;
return -1;
} else if (tag == 0) {
}
int8_t exist = 0;
if (tag == 0 && hasIdx == 1) {
exist = mndCheckIndexNameByTagName(pMnode, &idxObj);
if (exist) {
terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST;
return -1;
}
} else if (hasIdx == 1) {
terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST;
return -1;
}
@ -695,11 +743,11 @@ static int32_t mndAddIndex(SMnode *pMnode, SRpcMsg *pReq, SCreateTagIndexReq *re
return -1;
}
SSchema *pTag = pStb->pTags + tag;
if (IS_IDX_ON(pTag)) {
terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST;
return -1;
}
// SSchema *pTag = pStb->pTags + tag;
// if (IS_IDX_ON(pTag)) {
// terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST;
// return -1;
// }
code = mndAddIndexImpl(pMnode, pReq, pDb, pStb, &idxObj);
return code;
@ -806,8 +854,8 @@ int32_t mndDropIdxsByStb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *p
if (pIdx->stbUid == pStb->uid) {
if (mndSetDropIdxCommitLogs(pMnode, pTrans, pIdx) != 0) {
sdbCancelFetch(pSdb, pIter);
sdbRelease(pSdb, pIdx);
sdbCancelFetch(pSdb, pIdx);
return -1;
}
}

5
source/dnode/mnode/impl/src/mndSma.c
View File

@ -705,7 +705,10 @@ static int32_t mndProcessCreateSmaReq(SRpcMsg *pReq) {
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
#ifdef WINDOWS
terrno = TSDB_CODE_MND_INVALID_PLATFORM;
goto _OVER;
#endif
mInfo("sma:%s, start to create", createReq.name);
if (mndCheckCreateSmaReq(&createReq) != 0) {
goto _OVER;

46
source/dnode/mnode/impl/src/mndStb.c
View File

@ -18,6 +18,7 @@
#include "mndDb.h"
#include "mndDnode.h"
#include "mndIndex.h"
#include "mndIndexComm.h"
#include "mndInfoSchema.h"
#include "mndMnode.h"
#include "mndPerfSchema.h"
@ -822,7 +823,7 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat
return -1;
}
if(pDst->nextColId < 0 || pDst->nextColId >= 0x7fff - pDst->numOfColumns - pDst->numOfTags){
if (pDst->nextColId < 0 || pDst->nextColId >= 0x7fff - pDst->numOfColumns - pDst->numOfTags) {
terrno = TSDB_CODE_MND_FIELD_VALUE_OVERFLOW;
return -1;
}
@ -857,11 +858,39 @@ static int32_t mndCreateStb(SMnode *pMnode, SRpcMsg *pReq, SMCreateStbReq *pCrea
SStbObj stbObj = {0};
int32_t code = -1;
char fullIdxName[TSDB_INDEX_FNAME_LEN * 2] = {0};
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pReq, "create-stb");
if (pTrans == NULL) goto _OVER;
mInfo("trans:%d, used to create stb:%s", pTrans->id, pCreate->name);
if (mndBuildStbFromReq(pMnode, &stbObj, pCreate, pDb) != 0) goto _OVER;
char randStr[24] = {0};
taosRandStr2(randStr, tListLen(randStr) - 1);
SSchema *pSchema = &(stbObj.pTags[0]);
sprintf(fullIdxName, "%s.%s_%s", pDb->name, pSchema->name, randStr);
SSIdx idx = {0};
if (mndAcquireGlobalIdx(pMnode, fullIdxName, SDB_IDX, &idx) == 0 && idx.pIdx != NULL) {
terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST;
mndReleaseIdx(pMnode, idx.pIdx);
goto _OVER;
}
SIdxObj idxObj = {0};
memcpy(idxObj.name, fullIdxName, TSDB_INDEX_FNAME_LEN);
memcpy(idxObj.stb, stbObj.name, TSDB_TABLE_FNAME_LEN);
memcpy(idxObj.db, stbObj.db, TSDB_DB_FNAME_LEN);
memcpy(idxObj.colName, pSchema->name, TSDB_COL_NAME_LEN);
idxObj.createdTime = taosGetTimestampMs();
idxObj.uid = mndGenerateUid(fullIdxName, strlen(fullIdxName));
idxObj.stbUid = stbObj.uid;
idxObj.dbUid = stbObj.dbUid;
if (mndSetCreateIdxCommitLogs(pMnode, pTrans, &idxObj) < 0) goto _OVER;
if (mndAddStbToTrans(pMnode, pTrans, pDb, &stbObj) < 0) goto _OVER;
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
code = 0;
@ -956,7 +985,7 @@ static int32_t mndBuildStbFromAlter(SStbObj *pStb, SStbObj *pDst, SMCreateStbReq
return -1;
}
if(pDst->nextColId < 0 || pDst->nextColId >= 0x7fff - pDst->numOfColumns - pDst->numOfTags){
if (pDst->nextColId < 0 || pDst->nextColId >= 0x7fff - pDst->numOfColumns - pDst->numOfTags) {
terrno = TSDB_CODE_MND_FIELD_VALUE_OVERFLOW;
return -1;
}
@ -1188,7 +1217,7 @@ static int32_t mndAddSuperTableTag(const SStbObj *pOld, SStbObj *pNew, SArray *p
return -1;
}
if(pNew->nextColId < 0 || pNew->nextColId >= 0x7fff - ntags){
if (pNew->nextColId < 0 || pNew->nextColId >= 0x7fff - ntags) {
terrno = TSDB_CODE_MND_FIELD_VALUE_OVERFLOW;
return -1;
}
@ -1478,7 +1507,8 @@ static int32_t mndAlterStbTagBytes(SMnode *pMnode, const SStbObj *pOld, SStbObj
SSchema *pTag = pNew->pTags + tag;
if (!(pTag->type == TSDB_DATA_TYPE_BINARY || pTag->type == TSDB_DATA_TYPE_NCHAR || pTag->type == TSDB_DATA_TYPE_GEOMETRY)) {
if (!(pTag->type == TSDB_DATA_TYPE_BINARY || pTag->type == TSDB_DATA_TYPE_NCHAR ||
pTag->type == TSDB_DATA_TYPE_GEOMETRY)) {
terrno = TSDB_CODE_MND_INVALID_STB_OPTION;
return -1;
}
@ -1506,7 +1536,7 @@ static int32_t mndAddSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, SArray
return -1;
}
if(pNew->nextColId < 0 || pNew->nextColId >= 0x7fff - ncols){
if (pNew->nextColId < 0 || pNew->nextColId >= 0x7fff - ncols) {
terrno = TSDB_CODE_MND_FIELD_VALUE_OVERFLOW;
return -1;
}
@ -1598,7 +1628,8 @@ static int32_t mndAlterStbColumnBytes(SMnode *pMnode, const SStbObj *pOld, SStbO
}
SSchema *pCol = pNew->pColumns + col;
if (!(pCol->type == TSDB_DATA_TYPE_BINARY || pCol->type == TSDB_DATA_TYPE_NCHAR || pCol->type == TSDB_DATA_TYPE_GEOMETRY)) {
if (!(pCol->type == TSDB_DATA_TYPE_BINARY || pCol->type == TSDB_DATA_TYPE_NCHAR ||
pCol->type == TSDB_DATA_TYPE_GEOMETRY)) {
terrno = TSDB_CODE_MND_INVALID_STB_OPTION;
return -1;
}
@ -3182,7 +3213,6 @@ static int32_t mndRetrieveStbCol(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB
SSdb *pSdb = pMnode->pSdb;
SStbObj *pStb = NULL;
int32_t numOfRows = 0;
if (!pShow->sysDbRsp) {
numOfRows = buildSysDbColsInfo(pBlock, pShow->db, pShow->filterTb);
@ -3206,7 +3236,7 @@ static int32_t mndRetrieveStbCol(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB
if (pShow->pIter == NULL) break;
} else {
fetch = true;
void *pKey = taosHashGetKey(pShow->pIter, NULL);
void *pKey = taosHashGetKey(pShow->pIter, NULL);
pStb = sdbAcquire(pSdb, SDB_STB, pKey);
if (!pStb) continue;
}

5
source/dnode/mnode/impl/src/mndStream.c
View File

@ -692,7 +692,10 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
#ifdef WINDOWS
terrno = TSDB_CODE_MND_INVALID_PLATFORM;
goto _OVER;
#endif
mInfo("stream:%s, start to create, sql:%s", createStreamReq.name, createStreamReq.sql);
if (mndCheckCreateStreamReq(&createStreamReq) != 0) {

42
source/dnode/vnode/CMakeLists.txt
View File

@ -8,6 +8,7 @@ set(
"src/vnd/vnodeCommit.c"
"src/vnd/vnodeQuery.c"
"src/vnd/vnodeModule.c"
"src/vnd/vnodeCos.c"
"src/vnd/vnodeSvr.c"
"src/vnd/vnodeSync.c"
"src/vnd/vnodeSnapshot.c"
@ -155,6 +156,45 @@ target_link_libraries(
PUBLIC index
)
if(${TD_LINUX})
set(CMAKE_FIND_LIBRARY_SUFFIXES ".a")
find_library(APR_LIBRARY apr-1 PATHS /usr/local/apr/lib/)
find_library(APR_UTIL_LIBRARY aprutil-1 PATHS /usr/local/apr/lib/)
find_library(MINIXML_LIBRARY mxml)
find_library(CURL_LIBRARY curl)
target_link_libraries(
vnode
# s3
PUBLIC cos_c_sdk_static
PUBLIC ${APR_UTIL_LIBRARY}
PUBLIC ${APR_LIBRARY}
PUBLIC ${MINIXML_LIBRARY}
PUBLIC ${CURL_LIBRARY}
)
# s3
FIND_PROGRAM(APR_CONFIG_BIN NAMES apr-config apr-1-config PATHS /usr/bin /usr/local/bin /usr/local/apr/bin/)
IF (APR_CONFIG_BIN)
EXECUTE_PROCESS(
COMMAND ${APR_CONFIG_BIN} --includedir
OUTPUT_VARIABLE APR_INCLUDE_DIR
OUTPUT_STRIP_TRAILING_WHITESPACE
)
ENDIF()
include_directories (${APR_INCLUDE_DIR})
target_include_directories(
vnode
PUBLIC "${TD_SOURCE_DIR}/contrib/cos-c-sdk-v5/cos_c_sdk"
PUBLIC "$ENV{HOME}/.cos-local.1/include"
)
if(${BUILD_WITH_COS})
add_definitions(-DUSE_COS)
endif(${BUILD_WITH_COS})
endif(${TD_LINUX})
IF (TD_GRANT)
TARGET_LINK_LIBRARIES(vnode PUBLIC grant)
ENDIF ()
@ -169,8 +209,6 @@ if(${BUILD_WITH_ROCKSDB})
add_definitions(-DUSE_ROCKSDB)
endif(${BUILD_WITH_ROCKSDB})
if(${BUILD_TEST})
add_subdirectory(test)
endif(${BUILD_TEST})

40
source/dnode/vnode/src/inc/vndCos.h Normal file
View File

@ -0,0 +1,40 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_VND_COS_H_
#define _TD_VND_COS_H_
#include "vnd.h"
#ifdef __cplusplus
extern "C" {
#endif
extern int8_t tsS3Enabled;
int32_t s3Init();
void s3CleanUp();
int32_t s3PutObjectFromFile(const char *file, const char *object);
void s3DeleteObjects(const char *object_name[], int nobject);
bool s3Exists(const char *object_name);
bool s3Get(const char *object_name, const char *path);
void s3EvictCache(const char *path, long object_size);
long s3Size(const char *object_name);
#ifdef __cplusplus
}
#endif
#endif /*_TD_VND_COS_H_*/

2
source/dnode/vnode/src/inc/vnodeInt.h
View File

@ -167,7 +167,7 @@ int metaAddIndexToSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq);
int metaDropIndexFromSTable(SMeta* pMeta, int64_t version, SDropIndexReq* pReq);
int64_t metaGetTimeSeriesNum(SMeta* pMeta);
SMCtbCursor* metaOpenCtbCursor(SMeta* pMeta, tb_uid_t uid, int lock);
SMCtbCursor* metaOpenCtbCursor(void* pVnode, tb_uid_t uid, int lock);
void metaCloseCtbCursor(SMCtbCursor* pCtbCur, int lock);
tb_uid_t metaCtbCursorNext(SMCtbCursor* pCtbCur);
SMStbCursor* metaOpenStbCursor(SMeta* pMeta, tb_uid_t uid);

30
source/dnode/vnode/src/meta/metaQuery.c
View File

@ -17,8 +17,8 @@
#include "osMemory.h"
#include "tencode.h"
void _metaReaderInit(SMetaReader* pReader, void* pVnode, int32_t flags, SStoreMeta* pAPI) {
SMeta* pMeta = ((SVnode*)pVnode)->pMeta;
void _metaReaderInit(SMetaReader *pReader, void *pVnode, int32_t flags, SStoreMeta *pAPI) {
SMeta *pMeta = ((SVnode *)pVnode)->pMeta;
metaReaderDoInit(pReader, pMeta, flags);
pReader->pAPI = pAPI;
}
@ -143,7 +143,7 @@ tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name) {
int metaGetTableNameByUid(void *pVnode, uint64_t uid, char *tbName) {
int code = 0;
SMetaReader mr = {0};
metaReaderDoInit(&mr, ((SVnode*)pVnode)->pMeta, 0);
metaReaderDoInit(&mr, ((SVnode *)pVnode)->pMeta, 0);
code = metaReaderGetTableEntryByUid(&mr, uid);
if (code < 0) {
metaReaderClear(&mr);
@ -195,7 +195,7 @@ int metaGetTableUidByName(void *pVnode, char *tbName, uint64_t *uid) {
int metaGetTableTypeByName(void *pVnode, char *tbName, ETableType *tbType) {
int code = 0;
SMetaReader mr = {0};
metaReaderDoInit(&mr, ((SVnode*)pVnode)->pMeta, 0);
metaReaderDoInit(&mr, ((SVnode *)pVnode)->pMeta, 0);
code = metaGetTableEntryByName(&mr, tbName);
if (code == 0) *tbType = mr.me.type;
@ -244,7 +244,7 @@ SMTbCursor *metaOpenTbCursor(void *pVnode) {
return NULL;
}
SVnode* pVnodeObj = pVnode;
SVnode *pVnodeObj = pVnode;
// tdbTbcMoveToFirst((TBC *)pTbCur->pDbc);
pTbCur->pMeta = pVnodeObj->pMeta;
pTbCur->paused = 1;
@ -408,17 +408,9 @@ _err:
return NULL;
}
struct SMCtbCursor {
SMeta *pMeta;
TBC *pCur;
tb_uid_t suid;
void *pKey;
void *pVal;
int kLen;
int vLen;
};
SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid, int lock) {
SMCtbCursor *metaOpenCtbCursor(void* pVnode, tb_uid_t uid, int lock) {
SMeta* pMeta = ((SVnode*)pVnode)->pMeta;
SMCtbCursor *pCtbCur = NULL;
SCtbIdxKey ctbIdxKey;
int ret = 0;
@ -435,7 +427,7 @@ SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid, int lock) {
metaRLock(pMeta);
}
ret = tdbTbcOpen(pMeta->pCtbIdx, &pCtbCur->pCur, NULL);
ret = tdbTbcOpen(pMeta->pCtbIdx, (TBC**)&pCtbCur->pCur, NULL);
if (ret < 0) {
metaULock(pMeta);
taosMemoryFree(pCtbCur);
@ -1139,7 +1131,7 @@ int32_t metaFilterTtl(void *pVnode, SMetaFltParam *arg, SArray *pUids) {
pCursor->type = param->type;
metaRLock(pMeta);
//ret = tdbTbcOpen(pMeta->pTtlIdx, &pCursor->pCur, NULL);
// ret = tdbTbcOpen(pMeta->pTtlIdx, &pCursor->pCur, NULL);
END:
if (pCursor->pMeta) metaULock(pCursor->pMeta);
@ -1194,7 +1186,7 @@ int32_t metaFilterTableIds(void *pVnode, SMetaFltParam *arg, SArray *pUids) {
ret = -1;
for (int i = 0; i < oStbEntry.stbEntry.schemaTag.nCols; i++) {
SSchema *schema = oStbEntry.stbEntry.schemaTag.pSchema + i;
if (schema->colId == param->cid && param->type == schema->type && (IS_IDX_ON(schema) || i == 0)) {
if (schema->colId == param->cid && param->type == schema->type && (IS_IDX_ON(schema))) {
ret = 0;
}
}
@ -1373,7 +1365,7 @@ int32_t metaGetTableTagsByUids(void *pVnode, int64_t suid, SArray *uidList) {
}
int32_t metaGetTableTags(void *pVnode, uint64_t suid, SArray *pUidTagInfo) {
SMCtbCursor *pCur = metaOpenCtbCursor(((SVnode *)pVnode)->pMeta, suid, 1);
SMCtbCursor *pCur = metaOpenCtbCursor(pVnode, suid, 1);
// If len > 0 means there already have uids, and we only want the
// tags of the specified tables, of which uid in the uid list. Otherwise, all table tags are retrieved and kept

9
source/dnode/vnode/src/meta/metaTable.c
View File

@ -450,12 +450,13 @@ int metaAddIndexToSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
goto _err;
}
if (IS_IDX_ON(pNew) && !IS_IDX_ON(pOld)) {
if (diffIdx != -1) goto _err;
// if (diffIdx != -1) goto _err;
diffIdx = i;
break;
}
}
if (diffIdx == -1 || diffIdx == 0) {
if (diffIdx == -1) {
goto _err;
}
@ -586,7 +587,7 @@ int metaDropIndexFromSTable(SMeta *pMeta, int64_t version, SDropIndexReq *pReq)
for (int i = 0; i < oStbEntry.stbEntry.schemaTag.nCols; i++) {
SSchema *schema = oStbEntry.stbEntry.schemaTag.pSchema + i;
if (0 == strncmp(schema->name, pReq->colName, sizeof(pReq->colName))) {
if (i != 0 || IS_IDX_ON(schema)) {
if (IS_IDX_ON(schema)) {
pCol = schema;
}
break;
@ -2094,7 +2095,7 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) {
} else {
for (int i = 0; i < pTagSchema->nCols; i++) {
pTagColumn = &pTagSchema->pSchema[i];
if (i != 0 && !IS_IDX_ON(pTagColumn)) continue;
if (!IS_IDX_ON(pTagColumn)) continue;
STagVal tagVal = {.cid = pTagColumn->colId};
tTagGet((const STag *)pCtbEntry->ctbEntry.pTags, &tagVal);

2
source/dnode/vnode/src/tq/tqOffsetSnapshot.c
View File

@ -60,7 +60,7 @@ int32_t tqOffsetSnapRead(STqOffsetReader* pReader, uint8_t** ppData) {
}
int64_t sz = 0;
if (taosStatFile(fname, &sz, NULL) < 0) {
if (taosStatFile(fname, &sz, NULL, NULL) < 0) {
taosCloseFile(&pFile);
taosMemoryFree(fname);
return -1;

10
source/dnode/vnode/src/tsdb/tsdbFS.c
View File

@ -176,7 +176,7 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) {
// SDelFile
if (pTsdb->fs.pDelFile) {
tsdbDelFileName(pTsdb, pTsdb->fs.pDelFile, fname);
if (taosStatFile(fname, &size, NULL)) {
if (taosStatFile(fname, &size, NULL, NULL)) {
code = TAOS_SYSTEM_ERROR(errno);
TSDB_CHECK_CODE(code, lino, _exit);
}
@ -195,7 +195,7 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) {
// head =========
tsdbHeadFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pHeadF, fname);
if (taosStatFile(fname, &size, NULL)) {
if (taosStatFile(fname, &size, NULL, NULL)) {
code = TAOS_SYSTEM_ERROR(errno);
TSDB_CHECK_CODE(code, lino, _exit);
}
@ -206,7 +206,7 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) {
// data =========
tsdbDataFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pDataF, fname);
if (taosStatFile(fname, &size, NULL)) {
if (taosStatFile(fname, &size, NULL, NULL)) {
code = TAOS_SYSTEM_ERROR(errno);
TSDB_CHECK_CODE(code, lino, _exit);
}
@ -221,7 +221,7 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) {
// sma =============
tsdbSmaFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pSmaF, fname);
if (taosStatFile(fname, &size, NULL)) {
if (taosStatFile(fname, &size, NULL, NULL)) {
code = TAOS_SYSTEM_ERROR(errno);
TSDB_CHECK_CODE(code, lino, _exit);
}
@ -237,7 +237,7 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) {
// stt ===========
for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) {
tsdbSttFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSttF[iStt], fname);
if (taosStatFile(fname, &size, NULL)) {
if (taosStatFile(fname, &size, NULL, NULL)) {
code = TAOS_SYSTEM_ERROR(errno);
TSDB_CHECK_CODE(code, lino, _exit);
}

24
source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
View File

@ -14,6 +14,7 @@
*/
#include "tsdb.h"
#include "vndCos.h"
// =============== PAGE-WISE FILE ===============
int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t flag, STsdbFD **ppFD) {
@ -34,9 +35,24 @@ int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t flag, STsdbFD **p
pFD->flag = flag;
pFD->pFD = taosOpenFile(path, flag);
if (pFD->pFD == NULL) {
code = TAOS_SYSTEM_ERROR(errno);
taosMemoryFree(pFD);
goto _exit;
const char *object_name = taosDirEntryBaseName((char *)path);
long s3_size = s3Size(object_name);
if (!strncmp(path + strlen(path) - 5, ".data", 5) && s3_size > 0) {
s3EvictCache(path, s3_size);
s3Get(object_name, path);
pFD->pFD = taosOpenFile(path, flag);
if (pFD->pFD == NULL) {
code = TAOS_SYSTEM_ERROR(errno);
taosMemoryFree(pFD);
goto _exit;
}
} else {
code = TAOS_SYSTEM_ERROR(errno);
taosMemoryFree(pFD);
goto _exit;
}
}
pFD->szPage = szPage;
pFD->pgno = 0;
@ -50,7 +66,7 @@ int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t flag, STsdbFD **p
// not check file size when reading data files.
if (flag != TD_FILE_READ) {
if (taosStatFile(path, &pFD->szFile, NULL) < 0) {
if (taosStatFile(path, &pFD->szFile, NULL, NULL) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
taosMemoryFree(pFD->pBuf);
taosCloseFile(&pFD->pFD);

121
source/dnode/vnode/src/tsdb/tsdbRetention.c
View File

@ -15,6 +15,7 @@
#include "tsdb.h"
#include "tsdbFS2.h"
#include "vndCos.h"
typedef struct {
STsdb *tsdb;
@ -41,6 +42,28 @@ static int32_t tsdbDoRemoveFileObject(SRTNer *rtner, const STFileObj *fobj) {
return TARRAY2_APPEND(rtner->fopArr, op);
}
static int32_t tsdbRemoveFileObjectS3(SRTNer *rtner, const STFileObj *fobj) {
int32_t code = 0, lino = 0;
STFileOp op = {
.optype = TSDB_FOP_REMOVE,
.fid = fobj->f->fid,
.of = fobj->f[0],
};
code = TARRAY2_APPEND(rtner->fopArr, op);
TSDB_CHECK_CODE(code, lino, _exit);
const char *object_name = taosDirEntryBaseName((char *)fobj->fname);
s3DeleteObjects(&object_name, 1);
_exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
}
return code;
}
static int32_t tsdbDoCopyFile(SRTNer *rtner, const STFileObj *from, const STFile *to) {
int32_t code = 0;
int32_t lino = 0;
@ -76,6 +99,34 @@ _exit:
return code;
}
static int32_t tsdbCopyFileS3(SRTNer *rtner, const STFileObj *from, const STFile *to) {
int32_t code = 0;
int32_t lino = 0;
char fname[TSDB_FILENAME_LEN];
TdFilePtr fdFrom = NULL;
TdFilePtr fdTo = NULL;
tsdbTFileName(rtner->tsdb, to, fname);
fdFrom = taosOpenFile(from->fname, TD_FILE_READ);
if (fdFrom == NULL) code = terrno;
TSDB_CHECK_CODE(code, lino, _exit);
char *object_name = taosDirEntryBaseName(fname);
code = s3PutObjectFromFile(from->fname, object_name);
TSDB_CHECK_CODE(code, lino, _exit);
taosCloseFile(&fdFrom);
_exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
taosCloseFile(&fdFrom);
}
return code;
}
static int32_t tsdbDoMigrateFileObj(SRTNer *rtner, const STFileObj *fobj, const SDiskID *did) {
int32_t code = 0;
int32_t lino = 0;
@ -123,6 +174,53 @@ _exit:
return code;
}
static int32_t tsdbMigrateDataFileS3(SRTNer *rtner, const STFileObj *fobj, const SDiskID *did) {
int32_t code = 0;
int32_t lino = 0;
STFileOp op = {0};
// remove old
op = (STFileOp){
.optype = TSDB_FOP_REMOVE,
.fid = fobj->f->fid,
.of = fobj->f[0],
};
code = TARRAY2_APPEND(rtner->fopArr, op);
TSDB_CHECK_CODE(code, lino, _exit);
// create new
op = (STFileOp){
.optype = TSDB_FOP_CREATE,
.fid = fobj->f->fid,
.nf =
{
.type = fobj->f->type,
.did = did[0],
.fid = fobj->f->fid,
.cid = fobj->f->cid,
.size = fobj->f->size,
.stt[0] =
{
.level = fobj->f->stt[0].level,
},
},
};
code = TARRAY2_APPEND(rtner->fopArr, op);
TSDB_CHECK_CODE(code, lino, _exit);
// do copy the file
code = tsdbCopyFileS3(rtner, fobj, &op.nf);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
}
return code;
}
typedef struct {
STsdb *tsdb;
int32_t sync;
@ -201,8 +299,14 @@ static int32_t tsdbDoRetention2(void *arg) {
for (int32_t ftype = 0; (ftype < TSDB_FTYPE_MAX) && (fobj = rtner->ctx->fset->farr[ftype], 1); ++ftype) {
if (fobj == NULL) continue;
code = tsdbDoRemoveFileObject(rtner, fobj);
TSDB_CHECK_CODE(code, lino, _exit);
int32_t nlevel = tfsGetLevel(rtner->tsdb->pVnode->pTfs);
if (tsS3Enabled && nlevel > 1 && TSDB_FTYPE_DATA == ftype && fobj->f->did.level == nlevel - 1) {
code = tsdbRemoveFileObjectS3(rtner, fobj);
TSDB_CHECK_CODE(code, lino, _exit);
} else {
code = tsdbDoRemoveFileObject(rtner, fobj);
TSDB_CHECK_CODE(code, lino, _exit);
}
}
SSttLvl *lvl;
@ -228,8 +332,15 @@ static int32_t tsdbDoRetention2(void *arg) {
if (fobj == NULL) continue;
if (fobj->f->did.level == did.level) continue;
code = tsdbDoMigrateFileObj(rtner, fobj, &did);
TSDB_CHECK_CODE(code, lino, _exit);
int32_t nlevel = tfsGetLevel(rtner->tsdb->pVnode->pTfs);
if (tsS3Enabled && nlevel > 1 && TSDB_FTYPE_DATA == ftype && did.level == nlevel - 1) {
code = tsdbMigrateDataFileS3(rtner, fobj, &did);
TSDB_CHECK_CODE(code, lino, _exit);
} else {
code = tsdbDoMigrateFileObj(rtner, fobj, &did);
TSDB_CHECK_CODE(code, lino, _exit);
}
}
// stt
@ -281,4 +392,4 @@ int32_t tsdbRetention(STsdb *tsdb, int64_t now, int32_t sync) {
tsdbFreeRtnArg(arg);
}
return code;
}
}

4
source/dnode/vnode/src/tsdb/tsdbWrite.c
View File

@ -76,7 +76,7 @@ int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq2 *pMsg) {
int32_t code = 0;
STsdbKeepCfg *pCfg = &pTsdb->keepCfg;
TSKEY now = taosGetTimestamp(pCfg->precision);
TSKEY minKey = now - tsTickPerMin[pCfg->precision] * pCfg->keep2;
TSKEY minKey = now - tsTickPerMin[pCfg->precision] * pCfg->keep1;
TSKEY maxKey = tsMaxKeyByPrecision[pCfg->precision];
int32_t size = taosArrayGetSize(pMsg->aSubmitTbData);
@ -107,4 +107,4 @@ int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq2 *pMsg) {
_exit:
return code;
}
}

323
source/dnode/vnode/src/vnd/vnodeCos.c Normal file
View File

@ -0,0 +1,323 @@
#define ALLOW_FORBID_FUNC
#include "vndCos.h"
extern char tsS3Endpoint[];
extern char tsS3AccessKeyId[];
extern char tsS3AccessKeySecret[];
extern char tsS3BucketName[];
extern char tsS3AppId[];
#ifdef USE_COS
#include "cos_api.h"
#include "cos_http_io.h"
#include "cos_log.h"
int32_t s3Init() {
if (cos_http_io_initialize(NULL, 0) != COSE_OK) {
return -1;
}
// set log level, default COS_LOG_WARN
cos_log_set_level(COS_LOG_WARN);
// set log output, default stderr
cos_log_set_output(NULL);
return 0;
}
void s3CleanUp() { cos_http_io_deinitialize(); }
static void log_status(cos_status_t *s) {
cos_warn_log("status->code: %d", s->code);
if (s->error_code) cos_warn_log("status->error_code: %s", s->error_code);
if (s->error_msg) cos_warn_log("status->error_msg: %s", s->error_msg);
if (s->req_id) cos_warn_log("status->req_id: %s", s->req_id);
}
static void s3InitRequestOptions(cos_request_options_t *options, int is_cname) {
options->config = cos_config_create(options->pool);
cos_config_t *config = options->config;
cos_str_set(&config->endpoint, tsS3Endpoint);
cos_str_set(&config->access_key_id, tsS3AccessKeyId);
cos_str_set(&config->access_key_secret, tsS3AccessKeySecret);
cos_str_set(&config->appid, tsS3AppId);
config->is_cname = is_cname;
options->ctl = cos_http_controller_create(options->pool, 0);
}
int32_t s3PutObjectFromFile(const char *file_str, const char *object_str) {
int32_t code = 0;
cos_pool_t *p = NULL;
int is_cname = 0;
cos_status_t *s = NULL;
cos_request_options_t *options = NULL;
cos_string_t bucket, object, file;
cos_table_t *resp_headers;
int traffic_limit = 0;
cos_pool_create(&p, NULL);
options = cos_request_options_create(p);
s3InitRequestOptions(options, is_cname);
cos_table_t *headers = NULL;
if (traffic_limit) {
// 限速值设置范围为819200 - 838860800，即100KB/s - 100MB/s，如果超出该范围将返回400错误
headers = cos_table_make(p, 1);
cos_table_add_int(headers, "x-cos-traffic-limit", 819200);
}
cos_str_set(&bucket, tsS3BucketName);
cos_str_set(&file, file_str);
cos_str_set(&object, object_str);
s = cos_put_object_from_file(options, &bucket, &object, &file, headers, &resp_headers);
log_status(s);
cos_pool_destroy(p);
if (s->code != 200) {
return code = s->code;
}
return code;
}
void s3DeleteObjects(const char *object_name[], int nobject) {
cos_pool_t *p = NULL;
int is_cname = 0;
cos_string_t bucket;
cos_table_t *resp_headers = NULL;
cos_request_options_t *options = NULL;
cos_list_t object_list;
cos_list_t deleted_object_list;
int is_quiet = COS_TRUE;
cos_pool_create(&p, NULL);
options = cos_request_options_create(p);
s3InitRequestOptions(options, is_cname);
cos_str_set(&bucket, tsS3BucketName);
cos_list_init(&object_list);
cos_list_init(&deleted_object_list);
for (int i = 0; i < nobject; ++i) {
cos_object_key_t *content = cos_create_cos_object_key(p);
cos_str_set(&content->key, object_name[i]);
cos_list_add_tail(&content->node, &object_list);
}
cos_status_t *s = cos_delete_objects(options, &bucket, &object_list, is_quiet, &resp_headers, &deleted_object_list);
log_status(s);
cos_pool_destroy(p);
if (cos_status_is_ok(s)) {
cos_warn_log("delete objects succeeded\n");
} else {
cos_warn_log("delete objects failed\n");
}
}
bool s3Exists(const char *object_name) {
bool ret = false;
cos_pool_t *p = NULL;
int is_cname = 0;
cos_status_t *s = NULL;
cos_request_options_t *options = NULL;
cos_string_t bucket;
cos_string_t object;
cos_table_t *resp_headers;
cos_table_t *headers = NULL;
cos_object_exist_status_e object_exist;
cos_pool_create(&p, NULL);
options = cos_request_options_create(p);
s3InitRequestOptions(options, is_cname);
cos_str_set(&bucket, tsS3BucketName);
cos_str_set(&object, object_name);
s = cos_check_object_exist(options, &bucket, &object, headers, &object_exist, &resp_headers);
if (object_exist == COS_OBJECT_NON_EXIST) {
cos_warn_log("object: %.*s non exist.\n", object.len, object.data);
} else if (object_exist == COS_OBJECT_EXIST) {
ret = true;
cos_warn_log("object: %.*s exist.\n", object.len, object.data);
} else {
cos_warn_log("object: %.*s unknown status.\n", object.len, object.data);
log_status(s);
}
cos_pool_destroy(p);
return ret;
}
bool s3Get(const char *object_name, const char *path) {
bool ret = false;
cos_pool_t *p = NULL;
int is_cname = 0;
cos_status_t *s = NULL;
cos_request_options_t *options = NULL;
cos_string_t bucket;
cos_string_t object;
cos_string_t file;
cos_table_t *resp_headers = NULL;
cos_table_t *headers = NULL;
int traffic_limit = 0;
//创建内存池
cos_pool_create(&p, NULL);
//初始化请求选项
options = cos_request_options_create(p);
s3InitRequestOptions(options, is_cname);
cos_str_set(&bucket, tsS3BucketName);
if (traffic_limit) {
//限速值设置范围为819200 - 838860800，即100KB/s - 100MB/s，如果超出该范围将返回400错误
headers = cos_table_make(p, 1);
cos_table_add_int(headers, "x-cos-traffic-limit", 819200);
}
//下载对象
cos_str_set(&file, path);
cos_str_set(&object, object_name);
s = cos_get_object_to_file(options, &bucket, &object, headers, NULL, &file, &resp_headers);
if (cos_status_is_ok(s)) {
ret = true;
cos_warn_log("get object succeeded\n");
} else {
cos_warn_log("get object failed\n");
}
//销毁内存池
cos_pool_destroy(p);
return ret;
}
typedef struct {
int64_t size;
int32_t atime;
char name[TSDB_FILENAME_LEN];
} SEvictFile;
static int32_t evictFileCompareAsce(const void *pLeft, const void *pRight) {
SEvictFile *lhs = (SEvictFile *)pLeft;
SEvictFile *rhs = (SEvictFile *)pRight;
return lhs->atime < rhs->atime ? -1 : 1;
}
void s3EvictCache(const char *path, long object_size) {
SDiskSize disk_size = {0};
char dir_name[TSDB_FILENAME_LEN] = "\0";
tstrncpy(dir_name, path, TSDB_FILENAME_LEN);
taosDirName(dir_name);
if (taosGetDiskSize((char *)dir_name, &disk_size) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
vError("failed to get disk:%s size since %s", path, terrstr());
return;
}
if (object_size >= disk_size.avail - (1 << 30)) {
// evict too old files
// 1, list data files' atime under dir(path)
tdbDirPtr pDir = taosOpenDir(dir_name);
if (pDir == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
vError("failed to open %s since %s", dir_name, terrstr());
}
SArray *evict_files = taosArrayInit(16, sizeof(SEvictFile));
tdbDirEntryPtr pDirEntry;
while ((pDirEntry = taosReadDir(pDir)) != NULL) {
char *name = taosGetDirEntryName(pDirEntry);
if (!strncmp(name + strlen(name) - 5, ".data", 5)) {
SEvictFile e_file = {0};
char entry_name[TSDB_FILENAME_LEN] = "\0";
int dir_len = strlen(dir_name);
memcpy(e_file.name, dir_name, dir_len);
e_file.name[dir_len] = '/';
memcpy(e_file.name + dir_len + 1, name, strlen(name));
taosStatFile(e_file.name, &e_file.size, NULL, &e_file.atime);
taosArrayPush(evict_files, &e_file);
}
}
taosCloseDir(&pDir);
// 2, sort by atime
taosArraySort(evict_files, evictFileCompareAsce);
// 3, remove files ascendingly until we get enough object_size space
long evict_size = 0;
size_t ef_size = TARRAY_SIZE(evict_files);
for (size_t i = 0; i < ef_size; ++i) {
SEvictFile *evict_file = taosArrayGet(evict_files, i);
taosRemoveFile(evict_file->name);
evict_size += evict_file->size;
if (evict_size >= object_size) {
break;
}
}
taosArrayDestroy(evict_files);
}
}
long s3Size(const char *object_name) {
long size = 0;
cos_pool_t *p = NULL;
int is_cname = 0;
cos_status_t *s = NULL;
cos_request_options_t *options = NULL;
cos_string_t bucket;
cos_string_t object;
cos_table_t *resp_headers = NULL;
//创建内存池
cos_pool_create(&p, NULL);
//初始化请求选项
options = cos_request_options_create(p);
s3InitRequestOptions(options, is_cname);
cos_str_set(&bucket, tsS3BucketName);
//获取对象元数据
cos_str_set(&object, object_name);
s = cos_head_object(options, &bucket, &object, NULL, &resp_headers);
// print_headers(resp_headers);
if (cos_status_is_ok(s)) {
char *content_length_str = (char *)apr_table_get(resp_headers, COS_CONTENT_LENGTH);
if (content_length_str != NULL) {
size = atol(content_length_str);
}
cos_warn_log("head object succeeded: %ld\n", size);
} else {
cos_warn_log("head object failed\n");
}
//销毁内存池
cos_pool_destroy(p);
return size;
}
#else
int32_t s3Init() { return 0; }
void s3CleanUp() {}
int32_t s3PutObjectFromFile(const char *file, const char *object) { return 0; }
void s3DeleteObjects(const char *object_name[], int nobject) {}
bool s3Exists(const char *object_name) { return false; }
bool s3Get(const char *object_name, const char *path) { return false; }
void s3EvictCache(const char *path, long object_size) {}
long s3Size(const char *object_name) { return 0; }
#endif

4
source/dnode/vnode/src/vnd/vnodeInitApi.c
View File

@ -96,6 +96,10 @@ void initMetadataAPI(SStoreMeta* pMeta) {
pMeta->metaGetCachedTbGroup = metaGetCachedTbGroup;
pMeta->metaPutTbGroupToCache = metaPutTbGroupToCache;
pMeta->openCtbCursor = metaOpenCtbCursor;
pMeta->closeCtbCursor = metaCloseCtbCursor;
pMeta->ctbCursorNext = metaCtbCursorNext;
}
void initTqAPI(SStoreTqReader* pTq) {

5
source/dnode/vnode/src/vnd/vnodeModule.c
View File

@ -14,6 +14,7 @@
*/
#include "vnd.h"
#include "vndCos.h"
typedef struct SVnodeTask SVnodeTask;
struct SVnodeTask {
@ -81,6 +82,9 @@ int vnodeInit(int nthreads) {
if (tqInit() < 0) {
return -1;
}
if (s3Init() < 0) {
return -1;
}
return 0;
}
@ -112,6 +116,7 @@ void vnodeCleanup() {
walCleanUp();
tqCleanUp();
smaCleanUp();
s3CleanUp();
}
int vnodeScheduleTaskEx(int tpid, int (*execute)(void*), void* arg) {

6
source/dnode/vnode/src/vnd/vnodeQuery.c
View File

@ -440,7 +440,7 @@ int32_t vnodeGetTableList(void* pVnode, int8_t type, SArray* pList) {
}
int32_t vnodeGetAllTableList(SVnode *pVnode, uint64_t uid, SArray *list) {
SMCtbCursor *pCur = metaOpenCtbCursor(pVnode->pMeta, uid, 1);
SMCtbCursor *pCur = metaOpenCtbCursor(pVnode, uid, 1);
while (1) {
tb_uid_t id = metaCtbCursorNext(pCur);
@ -462,7 +462,7 @@ int32_t vnodeGetCtbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bo
int32_t vnodeGetCtbIdList(void *pVnode, int64_t suid, SArray *list) {
SVnode *pVnodeObj = pVnode;
SMCtbCursor *pCur = metaOpenCtbCursor(pVnodeObj->pMeta, suid, 1);
SMCtbCursor *pCur = metaOpenCtbCursor(pVnodeObj, suid, 1);
while (1) {
tb_uid_t id = metaCtbCursorNext(pCur);
@ -521,7 +521,7 @@ int32_t vnodeGetStbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bo
}
int32_t vnodeGetCtbNum(SVnode *pVnode, int64_t suid, int64_t *num) {
SMCtbCursor *pCur = metaOpenCtbCursor(pVnode->pMeta, suid, 0);
SMCtbCursor *pCur = metaOpenCtbCursor(pVnode, suid, 0);
if (!pCur) {
return TSDB_CODE_FAILED;
}

16
source/libs/command/src/explain.c
View File

@ -302,17 +302,17 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
switch (pNode->type) {
case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: {
STagScanPhysiNode *pTagScanNode = (STagScanPhysiNode *)pNode;
EXPLAIN_ROW_NEW(level, EXPLAIN_TAG_SCAN_FORMAT, pTagScanNode->tableName.tname);
EXPLAIN_ROW_NEW(level, EXPLAIN_TAG_SCAN_FORMAT, pTagScanNode->scan.tableName.tname);
EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT);
if (pResNode->pExecInfo) {
QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen));
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
}
if (pTagScanNode->pScanPseudoCols) {
EXPLAIN_ROW_APPEND(EXPLAIN_PSEUDO_COLUMNS_FORMAT, pTagScanNode->pScanPseudoCols->length);
if (pTagScanNode->scan.pScanPseudoCols) {
EXPLAIN_ROW_APPEND(EXPLAIN_PSEUDO_COLUMNS_FORMAT, pTagScanNode->scan.pScanPseudoCols->length);
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
}
EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pTagScanNode->node.pOutputDataBlockDesc->totalRowSize);
EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pTagScanNode->scan.node.pOutputDataBlockDesc->totalRowSize);
EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT);
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
@ -320,11 +320,11 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
if (verbose) {
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT);
EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT,
nodesGetOutputNumFromSlotList(pTagScanNode->node.pOutputDataBlockDesc->pSlots));
nodesGetOutputNumFromSlotList(pTagScanNode->scan.node.pOutputDataBlockDesc->pSlots));
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pTagScanNode->node.pOutputDataBlockDesc->outputRowSize);
EXPLAIN_ROW_APPEND_LIMIT(pTagScanNode->node.pLimit);
EXPLAIN_ROW_APPEND_SLIMIT(pTagScanNode->node.pSlimit);
EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pTagScanNode->scan.node.pOutputDataBlockDesc->outputRowSize);
EXPLAIN_ROW_APPEND_LIMIT(pTagScanNode->scan.node.pLimit);
EXPLAIN_ROW_APPEND_SLIMIT(pTagScanNode->scan.node.pSlimit);
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));

2
source/libs/executor/inc/executil.h
View File

@ -190,4 +190,6 @@ void printDataBlock(SSDataBlock* pBlock, const char* flag);
void getNextTimeWindow(const SInterval* pInterval, STimeWindow* tw, int32_t order);
void getInitialStartTimeWindow(SInterval* pInterval, TSKEY ts, STimeWindow* w, bool ascQuery);
SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, SArray* pUidTagList, void* pVnode,
SStorageAPI* pStorageAPI);
#endif // TDENGINE_EXECUTIL_H

14
source/libs/executor/inc/executorInt.h
View File

@ -302,6 +302,12 @@ typedef struct STableMergeScanInfo {
SSortExecInfo sortExecInfo;
} STableMergeScanInfo;
typedef struct STagScanFilterContext {
SHashObj* colHash;
int32_t index;
SArray* cInfoList;
} STagScanFilterContext;
typedef struct STagScanInfo {
SColumnInfo* pCols;
SSDataBlock* pRes;
@ -310,6 +316,14 @@ typedef struct STagScanInfo {
SLimitNode* pSlimit;
SReadHandle readHandle;
STableListInfo* pTableListInfo;
uint64_t suid;
void* pCtbCursor;
SNode* pTagCond;
SNode* pTagIndexCond;
STagScanFilterContext filterCtx;
SArray* aUidTags; // SArray<STUidTagInfo>
SArray* aFilterIdxs; // SArray<int32_t>
SStorageAPI* pStorageAPI;
} STagScanInfo;
typedef enum EStreamScanMode {

2
source/libs/executor/inc/operator.h
View File

@ -92,7 +92,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode,
SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* readHandle, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode, STableListInfo* pTableListInfo, SNode* pTagCond, SNode*pTagIndexCond, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode* pScanPhyNode, const char* pUser, SExecTaskInfo* pTaskInfo);

2
source/libs/executor/inc/querytask.h
View File

@ -104,7 +104,7 @@ struct SExecTaskInfo {
void buildTaskId(uint64_t taskId, uint64_t queryId, char* dst);
SExecTaskInfo* doCreateTask(uint64_t queryId, uint64_t taskId, int32_t vgId, EOPTR_EXEC_MODEL model, SStorageAPI* pAPI);
void doDestroyTask(SExecTaskInfo* pTaskInfo);
bool isTaskKilled(SExecTaskInfo* pTaskInfo);
bool isTaskKilled(void* pTaskInfo);
void setTaskKilled(SExecTaskInfo* pTaskInfo, int32_t rspCode);
void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status);
int32_t createExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId,

3
source/libs/executor/inc/tsort.h
View File

@ -191,7 +191,8 @@ int32_t getProperSortPageSize(size_t rowSize, uint32_t numOfCols);
bool tsortIsClosed(SSortHandle* pHandle);
void tsortSetClosed(SSortHandle* pHandle);
void setSingleTableMerge(SSortHandle* pHandle);
void tsortSetSingleTableMerge(SSortHandle* pHandle);
void tsortSetAbortCheckFn(SSortHandle* pHandle, bool (*checkFn)(void* param), void* param);
#ifdef __cplusplus
}

5
source/libs/executor/src/executil.c
View File

@ -47,8 +47,6 @@ static int32_t optimizeTbnameInCondImpl(void* metaHandle, SArray* list, SNode* p
static int32_t getTableList(void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, SNode* pTagIndexCond,
STableListInfo* pListInfo, uint8_t* digest, const char* idstr, SStorageAPI* pStorageAPI);
static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, SArray* pUidTagList, void* pVnode,
SStorageAPI* pStorageAPI);
static int64_t getLimit(const SNode* pLimit) { return NULL == pLimit ? -1 : ((SLimitNode*)pLimit)->limit; }
static int64_t getOffset(const SNode* pLimit) { return NULL == pLimit ? -1 : ((SLimitNode*)pLimit)->offset; }
@ -846,7 +844,7 @@ static int32_t optimizeTbnameInCondImpl(void* pVnode, SArray* pExistedUidList, S
return -1;
}
static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, SArray* pUidTagList, void* pVnode,
SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, SArray* pUidTagList, void* pVnode,
SStorageAPI* pStorageAPI) {
SSDataBlock* pResBlock = createDataBlock();
if (pResBlock == NULL) {
@ -1697,6 +1695,7 @@ SInterval extractIntervalInfo(const STableScanPhysiNode* pTableScanNode) {
.intervalUnit = pTableScanNode->intervalUnit,
.slidingUnit = pTableScanNode->slidingUnit,
.offset = pTableScanNode->offset,
.precision = pTableScanNode->scan.node.pOutputDataBlockDesc->precision,
};
return interval;

9
source/libs/executor/src/executor.c
View File

@ -615,6 +615,10 @@ int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, bo
pRes = pTaskInfo->pRoot->fpSet.getNextFn(pTaskInfo->pRoot);
}
int32_t rowsThreshold = pTaskInfo->pSubplan->rowsThreshold;
if (!pTaskInfo->pSubplan->dynamicRowThreshold || 4096 <= pTaskInfo->pSubplan->rowsThreshold) {
rowsThreshold = 4096;
}
int32_t blockIndex = 0;
while (pRes != NULL) {
SSDataBlock* p = NULL;
@ -633,12 +637,15 @@ int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, bo
ASSERT(p->info.rows > 0);
taosArrayPush(pResList, &p);
if (current >= 4096) {
if (current >= rowsThreshold) {
break;
}
pRes = pTaskInfo->pRoot->fpSet.getNextFn(pTaskInfo->pRoot);
}
if (pTaskInfo->pSubplan->dynamicRowThreshold) {
pTaskInfo->pSubplan->rowsThreshold -= current;
}
*hasMore = (pRes != NULL);
uint64_t el = (taosGetTimestampUs() - st);

19
source/libs/executor/src/operator.c
View File

@ -378,17 +378,18 @@ SOperatorInfo* createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SR
STableCountScanPhysiNode* pTblCountScanNode = (STableCountScanPhysiNode*)pPhyNode;
pOperator = createTableCountScanOperatorInfo(pHandle, pTblCountScanNode, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN == type) {
STagScanPhysiNode* pScanPhyNode = (STagScanPhysiNode*)pPhyNode;
STagScanPhysiNode* pTagScanPhyNode = (STagScanPhysiNode*)pPhyNode;
STableListInfo* pTableListInfo = tableListCreate();
int32_t code = createScanTableListInfo(pScanPhyNode, NULL, false, pHandle, pTableListInfo, pTagCond,
pTagIndexCond, pTaskInfo);
if (code != TSDB_CODE_SUCCESS) {
pTaskInfo->code = code;
qError("failed to getTableList, code: %s", tstrerror(code));
return NULL;
if (!pTagScanPhyNode->onlyMetaCtbIdx) {
int32_t code = createScanTableListInfo((SScanPhysiNode*)pTagScanPhyNode, NULL, false, pHandle, pTableListInfo, pTagCond,
pTagIndexCond, pTaskInfo);
if (code != TSDB_CODE_SUCCESS) {
pTaskInfo->code = code;
qError("failed to getTableList, code: %s", tstrerror(code));
return NULL;
}
}
pOperator = createTagScanOperatorInfo(pHandle, pScanPhyNode, pTableListInfo, pTaskInfo);
pOperator = createTagScanOperatorInfo(pHandle, pTagScanPhyNode, pTableListInfo, pTagCond, pTagIndexCond, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN == type) {
SBlockDistScanPhysiNode* pBlockNode = (SBlockDistScanPhysiNode*)pPhyNode;
STableListInfo* pTableListInfo = tableListCreate();

2
source/libs/executor/src/querytask.c
View File

@ -62,7 +62,7 @@ SExecTaskInfo* doCreateTask(uint64_t queryId, uint64_t taskId, int32_t vgId, EOP
return pTaskInfo;
}
bool isTaskKilled(SExecTaskInfo* pTaskInfo) { return (0 != pTaskInfo->code); }
bool isTaskKilled(void* pTaskInfo) { return (0 != ((SExecTaskInfo*)pTaskInfo)->code); }
void setTaskKilled(SExecTaskInfo* pTaskInfo, int32_t rspCode) {
pTaskInfo->code = rspCode;

266
source/libs/executor/src/scanoperator.c
View File

@ -2799,7 +2799,236 @@ static void doTagScanOneTable(SOperatorInfo* pOperator, const SSDataBlock* pRes,
}
}
static SSDataBlock* doTagScan(SOperatorInfo* pOperator) {
static void tagScanFreeUidTag(void* p) {
STUidTagInfo* pInfo = p;
if (pInfo->pTagVal != NULL) {
taosMemoryFree(pInfo->pTagVal);
}
}
static int32_t tagScanCreateResultData(SDataType* pType, int32_t numOfRows, SScalarParam* pParam) {
SColumnInfoData* pColumnData = taosMemoryCalloc(1, sizeof(SColumnInfoData));
if (pColumnData == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
pColumnData->info.type = pType->type;
pColumnData->info.bytes = pType->bytes;
pColumnData->info.scale = pType->scale;
pColumnData->info.precision = pType->precision;
int32_t code = colInfoDataEnsureCapacity(pColumnData, numOfRows, true);
if (code != TSDB_CODE_SUCCESS) {
terrno = code;
taosMemoryFree(pColumnData);
return terrno;
}
pParam->columnData = pColumnData;
pParam->colAlloced = true;
return TSDB_CODE_SUCCESS;
}
static EDealRes tagScanRewriteTagColumn(SNode** pNode, void* pContext) {
SColumnNode* pSColumnNode = NULL;
if (QUERY_NODE_COLUMN == nodeType((*pNode))) {
pSColumnNode = *(SColumnNode**)pNode;
} else if (QUERY_NODE_FUNCTION == nodeType((*pNode))) {
SFunctionNode* pFuncNode = *(SFunctionNode**)(pNode);
if (pFuncNode->funcType == FUNCTION_TYPE_TBNAME) {
pSColumnNode = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN);
if (NULL == pSColumnNode) {
return DEAL_RES_ERROR;
}
pSColumnNode->colId = -1;
pSColumnNode->colType = COLUMN_TYPE_TBNAME;
pSColumnNode->node.resType.type = TSDB_DATA_TYPE_VARCHAR;
pSColumnNode->node.resType.bytes = TSDB_TABLE_FNAME_LEN - 1 + VARSTR_HEADER_SIZE;
nodesDestroyNode(*pNode);
*pNode = (SNode*)pSColumnNode;
} else {
return DEAL_RES_CONTINUE;
}
} else {
return DEAL_RES_CONTINUE;
}
STagScanFilterContext* pCtx = (STagScanFilterContext*)pContext;
void* data = taosHashGet(pCtx->colHash, &pSColumnNode->colId, sizeof(pSColumnNode->colId));
if (!data) {
taosHashPut(pCtx->colHash, &pSColumnNode->colId, sizeof(pSColumnNode->colId), pNode, sizeof((*pNode)));
pSColumnNode->slotId = pCtx->index++;
SColumnInfo cInfo = {.colId = pSColumnNode->colId,
.type = pSColumnNode->node.resType.type,
.bytes = pSColumnNode->node.resType.bytes};
taosArrayPush(pCtx->cInfoList, &cInfo);
} else {
SColumnNode* col = *(SColumnNode**)data;
pSColumnNode->slotId = col->slotId;
}
return DEAL_RES_CONTINUE;
}
static void tagScanFilterByTagCond(SArray* aUidTags, SNode* pTagCond, SArray* aFilterIdxs, void* pVnode, SStorageAPI* pAPI, STagScanInfo* pInfo) {
int32_t code = 0;
int32_t numOfTables = taosArrayGetSize(aUidTags);
SSDataBlock* pResBlock = createTagValBlockForFilter(pInfo->filterCtx.cInfoList, numOfTables, aUidTags, pVnode, pAPI);
SArray* pBlockList = taosArrayInit(1, POINTER_BYTES);
taosArrayPush(pBlockList, &pResBlock);
SDataType type = {.type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)};
SScalarParam output = {0};
tagScanCreateResultData(&type, numOfTables, &output);
scalarCalculate(pTagCond, pBlockList, &output);
bool* result = (bool*)output.columnData->pData;
for (int32_t i = 0 ; i < numOfTables; ++i) {
if (result[i]) {
taosArrayPush(aFilterIdxs, &i);
}
}
colDataDestroy(output.columnData);
taosMemoryFreeClear(output.columnData);
blockDataDestroy(pResBlock);
taosArrayDestroy(pBlockList);
}
static void tagScanFillOneCellWithTag(const STUidTagInfo* pUidTagInfo, SExprInfo* pExprInfo, SColumnInfoData* pColInfo, int rowIndex, const SStorageAPI* pAPI, void* pVnode) {
if (fmIsScanPseudoColumnFunc(pExprInfo->pExpr->_function.functionId)) { // tbname
char str[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
// if (pUidTagInfo->name != NULL) {
// STR_TO_VARSTR(str, pUidTagInfo->name);
// } else { // name is not retrieved during filter
// pAPI->metaFn.getTableNameByUid(pVnode, pUidTagInfo->uid, str);
// }
STR_TO_VARSTR(str, "ctbidx");
colDataSetVal(pColInfo, rowIndex, str, false);
} else {
STagVal tagVal = {0};
tagVal.cid = pExprInfo->base.pParam[0].pCol->colId;
if (pUidTagInfo->pTagVal == NULL) {
colDataSetNULL(pColInfo, rowIndex);
} else {
const char* p = pAPI->metaFn.extractTagVal(pUidTagInfo->pTagVal, pColInfo->info.type, &tagVal);
if (p == NULL || (pColInfo->info.type == TSDB_DATA_TYPE_JSON && ((STag*)p)->nTag == 0)) {
colDataSetNULL(pColInfo, rowIndex);
} else if (pColInfo->info.type == TSDB_DATA_TYPE_JSON) {
colDataSetVal(pColInfo, rowIndex, p, false);
} else if (IS_VAR_DATA_TYPE(pColInfo->info.type)) {
char* tmp = taosMemoryMalloc(tagVal.nData + VARSTR_HEADER_SIZE + 1);
varDataSetLen(tmp, tagVal.nData);
memcpy(tmp + VARSTR_HEADER_SIZE, tagVal.pData, tagVal.nData);
colDataSetVal(pColInfo, rowIndex, tmp, false);
taosMemoryFree(tmp);
} else {
colDataSetVal(pColInfo, rowIndex, (const char*)&tagVal.i64, false);
}
}
}
}
static int32_t tagScanFillResultBlock(SOperatorInfo* pOperator, SSDataBlock* pRes, SArray* aUidTags, SArray* aFilterIdxs, bool ignoreFilterIdx,
SStorageAPI* pAPI) {
STagScanInfo* pInfo = pOperator->info;
SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[0];
if (!ignoreFilterIdx) {
size_t szTables = taosArrayGetSize(aFilterIdxs);
for (int i = 0; i < szTables; ++i) {
int32_t idx = *(int32_t*)taosArrayGet(aFilterIdxs, i);
STUidTagInfo* pUidTagInfo = taosArrayGet(aUidTags, idx);
for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) {
SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, pExprInfo[j].base.resSchema.slotId);
tagScanFillOneCellWithTag(pUidTagInfo, &pExprInfo[j], pDst, i, pAPI, pInfo->readHandle.vnode);
}
}
} else {
size_t szTables = taosArrayGetSize(aUidTags);
for (int i = 0; i < szTables; ++i) {
STUidTagInfo* pUidTagInfo = taosArrayGet(aUidTags, i);
for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) {
SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, pExprInfo[j].base.resSchema.slotId);
tagScanFillOneCellWithTag(pUidTagInfo, &pExprInfo[j], pDst, i, pAPI, pInfo->readHandle.vnode);
}
}
}
return 0;
}
static SSDataBlock* doTagScanFromCtbIdx(SOperatorInfo* pOperator) {
if (pOperator->status == OP_EXEC_DONE) {
return NULL;
}
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStorageAPI* pAPI = &pTaskInfo->storageAPI;
STagScanInfo* pInfo = pOperator->info;
SSDataBlock* pRes = pInfo->pRes;
blockDataCleanup(pRes);
if (pInfo->pCtbCursor == NULL) {
pInfo->pCtbCursor = pAPI->metaFn.openCtbCursor(pInfo->readHandle.vnode, pInfo->suid, 1);
}
SArray* aUidTags = pInfo->aUidTags;
SArray* aFilterIdxs = pInfo->aFilterIdxs;
int32_t count = 0;
while (1) {
taosArrayClearEx(aUidTags, tagScanFreeUidTag);
taosArrayClear(aFilterIdxs);
int32_t numTables = 0;
while (numTables < pOperator->resultInfo.capacity) {
SMCtbCursor* pCur = pInfo->pCtbCursor;
tb_uid_t uid = pAPI->metaFn.ctbCursorNext(pInfo->pCtbCursor);
if (uid == 0) {
break;
}
STUidTagInfo info = {.uid = uid, .pTagVal = pCur->pVal};
info.pTagVal = taosMemoryMalloc(pCur->vLen);
memcpy(info.pTagVal, pCur->pVal, pCur->vLen);
taosArrayPush(aUidTags, &info);
++numTables;
}
if (numTables == 0) {
break;
}
bool ignoreFilterIdx = true;
if (pInfo->pTagCond != NULL) {
ignoreFilterIdx = false;
tagScanFilterByTagCond(aUidTags, pInfo->pTagCond, aFilterIdxs, pInfo->readHandle.vnode, pAPI, pInfo);
} else {
ignoreFilterIdx = true;
}
tagScanFillResultBlock(pOperator, pRes, aUidTags, aFilterIdxs, ignoreFilterIdx, pAPI);
count = ignoreFilterIdx ? taosArrayGetSize(aUidTags): taosArrayGetSize(aFilterIdxs);
if (count != 0) {
break;
}
}
pRes->info.rows = count;
pOperator->resultInfo.totalRows += count;
return (pRes->info.rows == 0) ? NULL : pInfo->pRes;
}
static SSDataBlock* doTagScanFromMetaEntry(SOperatorInfo* pOperator) {
if (pOperator->status == OP_EXEC_DONE) {
return NULL;
}
@ -2857,14 +3086,23 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) {
static void destroyTagScanOperatorInfo(void* param) {
STagScanInfo* pInfo = (STagScanInfo*)param;
if (pInfo->pCtbCursor != NULL) {
pInfo->pStorageAPI->metaFn.closeCtbCursor(pInfo->pCtbCursor, 1);
}
taosHashCleanup(pInfo->filterCtx.colHash);
taosArrayDestroy(pInfo->filterCtx.cInfoList);
taosArrayDestroy(pInfo->aFilterIdxs);
taosArrayDestroyEx(pInfo->aUidTags, tagScanFreeUidTag);
pInfo->pRes = blockDataDestroy(pInfo->pRes);
taosArrayDestroy(pInfo->matchInfo.pList);
pInfo->pTableListInfo = tableListDestroy(pInfo->pTableListInfo);
taosMemoryFreeClear(param);
}
SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode,
STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo) {
SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pTagScanNode,
STableListInfo* pTableListInfo, SNode* pTagCond, SNode* pTagIndexCond, SExecTaskInfo* pTaskInfo) {
SScanPhysiNode* pPhyNode = (SScanPhysiNode*)pTagScanNode;
STagScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STagScanInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
@ -2886,6 +3124,11 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi
goto _error;
}
pInfo->pTagCond = pTagCond;
pInfo->pTagIndexCond = pTagIndexCond;
pInfo->suid = pPhyNode->suid;
pInfo->pStorageAPI = &pTaskInfo->storageAPI;
pInfo->pTableListInfo = pTableListInfo;
pInfo->pRes = createDataBlockFromDescNode(pDescNode);
pInfo->readHandle = *pReadHandle;
@ -2897,8 +3140,18 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi
initResultSizeInfo(&pOperator->resultInfo, 4096);
blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);
if (pTagScanNode->onlyMetaCtbIdx) {
pInfo->aUidTags = taosArrayInit(pOperator->resultInfo.capacity, sizeof(STUidTagInfo));
pInfo->aFilterIdxs = taosArrayInit(pOperator->resultInfo.capacity, sizeof(int32_t));
pInfo->filterCtx.colHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_SMALLINT), false, HASH_NO_LOCK);
pInfo->filterCtx.cInfoList = taosArrayInit(4, sizeof(SColumnInfo));
if (pInfo->pTagCond != NULL) {
nodesRewriteExprPostOrder(&pTagCond, tagScanRewriteTagColumn, (void*)&pInfo->filterCtx);
}
}
__optr_fn_t tagScanNextFn = (pTagScanNode->onlyMetaCtbIdx) ? doTagScanFromCtbIdx : doTagScanFromMetaEntry;
pOperator->fpSet =
createOperatorFpSet(optrDummyOpenFn, doTagScan, NULL, destroyTagScanOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
createOperatorFpSet(optrDummyOpenFn, tagScanNextFn, NULL, destroyTagScanOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
return pOperator;
@ -3039,8 +3292,9 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) {
int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize;
pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_BLOCK_TS_MERGE, pInfo->bufPageSize, numOfBufPage,
pInfo->pSortInputBlock, pTaskInfo->id.str, 0, 0, 0);
tsortSetMergeLimit(pInfo->pSortHandle, mergeLimit);
tsortSetAbortCheckFn(pInfo->pSortHandle, isTaskKilled, pOperator->pTaskInfo);
}
tsortSetFetchRawDataFp(pInfo->pSortHandle, getBlockForTableMergeScan, NULL, NULL);
@ -3060,7 +3314,7 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) {
int32_t code = TSDB_CODE_SUCCESS;
if (numOfTable == 1) {
setSingleTableMerge(pInfo->pSortHandle);
tsortSetSingleTableMerge(pInfo->pSortHandle);
} else {
code = tsortOpen(pInfo->pSortHandle);
}

5
source/libs/executor/src/sysscanoperator.c
View File

@ -1601,6 +1601,11 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) {
SSysTableScanInfo* pInfo = pOperator->info;
char dbName[TSDB_DB_NAME_LEN] = {0};
if (isTaskKilled(pOperator->pTaskInfo)) {
setOperatorCompleted(pOperator);
return NULL;
}
blockDataCleanup(pInfo->pRes);
const char* name = tNameGetTableName(&pInfo->name);

13
source/libs/executor/src/tsort.c
View File

@ -71,12 +71,20 @@ struct SSortHandle {
SMultiwayMergeTreeInfo* pMergeTree;
bool singleTableMerge;
bool (*abortCheckFn)(void* param);
void* abortCheckParam;
};
void setSingleTableMerge(SSortHandle* pHandle) {
void tsortSetSingleTableMerge(SSortHandle* pHandle) {
pHandle->singleTableMerge = true;
}
void tsortSetAbortCheckFn(SSortHandle *pHandle, bool (*checkFn)(void *), void* param) {
pHandle->abortCheckFn = checkFn;
pHandle->abortCheckParam = param;
}
static int32_t msortComparFn(const void* pLeft, const void* pRight, void* param);
// | offset[0] | offset[1] |....| nullbitmap | data |...|
@ -726,11 +734,10 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) {
SArray* pPageIdList = taosArrayInit(4, sizeof(int32_t));
while (1) {
if (tsortIsClosed(pHandle)) {
if (tsortIsClosed(pHandle) || (pHandle->abortCheckFn && pHandle->abortCheckFn(pHandle->abortCheckParam))) {
code = terrno = TSDB_CODE_TSC_QUERY_CANCELLED;
return code;
}
SSDataBlock* pDataBlock = getSortedBlockDataInner(pHandle, &pHandle->cmpParam, numOfRows);
if (pDataBlock == NULL) {
break;

8
source/libs/function/src/udfd.c
View File

@ -378,9 +378,9 @@ int32_t udfdInitializePythonPlugin(SUdfScriptPlugin *plugin) {
"pyUdfDestroy", "pyUdfScalarProc", "pyUdfAggStart",
"pyUdfAggFinish", "pyUdfAggProc", "pyUdfAggMerge"};
void **funcs[UDFD_MAX_PLUGIN_FUNCS] = {
(void **)&plugin->openFunc, (void **)&plugin->closeFunc, (void **)&plugin->udfInitFunc,
(void **)&plugin->udfDestroyFunc, (void **)&plugin->udfScalarProcFunc, (void **)&plugin->udfAggStartFunc,
(void **)&plugin->udfAggFinishFunc, (void **)&plugin->udfAggProcFunc, (void **)&plugin->udfAggMergeFunc};
(void **)&plugin->openFunc, (void **)&plugin->closeFunc, (void **)&plugin->udfInitFunc,
(void **)&plugin->udfDestroyFunc, (void **)&plugin->udfScalarProcFunc, (void **)&plugin->udfAggStartFunc,
(void **)&plugin->udfAggFinishFunc, (void **)&plugin->udfAggProcFunc, (void **)&plugin->udfAggMergeFunc};
int32_t err = udfdLoadSharedLib(plugin->libPath, &plugin->lib, funcName, funcs, UDFD_MAX_PLUGIN_FUNCS);
if (err != 0) {
fnError("can not load python plugin. lib path %s", plugin->libPath);
@ -848,7 +848,7 @@ int32_t udfdSaveFuncBodyToFile(SFuncInfo *pFuncInfo, SUdf *udf) {
char path[PATH_MAX] = {0};
udfdGetFuncBodyPath(udf, path);
bool fileExist = !(taosStatFile(path, NULL, NULL) < 0);
bool fileExist = !(taosStatFile(path, NULL, NULL, NULL) < 0);
if (fileExist) {
strncpy(udf->path, path, PATH_MAX);
fnInfo("udfd func body file. reuse existing file %s", path);

4
source/libs/index/src/indexFstFile.c
View File

@ -162,7 +162,7 @@ static FORCE_INLINE int idxFileCtxGetSize(IFileCtx* ctx) {
return ctx->offset;
} else {
int64_t file_size = 0;
taosStatFile(ctx->file.buf, &file_size, NULL);
taosStatFile(ctx->file.buf, &file_size, NULL, NULL);
return (int)file_size;
}
}
@ -199,7 +199,7 @@ IFileCtx* idxFileCtxCreate(WriterType type, const char* path, bool readOnly, int
code = taosFtruncateFile(ctx->file.pFile, 0);
UNUSED(code);
code = taosStatFile(path, &ctx->file.size, NULL);
code = taosStatFile(path, &ctx->file.size, NULL, NULL);
UNUSED(code);
ctx->file.wBufOffset = 0;

4
source/libs/nodes/src/nodesCloneFuncs.c
View File

@ -609,7 +609,9 @@ static int32_t physiScanCopy(const SScanPhysiNode* pSrc, SScanPhysiNode* pDst) {
}
static int32_t physiTagScanCopy(const STagScanPhysiNode* pSrc, STagScanPhysiNode* pDst) {
return physiScanCopy(pSrc, pDst);
COPY_BASE_OBJECT_FIELD(scan, physiScanCopy);
COPY_SCALAR_FIELD(onlyMetaCtbIdx);
return TSDB_CODE_SUCCESS;
}
static int32_t physiTableScanCopy(const STableScanPhysiNode* pSrc, STableScanPhysiNode* pDst) {

54
source/libs/nodes/src/nodesCodeFuncs.c
View File

@ -672,6 +672,7 @@ static const char* jkScanLogicPlanDynamicScanFuncs = "DynamicScanFuncs";
static const char* jkScanLogicPlanDataRequired = "DataRequired";
static const char* jkScanLogicPlanTagCond = "TagCond";
static const char* jkScanLogicPlanGroupTags = "GroupTags";
static const char* jkScanLogicPlanOnlyMetaCtbIdx = "OnlyMetaCtbIdx";
static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) {
const SScanLogicNode* pNode = (const SScanLogicNode*)pObj;
@ -713,7 +714,9 @@ static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) {
if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkScanLogicPlanGroupTags, pNode->pGroupTags);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddBoolToObject(pJson, jkScanLogicPlanOnlyMetaCtbIdx, pNode->onlyMetaCtbIdx);
}
return code;
}
@ -758,7 +761,10 @@ static int32_t jsonToLogicScanNode(const SJson* pJson, void* pObj) {
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkScanLogicPlanGroupTags, &pNode->pGroupTags);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBoolValue(pJson, jkScanLogicPlanOnlyMetaCtbIdx, &pNode->onlyMetaCtbIdx);
}
return code;
}
@ -1680,7 +1686,7 @@ static const char* jkScanPhysiPlanTableName = "TableName";
static const char* jkScanPhysiPlanGroupOrderScan = "GroupOrderScan";
static int32_t physiScanNodeToJson(const void* pObj, SJson* pJson) {
const STagScanPhysiNode* pNode = (const STagScanPhysiNode*)pObj;
const SScanPhysiNode* pNode = (const SScanPhysiNode*)pObj;
int32_t code = physicPlanNodeToJson(pObj, pJson);
if (TSDB_CODE_SUCCESS == code) {
@ -1709,7 +1715,7 @@ static int32_t physiScanNodeToJson(const void* pObj, SJson* pJson) {
}
static int32_t jsonToPhysiScanNode(const SJson* pJson, void* pObj) {
STagScanPhysiNode* pNode = (STagScanPhysiNode*)pObj;
SScanPhysiNode* pNode = (SScanPhysiNode*)pObj;
int32_t code = jsonToPhysicPlanNode(pJson, pObj);
if (TSDB_CODE_SUCCESS == code) {
@ -1737,6 +1743,30 @@ static int32_t jsonToPhysiScanNode(const SJson* pJson, void* pObj) {
return code;
}
static const char* jkTagScanPhysiOnlyMetaCtbIdx = "OnlyMetaCtbIdx";
static int32_t physiTagScanNodeToJson(const void* pObj, SJson* pJson) {
const STagScanPhysiNode* pNode = (const STagScanPhysiNode*)pObj;
int32_t code = physiScanNodeToJson(pObj, pJson);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddBoolToObject(pJson, jkTagScanPhysiOnlyMetaCtbIdx, pNode->onlyMetaCtbIdx);
}
return code;
}
static int32_t jsonToPhysiTagScanNode(const SJson* pJson, void* pObj) {
STagScanPhysiNode* pNode = (STagScanPhysiNode*)pObj;
int32_t code = jsonToPhysiScanNode(pJson, pObj);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBoolValue(pJson, jkTagScanPhysiOnlyMetaCtbIdx, &pNode->onlyMetaCtbIdx);
}
return code;
}
static const char* jkLastRowScanPhysiPlanGroupTags = "GroupTags";
static const char* jkLastRowScanPhysiPlanGroupSort = "GroupSort";
@ -3148,6 +3178,8 @@ static const char* jkSubplanDataSink = "DataSink";
static const char* jkSubplanTagCond = "TagCond";
static const char* jkSubplanTagIndexCond = "TagIndexCond";
static const char* jkSubplanShowRewrite = "ShowRewrite";
static const char* jkSubplanRowsThreshold = "RowThreshold";
static const char* jkSubplanDynamicRowsThreshold = "DyRowThreshold";
static int32_t subplanToJson(const void* pObj, SJson* pJson) {
const SSubplan* pNode = (const SSubplan*)pObj;
@ -3186,6 +3218,12 @@ static int32_t subplanToJson(const void* pObj, SJson* pJson) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddBoolToObject(pJson, jkSubplanShowRewrite, pNode->showRewrite);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkSubplanRowsThreshold, pNode->rowsThreshold);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddBoolToObject(pJson, jkSubplanDynamicRowsThreshold, pNode->dynamicRowThreshold);
}
return code;
}
@ -3227,6 +3265,12 @@ static int32_t jsonToSubplan(const SJson* pJson, void* pObj) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBoolValue(pJson, jkSubplanShowRewrite, &pNode->showRewrite);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetIntValue(pJson, jkSubplanRowsThreshold, &pNode->rowsThreshold);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBoolValue(pJson, jkSubplanDynamicRowsThreshold, &pNode->dynamicRowThreshold);
}
return code;
}
@ -6928,6 +6972,7 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
case QUERY_NODE_LOGIC_PLAN:
return logicPlanToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN:
return physiTagScanNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN:
return physiScanNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN:
@ -7256,6 +7301,7 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
case QUERY_NODE_LOGIC_PLAN:
return jsonToLogicPlan(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN:
return jsonToPhysiTagScanNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN:
case QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN:
return jsonToPhysiScanNode(pJson, pObj);

53
source/libs/nodes/src/nodesMsgFuncs.c
View File

@ -2011,6 +2011,43 @@ static int32_t msgToPhysiScanNode(STlvDecoder* pDecoder, void* pObj) {
return code;
}
enum {
PHY_TAG_SCAN_CODE_SCAN = 1,
PHY_TAG_SCAN_CODE_ONLY_META_CTB_IDX
};
static int32_t physiTagScanNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
const STagScanPhysiNode* pNode = (const STagScanPhysiNode*)pObj;
int32_t code = tlvEncodeObj(pEncoder, PHY_TAG_SCAN_CODE_SCAN, physiScanNodeToMsg, &pNode->scan);
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeBool(pEncoder, PHY_TAG_SCAN_CODE_ONLY_META_CTB_IDX, pNode->onlyMetaCtbIdx);
}
return code;
}
static int32_t msgToPhysiTagScanNode(STlvDecoder* pDecoder, void* pObj) {
STagScanPhysiNode* pNode = (STagScanPhysiNode*)pObj;
int32_t code = TSDB_CODE_SUCCESS;
STlv* pTlv = NULL;
tlvForEach(pDecoder, pTlv, code) {
switch (pTlv->type) {
case PHY_TAG_SCAN_CODE_SCAN:
code = tlvDecodeObjFromTlv(pTlv, msgToPhysiScanNode, &pNode->scan);
break;
case PHY_TAG_SCAN_CODE_ONLY_META_CTB_IDX:
code = tlvDecodeBool(pTlv, &pNode->onlyMetaCtbIdx);
break;
default:
break;
}
}
return code;
}
enum {
PHY_LAST_ROW_SCAN_CODE_SCAN = 1,
PHY_LAST_ROW_SCAN_CODE_GROUP_TAGS,
@ -3800,6 +3837,12 @@ static int32_t subplanInlineToMsg(const void* pObj, STlvEncoder* pEncoder) {
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeValueBool(pEncoder, pNode->showRewrite);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeValueI32(pEncoder, pNode->rowsThreshold);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeValueBool(pEncoder, pNode->dynamicRowThreshold);
}
return code;
}
@ -3849,6 +3892,12 @@ static int32_t msgToSubplanInline(STlvDecoder* pDecoder, void* pObj) {
if (TSDB_CODE_SUCCESS == code) {
code = tlvDecodeValueBool(pDecoder, &pNode->showRewrite);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvDecodeValueI32(pDecoder, &pNode->rowsThreshold);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvDecodeValueBool(pDecoder, &pNode->dynamicRowThreshold);
}
return code;
}
@ -3988,6 +4037,8 @@ static int32_t specificNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
code = caseWhenNodeToMsg(pObj, pEncoder);
break;
case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN:
code = physiTagScanNodeToMsg(pObj, pEncoder);
break;
case QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN:
code = physiScanNodeToMsg(pObj, pEncoder);
break;
@ -4140,6 +4191,8 @@ static int32_t msgToSpecificNode(STlvDecoder* pDecoder, void* pObj) {
code = msgToCaseWhenNode(pDecoder, pObj);
break;
case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN:
code = msgToPhysiTagScanNode(pDecoder, pObj);
break;
case QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN:
code = msgToPhysiScanNode(pDecoder, pObj);
break;

7
source/libs/nodes/test/nodesCloneTest.cpp
View File

@ -199,9 +199,10 @@ TEST_F(NodesCloneTest, physiScan) {
ASSERT_EQ(nodeType(pSrc), nodeType(pDst));
STagScanPhysiNode* pSrcNode = (STagScanPhysiNode*)pSrc;
STagScanPhysiNode* pDstNode = (STagScanPhysiNode*)pDst;
ASSERT_EQ(pSrcNode->uid, pDstNode->uid);
ASSERT_EQ(pSrcNode->suid, pDstNode->suid);
ASSERT_EQ(pSrcNode->tableType, pDstNode->tableType);
ASSERT_EQ(pSrcNode->scan.uid, pDstNode->scan.uid);
ASSERT_EQ(pSrcNode->scan.suid, pDstNode->scan.suid);
ASSERT_EQ(pSrcNode->scan.tableType, pDstNode->scan.tableType);
ASSERT_EQ(pSrcNode->onlyMetaCtbIdx, pDstNode->onlyMetaCtbIdx);
});
std::unique_ptr<SNode, void (*)(SNode*)> srcNode(nullptr, nodesDestroyNode);

231
source/libs/parser/src/parTranslater.c
View File

@ -31,8 +31,8 @@
#define SYSTABLE_SHOW_TYPE_OFFSET QUERY_NODE_SHOW_DNODES_STMT
typedef struct SRewriteTbNameContext {
int32_t errCode;
char* pTbName;
int32_t errCode;
char* pTbName;
} SRewriteTbNameContext;
typedef struct STranslateContext {
@ -54,7 +54,7 @@ typedef struct STranslateContext {
bool stableQuery;
bool showRewrite;
SNode* pPrevRoot;
SNode* pPostRoot;
SNode* pPostRoot;
} STranslateContext;
typedef struct SBuildTopicContext {
@ -280,10 +280,11 @@ static const SSysTableShowAdapter sysTableShowAdapter[] = {
static int32_t translateSubquery(STranslateContext* pCxt, SNode* pNode);
static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode);
static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal);
static int32_t createSimpleSelectStmtFromProjList(const char* pDb, const char* pTable, SNodeList* pProjectionList, SSelectStmt** pStmt);
static int32_t createLastTsSelectStmt(char* pDb, char* pTable, STableMeta* pMeta, SNode** pQuery);
static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery);
static int32_t setRefreshMate(STranslateContext* pCxt, SQuery* pQuery);
static int32_t createSimpleSelectStmtFromProjList(const char* pDb, const char* pTable, SNodeList* pProjectionList,
SSelectStmt** pStmt);
static int32_t createLastTsSelectStmt(char* pDb, char* pTable, STableMeta* pMeta, SNode** pQuery);
static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery);
static int32_t setRefreshMate(STranslateContext* pCxt, SQuery* pQuery);
static bool afterGroupBy(ESqlClause clause) { return clause > SQL_CLAUSE_GROUP_BY; }
@ -774,7 +775,8 @@ static SNodeList* getProjectList(const SNode* pNode) {
static bool isTimeLineQuery(SNode* pStmt) {
if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) {
return (TIME_LINE_MULTI == ((SSelectStmt*)pStmt)->timeLineResMode) || (TIME_LINE_GLOBAL == ((SSelectStmt*)pStmt)->timeLineResMode);
return (TIME_LINE_MULTI == ((SSelectStmt*)pStmt)->timeLineResMode) ||
(TIME_LINE_GLOBAL == ((SSelectStmt*)pStmt)->timeLineResMode);
} else if (QUERY_NODE_SET_OPERATOR == nodeType(pStmt)) {
return TIME_LINE_GLOBAL == ((SSetOperator*)pStmt)->timeLineResMode;
} else {
@ -793,7 +795,7 @@ static bool isGlobalTimeLineQuery(SNode* pStmt) {
}
static bool isTimeLineAlignedQuery(SNode* pStmt) {
SSelectStmt *pSelect = (SSelectStmt *)pStmt;
SSelectStmt* pSelect = (SSelectStmt*)pStmt;
if (isGlobalTimeLineQuery(((STempTableNode*)pSelect->pFromTable)->pSubquery)) {
return true;
}
@ -803,7 +805,7 @@ static bool isTimeLineAlignedQuery(SNode* pStmt) {
if (QUERY_NODE_SELECT_STMT != nodeType(((STempTableNode*)pSelect->pFromTable)->pSubquery)) {
return false;
}
SSelectStmt *pSub = (SSelectStmt *)((STempTableNode*)pSelect->pFromTable)->pSubquery;
SSelectStmt* pSub = (SSelectStmt*)((STempTableNode*)pSelect->pFromTable)->pSubquery;
if (nodesListMatch(pSelect->pPartitionByList, pSub->pPartitionByList)) {
return true;
}
@ -824,18 +826,18 @@ static bool isPrimaryKeyImpl(SNode* pExpr) {
return true;
}
} else if (QUERY_NODE_OPERATOR == nodeType(pExpr)) {
SOperatorNode* pOper = (SOperatorNode*)pExpr;
if (OP_TYPE_ADD != pOper->opType && OP_TYPE_SUB != pOper->opType) {
return false;
}
if (!isPrimaryKeyImpl(pOper->pLeft)) {
return false;
}
if (QUERY_NODE_VALUE != nodeType(pOper->pRight)) {
return false;
}
return true;
SOperatorNode* pOper = (SOperatorNode*)pExpr;
if (OP_TYPE_ADD != pOper->opType && OP_TYPE_SUB != pOper->opType) {
return false;
}
if (!isPrimaryKeyImpl(pOper->pLeft)) {
return false;
}
if (QUERY_NODE_VALUE != nodeType(pOper->pRight)) {
return false;
}
return true;
}
return false;
}
@ -862,7 +864,7 @@ static void setColumnInfoBySchema(const SRealTableNode* pTable, const SSchema* p
pCol->tableType = pTable->pMeta->tableType;
pCol->colId = pColSchema->colId;
pCol->colType = (tagFlag >= 0 ? COLUMN_TYPE_TAG : COLUMN_TYPE_COLUMN);
pCol->hasIndex = ((0 == tagFlag) || (pColSchema != NULL && IS_IDX_ON(pColSchema)));
pCol->hasIndex = (pColSchema != NULL && IS_IDX_ON(pColSchema));
pCol->node.resType.type = pColSchema->type;
pCol->node.resType.bytes = pColSchema->bytes;
if (TSDB_DATA_TYPE_TIMESTAMP == pCol->node.resType.type) {
@ -1408,7 +1410,7 @@ static bool isCountStar(SFunctionNode* pFunc) {
}
static int32_t rewriteCountStarAsCount1(STranslateContext* pCxt, SFunctionNode* pCount) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t code = TSDB_CODE_SUCCESS;
SValueNode* pVal = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
if (NULL == pVal) {
return TSDB_CODE_OUT_OF_MEMORY;
@ -1610,9 +1612,11 @@ static int32_t translateInterpFunc(STranslateContext* pCxt, SFunctionNode* pFunc
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC);
}
if (pSelect->hasInterpFunc && (FUNC_RETURN_ROWS_INDEFINITE == pSelect->returnRows || pSelect->returnRows != fmGetFuncReturnRows(pFunc))) {
if (pSelect->hasInterpFunc &&
(FUNC_RETURN_ROWS_INDEFINITE == pSelect->returnRows || pSelect->returnRows != fmGetFuncReturnRows(pFunc))) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC,
"%s ignoring null value options cannot be used when applying to multiple columns", pFunc->functionName);
"%s ignoring null value options cannot be used when applying to multiple columns",
pFunc->functionName);
}
if (NULL != pSelect->pWindow || NULL != pSelect->pGroupByList) {
@ -1650,7 +1654,8 @@ static int32_t translateTimelineFunc(STranslateContext* pCxt, SFunctionNode* pFu
}
SSelectStmt* pSelect = (SSelectStmt*)pCxt->pCurrStmt;
if (NULL != pSelect->pFromTable && QUERY_NODE_TEMP_TABLE == nodeType(pSelect->pFromTable) &&
!isGlobalTimeLineQuery(((STempTableNode*)pSelect->pFromTable)->pSubquery) && !isTimeLineAlignedQuery(pCxt->pCurrStmt)) {
!isGlobalTimeLineQuery(((STempTableNode*)pSelect->pFromTable)->pSubquery) &&
!isTimeLineAlignedQuery(pCxt->pCurrStmt)) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC,
"%s function requires valid time series input", pFunc->functionName);
}
@ -1720,8 +1725,8 @@ static int32_t translateForbidSysTableFunc(STranslateContext* pCxt, SFunctionNod
return TSDB_CODE_SUCCESS;
}
SSelectStmt* pSelect = (SSelectStmt*)pCxt->pCurrStmt;
SNode* pTable = pSelect->pFromTable;
SSelectStmt* pSelect = (SSelectStmt*)pCxt->pCurrStmt;
SNode* pTable = pSelect->pFromTable;
if (NULL != pTable && QUERY_NODE_REAL_TABLE == nodeType(pTable) &&
TSDB_SYSTEM_TABLE == ((SRealTableNode*)pTable)->pMeta->tableType) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_SYSTABLE_NOT_ALLOWED_FUNC, pFunc->functionName);
@ -2310,7 +2315,8 @@ static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) {
}
}
if (isScanPseudoColumnFunc(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) {
if (pSelect->selectFuncNum > 1 || pSelect->hasOtherVectorFunc || !pSelect->hasSelectFunc || (isDistinctOrderBy(pCxt) && pCxt->currClause == SQL_CLAUSE_ORDER_BY)) {
if (pSelect->selectFuncNum > 1 || pSelect->hasOtherVectorFunc || !pSelect->hasSelectFunc ||
(isDistinctOrderBy(pCxt) && pCxt->currClause == SQL_CLAUSE_ORDER_BY)) {
return generateDealNodeErrMsg(pCxt, getGroupByErrorCode(pCxt), ((SExprNode*)(*pNode))->userAlias);
} else {
return rewriteColToSelectValFunc(pCxt, pNode);
@ -2405,14 +2411,14 @@ static int32_t checkHavingGroupBy(STranslateContext* pCxt, SSelectStmt* pSelect)
if (NULL != pSelect->pHaving) {
code = checkExprForGroupBy(pCxt, &pSelect->pHaving);
}
/*
if (TSDB_CODE_SUCCESS == code && NULL != pSelect->pProjectionList) {
code = checkExprListForGroupBy(pCxt, pSelect, pSelect->pProjectionList);
}
if (TSDB_CODE_SUCCESS == code && NULL != pSelect->pOrderByList) {
code = checkExprListForGroupBy(pCxt, pSelect, pSelect->pOrderByList);
}
*/
/*
if (TSDB_CODE_SUCCESS == code && NULL != pSelect->pProjectionList) {
code = checkExprListForGroupBy(pCxt, pSelect, pSelect->pProjectionList);
}
if (TSDB_CODE_SUCCESS == code && NULL != pSelect->pOrderByList) {
code = checkExprListForGroupBy(pCxt, pSelect, pSelect->pOrderByList);
}
*/
return code;
}
@ -2671,10 +2677,10 @@ static int32_t setTableCacheLastMode(STranslateContext* pCxt, SSelectStmt* pSele
static EDealRes doTranslateTbName(SNode** pNode, void* pContext) {
switch (nodeType(*pNode)) {
case QUERY_NODE_FUNCTION: {
SFunctionNode *pFunc = (SFunctionNode *)*pNode;
SFunctionNode* pFunc = (SFunctionNode*)*pNode;
if (FUNCTION_TYPE_TBNAME == pFunc->funcType) {
SRewriteTbNameContext *pCxt = (SRewriteTbNameContext*)pContext;
SValueNode* pVal = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
SRewriteTbNameContext* pCxt = (SRewriteTbNameContext*)pContext;
SValueNode* pVal = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
if (NULL == pVal) {
pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY;
return DEAL_RES_ERROR;
@ -2713,11 +2719,12 @@ static int32_t replaceTbName(STranslateContext* pCxt, SSelectStmt* pSelect) {
}
SRealTableNode* pTable = (SRealTableNode*)pSelect->pFromTable;
if (TSDB_CHILD_TABLE != pTable->pMeta->tableType && TSDB_NORMAL_TABLE != pTable->pMeta->tableType && TSDB_SYSTEM_TABLE != pTable->pMeta->tableType) {
if (TSDB_CHILD_TABLE != pTable->pMeta->tableType && TSDB_NORMAL_TABLE != pTable->pMeta->tableType &&
TSDB_SYSTEM_TABLE != pTable->pMeta->tableType) {
return TSDB_CODE_SUCCESS;
}
SNode** pNode = NULL;
SNode** pNode = NULL;
SRewriteTbNameContext pRewriteCxt = {0};
pRewriteCxt.pTbName = pTable->table.tableName;
@ -3125,7 +3132,8 @@ static int32_t convertFillValue(STranslateContext* pCxt, SDataType dt, SNodeList
code = scalarCalculateConstants(pCastFunc, &pCell->pNode);
}
if (TSDB_CODE_SUCCESS == code && QUERY_NODE_VALUE != nodeType(pCell->pNode)) {
code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Fill value can only accept constant");
code =
generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Fill value can only accept constant");
} else if (TSDB_CODE_SUCCESS != code) {
code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Filled data type mismatch");
}
@ -3591,7 +3599,6 @@ static int32_t createDefaultEveryNode(STranslateContext* pCxt, SNode** pOutput)
pEvery->isDuration = true;
pEvery->literal = taosStrdup("1s");
*pOutput = (SNode*)pEvery;
return TSDB_CODE_SUCCESS;
}
@ -3686,15 +3693,15 @@ static int32_t translateInterp(STranslateContext* pCxt, SSelectStmt* pSelect) {
static int32_t translatePartitionBy(STranslateContext* pCxt, SSelectStmt* pSelect) {
pCxt->currClause = SQL_CLAUSE_PARTITION_BY;
int32_t code = TSDB_CODE_SUCCESS;
if (pSelect->pPartitionByList) {
int8_t typeType = getTableTypeFromTableNode(pSelect->pFromTable);
SNode* pPar = nodesListGetNode(pSelect->pPartitionByList, 0);
if (!((TSDB_NORMAL_TABLE == typeType || TSDB_CHILD_TABLE == typeType) &&
1 == pSelect->pPartitionByList->length && (QUERY_NODE_FUNCTION == nodeType(pPar) && FUNCTION_TYPE_TBNAME == ((SFunctionNode*)pPar)->funcType))) {
if (!((TSDB_NORMAL_TABLE == typeType || TSDB_CHILD_TABLE == typeType) && 1 == pSelect->pPartitionByList->length &&
(QUERY_NODE_FUNCTION == nodeType(pPar) && FUNCTION_TYPE_TBNAME == ((SFunctionNode*)pPar)->funcType))) {
pSelect->timeLineResMode = TIME_LINE_MULTI;
}
code = translateExprList(pCxt, pSelect->pPartitionByList);
}
if (TSDB_CODE_SUCCESS == code) {
@ -3958,9 +3965,9 @@ static int32_t translateSetOperProject(STranslateContext* pCxt, SSetOperator* pS
}
snprintf(pRightExpr->aliasName, sizeof(pRightExpr->aliasName), "%s", pLeftExpr->aliasName);
SNode* pProj = createSetOperProject(pSetOperator->stmtName, pLeft);
if (QUERY_NODE_COLUMN == nodeType(pLeft) && QUERY_NODE_COLUMN == nodeType(pRight)
&& ((SColumnNode*)pLeft)->colId == PRIMARYKEY_TIMESTAMP_COL_ID
&& ((SColumnNode*)pRight)->colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
if (QUERY_NODE_COLUMN == nodeType(pLeft) && QUERY_NODE_COLUMN == nodeType(pRight) &&
((SColumnNode*)pLeft)->colId == PRIMARYKEY_TIMESTAMP_COL_ID &&
((SColumnNode*)pRight)->colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
((SColumnNode*)pProj)->colId = PRIMARYKEY_TIMESTAMP_COL_ID;
}
if (TSDB_CODE_SUCCESS != nodesListMakeStrictAppend(&pSetOperator->pProjectionList, pProj)) {
@ -5740,7 +5747,6 @@ static int32_t translateRestoreDnode(STranslateContext* pCxt, SRestoreComponentN
return buildCmdMsg(pCxt, TDMT_MND_RESTORE_DNODE, (FSerializeFunc)tSerializeSRestoreDnodeReq, &restoreReq);
}
static int32_t getSmaIndexDstVgId(STranslateContext* pCxt, const char* pDbName, const char* pTableName,
int32_t* pVgId) {
SVgroupInfo vg = {0};
@ -5868,7 +5874,7 @@ static int32_t checkCreateSmaIndex(STranslateContext* pCxt, SCreateIndexStmt* pS
}
static int32_t translateCreateSmaIndex(STranslateContext* pCxt, SCreateIndexStmt* pStmt) {
int32_t code = checkCreateSmaIndex(pCxt, pStmt);
int32_t code = checkCreateSmaIndex(pCxt, pStmt);
pStmt->pReq = taosMemoryCalloc(1, sizeof(SMCreateSmaReq));
if (pStmt->pReq == NULL) code = TSDB_CODE_OUT_OF_MEMORY;
if (TSDB_CODE_SUCCESS == code) {
@ -5882,13 +5888,15 @@ int32_t createIntervalFromCreateSmaIndexStmt(SCreateIndexStmt* pStmt, SInterval*
pInterval->interval = ((SValueNode*)pStmt->pOptions->pInterval)->datum.i;
pInterval->intervalUnit = ((SValueNode*)pStmt->pOptions->pInterval)->unit;
pInterval->offset = NULL != pStmt->pOptions->pOffset ? ((SValueNode*)pStmt->pOptions->pOffset)->datum.i : 0;
pInterval->sliding = NULL != pStmt->pOptions->pSliding ? ((SValueNode*)pStmt->pOptions->pSliding)->datum.i : pInterval->interval;
pInterval->slidingUnit = NULL != pStmt->pOptions->pSliding ? ((SValueNode*)pStmt->pOptions->pSliding)->unit : pInterval->intervalUnit;
pInterval->sliding =
NULL != pStmt->pOptions->pSliding ? ((SValueNode*)pStmt->pOptions->pSliding)->datum.i : pInterval->interval;
pInterval->slidingUnit =
NULL != pStmt->pOptions->pSliding ? ((SValueNode*)pStmt->pOptions->pSliding)->unit : pInterval->intervalUnit;
pInterval->precision = pStmt->pOptions->tsPrecision;
return TSDB_CODE_SUCCESS;
}
int32_t translatePostCreateSmaIndex(SParseContext* pParseCxt, SQuery* pQuery, void ** pResRow) {
int32_t translatePostCreateSmaIndex(SParseContext* pParseCxt, SQuery* pQuery, void** pResRow) {
int32_t code = TSDB_CODE_SUCCESS;
SCreateIndexStmt* pStmt = (SCreateIndexStmt*)pQuery->pRoot;
int64_t lastTs = 0;
@ -6056,7 +6064,7 @@ static int32_t buildCreateTopicReq(STranslateContext* pCxt, SCreateTopicStmt* pS
toName(pCxt->pParseCxt->acctId, pStmt->subDbName, pStmt->subSTbName, &name);
tNameGetFullDbName(&name, pReq->subDbName);
tNameExtractFullName(&name, pReq->subStbName);
if(pStmt->pQuery != NULL) {
if (pStmt->pQuery != NULL) {
code = nodesNodeToString(pStmt->pQuery, false, &pReq->ast, NULL);
}
} else if ('\0' != pStmt->subDbName[0]) {
@ -6111,11 +6119,12 @@ static EDealRes checkColumnTagsInCond(SNode* pNode, void* pContext) {
addTagList(&pCxt->pTags, nodesCloneNode(pNode));
}
}
return DEAL_RES_CONTINUE;
}
static int32_t checkCollectTopicTags(STranslateContext* pCxt, SCreateTopicStmt* pStmt, STableMeta* pMeta, SNodeList** ppProjection) {
static int32_t checkCollectTopicTags(STranslateContext* pCxt, SCreateTopicStmt* pStmt, STableMeta* pMeta,
SNodeList** ppProjection) {
SBuildTopicContext colCxt = {.colExists = false, .colNotFound = false, .pMeta = pMeta, .pTags = NULL};
nodesWalkExprPostOrder(pStmt->pWhere, checkColumnTagsInCond, &colCxt);
if (colCxt.colNotFound) {
@ -6125,18 +6134,18 @@ static int32_t checkCollectTopicTags(STranslateContext* pCxt, SCreateTopicStmt*
nodesDestroyList(colCxt.pTags);
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, "Columns are forbidden in where clause");
}
if (NULL == colCxt.pTags) { // put one column to select
// for (int32_t i = 0; i < pMeta->tableInfo.numOfColumns; ++i) {
SSchema* column = &pMeta->schema[0];
SColumnNode* col = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN);
if (NULL == col) {
return TSDB_CODE_OUT_OF_MEMORY;
}
strcpy(col->colName, column->name);
strcpy(col->node.aliasName, col->colName);
strcpy(col->node.userAlias, col->colName);
addTagList(&colCxt.pTags, (SNode*)col);
// }
if (NULL == colCxt.pTags) { // put one column to select
// for (int32_t i = 0; i < pMeta->tableInfo.numOfColumns; ++i) {
SSchema* column = &pMeta->schema[0];
SColumnNode* col = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN);
if (NULL == col) {
return TSDB_CODE_OUT_OF_MEMORY;
}
strcpy(col->colName, column->name);
strcpy(col->node.aliasName, col->colName);
strcpy(col->node.userAlias, col->colName);
addTagList(&colCxt.pTags, (SNode*)col);
// }
}
*ppProjection = colCxt.pTags;
@ -6144,13 +6153,13 @@ static int32_t checkCollectTopicTags(STranslateContext* pCxt, SCreateTopicStmt*
}
static int32_t buildQueryForTableTopic(STranslateContext* pCxt, SCreateTopicStmt* pStmt, SNode** pSelect) {
SParseContext* pParCxt = pCxt->pParseCxt;
SRequestConnInfo connInfo = {.pTrans = pParCxt->pTransporter,
.requestId = pParCxt->requestId,
SParseContext* pParCxt = pCxt->pParseCxt;
SRequestConnInfo connInfo = {.pTrans = pParCxt->pTransporter,
.requestId = pParCxt->requestId,
.requestObjRefId = pParCxt->requestRid,
.mgmtEps = pParCxt->mgmtEpSet};
SName name;
STableMeta* pMeta = NULL;
SName name;
STableMeta* pMeta = NULL;
int32_t code = getTableMetaImpl(pCxt, toName(pParCxt->acctId, pStmt->subDbName, pStmt->subSTbName, &name), &pMeta);
if (code) {
taosMemoryFree(pMeta);
@ -6159,7 +6168,7 @@ static int32_t buildQueryForTableTopic(STranslateContext* pCxt, SCreateTopicStmt
if (TSDB_SUPER_TABLE != pMeta->tableType) {
taosMemoryFree(pMeta);
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, "Only supertable table can be used");
}
}
SNodeList* pProjection = NULL;
code = checkCollectTopicTags(pCxt, pStmt, pMeta, &pProjection);
@ -6557,7 +6566,8 @@ static int32_t checkStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStm
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY,
"SUBTABLE expression must be of VARCHAR type");
}
if (NULL != pSelect->pSubtable && 0 == LIST_LENGTH(pSelect->pPartitionByList) && subtableExprHasColumnOrPseudoColumn(pSelect->pSubtable)) {
if (NULL != pSelect->pSubtable && 0 == LIST_LENGTH(pSelect->pPartitionByList) &&
subtableExprHasColumnOrPseudoColumn(pSelect->pSubtable)) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY,
"SUBTABLE expression must not has column when no partition by clause");
}
@ -6913,28 +6923,28 @@ static int32_t createLastTsSelectStmt(char* pDb, char* pTable, STableMeta* pMeta
if (NULL == col) {
return TSDB_CODE_OUT_OF_MEMORY;
}
strcpy(col->tableAlias, pTable);
strcpy(col->colName, pMeta->schema[0].name);
SNodeList* pParamterList = nodesMakeList();
if (NULL == pParamterList) {
nodesDestroyNode((SNode *)col);
nodesDestroyNode((SNode*)col);
return TSDB_CODE_OUT_OF_MEMORY;
}
int32_t code = nodesListStrictAppend(pParamterList, (SNode *)col);
int32_t code = nodesListStrictAppend(pParamterList, (SNode*)col);
if (code) {
nodesDestroyNode((SNode *)col);
nodesDestroyNode((SNode*)col);
nodesDestroyList(pParamterList);
return code;
}
SNode* pFunc = (SNode*)createFunction("last", pParamterList);
if (NULL == pFunc) {
nodesDestroyList(pParamterList);
return TSDB_CODE_OUT_OF_MEMORY;
}
SNodeList* pProjectionList = nodesMakeList();
if (NULL == pProjectionList) {
nodesDestroyList(pParamterList);
@ -6946,8 +6956,8 @@ static int32_t createLastTsSelectStmt(char* pDb, char* pTable, STableMeta* pMeta
nodesDestroyList(pProjectionList);
return code;
}
code = createSimpleSelectStmtFromProjList(pDb, pTable, pProjectionList, (SSelectStmt **)pQuery);
code = createSimpleSelectStmtFromProjList(pDb, pTable, pProjectionList, (SSelectStmt**)pQuery);
if (code) {
nodesDestroyList(pProjectionList);
return code;
@ -6985,14 +6995,14 @@ static int32_t buildCreateStreamQuery(STranslateContext* pCxt, SCreateStreamStmt
if (TSDB_CODE_SUCCESS == code && pStmt->pOptions->fillHistory) {
SRealTableNode* pTable = (SRealTableNode*)(((SSelectStmt*)pStmt->pQuery)->pFromTable);
code = createLastTsSelectStmt(pTable->table.dbName, pTable->table.tableName, pTable->pMeta, &pStmt->pPrevQuery);
/*
if (TSDB_CODE_SUCCESS == code) {
STranslateContext cxt = {0};
int32_t code = initTranslateContext(pCxt->pParseCxt, pCxt->pMetaCache, &cxt);
code = translateQuery(&cxt, pStmt->pPrevQuery);
destroyTranslateContext(&cxt);
}
*/
/*
if (TSDB_CODE_SUCCESS == code) {
STranslateContext cxt = {0};
int32_t code = initTranslateContext(pCxt->pParseCxt, pCxt->pMetaCache, &cxt);
code = translateQuery(&cxt, pStmt->pPrevQuery);
destroyTranslateContext(&cxt);
}
*/
}
taosMemoryFree(pMeta);
return code;
@ -7087,7 +7097,7 @@ static int32_t buildIntervalForCreateStream(SCreateStreamStmt* pStmt, SInterval*
if (NULL == pSelect->pWindow || QUERY_NODE_INTERVAL_WINDOW != nodeType(pSelect->pWindow)) {
return code;
}
SIntervalWindowNode* pWindow = (SIntervalWindowNode*)pSelect->pWindow;
pInterval->interval = ((SValueNode*)pWindow->pInterval)->datum.i;
pInterval->intervalUnit = ((SValueNode*)pWindow->pInterval)->unit;
@ -7095,16 +7105,16 @@ static int32_t buildIntervalForCreateStream(SCreateStreamStmt* pStmt, SInterval*
pInterval->sliding = (NULL != pWindow->pSliding ? ((SValueNode*)pWindow->pSliding)->datum.i : pInterval->interval);
pInterval->slidingUnit =
(NULL != pWindow->pSliding ? ((SValueNode*)pWindow->pSliding)->unit : pInterval->intervalUnit);
pInterval->precision = ((SColumnNode*)pWindow->pCol)->node.resType.precision;
pInterval->precision = ((SColumnNode*)pWindow->pCol)->node.resType.precision;
return code;
}
int32_t translatePostCreateStream(SParseContext* pParseCxt, SQuery* pQuery, void** pResRow) {
SCreateStreamStmt* pStmt = (SCreateStreamStmt*)pQuery->pRoot;
STranslateContext cxt = {0};
SInterval interval = {0};
int64_t lastTs = 0;
STranslateContext cxt = {0};
SInterval interval = {0};
int64_t lastTs = 0;
int32_t code = initTranslateContext(pParseCxt, NULL, &cxt);
if (TSDB_CODE_SUCCESS == code) {
@ -7139,7 +7149,6 @@ int32_t translatePostCreateStream(SParseContext* pParseCxt, SQuery* pQuery, void
return code;
}
static int32_t translateDropStream(STranslateContext* pCxt, SDropStreamStmt* pStmt) {
SMDropStreamReq dropReq = {0};
SName name;
@ -7170,7 +7179,7 @@ static int32_t translateResumeStream(STranslateContext* pCxt, SResumeStreamStmt*
static int32_t readFromFile(char* pName, int32_t* len, char** buf) {
int64_t filesize = 0;
if (taosStatFile(pName, &filesize, NULL) < 0) {
if (taosStatFile(pName, &filesize, NULL, NULL) < 0) {
return TAOS_SYSTEM_ERROR(errno);
}
@ -7264,7 +7273,7 @@ static int32_t translateGrantTagCond(STranslateContext* pCxt, SGrantStmt* pStmt,
}
}
int32_t code = createRealTableForGrantTable(pStmt, &pTable);
int32_t code = createRealTableForGrantTable(pStmt, &pTable);
if (TSDB_CODE_SUCCESS == code) {
SName name;
code = getTableMetaImpl(pCxt, toName(pCxt->pParseCxt->acctId, pTable->table.dbName, pTable->table.tableName, &name),
@ -7824,7 +7833,8 @@ static SNodeList* createProjectCols(int32_t ncols, const char* const pCols[]) {
return pProjections;
}
static int32_t createSimpleSelectStmtImpl(const char* pDb, const char* pTable, SNodeList* pProjectionList, SSelectStmt** pStmt) {
static int32_t createSimpleSelectStmtImpl(const char* pDb, const char* pTable, SNodeList* pProjectionList,
SSelectStmt** pStmt) {
SSelectStmt* pSelect = (SSelectStmt*)nodesMakeNode(QUERY_NODE_SELECT_STMT);
if (NULL == pSelect) {
return TSDB_CODE_OUT_OF_MEMORY;
@ -7847,9 +7857,8 @@ static int32_t createSimpleSelectStmtImpl(const char* pDb, const char* pTable, S
return TSDB_CODE_SUCCESS;
}
static int32_t createSimpleSelectStmtFromCols(const char* pDb, const char* pTable, int32_t numOfProjs,
const char* const pProjCol[], SSelectStmt** pStmt) {
const char* const pProjCol[], SSelectStmt** pStmt) {
SNodeList* pProjectionList = NULL;
if (numOfProjs >= 0) {
pProjectionList = createProjectCols(numOfProjs, pProjCol);
@ -7861,13 +7870,15 @@ static int32_t createSimpleSelectStmtFromCols(const char* pDb, const char* pTabl
return createSimpleSelectStmtImpl(pDb, pTable, pProjectionList, pStmt);
}
static int32_t createSimpleSelectStmtFromProjList(const char* pDb, const char* pTable, SNodeList* pProjectionList, SSelectStmt** pStmt) {
static int32_t createSimpleSelectStmtFromProjList(const char* pDb, const char* pTable, SNodeList* pProjectionList,
SSelectStmt** pStmt) {
return createSimpleSelectStmtImpl(pDb, pTable, pProjectionList, pStmt);
}
static int32_t createSelectStmtForShow(ENodeType showType, SSelectStmt** pStmt) {
const SSysTableShowAdapter* pShow = &sysTableShowAdapter[showType - SYSTABLE_SHOW_TYPE_OFFSET];
return createSimpleSelectStmtFromCols(pShow->pDbName, pShow->pTableName, pShow->numOfShowCols, pShow->pShowCols, pStmt);
return createSimpleSelectStmtFromCols(pShow->pDbName, pShow->pTableName, pShow->numOfShowCols, pShow->pShowCols,
pStmt);
}
static int32_t createSelectStmtForShowTableDist(SShowTableDistributedStmt* pStmt, SSelectStmt** pOutput) {
@ -8005,8 +8016,8 @@ static int32_t createShowTableTagsProjections(SNodeList** pProjections, SNodeLis
static int32_t rewriteShowStableTags(STranslateContext* pCxt, SQuery* pQuery) {
SShowTableTagsStmt* pShow = (SShowTableTagsStmt*)pQuery->pRoot;
SSelectStmt* pSelect = NULL;
int32_t code = createSimpleSelectStmtFromCols(((SValueNode*)pShow->pDbName)->literal, ((SValueNode*)pShow->pTbName)->literal,
-1, NULL, &pSelect);
int32_t code = createSimpleSelectStmtFromCols(((SValueNode*)pShow->pDbName)->literal,
((SValueNode*)pShow->pTbName)->literal, -1, NULL, &pSelect);
if (TSDB_CODE_SUCCESS == code) {
code = createShowTableTagsProjections(&pSelect->pProjectionList, &pShow->pTags);
}

10
source/libs/planner/inc/planInt.h
View File

@ -44,11 +44,17 @@ int32_t splitLogicPlan(SPlanContext* pCxt, SLogicSubplan* pLogicSubplan);
int32_t scaleOutLogicPlan(SPlanContext* pCxt, SLogicSubplan* pLogicSubplan, SQueryLogicPlan** pLogicPlan);
int32_t createPhysiPlan(SPlanContext* pCxt, SQueryLogicPlan* pLogicPlan, SQueryPlan** pPlan, SArray* pExecNodeList);
bool isPartTableAgg(SAggLogicNode* pAgg);
bool isPartTableWinodw(SWindowLogicNode* pWindow);
bool getBatchScanOptionFromHint(SNodeList* pList);
SLogicNode* getLogicNodeRootNode(SLogicNode* pCurr);
int32_t collectTableAliasFromNodes(SNode* pNode, SSHashObj** ppRes);
bool isPartTableAgg(SAggLogicNode* pAgg);
bool isPartTagAgg(SAggLogicNode* pAgg);
bool isPartTableWinodw(SWindowLogicNode* pWindow);
#define CLONE_LIMIT 1
#define CLONE_SLIMIT 1 << 1
#define CLONE_LIMIT_SLIMIT (CLONE_LIMIT | CLONE_SLIMIT)
bool cloneLimit(SLogicNode* pParent, SLogicNode* pChild, uint8_t cloneWhat);
#ifdef __cplusplus
}

37
source/libs/planner/src/planOptimizer.c
View File

@ -368,8 +368,8 @@ static void scanPathOptSetGroupOrderScan(SScanLogicNode* pScan) {
if (pScan->node.pParent && nodeType(pScan->node.pParent) == QUERY_NODE_LOGIC_PLAN_AGG) {
SAggLogicNode* pAgg = (SAggLogicNode*)pScan->node.pParent;
bool withSlimit = pAgg->node.pSlimit != NULL || (pAgg->node.pParent && pAgg->node.pParent->pSlimit);
if (withSlimit && isPartTableAgg(pAgg)) {
bool withSlimit = pAgg->node.pSlimit != NULL;
if (withSlimit && (isPartTableAgg(pAgg) || isPartTagAgg(pAgg))) {
pScan->groupOrderScan = pAgg->node.forceCreateNonBlockingOptr = true;
}
}
@ -2793,6 +2793,9 @@ static int32_t tagScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubp
}
nodesDestroyNode((SNode*)pAgg);
tagScanOptCloneAncestorSlimit((SLogicNode*)pScanNode);
pScanNode->onlyMetaCtbIdx = false;
pCxt->optimized = true;
return TSDB_CODE_SUCCESS;
}
@ -2812,39 +2815,31 @@ static void swapLimit(SLogicNode* pParent, SLogicNode* pChild) {
pParent->pLimit = NULL;
}
static void cloneLimit(SLogicNode* pParent, SLogicNode* pChild) {
SLimitNode* pLimit = NULL;
if (pParent->pLimit) {
pChild->pLimit = nodesCloneNode(pParent->pLimit);
pLimit = (SLimitNode*)pChild->pLimit;
pLimit->limit += pLimit->offset;
pLimit->offset = 0;
}
if (pParent->pSlimit) {
pChild->pSlimit = nodesCloneNode(pParent->pSlimit);
pLimit = (SLimitNode*)pChild->pSlimit;
pLimit->limit += pLimit->offset;
pLimit->offset = 0;
}
}
static bool pushDownLimitHow(SLogicNode* pNodeWithLimit, SLogicNode* pNodeLimitPushTo);
static bool pushDownLimitTo(SLogicNode* pNodeWithLimit, SLogicNode* pNodeLimitPushTo) {
switch (nodeType(pNodeLimitPushTo)) {
case QUERY_NODE_LOGIC_PLAN_WINDOW: {
SWindowLogicNode* pWindow = (SWindowLogicNode*)pNodeLimitPushTo;
if (pWindow->winType != WINDOW_TYPE_INTERVAL) break;
cloneLimit(pNodeWithLimit, pNodeLimitPushTo);
cloneLimit(pNodeWithLimit, pNodeLimitPushTo, CLONE_LIMIT_SLIMIT);
return true;
}
case QUERY_NODE_LOGIC_PLAN_FILL:
case QUERY_NODE_LOGIC_PLAN_SORT: {
cloneLimit(pNodeWithLimit, pNodeLimitPushTo);
cloneLimit(pNodeWithLimit, pNodeLimitPushTo, CLONE_LIMIT_SLIMIT);
SNode* pChild = NULL;
FOREACH(pChild, pNodeLimitPushTo->pChildren) { pushDownLimitHow(pNodeLimitPushTo, (SLogicNode*)pChild); }
return true;
}
case QUERY_NODE_LOGIC_PLAN_AGG: {
if (nodeType(pNodeWithLimit) == QUERY_NODE_LOGIC_PLAN_PROJECT &&
(isPartTagAgg((SAggLogicNode*)pNodeLimitPushTo) || isPartTableAgg((SAggLogicNode*)pNodeLimitPushTo))) {
// when part by tag, slimit will be cloned to agg, and it will be pipelined.
// The scan below will do scanning with group order
return cloneLimit(pNodeWithLimit, pNodeLimitPushTo, CLONE_SLIMIT);
}
break;
}
case QUERY_NODE_LOGIC_PLAN_SCAN:
if (nodeType(pNodeWithLimit) == QUERY_NODE_LOGIC_PLAN_PROJECT && pNodeWithLimit->pLimit) {
swapLimit(pNodeWithLimit, pNodeLimitPushTo);

25
source/libs/planner/src/planPhysiCreater.c
View File

@ -512,6 +512,20 @@ static int32_t createSimpleScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSub
return createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, pScan, pPhyNode);
}
static int32_t createTagScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode,
SPhysiNode** pPhyNode) {
STagScanPhysiNode* pScan =
(STagScanPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pScanLogicNode, QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN);
if (NULL == pScan) {
return TSDB_CODE_OUT_OF_MEMORY;
}
vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode);
pScan->onlyMetaCtbIdx = pScanLogicNode->onlyMetaCtbIdx;
return createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pScan, pPhyNode);
}
static int32_t createLastRowScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode,
SPhysiNode** pPhyNode) {
SLastRowScanPhysiNode* pScan =
@ -647,6 +661,7 @@ static int32_t createScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan,
pCxt->hasScan = true;
switch (pScanLogicNode->scanType) {
case SCAN_TYPE_TAG:
return createTagScanPhysiNode(pCxt, pSubplan, pScanLogicNode, pPhyNode);
case SCAN_TYPE_BLOCK_INFO:
return createSimpleScanPhysiNode(pCxt, pSubplan, pScanLogicNode, pPhyNode);
case SCAN_TYPE_TABLE_COUNT:
@ -1203,12 +1218,16 @@ static int32_t rewritePrecalcExpr(SPhysiPlanContext* pCxt, SNode* pNode, SNodeLi
}
static int32_t createAggPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SAggLogicNode* pAggLogicNode,
SPhysiNode** pPhyNode) {
SPhysiNode** pPhyNode, SSubplan* pSubPlan) {
SAggPhysiNode* pAgg =
(SAggPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pAggLogicNode, QUERY_NODE_PHYSICAL_PLAN_HASH_AGG);
if (NULL == pAgg) {
return TSDB_CODE_OUT_OF_MEMORY;
}
if (pAgg->node.pSlimit) {
pSubPlan->dynamicRowThreshold = true;
pSubPlan->rowsThreshold = ((SLimitNode*)pAgg->node.pSlimit)->limit;
}
pAgg->mergeDataBlock = (GROUP_ACTION_KEEP == pAggLogicNode->node.groupAction ? false : true);
pAgg->groupKeyOptimized = pAggLogicNode->hasGroupKeyOptimized;
@ -1948,7 +1967,7 @@ static int32_t doCreatePhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicNode
case QUERY_NODE_LOGIC_PLAN_JOIN:
return createJoinPhysiNode(pCxt, pChildren, (SJoinLogicNode*)pLogicNode, pPhyNode);
case QUERY_NODE_LOGIC_PLAN_AGG:
return createAggPhysiNode(pCxt, pChildren, (SAggLogicNode*)pLogicNode, pPhyNode);
return createAggPhysiNode(pCxt, pChildren, (SAggLogicNode*)pLogicNode, pPhyNode, pSubplan);
case QUERY_NODE_LOGIC_PLAN_PROJECT:
return createProjectPhysiNode(pCxt, pChildren, (SProjectLogicNode*)pLogicNode, pPhyNode);
case QUERY_NODE_LOGIC_PLAN_EXCHANGE:
@ -2056,6 +2075,8 @@ static SSubplan* makeSubplan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogicSubpl
pSubplan->id = pLogicSubplan->id;
pSubplan->subplanType = pLogicSubplan->subplanType;
pSubplan->level = pLogicSubplan->level;
pSubplan->rowsThreshold = 4096;
pSubplan->dynamicRowThreshold = false;
if (NULL != pCxt->pPlanCxt->pUser) {
snprintf(pSubplan->user, sizeof(pSubplan->user), "%s", pCxt->pPlanCxt->pUser);
}

10
source/libs/planner/src/planSpliter.c
View File

@ -868,8 +868,16 @@ static int32_t stbSplSplitAggNodeForPartTable(SSplitContext* pCxt, SStableSplitI
static int32_t stbSplSplitAggNodeForCrossTable(SSplitContext* pCxt, SStableSplitInfo* pInfo) {
SLogicNode* pPartAgg = NULL;
int32_t code = stbSplCreatePartAggNode((SAggLogicNode*)pInfo->pSplitNode, &pPartAgg);
if (TSDB_CODE_SUCCESS == code) {
code = stbSplCreateExchangeNode(pCxt, pInfo->pSplitNode, pPartAgg);
// if slimit was pushed down to agg, agg will be pipelined mode, add sort merge before parent agg
if ((SAggLogicNode*)pInfo->pSplitNode->pSlimit)
code = stbSplCreateMergeNode(pCxt, NULL, pInfo->pSplitNode, NULL, pPartAgg, true);
else
code = stbSplCreateExchangeNode(pCxt, pInfo->pSplitNode, pPartAgg);
} else {
nodesDestroyNode((SNode*)pPartAgg);
}
if (TSDB_CODE_SUCCESS == code) {
code = nodesListMakeStrictAppend(&pInfo->pSubplan->pChildren,

54
source/libs/planner/src/planUtil.c
View File

@ -363,7 +363,7 @@ static bool stbHasPartTbname(SNodeList* pPartKeys) {
return false;
}
static SNodeList* stbSplGetPartKeys(SLogicNode* pNode) {
static SNodeList* stbGetPartKeys(SLogicNode* pNode) {
if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) {
return ((SScanLogicNode*)pNode)->pGroupTags;
} else if (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode)) {
@ -381,11 +381,24 @@ bool isPartTableAgg(SAggLogicNode* pAgg) {
return stbHasPartTbname(pAgg->pGroupKeys) &&
stbNotSystemScan((SLogicNode*)nodesListGetNode(pAgg->node.pChildren, 0));
}
return stbHasPartTbname(stbSplGetPartKeys((SLogicNode*)nodesListGetNode(pAgg->node.pChildren, 0)));
return stbHasPartTbname(stbGetPartKeys((SLogicNode*)nodesListGetNode(pAgg->node.pChildren, 0)));
}
bool isPartTableWinodw(SWindowLogicNode* pWindow) {
return stbHasPartTbname(stbSplGetPartKeys((SLogicNode*)nodesListGetNode(pWindow->node.pChildren, 0)));
static bool stbHasPartTag(SNodeList* pPartKeys) {
if (NULL == pPartKeys) {
return false;
}
SNode* pPartKey = NULL;
FOREACH(pPartKey, pPartKeys) {
if (QUERY_NODE_GROUPING_SET == nodeType(pPartKey)) {
pPartKey = nodesListGetNode(((SGroupingSetNode*)pPartKey)->pParameterList, 0);
}
if ((QUERY_NODE_FUNCTION == nodeType(pPartKey) && FUNCTION_TYPE_TAGS == ((SFunctionNode*)pPartKey)->funcType) ||
(QUERY_NODE_COLUMN == nodeType(pPartKey) && COLUMN_TYPE_TAG == ((SColumnNode*)pPartKey)->colType)) {
return true;
}
}
return false;
}
bool getBatchScanOptionFromHint(SNodeList* pList) {
@ -430,5 +443,38 @@ int32_t collectTableAliasFromNodes(SNode* pNode, SSHashObj** ppRes) {
return TSDB_CODE_SUCCESS;
}
bool isPartTagAgg(SAggLogicNode* pAgg) {
if (1 != LIST_LENGTH(pAgg->node.pChildren)) {
return false;
}
if (pAgg->pGroupKeys) {
return stbHasPartTag(pAgg->pGroupKeys) &&
stbNotSystemScan((SLogicNode*)nodesListGetNode(pAgg->node.pChildren, 0));
}
return stbHasPartTag(stbGetPartKeys((SLogicNode*)nodesListGetNode(pAgg->node.pChildren, 0)));
}
bool isPartTableWinodw(SWindowLogicNode* pWindow) {
return stbHasPartTbname(stbGetPartKeys((SLogicNode*)nodesListGetNode(pWindow->node.pChildren, 0)));
}
bool cloneLimit(SLogicNode* pParent, SLogicNode* pChild, uint8_t cloneWhat) {
SLimitNode* pLimit;
bool cloned = false;
if (pParent->pLimit && (cloneWhat & CLONE_LIMIT)) {
pChild->pLimit = nodesCloneNode(pParent->pLimit);
pLimit = (SLimitNode*)pChild->pLimit;
pLimit->limit += pLimit->offset;
pLimit->offset = 0;
cloned = true;
}
if (pParent->pSlimit && (cloneWhat & CLONE_SLIMIT)) {
pChild->pSlimit = nodesCloneNode(pParent->pSlimit);
pLimit = (SLimitNode*)pChild->pSlimit;
pLimit->limit += pLimit->offset;
pLimit->offset = 0;
cloned = true;
}
return cloned;
}

2
source/libs/sync/src/syncRaftStore.c
View File

@ -42,7 +42,7 @@ int32_t raftStoreReadFile(SSyncNode *pNode) {
const char *file = pNode->raftStorePath;
SRaftStore *pStore = &pNode->raftStore;
if (taosStatFile(file, NULL, NULL) < 0) {
if (taosStatFile(file, NULL, NULL, NULL) < 0) {
sInfo("vgId:%d, raft store file:%s not exist, use default value", pNode->vgId, file);
pStore->currentTerm = 0;
pStore->voteFor.addr = 0;

12
source/libs/wal/src/walMeta.c
View File

@ -53,7 +53,7 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) {
walBuildLogName(pWal, pFileInfo->firstVer, fnameStr);
int64_t fileSize = 0;
taosStatFile(fnameStr, &fileSize, NULL);
taosStatFile(fnameStr, &fileSize, NULL, NULL);
TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ | TD_FILE_WRITE);
if (pFile == NULL) {
@ -304,7 +304,7 @@ int walRepairLogFileTs(SWal* pWal, bool* updateMeta) {
walBuildLogName(pWal, pFileInfo->firstVer, fnameStr);
int32_t mtime = 0;
if (taosStatFile(fnameStr, NULL, &mtime) < 0) {
if (taosStatFile(fnameStr, NULL, &mtime, NULL) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, failed to stat file due to %s, file:%s", pWal->cfg.vgId, strerror(errno), fnameStr);
return -1;
@ -353,7 +353,7 @@ int walTrimIdxFile(SWal* pWal, int32_t fileIdx) {
walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr);
int64_t fileSize = 0;
taosStatFile(fnameStr, &fileSize, NULL);
taosStatFile(fnameStr, &fileSize, NULL, NULL);
int64_t records = TMAX(0, pFileInfo->lastVer - pFileInfo->firstVer + 1);
int64_t lastEndOffset = records * sizeof(SWalIdxEntry);
@ -436,7 +436,7 @@ int walCheckAndRepairMeta(SWal* pWal) {
SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx);
walBuildLogName(pWal, pFileInfo->firstVer, fnameStr);
int32_t code = taosStatFile(fnameStr, &fileSize, NULL);
int32_t code = taosStatFile(fnameStr, &fileSize, NULL, NULL);
if (code < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
wError("failed to stat file since %s. file:%s", terrstr(), fnameStr);
@ -522,7 +522,7 @@ int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) {
walBuildLogName(pWal, pFileInfo->firstVer, fLogNameStr);
int64_t fileSize = 0;
if (taosStatFile(fnameStr, &fileSize, NULL) < 0 && errno != ENOENT) {
if (taosStatFile(fnameStr, &fileSize, NULL, NULL) < 0 && errno != ENOENT) {
wError("vgId:%d, failed to stat file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr);
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
@ -935,7 +935,7 @@ int walLoadMeta(SWal* pWal) {
walBuildMetaName(pWal, metaVer, fnameStr);
// read metafile
int64_t fileSize = 0;
taosStatFile(fnameStr, &fileSize, NULL);
taosStatFile(fnameStr, &fileSize, NULL, NULL);
if (fileSize == 0) {
(void)taosRemoveFile(fnameStr);
wDebug("vgId:%d, wal find empty meta ver %d", pWal->cfg.vgId, metaVer);

28
source/os/src/osFile.c
View File

@ -191,7 +191,7 @@ int32_t taosRenameFile(const char *oldName, const char *newName) {
#endif
}
int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime) {
int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime, int32_t *atime) {
#ifdef WINDOWS
struct _stati64 fileStat;
int32_t code = _stati64(path, &fileStat);
@ -211,6 +211,10 @@ int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime) {
*mtime = fileStat.st_mtime;
}
if (atime != NULL) {
*atime = fileStat.st_mtime;
}
return 0;
}
int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno) {
@ -540,7 +544,7 @@ int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) {
#ifdef WINDOWS
struct __stat64 fileStat;
int32_t code = _fstat64(pFile->fd, &fileStat);
int32_t code = _fstat64(pFile->fd, &fileStat);
#else
struct stat fileStat;
int32_t code = fstat(pFile->fd, &fileStat);
@ -897,17 +901,17 @@ int32_t taosCompressFile(char *srcFileName, char *destFileName) {
goto cmp_end;
}
dstFp = gzdopen(pFile->fd, "wb6f");
if (dstFp == NULL) {
ret = -3;
taosCloseFile(&pFile);
goto cmp_end;
}
dstFp = gzdopen(pFile->fd, "wb6f");
if (dstFp == NULL) {
ret = -3;
taosCloseFile(&pFile);
goto cmp_end;
}
while (!feof(pSrcFile->fp)) {
len = (int32_t)fread(data, 1, compressSize, pSrcFile->fp);
(void)gzwrite(dstFp, data, len);
}
while (!feof(pSrcFile->fp)) {
len = (int32_t)fread(data, 1, compressSize, pSrcFile->fp);
(void)gzwrite(dstFp, data, len);
}
cmp_end:
if (pSrcFile) {

17
source/os/src/osRand.c
View File

@ -27,11 +27,11 @@ void taosSeedRand(uint32_t seed) { return srand(seed); }
uint32_t taosRand(void) {
#ifdef WINDOWS
unsigned int pSeed;
rand_s(&pSeed);
return pSeed;
unsigned int pSeed;
rand_s(&pSeed);
return pSeed;
#else
return rand();
return rand();
#endif
}
@ -80,6 +80,15 @@ void taosRandStr(char* str, int32_t size) {
const char* set = "abcdefghijklmnopqrstuvwxyz0123456789-_.";
int32_t len = 39;
for (int32_t i = 0; i < size; ++i) {
str[i] = set[taosRand() % len];
}
}
void taosRandStr2(char* str, int32_t size) {
const char* set = "abcdefghijklmnopqrstuvwxyz0123456789";
int32_t len = 36;
for (int32_t i = 0; i < size; ++i) {
str[i] = set[taosRand() % len];
}

1
source/util/src/terror.c
View File

@ -151,6 +151,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_SHOWOBJ, "Data expired")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_QUERY_ID, "Invalid query id")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CONN_ID, "Invalid connection id")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_USER_DISABLED, "User is disabled")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_PLATFORM, "Unsupported feature on this platform")
// mnode-sdb
TAOS_DEFINE_ERROR(TSDB_CODE_SDB_OBJ_ALREADY_THERE, "Object already there")

60
source/util/src/tlog.c
View File

@ -17,8 +17,8 @@
#include "tlog.h"
#include "os.h"
#include "tconfig.h"
#include "tjson.h"
#include "tglobal.h"
#include "tjson.h"
#define LOG_MAX_LINE_SIZE (10024)
#define LOG_MAX_LINE_BUFFER_SIZE (LOG_MAX_LINE_SIZE + 3)
@ -74,12 +74,12 @@ static SLogObj tsLogObj = {.fileNum = 1};
static int64_t tsAsyncLogLostLines = 0;
static int32_t tsDaylightActive; /* Currently in daylight saving time. */
bool tsLogEmbedded = 0;
bool tsAsyncLog = true;
bool tsLogEmbedded = 0;
bool tsAsyncLog = true;
#ifdef ASSERT_NOT_CORE
bool tsAssert = false;
bool tsAssert = false;
#else
bool tsAssert = true;
bool tsAssert = true;
#endif
int32_t tsNumOfLogLines = 10000000;
int32_t tsLogKeepDays = 0;
@ -160,7 +160,7 @@ int32_t taosInitSlowLog() {
tsLogObj.slowHandle = taosLogBuffNew(LOG_SLOW_BUF_SIZE);
if (tsLogObj.slowHandle == NULL) return -1;
taosUmaskFile(0);
tsLogObj.slowHandle->pFile = taosOpenFile(fullName, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (tsLogObj.slowHandle->pFile == NULL) {
@ -403,13 +403,13 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) {
strcpy(name, fn);
strcat(name, ".0");
}
bool log0Exist = taosStatFile(name, NULL, &logstat0_mtime) >= 0;
bool log0Exist = taosStatFile(name, NULL, &logstat0_mtime, NULL) >= 0;
if (strlen(fn) < LOG_FILE_NAME_LEN + 50 - 2) {
strcpy(name, fn);
strcat(name, ".1");
}
bool log1Exist = taosStatFile(name, NULL, &logstat1_mtime) >= 0;
bool log1Exist = taosStatFile(name, NULL, &logstat1_mtime, NULL) >= 0;
// if none of the log files exist, open 0, if both exists, open the old one
if (!log0Exist && !log1Exist) {
@ -576,7 +576,7 @@ void taosPrintSlowLog(const char *format, ...) {
} else {
taosWriteFile(tsLogObj.slowHandle->pFile, buffer, len);
}
taosMemoryFree(buffer);
}
@ -769,12 +769,12 @@ static void taosWriteLog(SLogBuff *pLogBuf) {
static void *taosAsyncOutputLog(void *param) {
SLogBuff *pLogBuf = (SLogBuff *)tsLogObj.logHandle;
SLogBuff *pSlowBuf = (SLogBuff *)tsLogObj.slowHandle;
setThreadName("log");
int32_t count = 0;
int32_t updateCron = 0;
int32_t writeInterval = 0;
while (1) {
writeInterval = TMIN(pLogBuf->writeInterval, pSlowBuf->writeInterval);
count += writeInterval;
@ -834,12 +834,12 @@ bool taosAssertDebug(bool condition, const char *file, int32_t line, const char
return true;
}
void taosLogCrashInfo(char* nodeType, char* pMsg, int64_t msgLen, int signum, void *sigInfo) {
void taosLogCrashInfo(char *nodeType, char *pMsg, int64_t msgLen, int signum, void *sigInfo) {
const char *flags = "UTL FATAL ";
ELogLevel level = DEBUG_FATAL;
int32_t dflag = 255;
char filepath[PATH_MAX] = {0};
TdFilePtr pFile = NULL;
char filepath[PATH_MAX] = {0};
TdFilePtr pFile = NULL;
if (pMsg && msgLen > 0) {
snprintf(filepath, sizeof(filepath), "%s%s.%sCrashLog", tsLogDir, TD_DIRSEP, nodeType);
@ -856,16 +856,16 @@ void taosLogCrashInfo(char* nodeType, char* pMsg, int64_t msgLen, int signum, vo
int64_t writeSize = taosWriteFile(pFile, &msgLen, sizeof(msgLen));
if (sizeof(msgLen) != writeSize) {
taosUnLockFile(pFile);
taosPrintLog(flags, level, dflag, "failed to write len to file:%s,%p wlen:%" PRId64 " tlen:%lu since %s",
filepath, pFile, writeSize, sizeof(msgLen), terrstr());
taosPrintLog(flags, level, dflag, "failed to write len to file:%s,%p wlen:%" PRId64 " tlen:%lu since %s",
filepath, pFile, writeSize, sizeof(msgLen), terrstr());
goto _return;
}
writeSize = taosWriteFile(pFile, pMsg, msgLen);
if (msgLen != writeSize) {
taosUnLockFile(pFile);
taosPrintLog(flags, level, dflag, "failed to write file:%s,%p wlen:%" PRId64 " tlen:%" PRId64 " since %s",
filepath, pFile, writeSize, msgLen, terrstr());
taosPrintLog(flags, level, dflag, "failed to write file:%s,%p wlen:%" PRId64 " tlen:%" PRId64 " since %s",
filepath, pFile, writeSize, msgLen, terrstr());
goto _return;
}
@ -883,7 +883,7 @@ _return:
taosPrintTrace(flags, level, dflag, 4);
#elif !defined(WINDOWS)
taosPrintLog(flags, level, dflag, "sender PID:%d cmdline:%s", ((siginfo_t *)sigInfo)->si_pid,
taosGetCmdlineByPID(((siginfo_t *)sigInfo)->si_pid));
taosGetCmdlineByPID(((siginfo_t *)sigInfo)->si_pid));
taosPrintTrace(flags, level, dflag, 3);
#else
taosPrintTrace(flags, level, dflag, 8);
@ -892,17 +892,17 @@ _return:
taosMemoryFree(pMsg);
}
void taosReadCrashInfo(char* filepath, char** pMsg, int64_t* pMsgLen, TdFilePtr* pFd) {
void taosReadCrashInfo(char *filepath, char **pMsg, int64_t *pMsgLen, TdFilePtr *pFd) {
const char *flags = "UTL FATAL ";
ELogLevel level = DEBUG_FATAL;
int32_t dflag = 255;
TdFilePtr pFile = NULL;
bool truncateFile = false;
char* buf = NULL;
char *buf = NULL;
if (NULL == *pFd) {
int64_t filesize = 0;
if (taosStatFile(filepath, &filesize, NULL) < 0) {
if (taosStatFile(filepath, &filesize, NULL, NULL) < 0) {
if (ENOENT == errno) {
return;
}
@ -916,7 +916,7 @@ void taosReadCrashInfo(char* filepath, char** pMsg, int64_t* pMsgLen, TdFilePtr*
return;
}
pFile = taosOpenFile(filepath, TD_FILE_READ|TD_FILE_WRITE);
pFile = taosOpenFile(filepath, TD_FILE_READ | TD_FILE_WRITE);
if (pFile == NULL) {
if (ENOENT == errno) {
return;
@ -926,7 +926,7 @@ void taosReadCrashInfo(char* filepath, char** pMsg, int64_t* pMsgLen, TdFilePtr*
taosPrintLog(flags, level, dflag, "failed to open file:%s since %s", filepath, terrstr());
return;
}
taosLockFile(pFile);
} else {
pFile = *pFd;
@ -937,8 +937,8 @@ void taosReadCrashInfo(char* filepath, char** pMsg, int64_t* pMsgLen, TdFilePtr*
if (sizeof(msgLen) != readSize) {
truncateFile = true;
if (readSize < 0) {
taosPrintLog(flags, level, dflag, "failed to read len from file:%s,%p wlen:%" PRId64 " tlen:%lu since %s",
filepath, pFile, readSize, sizeof(msgLen), terrstr());
taosPrintLog(flags, level, dflag, "failed to read len from file:%s,%p wlen:%" PRId64 " tlen:%lu since %s",
filepath, pFile, readSize, sizeof(msgLen), terrstr());
}
goto _return;
}
@ -948,12 +948,12 @@ void taosReadCrashInfo(char* filepath, char** pMsg, int64_t* pMsgLen, TdFilePtr*
taosPrintLog(flags, level, dflag, "failed to malloc buf, size:%" PRId64, msgLen);
goto _return;
}
readSize = taosReadFile(pFile, buf, msgLen);
if (msgLen != readSize) {
truncateFile = true;
taosPrintLog(flags, level, dflag, "failed to read file:%s,%p wlen:%" PRId64 " tlen:%" PRId64 " since %s",
filepath, pFile, readSize, msgLen, terrstr());
taosPrintLog(flags, level, dflag, "failed to read file:%s,%p wlen:%" PRId64 " tlen:%" PRId64 " since %s", filepath,
pFile, readSize, msgLen, terrstr());
goto _return;
}
@ -981,7 +981,7 @@ void taosReleaseCrashLogFile(TdFilePtr pFile, bool truncateFile) {
if (truncateFile) {
taosFtruncateFile(pFile, 0);
}
taosUnLockFile(pFile);
taosCloseFile(&pFile);
}

3
tests/parallel_test/cases.task
View File

@ -1204,6 +1204,9 @@
,,y,script,./test.sh -f tsim/tag/drop_tag.sim
,,y,script,./test.sh -f tsim/tag/tbNameIn.sim
,,y,script,./test.sh -f tmp/monitor.sim
,,y,script,./test.sh -f tsim/tagindex/add_index.sim
,,n,script,./test.sh -f tsim/tagindex/sma_and_tag_index.sim
#develop test
,,n,develop-test,python3 ./test.py -f 2-query/table_count_scan.py

2
tests/parallel_test/container_build.sh
View File

@ -60,6 +60,7 @@ docker run \
-v /root/.cargo/git:/root/.cargo/git \
-v /root/go/pkg/mod:/root/go/pkg/mod \
-v /root/.cache/go-build:/root/.cache/go-build \
-v /root/.cos-local.1:/root/.cos-local.1 \
-v ${REP_REAL_PATH}/enterprise/src/plugins/taosx/target:${REP_DIR}/enterprise/src/plugins/taosx/target \
-v ${REP_REAL_PATH}/community/tools/taosws-rs/target:${REP_DIR}/community/tools/taosws-rs/target \
-v ${REP_REAL_PATH}/community/contrib/cJson/:${REP_DIR}/community/contrib/cJson \
@ -88,6 +89,7 @@ docker run \
-v /root/.cargo/git:/root/.cargo/git \
-v /root/go/pkg/mod:/root/go/pkg/mod \
-v /root/.cache/go-build:/root/.cache/go-build \
-v /root/.cos-local.1:/root/.cos-local.1 \
-v ${REP_REAL_PATH}/enterprise/src/plugins/taosx/target:${REP_DIR}/enterprise/src/plugins/taosx/target \
-v ${REP_REAL_PATH}/community/tools/taosws-rs/target:${REP_DIR}/community/tools/taosws-rs/target \
-v ${REP_REAL_PATH}/community/contrib/cJson/:${REP_DIR}/community/contrib/cJson \

65
tests/script/tsim/sma/drop_sma.sim
View File

@ -52,19 +52,35 @@ sql create sma index sma_index_name1 on stb function(max(c1),max(c2),min(c1)) in
print --> show sma
sql show indexes from stb from d1;
if $rows != 1 then
if $rows != 2 then
return -1
endi
if $data[0][0] != sma_index_name1 then
return -1
endi
if $data[0][1] != d1 then
return -1
endi
if $data[0][2] != stb then
return -1
if $data[0][6] == tag_index then
if $data[1][0] != sma_index_name1 then
return -1
endi
if $data[1][1] != d1 then
return -1
endi
if $data[1][2] != stb then
return -1
endi
else
if $data[0][0] != sma_index_name1 then
return -1
endi
if $data[0][1] != d1 then
return -1
endi
if $data[0][2] != stb then
return -1
endi
endi
print --> drop stb
sql drop table stb;
@ -78,17 +94,30 @@ sql create sma index sma_index_name1 on stb function(max(c1),max(c2),min(c1)) in
print --> show sma
sql show indexes from stb from d1;
if $rows != 1 then
if $rows != 2 then
return -1
endi
if $data[0][0] != sma_index_name1 then
return -1
endi
if $data[0][1] != d1 then
return -1
endi
if $data[0][2] != stb then
return -1
if $data[0][6] == tag_index then
if $data[1][0] != sma_index_name1 then
return -1
endi
if $data[1][1] != d1 then
return -1
endi
if $data[1][2] != stb then
return -1
endi
else
if $data[0][0] != sma_index_name1 then
return -1
endi
if $data[0][1] != d1 then
return -1
endi
if $data[0][2] != stb then
return -1
endi
endi
print --> drop stb

48
tests/script/tsim/tagindex/add_index.sim
View File

@ -7,7 +7,7 @@ print ======== step0
$dbPrefix = ta_3_db
$tbPrefix = ta_3_tb
$mtPrefix = ta_3_mt
$tbNum = 500
$tbNum = 50
$rowNum = 20
$totalNum = 200
@ -48,12 +48,16 @@ while $i < $tbNum
$i = $i + 1
endw
sql_error create index ti1 on $mtPrefix (t1)
sql create index ti2 on $mtPrefix (t2)
sql create index ti5 on $mtPrefix (t5)
print ==== test name conflict
#
sql_error create index ti1 on $mtPrefix(t1)
sql_error create index ti11 on $mtPrefix(t1)
sql_error create index ti3 on $mtPrefix(t2)
sql_error create index ti2 on $mtPrefix(t2)
@ -73,6 +77,15 @@ while $i < $tbNum
$i = $i + 1
endw
$i = 0
while $i < $tbNum
sql select * from $mtPrefix where t1= $i ;
if $rows != 1 then
return -1
endi
$i = $i + 1
endw
print ===== test operator great equal
@ -250,7 +263,7 @@ endw
print === show index
sql select * from information_schema.ins_indexes
if $rows != 1 then
if $rows != 2 then
return -1
endi
@ -259,12 +272,41 @@ print === drop index ti2
sql drop index ti2
print === drop not exist index
sql select * from information_schema.ins_indexes
if $rows != 1 then
return -1
endi
sql drop index $data[0][0]
if $rows != 0 then
return -1
endi
sql_error drop index t2
sql_error drop index t3
sql create index ti0 on $mtPrefix (t1)
$i = $interval
while $i < $limit
sql select * from $mtPrefix where t1 <= $i ;
$tmp = $i - $interval
$tmp = $tmp + 1
if $rows != $tmp then
return -1
endi
$i = $i + 1
endw
sql_error create index ti0 on $mtPrefix (t1)
sql_error create index ti2 on $mtPrefix (t1)
sql_error create index t2i on ta_3_tb17 (t2)

22
tests/script/tsim/tagindex/sma_and_tag_index.sim
View File

@ -69,7 +69,7 @@ sql create sma index smat2i on $mtPrefix function(max(c1)) interval(6m,10s) slid
sql select * from information_schema.ins_indexes
if $rows != 2 then
if $rows != 3 then
return -1
endi
@ -84,7 +84,7 @@ while $i < 5
endw
sql select * from information_schema.ins_indexes
if $rows != 6 then
if $rows != 7 then
return -1
endi
@ -114,13 +114,13 @@ sql use $dbPrefix
sql create table if not exists $mtPrefix (ts timestamp, c1 int) tags (t1 int, t2 int, t3 int, t4 int, t5 int)
sql create index tagt2i on $mtPrefix (t2)
sql select * from information_schema.ins_indexes
if $rows != 1 then
if $rows != 2 then
return -1
endi
sql alter table $mtPrefix drop tag t2
sql select * from information_schema.ins_indexes
if $rows != 0 then
if $rows != 1 then
return -1
endi
@ -128,18 +128,22 @@ endi
print ==== rename tag name, and update index colName
sql create index tagt3i on $mtPrefix (t3)
sql select * from information_schema.ins_indexes
if $rows != 1 then
if $rows != 2 then
return -1
endi
sql alter table $mtPrefix rename tag t3 txxx
sql select * from information_schema.ins_indexes
if $rows != 1 then
if $rows != 2 then
return -1
endi
if $data05 != txxx then
return -1
if $data05 == txxx then
print "manual created index"
elif $data15 == txxx then
print "auto created index at tag0"
else
return -1;
endi
@ -153,7 +157,7 @@ sql create table if not exists $mtPrefix (ts timestamp, c1 int) tags (t1 int, t2
sql create index tagt3i on $mtPrefix (t3)
sql select * from information_schema.ins_indexes
if $rows != 2 then
if $rows != 4 then
return -1
endi

14
tests/script/wtest.bat
View File

@ -17,6 +17,9 @@ rem echo SIM_DIR: %SIM_DIR%
set "TSIM_DIR=%SIM_DIR%tsim\"
rem echo TSIM_DIR: %TSIM_DIR%
set "DATA_DIR=%TSIM_DIR%data\"
rem echo DATA_DIR: %DATA_DIR%
set "CFG_DIR=%TSIM_DIR%cfg\"
rem echo CFG_DIR: %CFG_DIR%
@ -30,25 +33,30 @@ if not exist %SIM_DIR% mkdir %SIM_DIR%
if not exist %TSIM_DIR% mkdir %TSIM_DIR%
if exist %CFG_DIR% rmdir /s/q %CFG_DIR%
if exist %LOG_DIR% rmdir /s/q %LOG_DIR%
if exist %DATA_DIR% rmdir /s/q %DATA_DIR%
if not exist %CFG_DIR% mkdir %CFG_DIR%
if not exist %LOG_DIR% mkdir %LOG_DIR%
if not exist %DATA_DIR% mkdir %DATA_DIR%
set "fqdn=localhost"
for /f "skip=1" %%A in (
'wmic computersystem get caption'
) do if not defined fqdn set "fqdn=%%A"
echo firstEp %fqdn% > %TAOS_CFG%
echo firstEp %fqdn%:7100 > %TAOS_CFG%
echo secondEp %fqdn%:7200 >> %TAOS_CFG%
echo fqdn %fqdn% >> %TAOS_CFG%
echo serverPort 7100 >> %TAOS_CFG%
echo dataDir %DATA_DIR% >> %TAOS_CFG%
echo logDir %LOG_DIR% >> %TAOS_CFG%
echo scriptDir %SCRIPT_DIR% >> %TAOS_CFG%
echo numOfLogLines 100000000 >> %TAOS_CFG%
echo rpcDebugFlag 143 >> %TAOS_CFG%
echo tmrDebugFlag 131 >> %TAOS_CFG%
echo cDebugFlag 135 >> %TAOS_CFG%
echo cDebugFlag 143 >> %TAOS_CFG%
echo qDebugFlag 143 >> %TAOS_CFG%
echo udebugFlag 135 >> %TAOS_CFG%
echo uDebugFlag 143 >> %TAOS_CFG%
echo debugFlag 143 >> %TAOS_CFG%
echo wal 0 >> %TAOS_CFG%
echo asyncLog 0 >> %TAOS_CFG%
echo locale en_US.UTF-8 >> %TAOS_CFG%

20
tests/system-test/0-others/show_tag_index.py
View File

@ -59,14 +59,18 @@ class TDTestCase:
tdSql.checkData(1, 2, 2)
def check_indexes(self):
tdSql.checkRows(1)
tdSql.checkCols(7)
tdSql.checkData(0, 0, 'idx1')
tdSql.checkData(0, 1, 'db')
tdSql.checkData(0, 2, 'stb')
tdSql.checkData(0, 3, None)
tdSql.checkData(0, 5, 't1')
tdSql.checkData(0, 6, 'tag_index')
tdSql.checkRows(2)
for i in range(2):
col_name = tdSql.getData(i, 5)
if col_name == "t0":
continue
tdSql.checkCols(7)
tdSql.checkData(i, 0, 'idx1')
tdSql.checkData(i, 1, 'db')
tdSql.checkData(i, 2, 'stb')
tdSql.checkData(i, 3, None)
tdSql.checkData(i, 5, 't1')
tdSql.checkData(i, 6, 'tag_index')
def run(self):
tdSql.execute(f'create database db')

7
tests/system-test/0-others/tag_index_basic.py
View File

@ -118,12 +118,15 @@ class TDTestCase:
def show_tagidx(self, stbname):
sql = f'select index_name,column_name from information_schema.ins_indexes where db_name="db"'
tdSql.query(sql)
rows = len(self.tag_dict.keys())-1
rows = len(self.tag_dict.keys())
tdSql.checkRows(rows)
for i in range(rows):
col_name = tdSql.getData(i, 1)
idx_name = f'idx_{col_name}'
# skip first tag
if col_name == "t1":
continue
tdSql.checkData(i, 0, idx_name)
tdLog.info(f' show {rows} tag indexs ok.')
@ -201,7 +204,7 @@ class TDTestCase:
# check idx result is 0
sql = f'select index_name,column_name from information_schema.ins_indexes where db_name="db"'
tdSql.query(sql)
tdSql.checkRows(0)
tdSql.checkRows(1)
tdLog.info(f' drop {cnt} tag indexs ok.')
# create long name idx

100
tools/shell/src/shellEngine.c
View File

@ -18,9 +18,9 @@
#define _GNU_SOURCE
#define _XOPEN_SOURCE
#define _DEFAULT_SOURCE
#include "shellInt.h"
#include "shellAuto.h"
#include "geosWrapper.h"
#include "shellAuto.h"
#include "shellInt.h"
static bool shellIsEmptyCommand(const char *cmd);
static int32_t shellRunSingleCommand(char *command);
@ -41,9 +41,9 @@ static bool shellIsCommentLine(char *line);
static void shellSourceFile(const char *file);
static void shellGetGrantInfo();
static void shellCleanup(void *arg);
static void *shellCancelHandler(void *arg);
static void *shellThreadLoop(void *arg);
static void shellCleanup(void *arg);
static void *shellCancelHandler(void *arg);
static void *shellThreadLoop(void *arg);
bool shellIsEmptyCommand(const char *cmd) {
for (char c = *cmd++; c != 0; c = *cmd++) {
@ -66,7 +66,7 @@ int32_t shellRunSingleCommand(char *command) {
if (shellRegexMatch(command, "^[\t ]*clear[ \t;]*$", REG_EXTENDED | REG_ICASE)) {
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-result"
system("clear");
system("clear");
#pragma GCC diagnostic pop
return 0;
}
@ -142,8 +142,8 @@ int32_t shellRunCommand(char *command, bool recordHistory) {
return 0;
}
// add help or help;
if(strncasecmp(command, "help;", 5) == 0) {
// add help or help;
if (strncasecmp(command, "help;", 5) == 0) {
showHelp();
return 0;
}
@ -223,14 +223,14 @@ void shellRunSingleCommandImp(char *command) {
}
// pre string
char * pre = "Query OK";
char *pre = "Query OK";
if (shellRegexMatch(command, "^\\s*delete\\s*from\\s*.*", REG_EXTENDED | REG_ICASE)) {
pre = "Delete OK";
} else if(shellRegexMatch(command, "^\\s*insert\\s*into\\s*.*", REG_EXTENDED | REG_ICASE)) {
} else if (shellRegexMatch(command, "^\\s*insert\\s*into\\s*.*", REG_EXTENDED | REG_ICASE)) {
pre = "Insert OK";
} else if(shellRegexMatch(command, "^\\s*create\\s*.*", REG_EXTENDED | REG_ICASE)) {
} else if (shellRegexMatch(command, "^\\s*create\\s*.*", REG_EXTENDED | REG_ICASE)) {
pre = "Create OK";
} else if(shellRegexMatch(command, "^\\s*drop\\s*.*", REG_EXTENDED | REG_ICASE)) {
} else if (shellRegexMatch(command, "^\\s*drop\\s*.*", REG_EXTENDED | REG_ICASE)) {
pre = "Drop OK";
}
@ -295,7 +295,7 @@ char *shellFormatTimestamp(char *buf, int64_t val, int32_t precision) {
if (taosLocalTime(&tt, &ptm, buf) == NULL) {
return buf;
}
size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", &ptm);
size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", &ptm);
if (precision == TSDB_TIME_PRECISION_NANO) {
sprintf(buf + pos, ".%09d", ms);
@ -387,22 +387,20 @@ void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, i
break;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_JSON:
{
int32_t bufIndex = 0;
for (int32_t i = 0; i < length; i++) {
case TSDB_DATA_TYPE_JSON: {
int32_t bufIndex = 0;
for (int32_t i = 0; i < length; i++) {
buf[bufIndex] = val[i];
bufIndex++;
if (val[i] == '\"') {
buf[bufIndex] = val[i];
bufIndex++;
if (val[i] == '\"') {
buf[bufIndex] = val[i];
bufIndex++;
}
}
buf[bufIndex] = 0;
taosFprintfFile(pFile, "%s%s%s", quotationStr, buf, quotationStr);
}
break;
buf[bufIndex] = 0;
taosFprintfFile(pFile, "%s%s%s", quotationStr, buf, quotationStr);
} break;
case TSDB_DATA_TYPE_GEOMETRY:
shellDumpHexValue(buf, val, length);
taosFprintfFile(pFile, "%s", buf);
@ -535,12 +533,10 @@ void shellPrintString(const char *str, int32_t width) {
if (width == 0) {
printf("%s", str);
}
else if (len > width) {
} else if (len > width) {
if (width <= 3) {
printf("%.*s.", width - 1, str);
}
else {
} else {
printf("%.*s...", width - 3, str);
}
} else {
@ -549,7 +545,7 @@ void shellPrintString(const char *str, int32_t width) {
}
void shellPrintGeometry(const unsigned char *val, int32_t length, int32_t width) {
if (length == 0) { //empty value
if (length == 0) { // empty value
shellPrintString("", width);
return;
}
@ -565,7 +561,7 @@ void shellPrintGeometry(const unsigned char *val, int32_t length, int32_t width)
char *outputWKT = NULL;
code = doAsText(val, length, &outputWKT);
if (code != TSDB_CODE_SUCCESS) {
shellPrintString(getThreadLocalGeosCtx()->errMsg, width); //should NOT happen
shellPrintString(getThreadLocalGeosCtx()->errMsg, width); // should NOT happen
return;
}
@ -612,27 +608,26 @@ void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t
break;
case TSDB_DATA_TYPE_FLOAT:
if (tsEnableScience) {
printf("%*.7e",width,GET_FLOAT_VAL(val));
printf("%*.7e", width, GET_FLOAT_VAL(val));
} else {
n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.7f", width, GET_FLOAT_VAL(val));
if (n > SHELL_FLOAT_WIDTH) {
printf("%*.7e", width,GET_FLOAT_VAL(val));
printf("%*.7e", width, GET_FLOAT_VAL(val));
} else {
printf("%s", buf);
printf("%s", buf);
}
}
break;
case TSDB_DATA_TYPE_DOUBLE:
if (tsEnableScience) {
snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.15e", width,GET_DOUBLE_VAL(val));
snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.15e", width, GET_DOUBLE_VAL(val));
printf("%s", buf);
} else {
n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.15f", width, GET_DOUBLE_VAL(val));
if (n > SHELL_DOUBLE_WIDTH) {
printf("%*.15e", width, GET_DOUBLE_VAL(val));
printf("%*.15e", width, GET_DOUBLE_VAL(val));
} else {
printf("%*s", width,buf);
printf("%*s", width, buf);
}
}
break;
@ -905,7 +900,7 @@ void shellReadHistory() {
TdFilePtr pFile = taosOpenFile(pHistory->file, TD_FILE_READ | TD_FILE_STREAM);
if (pFile == NULL) return;
char *line = taosMemoryMalloc(TSDB_MAX_ALLOWED_SQL_LEN + 1);
char *line = taosMemoryMalloc(TSDB_MAX_ALLOWED_SQL_LEN + 1);
int32_t read_size = 0;
while ((read_size = taosGetsFile(pFile, TSDB_MAX_ALLOWED_SQL_LEN, line)) != -1) {
line[read_size - 1] = '\0';
@ -922,8 +917,8 @@ void shellReadHistory() {
taosMemoryFreeClear(line);
taosCloseFile(&pFile);
int64_t file_size;
if (taosStatFile(pHistory->file, &file_size, NULL) == 0 && file_size > SHELL_MAX_COMMAND_SIZE) {
TdFilePtr pFile = taosOpenFile(pHistory->file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_STREAM | TD_FILE_TRUNC);
if (taosStatFile(pHistory->file, &file_size, NULL, NULL) == 0 && file_size > SHELL_MAX_COMMAND_SIZE) {
TdFilePtr pFile = taosOpenFile(pHistory->file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_STREAM | TD_FILE_TRUNC);
if (pFile == NULL) return;
int32_t endIndex = pHistory->hstart;
if (endIndex != 0) {
@ -945,7 +940,7 @@ void shellReadHistory() {
void shellWriteHistory() {
SShellHistory *pHistory = &shell.history;
if (pHistory->hend == pHistory->hstart) return;
TdFilePtr pFile = taosOpenFile(pHistory->file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_STREAM | TD_FILE_APPEND);
TdFilePtr pFile = taosOpenFile(pHistory->file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_STREAM | TD_FILE_APPEND);
if (pFile == NULL) return;
for (int32_t i = pHistory->hstart; i != pHistory->hend;) {
@ -991,7 +986,7 @@ void shellSourceFile(const char *file) {
tstrncpy(fullname, file, PATH_MAX);
}
sprintf(sourceFileCommand, "source %s;",fullname);
sprintf(sourceFileCommand, "source %s;", fullname);
shellRecordCommandToHistory(sourceFileCommand);
TdFilePtr pFile = taosOpenFile(fullname, TD_FILE_READ | TD_FILE_STREAM);
@ -1001,7 +996,7 @@ void shellSourceFile(const char *file) {
return;
}
char *line = taosMemoryMalloc(TSDB_MAX_ALLOWED_SQL_LEN + 1);
char *line = taosMemoryMalloc(TSDB_MAX_ALLOWED_SQL_LEN + 1);
while ((read_len = taosGetsFile(pFile, TSDB_MAX_ALLOWED_SQL_LEN, line)) != -1) {
if (read_len >= TSDB_MAX_ALLOWED_SQL_LEN) continue;
line[--read_len] = '\0';
@ -1044,7 +1039,8 @@ void shellGetGrantInfo() {
int32_t code = taos_errno(tres);
if (code != TSDB_CODE_SUCCESS) {
if (code != TSDB_CODE_OPS_NOT_SUPPORT && code != TSDB_CODE_MND_NO_RIGHTS && code != TSDB_CODE_PAR_PERMISSION_DENIED) {
if (code != TSDB_CODE_OPS_NOT_SUPPORT && code != TSDB_CODE_MND_NO_RIGHTS &&
code != TSDB_CODE_PAR_PERMISSION_DENIED) {
fprintf(stderr, "Failed to check Server Edition, Reason:0x%04x:%s\r\n\r\n", code, taos_errstr(tres));
}
return;
@ -1080,7 +1076,8 @@ void shellGetGrantInfo() {
} else if (strcmp(expiretime, "unlimited") == 0) {
fprintf(stdout, "Server is Enterprise %s Edition, %s and will never expire.\r\n", serverVersion, sinfo);
} else {
fprintf(stdout, "Server is Enterprise %s Edition, %s and will expire at %s.\r\n", serverVersion, sinfo, expiretime);
fprintf(stdout, "Server is Enterprise %s Edition, %s and will expire at %s.\r\n", serverVersion, sinfo,
expiretime);
}
taos_free_result(tres);
@ -1123,9 +1120,9 @@ void *shellCancelHandler(void *arg) {
#ifdef WEBSOCKET
}
#endif
#ifdef WINDOWS
#ifdef WINDOWS
printf("\n%s", shell.info.promptHeader);
#endif
#endif
}
return NULL;
@ -1165,8 +1162,7 @@ void *shellThreadLoop(void *arg) {
}
int32_t shellExecute() {
printf(shell.info.clientVersion, shell.info.cusName,
taos_get_client_info(), shell.info.cusName);
printf(shell.info.clientVersion, shell.info.cusName, taos_get_client_info(), shell.info.cusName);
fflush(stdout);
SShellArgs *pArgs = &shell.args;
@ -1233,13 +1229,13 @@ int32_t shellExecute() {
taosSetSignal(SIGTERM, shellQueryInterruptHandler);
taosSetSignal(SIGHUP, shellQueryInterruptHandler);
taosSetSignal(SIGINT, shellQueryInterruptHandler);
#ifdef WEBSOCKET
if (!shell.args.restful && !shell.args.cloud) {
#endif
#ifndef WINDOWS
printfIntroduction();
#endif
#endif
shellGetGrantInfo();
#ifdef WEBSOCKET
}

2
utils/test/c/CMakeLists.txt
View File

@ -31,7 +31,7 @@ target_link_libraries(
)
target_link_libraries(
tmq_sim
PUBLIC taos
PUBLIC ${TAOS_LIB}
PUBLIC util
PUBLIC common
PUBLIC os

3
utils/test/c/tmqDemo.c
View File

@ -221,7 +221,7 @@ int64_t getDirectorySize(char* dir) {
totalSize += subDirSize;
} else if (0 == strcmp(strchr(fileName, '.'), ".log")) { // only calc .log file size, and not include .idx file
int64_t file_size = 0;
taosStatFile(subdir, &file_size, NULL);
taosStatFile(subdir, &file_size, NULL, NULL);
totalSize += file_size;
}
}
@ -702,4 +702,3 @@ int main(int32_t argc, char* argv[]) {
taosCloseFile(&g_fp);
return 0;
}

2
utils/tsim/CMakeLists.txt
View File

@ -2,7 +2,7 @@ aux_source_directory(src TSIM_SRC)
add_executable(tsim ${TSIM_SRC})
target_link_libraries(
tsim
PUBLIC taos_static
PUBLIC ${TAOS_LIB}
PUBLIC util
PUBLIC common
PUBLIC os