xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' into fix/TD-29995-3.0

This commit is contained in:
Minglei Jin 2024-05-17 10:17:33 +08:00
parent 5c952fc922 56b2ccebf6
commit 3b8c27764d
87 changed files with 6347 additions and 2803 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
cmake/cmake.options
View File

@ -47,27 +47,20 @@ IF(${TD_WINDOWS})
MESSAGE("build wingetopt Win32")
option(
BUILD_WINGETOPT
BUILD_WINGETOPT
"If build wingetopt on Windows"
ON
)
option(
TDENGINE_3
"TDengine 3.x for taos-tools"
ON
)
option(
BUILD_CRASHDUMP
"If build crashdump on Windows"
ON
)
MESSAGE("build geos Win32")
option(
BUILD_GEOS
"If build geos on Windows"
"If build crashdump on Windows"
ON
)
@ -79,7 +72,7 @@ ENDIF ()
option(
BUILD_GEOS
"If build geos on Windows"
"If build with geos"
ON
)
@ -95,6 +88,12 @@ option(
ON
)
option(
BUILD_PCRE2
"If build with pcre2"
ON
)
option(
JEMALLOC_ENABLED
"If build with jemalloc"
@ -114,14 +113,14 @@ option(
)
option(
BUILD_WITH_LEVELDB
"If build with leveldb"
BUILD_WITH_LEVELDB
"If build with leveldb"
OFF
)
option(
BUILD_WITH_ROCKSDB
"If build with rocksdb"
BUILD_WITH_ROCKSDB
"If build with rocksdb"
ON
)
@ -170,46 +169,46 @@ ENDIF ()
option(
BUILD_WITH_SQLITE
"If build with sqlite"
"If build with sqlite"
OFF
)
option(
BUILD_WITH_BDB
"If build with BDB"
"If build with BDB"
OFF
)
option(
BUILD_WITH_LUCENE
"If build with lucene"
off
BUILD_WITH_LUCENE
"If build with lucene"
off
)
option(
BUILD_WITH_NURAFT
"If build with NuRaft"
"If build with NuRaft"
OFF
)
option(
BUILD_WITH_UV
"If build with libuv"
ON
"If build with libuv"
ON
)
option(
BUILD_WITH_UV_TRANS
"If build with libuv_trans "
ON
"If build with libuv_trans "
ON
)
IF(${TD_LINUX} MATCHES TRUE)
option(
BUILD_DEPENDENCY_TESTS
"If build dependency tests"
BUILD_DEPENDENCY_TESTS
"If build dependency tests"
ON
)
@ -217,14 +216,14 @@ ENDIF ()
option(
BUILD_DOCS
"If use doxygen build documents"
"If use doxygen build documents"
OFF
)
option(
BUILD_WITH_INVERTEDINDEX
"If use invertedIndex"
ON
"If use invertedIndex"
ON
)
option(

13
cmake/pcre2_CMakeLists.txt.in Normal file
View File

@ -0,0 +1,13 @@
# pcre2
ExternalProject_Add(pcre2
GIT_REPOSITORY https://github.com/PCRE2Project/pcre2.git
GIT_TAG pcre2-10.43
SOURCE_DIR "${TD_CONTRIB_DIR}/pcre2"
#BINARY_DIR "${TD_CONTRIB_DIR}/pcre2"
#BUILD_IN_SOURCE TRUE
CONFIGURE_COMMAND ""
BUILD_COMMAND ""
INSTALL_COMMAND ""
TEST_COMMAND ""
)

21
contrib/CMakeLists.txt
View File

@ -121,7 +121,7 @@ if (${BUILD_CONTRIB})
cat("${TD_SUPPORT_DIR}/rocksdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
add_definitions(-DUSE_ROCKSDB)
endif()
else()
else()
if (NOT ${TD_LINUX})
if(${BUILD_WITH_ROCKSDB})
cat("${TD_SUPPORT_DIR}/rocksdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
@ -183,6 +183,11 @@ if(${BUILD_GEOS})
cat("${TD_SUPPORT_DIR}/geos_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
endif()
#
if(${BUILD_PCRE2})
cat("${TD_SUPPORT_DIR}/pcre2_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
endif()
# download dependencies
configure_file(${CONTRIB_TMP_FILE} "${TD_CONTRIB_DIR}/deps-download/CMakeLists.txt")
execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" .
@ -270,8 +275,8 @@ unset(CMAKE_PROJECT_INCLUDE_BEFORE)
# add_subdirectory(xz EXCLUDE_FROM_ALL)
# target_include_directories(
# xz
# PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/xz
# PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/xz
# PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/xz
# PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/xz
# )
# leveldb
@ -351,7 +356,7 @@ if (${BUILD_WITH_ROCKSDB})
rocksdb
PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/rocksdb/include>
)
else()
else()
if (NOT ${TD_LINUX})
MESSAGE(STATUS "ROCKSDB CXX STATUS CONFIG: " ${CMAKE_CXX_FLAGS})
MESSAGE(STATUS "ROCKSDB C STATUS CONFIG: " ${CMAKE_C_FLAGS})
@ -400,8 +405,8 @@ if (${BUILD_WITH_ROCKSDB})
rocksdb
PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/rocksdb/include>
)
endif()
endif()
endif()
endif()
@ -605,6 +610,10 @@ if(${BUILD_GEOS})
)
endif(${BUILD_GEOS})
if (${BUILD_PCRE2})
add_subdirectory(pcre2 EXCLUDE_FROM_ALL)
endif(${BUILD_PCRE2})
# ================================================================================================
# Build test
# ================================================================================================

13
include/common/tdataformat.h
View File

@ -95,6 +95,15 @@ const static uint8_t BIT2_MAP[4] = {0b11111100, 0b11110011, 0b11001111, 0b001111
#define COL_VAL_IS_NULL(CV) ((CV)->flag == CV_FLAG_NULL)
#define COL_VAL_IS_VALUE(CV) ((CV)->flag == CV_FLAG_VALUE)
#define tRowGetKey(_pRow, _pKey) \
do { \
(_pKey)->ts = (_pRow)->ts; \
(_pKey)->numOfPKs = 0; \
if ((_pRow)->numOfPKs > 0) { \
tRowGetPrimaryKey((_pRow), (_pKey)); \
} \
} while (0)
// SValueColumn ================================
typedef struct {
int8_t cmprAlg; // filled by caller
@ -125,8 +134,8 @@ void tRowDestroy(SRow *pRow);
int32_t tRowSort(SArray *aRowP);
int32_t tRowMerge(SArray *aRowP, STSchema *pTSchema, int8_t flag);
int32_t tRowUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData, int32_t flag);
void tRowGetKey(SRow *pRow, SRowKey *key);
int32_t tRowKeyCompare(const void *p1, const void *p2);
void tRowGetPrimaryKey(SRow *pRow, SRowKey *key);
int32_t tRowKeyCompare(const SRowKey *key1, const SRowKey *key2);
int32_t tRowKeyAssign(SRowKey *pDst, SRowKey *pSrc);
// SRowIter ================================

2
include/common/tglobal.h
View File

@ -258,7 +258,7 @@ extern bool tsExperimental;
int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd,
const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc);
int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs,
bool tsc);
bool tsc, bool isDumpCfg);
void taosCleanupCfg();
int32_t taosCfgDynamicOptions(SConfig *pCfg, const char *name, bool forServer);

4
include/common/tgrant.h
View File

@ -56,6 +56,10 @@ typedef enum {
TSDB_GRANT_VIEW,
TSDB_GRANT_MULTI_TIER,
TSDB_GRANT_BACKUP_RESTORE,
TSDB_GRANT_OBJECT_STORAGE,
TSDB_GRANT_ACTIVE_ACTIVE,
TSDB_GRANT_DUAL_REPLICA_HA,
TSDB_GRANT_DB_ENCRYPTION,
} EGrantType;
int32_t checkAndGetCryptKey(const char *encryptCode, const char *machineId, char **key);

4
include/common/tmsg.h
View File

@ -329,6 +329,7 @@ typedef enum ENodeType {
QUERY_NODE_SHOW_DB_ALIVE_STMT,
QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT,
QUERY_NODE_BALANCE_VGROUP_LEADER_STMT,
QUERY_NODE_BALANCE_VGROUP_LEADER_DATABASE_STMT,
QUERY_NODE_RESTORE_DNODE_STMT,
QUERY_NODE_RESTORE_QNODE_STMT,
QUERY_NODE_RESTORE_MNODE_STMT,
@ -2425,10 +2426,11 @@ int32_t tDeserializeSRedistributeVgroupReq(void* buf, int32_t bufLen, SRedistrib
void tFreeSRedistributeVgroupReq(SRedistributeVgroupReq* pReq);
typedef struct {
int32_t useless;
int32_t reserved;
int32_t vgId;
int32_t sqlLen;
char* sql;
char db[TSDB_DB_FNAME_LEN];
} SBalanceVgroupLeaderReq;
int32_t tSerializeSBalanceVgroupLeaderReq(void* buf, int32_t bufLen, SBalanceVgroupLeaderReq* pReq);

1
include/libs/nodes/cmdnodes.h
View File

@ -582,6 +582,7 @@ typedef struct SBalanceVgroupStmt {
typedef struct SBalanceVgroupLeaderStmt {
ENodeType type;
int32_t vgId;
char dbName[TSDB_DB_NAME_LEN];
} SBalanceVgroupLeaderStmt;
typedef struct SMergeVgroupStmt {

28
include/os/osSemaphore.h
View File

@ -33,6 +33,13 @@ int tsem_timewait(tsem_t *sim, int64_t milis);
int tsem_post(tsem_t *sem);
int tsem_destroy(tsem_t *sem);
#define tsem2_t tsem_t
#define tsem2_init tsem_init
#define tsem2_wait tsem_wait
#define tsem2_timewait tsem_timewait
#define tsem2_post tsem_post
#define tsem2_destroy tsem_destroy
#elif defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
#include <windows.h>
@ -44,6 +51,13 @@ int tsem_timewait(tsem_t *sim, int64_t milis);
int tsem_post(tsem_t *sem);
int tsem_destroy(tsem_t *sem);
#define tsem2_t tsem_t
#define tsem2_init tsem_init
#define tsem2_wait tsem_wait
#define tsem2_timewait tsem_timewait
#define tsem2_post tsem_post
#define tsem2_destroy tsem_destroy
#else
#define tsem_t sem_t
@ -53,6 +67,20 @@ int tsem_timewait(tsem_t *sim, int64_t milis);
#define tsem_post sem_post
#define tsem_destroy sem_destroy
typedef struct tsem2_t {
TdThreadMutex mutex;
TdThreadCond cond;
TdThreadCondAttr attr;
int count;
} tsem2_t;
// #define tsem2_t sem_t
int tsem2_init(tsem2_t* sem, int pshared, unsigned int value);
int tsem2_wait(tsem2_t* sem);
int tsem2_timewait(tsem2_t* sem, int64_t milis);
int tsem2_post(tsem2_t* sem);
int tsem2_destroy(tsem2_t* sem);
#endif
#if defined(_TD_DARWIN_64)

1
include/os/osThread.h
View File

@ -222,6 +222,7 @@ int32_t taosThreadCondTimedWait(TdThreadCond *cond, TdThreadMutex *mutex, const
int32_t taosThreadCondAttrDestroy(TdThreadCondAttr *attr);
int32_t taosThreadCondAttrGetPshared(const TdThreadCondAttr *attr, int32_t *pshared);
int32_t taosThreadCondAttrInit(TdThreadCondAttr *attr);
int32_t taosThreadCondAttrSetclock(TdThreadCondAttr *attr, int clockId);
int32_t taosThreadCondAttrSetPshared(TdThreadCondAttr *attr, int32_t pshared);
int32_t taosThreadDetach(TdThread thread);
int32_t taosThreadEqual(TdThread t1, TdThread t2);

1
include/util/taoserror.h
View File

@ -327,6 +327,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_MND_DB_IN_CREATING TAOS_DEF_ERROR_CODE(0, 0x0396) //
#define TSDB_CODE_MND_INVALID_SYS_TABLENAME TAOS_DEF_ERROR_CODE(0, 0x039A)
#define TSDB_CODE_MND_ENCRYPT_NOT_ALLOW_CHANGE TAOS_DEF_ERROR_CODE(0, 0x039B)
#define TSDB_CODE_MND_DB_ENCRYPT_GRANT_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x039C)
// mnode-node
#define TSDB_CODE_MND_MNODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03A0)

4
include/util/tgeosctx.h
View File

@ -21,6 +21,7 @@ extern "C" {
#endif
#include <geos_c.h>
#include <tpcre2.h>
typedef struct SGeosContext {
GEOSContextHandle_t handle;
@ -31,6 +32,9 @@ typedef struct SGeosContext {
GEOSWKBReader *WKBReader;
GEOSWKBWriter *WKBWriter;
pcre2_code *WKTRegex;
pcre2_match_data *WKTMatchData;
char errMsg[512];
} SGeosContext;

34
include/util/tpcre2.h Normal file
View File

@ -0,0 +1,34 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_ULIT_PCRE2_H_
#define _TD_ULIT_PCRE2_H_
#define PCRE2_CODE_UNIT_WIDTH 8
#include "pcre2.h"
#ifdef __cplusplus
extern "C" {
#endif
int32_t doRegComp(pcre2_code** ppRegex, pcre2_match_data** ppMatchData, const char* pattern);
int32_t doRegExec(const char* pString, pcre2_code* pRegex, pcre2_match_data* pMatchData);
void destroyRegexes(pcre2_code* pWktRegex, pcre2_match_data* pWktMatchData);
#ifdef __cplusplus
}
#endif
#endif // _TD_UTIL_PAGEDBUF_H_

2
source/client/src/clientEnv.c
View File

@ -725,7 +725,7 @@ void taos_init_imp(void) {
return;
}
if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 1) != 0) {
if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 1, true) != 0) {
tscInitRes = -1;
return;
}

18
source/client/src/clientTmq.c
View File

@ -105,7 +105,7 @@ struct tmq_t {
STaosQueue* mqueue; // queue of rsp
STaosQall* qall;
STaosQueue* delayedTask; // delayed task queue for heartbeat and auto commit
tsem_t rspSem;
tsem2_t rspSem;
};
typedef struct SAskEpInfo {
@ -727,7 +727,7 @@ static void generateTimedTask(int64_t refId, int32_t type) {
*pTaskType = type;
taosWriteQitem(tmq->delayedTask, pTaskType);
tsem_post(&tmq->rspSem);
tsem2_post(&tmq->rspSem);
taosReleaseRef(tmqMgmt.rsetId, refId);
}
@ -742,7 +742,7 @@ void tmqReplayTask(void* param, void* tmrId) {
tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId);
if (tmq == NULL) goto END;
tsem_post(&tmq->rspSem);
tsem2_post(&tmq->rspSem);
taosReleaseRef(tmqMgmt.rsetId, refId);
END:
taosMemoryFree(param);
@ -1033,7 +1033,7 @@ void tmqFreeImpl(void* handle) {
}
taosFreeQall(tmq->qall);
tsem_destroy(&tmq->rspSem);
tsem2_destroy(&tmq->rspSem);
taosArrayDestroyEx(tmq->clientTopics, freeClientVgImpl);
taos_close_internal(tmq->pTscObj);
@ -1121,7 +1121,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
pTmq->consumerId = tGenIdPI64();
// init semaphore
if (tsem_init(&pTmq->rspSem, 0, 0) != 0) {
if (tsem2_init(&pTmq->rspSem, 0, 0) != 0) {
tscError("consumer:0x %" PRIx64 " setup failed since %s, consumer group %s", pTmq->consumerId, terrstr(),
pTmq->groupId);
SET_ERROR_MSG_TMQ("init t_sem failed")
@ -1132,7 +1132,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
pTmq->pTscObj = taos_connect_internal(conf->ip, user, pass, NULL, NULL, conf->port, CONN_TYPE__TMQ);
if (pTmq->pTscObj == NULL) {
tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, terrstr(), pTmq->groupId);
tsem_destroy(&pTmq->rspSem);
tsem2_destroy(&pTmq->rspSem);
SET_ERROR_MSG_TMQ("init tscObj failed")
goto _failed;
}
@ -1434,7 +1434,7 @@ END:
taosReleaseRef(tmqMgmt.rsetId, refId);
FAIL:
if (tmq) tsem_post(&tmq->rspSem);
if (tmq) tsem2_post(&tmq->rspSem);
taosMemoryFree(pParam);
if (pMsg) taosMemoryFreeClear(pMsg->pData);
if (pMsg) taosMemoryFreeClear(pMsg->pEpSet);
@ -2132,10 +2132,10 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) {
tmq->consumerId, tmq->epoch, startTime, currentTime);
return NULL;
}
tsem_timewait(&tmq->rspSem, (timeout - elapsedTime));
tsem2_timewait(&tmq->rspSem, (timeout - elapsedTime));
} else {
// use tsem_timewait instead of tsem_wait to avoid unexpected stuck
tsem_timewait(&tmq->rspSem, 1000);
tsem2_timewait(&tmq->rspSem, 1000);
}
}
}

5
source/common/src/cos.c
View File

@ -63,6 +63,11 @@ static int32_t s3ListBucket(char const *bucketname);
int32_t s3CheckCfg() {
int32_t code = 0;
if (!tsS3Enabled) {
fprintf(stderr, "s3 not configured.\n");
goto _exit;
}
code = s3Begin();
if (code != 0) {
fprintf(stderr, "failed to initialize s3.\n");

7
source/common/src/tcol.c
View File

@ -250,7 +250,7 @@ bool checkColumnEncodeOrSetDefault(uint8_t type, char encode[TSDB_CL_COMPRESS_OP
strncpy(encode, getDefaultEncodeStr(type), TSDB_CL_COMPRESS_OPTION_LEN);
return true;
}
return checkColumnEncode(encode);
return checkColumnEncode(encode) && validColEncode(type, columnEncodeVal(encode));
}
bool checkColumnCompress(char compress[TSDB_CL_COMPRESS_OPTION_LEN]) {
if (0 == strlen(compress)) return true;
@ -267,7 +267,8 @@ bool checkColumnCompressOrSetDefault(uint8_t type, char compress[TSDB_CL_COMPRES
strncpy(compress, getDefaultCompressStr(type), TSDB_CL_COMPRESS_OPTION_LEN);
return true;
}
return checkColumnCompress(compress);
return checkColumnCompress(compress) && validColCompress(type, columnCompressVal(compress));
}
bool checkColumnLevel(char level[TSDB_CL_COMPRESS_OPTION_LEN]) {
if (0 == strlen(level)) return true;
@ -288,7 +289,7 @@ bool checkColumnLevelOrSetDefault(uint8_t type, char level[TSDB_CL_COMPRESS_OPTI
strncpy(level, getDefaultLevelStr(type), TSDB_CL_COMPRESS_OPTION_LEN);
return true;
}
return checkColumnLevel(level);
return checkColumnLevel(level) && validColCompressLevel(type, columnLevelVal(level));
}
void setColEncode(uint32_t* compress, uint8_t l1) {

19
source/common/src/tdatablock.c
View File

@ -2411,13 +2411,24 @@ _end:
void buildCtbNameAddGroupId(const char* stbName, char* ctbName, uint64_t groupId) {
char tmp[TSDB_TABLE_NAME_LEN] = {0};
if (stbName == NULL) {
snprintf(tmp, TSDB_TABLE_NAME_LEN, "_%" PRIu64, groupId);
} else {
snprintf(tmp, TSDB_TABLE_NAME_LEN, "_%s_%" PRIu64, stbName, groupId);
if (stbName == NULL){
snprintf(tmp, TSDB_TABLE_NAME_LEN, "_%"PRIu64, groupId);
}else{
int32_t i = strlen(stbName) - 1;
for(; i >= 0; i--){
if (stbName[i] == '.'){
break;
}
}
snprintf(tmp, TSDB_TABLE_NAME_LEN, "_%s_%"PRIu64, stbName + i + 1, groupId);
}
ctbName[TSDB_TABLE_NAME_LEN - strlen(tmp) - 1] = 0; // put stbname + groupId to the end
strcat(ctbName, tmp);
for(int i = 0; i < strlen(ctbName); i++){
if(ctbName[i] == '.'){
ctbName[i] = '_';
}
}
}
// auto stream subtable name starts with 't_', followed by the first segment of MD5 digest for group vals.

8
source/common/src/tdataformat.c
View File

@ -1198,8 +1198,7 @@ int32_t tRowUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, in
}
}
void tRowGetKey(SRow *row, SRowKey *key) {
key->ts = row->ts;
void tRowGetPrimaryKey(SRow *row, SRowKey *key) {
key->numOfPKs = row->numOfPKs;
if (key->numOfPKs == 0) {
@ -1298,10 +1297,7 @@ int32_t tValueCompare(const SValue *tv1, const SValue *tv2) {
// NOTE:
// set key->numOfPKs to 0 as the smallest key with ts
// set key->numOfPKs to (TD_MAX_PK_COLS + 1) as the largest key with ts
int32_t tRowKeyCompare(const void *p1, const void *p2) {
SRowKey *key1 = (SRowKey *)p1;
SRowKey *key2 = (SRowKey *)p2;
FORCE_INLINE int32_t tRowKeyCompare(const SRowKey *key1, const SRowKey *key2) {
if (key1->ts < key2->ts) {
return -1;
} else if (key1->ts > key2->ts) {

12
source/common/src/tglobal.c
View File

@ -240,8 +240,8 @@ int64_t tsTickPerMin[] = {60000L, 60000000L, 60000000000L};
int64_t tsTickPerHour[] = {3600000L, 3600000000L, 3600000000000L};
// lossy compress 7
char tsLossyColumns[32] = ""; // "float|double" means all float and double columns can be lossy compressed. set empty
// can close lossy compress.
char tsLossyColumns[32] = "float|double"; // "float|double" means all float and double columns can be lossy compressed.
// set empty can close lossy compress.
// below option can take effect when tsLossyColumns not empty
float tsFPrecision = 1E-8; // float column precision
double tsDPrecision = 1E-16; // double column precision
@ -296,6 +296,7 @@ char tsS3AccessKeySecret[TSDB_FQDN_LEN] = "<accesskeysecrect>";
char tsS3BucketName[TSDB_FQDN_LEN] = "<bucketname>";
char tsS3AppId[TSDB_FQDN_LEN] = "<appid>";
int8_t tsS3Enabled = false;
int8_t tsS3EnabledCfg = false;
int8_t tsS3Oss = false;
int8_t tsS3StreamEnabled = false;
@ -375,6 +376,7 @@ int32_t taosSetS3Cfg(SConfig *pCfg) {
#if defined(USE_COS) || defined(USE_S3)
#ifdef TD_ENTERPRISE
/*if (tsDiskCfgNum > 1) */ tsS3Enabled = true;
tsS3EnabledCfg = true;
#endif
tsS3StreamEnabled = true;
#endif
@ -813,7 +815,6 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
CFG_DYN_ENT_SERVER) != 0)
return -1;
if (cfgAddString(pCfg, "lossyColumns", tsLossyColumns, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddFloat(pCfg, "fPrecision", tsFPrecision, 0.0f, 100000.0f, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddFloat(pCfg, "dPrecision", tsDPrecision, 0.0f, 1000000.0f, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddInt32(pCfg, "maxRange", tsMaxRange, 0, 65536, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
@ -1274,7 +1275,6 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
tsCacheLazyLoadThreshold = cfgGetItem(pCfg, "cacheLazyLoadThreshold")->i32;
tstrncpy(tsLossyColumns, cfgGetItem(pCfg, "lossyColumns")->str, sizeof(tsLossyColumns));
tsFPrecision = cfgGetItem(pCfg, "fPrecision")->fval;
tsDPrecision = cfgGetItem(pCfg, "dPrecision")->fval;
tsMaxRange = cfgGetItem(pCfg, "maxRange")->i32;
@ -1394,7 +1394,7 @@ static int32_t taosCheckGlobalCfg() {
}
int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs,
bool tsc) {
bool tsc, bool isDumpCfg) {
if (tsCfg != NULL) return 0;
tsCfg = cfgInit();
@ -1441,7 +1441,7 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile
taosSetAllDebugFlag(tsCfg, cfgGetItem(tsCfg, "debugFlag")->i32);
cfgDumpCfg(tsCfg, tsc, false);
if(isDumpCfg) cfgDumpCfg(tsCfg, tsc, false);
if (taosCheckGlobalCfg() != 0) {
return -1;

4
source/common/src/tmisce.c
View File

@ -228,6 +228,8 @@ int32_t dumpConfToDataBlock(SSDataBlock* pBlock, int32_t startCol) {
blockDataEnsureCapacity(pBlock, cfgGetSize(pConf));
SConfigIter* pIter = cfgCreateIter(pConf);
cfgLock(pConf);
while ((pItem = cfgNextIter(pIter)) != NULL) {
col = startCol;
@ -253,6 +255,8 @@ int32_t dumpConfToDataBlock(SSDataBlock* pBlock, int32_t startCol) {
numOfRows++;
}
cfgUnLock(pConf);
pBlock->info.rows = numOfRows;
cfgDestroyIter(pIter);

11
source/common/src/tmsg.c
View File

@ -5962,9 +5962,11 @@ int32_t tSerializeSBalanceVgroupLeaderReq(void *buf, int32_t bufLen, SBalanceVgr
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeI32(&encoder, pReq->useless) < 0) return -1;
if (tEncodeI32(&encoder, pReq->reserved) < 0) return -1;
if (tEncodeI32(&encoder, pReq->vgId) < 0) return -1;
ENCODESQL();
if (tEncodeCStr(&encoder, pReq->db) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
@ -5977,12 +5979,15 @@ int32_t tDeserializeSBalanceVgroupLeaderReq(void *buf, int32_t bufLen, SBalanceV
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->useless) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->reserved) < 0) return -1;
if (!tDecodeIsEnd(&decoder)) {
if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1;
}
DECODESQL();
if (!tDecodeIsEnd(&decoder)) {
if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1;
}
tEndDecode(&decoder);
tDecoderClear(&decoder);

40
source/common/test/dataformatTest.cpp
View File

@ -20,6 +20,7 @@
#include <tglobal.h>
#include <tmsg.h>
#include <iostream>
#include <tdatablock.h>
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wwrite-strings"
@ -475,6 +476,45 @@ TEST(testCase, AllNormTest) {
taosMemoryFree(pTSchema);
}
TEST(testCase, StreamAllNormTest) {
char ctbName[TSDB_TABLE_NAME_LEN] = {0};
uint64_t groupId = 12345;
buildCtbNameAddGroupId(NULL, ctbName, groupId);
ASSERT_STREQ("_12345", ctbName);
}
TEST(testCase, StreamWithStbName) {
char stbName[] = "1.table.stb";
char ctbName[TSDB_TABLE_NAME_LEN] = {0};
uint64_t groupId = 12345;
buildCtbNameAddGroupId(stbName, ctbName, groupId);
ASSERT_STREQ("_stb_12345", ctbName);
}
TEST(testCase, StreamWithoutDotInStbName) {
char stbName[] = "table";
char ctbName[TSDB_TABLE_NAME_LEN] = {0};
uint64_t groupId = 12345;
buildCtbNameAddGroupId(stbName, ctbName, groupId);
ASSERT_STREQ("_table_12345", ctbName);
}
TEST(testCase, StreamWithoutDotInStbName2) {
char stbName[] = "";
char ctbName[TSDB_TABLE_NAME_LEN] = {0};
uint64_t groupId = 12345;
buildCtbNameAddGroupId(stbName, ctbName, groupId);
ASSERT_STREQ("__12345", ctbName);
}
#if 1
TEST(testCase, NoneTest) {
const static int nCols = 14;

12
source/dnode/mgmt/exe/dmMain.c
View File

@ -205,11 +205,11 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) {
if(i < argc - 1) {
int32_t len = strlen(argv[++i]);
if (len < ENCRYPT_KEY_LEN_MIN) {
printf("encrypt key is too short, it should be great or equal to %d\n", ENCRYPT_KEY_LEN_MIN);
printf("Error: Encrypt key should be at least %d characters\n", ENCRYPT_KEY_LEN_MIN);
return -1;
}
if (len > ENCRYPT_KEY_LEN) {
printf("encrypt key overflow, it should be less or equal to %d\n", ENCRYPT_KEY_LEN);
printf("Error: Encrypt key overflow, it should be at most %d characters\n", ENCRYPT_KEY_LEN);
return -1;
}
tstrncpy(global.encryptKey, argv[i], ENCRYPT_KEY_LEN);
@ -325,7 +325,7 @@ int main(int argc, char const *argv[]) {
}
if (dmParseArgs(argc, argv) != 0) {
printf("failed to start since parse args error\n");
//printf("failed to start since parse args error\n");
taosCleanupArgs();
return -1;
}
@ -380,7 +380,11 @@ int mainWindows(int argc, char **argv) {
dmPrintArgs(argc, argv);
if (taosInitCfg(configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, 0) != 0) {
bool isDumpCfg = true;
if(global.generateCode) {
isDumpCfg = false;
}
if (taosInitCfg(configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, 0, isDumpCfg) != 0) {
dError("failed to start since read config error");
taosCloseLog();
taosCleanupArgs();

2
source/dnode/mgmt/node_mgmt/src/dmEnv.c
View File

@ -17,6 +17,7 @@
#include "dmMgmt.h"
#include "audit.h"
#include "libs/function/tudf.h"
#include "tgrant.h"
#define DM_INIT_AUDIT() \
do { \
@ -150,6 +151,7 @@ static bool dmCheckDataDirVersion() {
extern int32_t s3Begin();
extern void s3End();
extern int8_t tsS3Enabled;
#endif

10
source/dnode/mgmt/node_mgmt/src/dmMgmt.c
View File

@ -18,12 +18,10 @@
#include "dmNodes.h"
#include "index.h"
#include "qworker.h"
#include "tstream.h"
#ifdef TD_TSZ
#include "tcompression.h"
#include "tglobal.h"
#include "tgrant.h"
#endif
#include "tstream.h"
static bool dmRequireNode(SDnode *pDnode, SMgmtWrapper *pWrapper) {
SMgmtInputOpt input = dmBuildMgmtInputOpt(pWrapper);
@ -48,10 +46,8 @@ int32_t dmInitDnode(SDnode *pDnode) {
goto _OVER;
}
#ifdef TD_TSZ
// compress module init
tsCompressInit(tsLossyColumns, tsFPrecision, tsDPrecision, tsMaxRange, tsCurRange, (int)tsIfAdtFse, tsCompressor);
#endif
pDnode->wrappers[DNODE].func = dmGetMgmtFunc();
pDnode->wrappers[MNODE].func = mmGetMgmtFunc();
@ -88,7 +84,7 @@ int32_t dmInitDnode(SDnode *pDnode) {
streamMetaInit();
dmInitStatusClient(pDnode);
dmInitSyncClient(pDnode);
dmInitSyncClient(pDnode);
dmReportStartup("dnode-transport", "initialized");
dDebug("dnode is created, ptr:%p", pDnode);
@ -120,10 +116,8 @@ void dmCleanupDnode(SDnode *pDnode) {
indexCleanup();
taosConvDestroy();
#ifdef TD_TSZ
// compress destroy
tsCompressExit();
#endif
dDebug("dnode is closed, ptr:%p", pDnode);
}

17
source/dnode/mnode/impl/src/mndDb.c
View File

@ -821,6 +821,10 @@ static int32_t mndCheckDbEncryptKey(SMnode *pMnode, SCreateDbReq *pReq) {
#ifdef TD_ENTERPRISE
if (pReq->encryptAlgorithm == TSDB_ENCRYPT_ALGO_NONE) goto _exit;
if (grantCheck(TSDB_GRANT_DB_ENCRYPTION) != 0) {
code = TSDB_CODE_MND_DB_ENCRYPT_GRANT_EXPIRED;
goto _exit;
}
if (tsEncryptionKeyStat != ENCRYPT_KEY_STAT_LOADED) {
code = TSDB_CODE_MND_INVALID_ENCRYPT_KEY;
mError("db:%s, failed to check encryption key:%" PRIi8 " in mnode leader since it's not loaded", pReq->db,
@ -903,6 +907,13 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) {
goto _OVER;
}
if (createReq.replications == 2) {
if ((terrno = grantCheck(TSDB_GRANT_DUAL_REPLICA_HA)) != 0) {
code = terrno;
goto _OVER;
}
}
if ((code = mndCheckDbEncryptKey(pMnode, &createReq)) != 0) {
terrno = code;
goto _OVER;
@ -1163,6 +1174,12 @@ static int32_t mndProcessAlterDbReq(SRpcMsg *pReq) {
goto _OVER;
}
if (alterReq.replications == 2) {
if ((code = grantCheck(TSDB_GRANT_DUAL_REPLICA_HA)) != 0) {
goto _OVER;
}
}
int32_t numOfTopics = 0;
if (mndGetNumOfTopics(pMnode, pDb->name, &numOfTopics) != 0) {
goto _OVER;

45
source/dnode/vnode/src/inc/tsdb.h
View File

@ -120,12 +120,32 @@ static FORCE_INLINE int64_t tsdbLogicToFileSize(int64_t lSize, int32_t szPage) {
#define tsdbRowFromBlockData(BLOCKDATA, IROW) \
((TSDBROW){.type = TSDBROW_COL_FMT, .pBlockData = (BLOCKDATA), .iRow = (IROW)})
#define TSDBROW_INIT_KEY(_ROW, _KEY) \
{ \
if ((_ROW)->type == TSDBROW_ROW_FMT) { \
_KEY.version = (_ROW)->version; \
_KEY.ts = (_ROW)->pTSRow->ts; \
} else { \
_KEY.version = (_ROW)->pBlockData->aVersion[(_ROW)->iRow]; \
_KEY.ts = (_ROW)->pBlockData->aTSKEY[(_ROW)->iRow]; \
} \
}
#define tColRowGetKey(_pBlock, _irow, _key) \
{ \
(_key)->ts = (_pBlock)->aTSKEY[(_irow)]; \
(_key)->numOfPKs = 0; \
if ((_pBlock)->nColData > 0) { \
tColRowGetPrimaryKey((_pBlock), (_irow), (_key)); \
} \
}
void tsdbRowGetColVal(TSDBROW *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal);
int32_t tsdbRowCompare(const void *p1, const void *p2);
int32_t tsdbRowCompareWithoutVersion(const void *p1, const void *p2);
int32_t tsdbRowKeyCmpr(const STsdbRowKey *key1, const STsdbRowKey *key2);
void tsdbRowGetKey(TSDBROW *row, STsdbRowKey *key);
void tColRowGetKey(SBlockData *pBlock, int32_t irow, SRowKey *key);
void tColRowGetPrimaryKey(SBlockData *pBlock, int32_t irow, SRowKey *key);
// STSDBRowIter
@ -953,7 +973,28 @@ static FORCE_INLINE int32_t tsdbKeyCmprFn(const void *p1, const void *p2) {
// #define SL_NODE_FORWARD(n, l) ((n)->forwards[l])
// #define SL_NODE_BACKWARD(n, l) ((n)->forwards[(n)->level + (l)])
TSDBROW *tsdbTbDataIterGet(STbDataIter *pIter);
static FORCE_INLINE TSDBROW *tsdbTbDataIterGet(STbDataIter *pIter) {
if (pIter == NULL) return NULL;
if (pIter->pRow) {
return pIter->pRow;
}
if (pIter->backward) {
if (pIter->pNode == pIter->pTbData->sl.pHead) {
return NULL;
}
} else {
if (pIter->pNode == pIter->pTbData->sl.pTail) {
return NULL;
}
}
pIter->pRow = &pIter->row;
pIter->row = pIter->pNode->row;
return pIter->pRow;
}
typedef struct {
int64_t suid;

80
source/dnode/vnode/src/tsdb/tsdbCache.c
View File

@ -16,6 +16,7 @@
#include "functionMgt.h"
#include "tsdb.h"
#include "tsdbDataFileRW.h"
#include "tsdbIter.h"
#include "tsdbReadUtil.h"
#include "vnd.h"
@ -359,7 +360,10 @@ static int32_t tsdbCacheDeserializeV0(char const *value, SLastCol *pLastCol) {
if (IS_VAR_DATA_TYPE(pLastCol->colVal.value.type)) {
pLastCol->colVal.value.nData = pLastColV0->colVal.value.nData;
pLastCol->colVal.value.pData = (uint8_t *)(&pLastColV0[1]);
pLastCol->colVal.value.pData = NULL;
if (pLastCol->colVal.value.nData > 0) {
pLastCol->colVal.value.pData = (uint8_t *)(&pLastColV0[1]);
}
return sizeof(SLastColV0) + pLastColV0->colVal.value.nData;
} else {
pLastCol->colVal.value.val = pLastColV0->colVal.value.val;
@ -401,8 +405,11 @@ static SLastCol *tsdbCacheDeserialize(char const *value, size_t size) {
offset += sizeof(SValue);
if (IS_VAR_DATA_TYPE(pLastCol->rowKey.pks[i].type)) {
pLastCol->rowKey.pks[i].pData = (uint8_t *)value + offset;
offset += pLastCol->rowKey.pks[i].nData;
pLastCol->rowKey.pks[i].pData = NULL;
if (pLastCol->rowKey.pks[i].nData > 0) {
pLastCol->rowKey.pks[i].pData = (uint8_t *)value + offset;
offset += pLastCol->rowKey.pks[i].nData;
}
}
}
@ -940,10 +947,37 @@ typedef struct {
SLastKey key;
} SIdxKey;
static void tsdbCacheUpdateLastCol(SLastCol *pLastCol, SRowKey *pRowKey, SColVal *pColVal) {
uint8_t *pVal = NULL;
static int32_t tsdbCacheUpdateValue(SValue *pOld, SValue *pNew) {
uint8_t *pFree = NULL;
int nData = 0;
if (IS_VAR_DATA_TYPE(pOld->type)) {
pFree = pOld->pData;
nData = pOld->nData;
}
*pOld = *pNew;
if (IS_VAR_DATA_TYPE(pNew->type)) {
if (nData < pNew->nData) {
pOld->pData = taosMemoryCalloc(1, pNew->nData);
} else {
pOld->pData = pFree;
pFree = NULL;
}
if (pNew->nData) {
memcpy(pOld->pData, pNew->pData, pNew->nData);
} else {
pFree = pOld->pData;
pOld->pData = NULL;
}
}
taosMemoryFreeClear(pFree);
return 0;
}
static void tsdbCacheUpdateLastCol(SLastCol *pLastCol, SRowKey *pRowKey, SColVal *pColVal) {
// update rowkey
pLastCol->rowKey.ts = pRowKey->ts;
pLastCol->rowKey.numOfPKs = pRowKey->numOfPKs;
@ -951,41 +985,13 @@ static void tsdbCacheUpdateLastCol(SLastCol *pLastCol, SRowKey *pRowKey, SColVal
SValue *pPKValue = &pLastCol->rowKey.pks[i];
SValue *pNewPKValue = &pRowKey->pks[i];
if (IS_VAR_DATA_TYPE(pPKValue->type)) {
pVal = pPKValue->pData;
nData = pPKValue->nData;
}
*pPKValue = *pNewPKValue;
if (IS_VAR_DATA_TYPE(pPKValue->type)) {
if (nData < pPKValue->nData) {
taosMemoryFree(pVal);
pPKValue->pData = taosMemoryCalloc(1, pNewPKValue->nData);
} else {
pPKValue->pData = pVal;
}
if (pNewPKValue->nData) {
memcpy(pPKValue->pData, pNewPKValue->pData, pNewPKValue->nData);
}
}
(void)tsdbCacheUpdateValue(pPKValue, pNewPKValue);
}
// update colval
if (IS_VAR_DATA_TYPE(pColVal->value.type)) {
nData = pLastCol->colVal.value.nData;
pVal = pLastCol->colVal.value.pData;
}
pLastCol->colVal = *pColVal;
if (IS_VAR_DATA_TYPE(pColVal->value.type)) {
if (nData < pColVal->value.nData) {
taosMemoryFree(pVal);
pLastCol->colVal.value.pData = taosMemoryCalloc(1, pColVal->value.nData);
} else {
pLastCol->colVal.value.pData = pVal;
}
if (pColVal->value.nData) {
memcpy(pLastCol->colVal.value.pData, pColVal->value.pData, pColVal->value.nData);
}
}
pLastCol->colVal.cid = pColVal->cid;
pLastCol->colVal.flag = pColVal->flag;
(void)tsdbCacheUpdateValue(&pLastCol->colVal.value, &pColVal->value);
if (!pLastCol->dirty) {
pLastCol->dirty = 1;

23
source/dnode/vnode/src/tsdb/tsdbMemTable.c
View File

@ -799,26 +799,3 @@ SArray *tsdbMemTableGetTbDataArray(SMemTable *pMemTable) {
_exit:
return aTbDataP;
}
TSDBROW *tsdbTbDataIterGet(STbDataIter *pIter) {
if (pIter == NULL) return NULL;
if (pIter->pRow) {
return pIter->pRow;
}
if (pIter->backward) {
if (pIter->pNode == pIter->pTbData->sl.pHead) {
return NULL;
}
} else {
if (pIter->pNode == pIter->pTbData->sl.pTail) {
return NULL;
}
}
pIter->pRow = &pIter->row;
pIter->row = pIter->pNode->row;
return pIter->pRow;
}

2
source/dnode/vnode/src/tsdb/tsdbMergeTree.c
View File

@ -828,7 +828,7 @@ static FORCE_INLINE int32_t tLDataIterCmprFn(const SRBTreeNode *p1, const SRBTre
SLDataIter *pIter1 = (SLDataIter *)(((uint8_t *)p1) - offsetof(SLDataIter, node));
SLDataIter *pIter2 = (SLDataIter *)(((uint8_t *)p2) - offsetof(SLDataIter, node));
SRowKey rkey1, rkey2;
SRowKey rkey1 = {0}, rkey2 = {0};
tRowGetKeyEx(&pIter1->rInfo.row, &rkey1);
tRowGetKeyEx(&pIter2->rInfo.row, &rkey2);

127
source/dnode/vnode/src/tsdb/tsdbRead2.c
View File

@ -24,6 +24,16 @@
#define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC)
#define getCurrentKeyInSttBlock(_r) (&((_r)->currentKey))
#define tColRowGetKeyDeepCopy(_pBlock, _irow, _slotId, _pKey) \
do { \
(_pKey)->ts = (_pBlock)->aTSKEY[(_irow)]; \
(_pKey)->numOfPKs = 0; \
if ((_slotId) != -1) { \
tColRowGetPriamyKeyDeepCopy(_pBlock, _irow, _slotId, _pKey); \
} \
} while (0)
#define outOfTimeWindow(_ts, _window) (((_ts) > (_window)->ekey) || ((_ts) < (_window)->skey))
typedef struct {
bool overlapWithNeighborBlock;
@ -75,11 +85,9 @@ static void getMemTableTimeRange(STsdbReader* pReader, int64_t* pMaxKey
static void updateComposedBlockInfo(STsdbReader* pReader, double el, STableBlockScanInfo* pBlockScanInfo);
static int32_t buildFromPreFilesetBuffer(STsdbReader* pReader);
static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); }
static void resetPreFilesetMemTableListIndex(SReaderStatus* pStatus);
int32_t pkCompEx(SRowKey* p1, SRowKey* p2) {
FORCE_INLINE int32_t pkCompEx(SRowKey* p1, SRowKey* p2) {
if (p2 == NULL) {
return 1;
}
@ -101,13 +109,7 @@ int32_t pkCompEx(SRowKey* p1, SRowKey* p2) {
}
}
static void tColRowGetKeyDeepCopy(SBlockData* pBlock, int32_t irow, int32_t slotId, SRowKey* pKey) {
pKey->ts = pBlock->aTSKEY[irow];
if (slotId == -1) {
pKey->numOfPKs = 0;
return;
}
static void tColRowGetPriamyKeyDeepCopy(SBlockData* pBlock, int32_t irow, int32_t slotId, SRowKey* pKey) {
SColData* pColData = &pBlock->aColData[slotId];
SColVal cv;
tColDataGetValue(pColData, irow, &cv);
@ -131,13 +133,7 @@ static int32_t tGetPrimaryKeyIndex(uint8_t *p, SPrimaryKeyIndex *index) {
return n;
}
static void tRowGetKeyDeepCopy(SRow* pRow, SRowKey* pKey) {
pKey->ts = pRow->ts;
pKey->numOfPKs = pRow->numOfPKs;
if (pKey->numOfPKs == 0) {
return;
}
static void tRowGetPrimaryKeyDeepCopy(SRow* pRow, SRowKey* pKey) {
SPrimaryKeyIndex indices[TD_MAX_PK_COLS];
ASSERT(pKey->numOfPKs <= TD_MAX_PK_COLS);
@ -1735,7 +1731,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo*
pSttKey = getCurrentKeyInSttBlock(pSttBlockReader);
}
SRowKey k;
SRowKey k = {0};
tRowGetKeyEx(pRow, &k);
STSchema* pSchema = NULL;
@ -1939,7 +1935,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo*
pfKey = NULL;
}
SRowKey k, ik;
SRowKey k = {0}, ik = {0};
tRowGetKeyEx(pRow, &k);
tRowGetKeyEx(piRow, &ik);
@ -2086,7 +2082,7 @@ int32_t doInitMemDataIter(STsdbReader* pReader, STbData** pData, STableBlockScan
}
static void doForwardDataIter(SRowKey* pKey, SIterInfo* pIter, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) {
SRowKey rowKey;
SRowKey rowKey = {0};
while (1) {
TSDBROW* pRow = getValidMemRow(pIter, pBlockScanInfo->delSkyline, pReader);
@ -3559,7 +3555,7 @@ bool hasBeenDropped(const SArray* pDelList, int32_t* index, int64_t key, int64_t
return false;
}
TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pReader) {
FORCE_INLINE TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pReader) {
if (!pIter->hasVal) {
return NULL;
}
@ -3567,7 +3563,8 @@ TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* p
int32_t order = pReader->info.order;
TSDBROW* pRow = tsdbTbDataIterGet(pIter->iter);
TSDBKEY key = TSDBROW_KEY(pRow);
TSDBKEY key;
TSDBROW_INIT_KEY(pRow, key);
if (outOfTimeWindow(key.ts, &pReader->info.window)) {
pIter->hasVal = false;
return NULL;
@ -3593,7 +3590,7 @@ TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* p
pRow = tsdbTbDataIterGet(pIter->iter);
key = TSDBROW_KEY(pRow);
TSDBROW_INIT_KEY(pRow, key);
if (outOfTimeWindow(key.ts, &pReader->info.window)) {
pIter->hasVal = false;
return NULL;
@ -3632,10 +3629,12 @@ int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, SRowKey *pCurKey, SArra
break;
}
SRowKey nextKey = {0};
tRowGetKeyEx(pRow, &nextKey);
if (pkCompEx(pCurKey, &nextKey) != 0) {
break;
if (pCurKey->numOfPKs > 0) {
SRowKey nextKey = {0};
tRowGetKeyEx(pRow, &nextKey);
if (pkCompEx(pCurKey, &nextKey) != 0) {
break;
}
}
STSchema* pTSchema = NULL;
@ -3785,12 +3784,14 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, SRowKey* pKey, uint64_t uid, SIt
return TSDB_CODE_SUCCESS;
}
SRowKey nextRowKey = {0};
tRowGetKeyEx(pNextRow, &nextRowKey);
if (pKey->numOfPKs > 0 && pkCompEx(pKey, &nextRowKey) != 0) {
*pResRow = current;
*freeTSRow = false;
return TSDB_CODE_SUCCESS;
if (pKey->numOfPKs > 0) {
SRowKey nextRowKey = {0};
tRowGetKeyEx(pNextRow, &nextRowKey);
if (pkCompEx(pKey, &nextRowKey) != 0) {
*pResRow = current;
*freeTSRow = false;
return TSDB_CODE_SUCCESS;
}
}
}
}
@ -3801,7 +3802,7 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, SRowKey* pKey, uint64_t uid, SIt
// start to merge duplicated rows
STSchema* pTSchema = NULL;
if (current.type == TSDBROW_ROW_FMT) { // get the correct schema for row-wise data in memory
pTSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(&current), pReader, uid);
pTSchema = doGetSchemaForTSRow(current.pTSRow->sver, pReader, uid);
if (pTSchema == NULL) {
return terrno;
}
@ -3814,7 +3815,7 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, SRowKey* pKey, uint64_t uid, SIt
STSchema* pTSchema1 = NULL;
if (pNextRow->type == TSDBROW_ROW_FMT) { // get the correct schema for row-wise data in memory
pTSchema1 = doGetSchemaForTSRow(TSDBROW_SVERSION(pNextRow), pReader, uid);
pTSchema1 = doGetSchemaForTSRow(pNextRow->pTSRow->sver, pReader, uid);
if (pTSchema1 == NULL) {
return terrno;
}
@ -3926,37 +3927,49 @@ static int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbRea
// todo refactor
bool asc = ASCENDING_TRAVERSE(pReader->info.order);
if (piter->hasVal) {
TSDBKEY k = TSDBROW_KEY(pRow);
if ((k.ts >= endKey && asc) || (k.ts <= endKey && !asc)) {
tRowGetKeyEx(pRow, &rowKey);
if ((rowKey.ts >= endKey && asc) || (rowKey.ts <= endKey && !asc)) {
pRow = NULL;
}
}
if (piiter->hasVal) {
TSDBKEY k = TSDBROW_KEY(piRow);
if ((k.ts >= endKey && asc) || (k.ts <= endKey && !asc)) {
tRowGetKeyEx(piRow, &irowKey);
if ((irowKey.ts >= endKey && asc) || (irowKey.ts <= endKey && !asc)) {
piRow = NULL;
}
}
if (piter->hasVal && piiter->hasVal && pRow != NULL && piRow != NULL) {
tRowGetKeyEx(pRow, &rowKey);
tRowGetKeyEx(piRow, &irowKey);
if (pRow != NULL && piRow != NULL) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t ret = pkCompEx(&rowKey, &irowKey);
if (ret != 0) {
if ((ret > 0 && asc) || (ret < 0 && (!asc))) { // ik.ts < k.ts
if (rowKey.numOfPKs == 0) {
if ((rowKey.ts > irowKey.ts && asc) || (rowKey.ts < irowKey.ts && (!asc))) { // ik.ts < k.ts
code = doMergeMemTableMultiRows(piRow, &irowKey, uid, piiter, pDelList, pResRow, pReader, freeTSRow);
} else if ((ret < 0 && asc) || (ret > 0 && (!asc))) {
} else if ((rowKey.ts < irowKey.ts && asc) || (rowKey.ts > irowKey.ts && (!asc))) {
code = doMergeMemTableMultiRows(pRow, &rowKey, uid, piter, pDelList, pResRow, pReader, freeTSRow);
} else { // ik.ts == k.ts
*freeTSRow = true;
pResRow->type = TSDBROW_ROW_FMT;
code = doMergeMemIMemRows(pRow, &rowKey, piRow, &irowKey, pBlockScanInfo, pReader, &pResRow->pTSRow);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
}
} else { // ik.ts == k.ts
*freeTSRow = true;
pResRow->type = TSDBROW_ROW_FMT;
code = doMergeMemIMemRows(pRow, &rowKey, piRow, &irowKey, pBlockScanInfo, pReader, &pResRow->pTSRow);
if (code != TSDB_CODE_SUCCESS) {
return code;
} else {
int32_t ret = pkCompEx(&rowKey, &irowKey);
if (ret != 0) {
if ((ret > 0 && asc) || (ret < 0 && (!asc))) { // ik.ts < k.ts
code = doMergeMemTableMultiRows(piRow, &irowKey, uid, piiter, pDelList, pResRow, pReader, freeTSRow);
} else if ((ret < 0 && asc) || (ret > 0 && (!asc))) {
code = doMergeMemTableMultiRows(pRow, &rowKey, uid, piter, pDelList, pResRow, pReader, freeTSRow);
}
} else { // ik.ts == k.ts
*freeTSRow = true;
pResRow->type = TSDBROW_ROW_FMT;
code = doMergeMemIMemRows(pRow, &rowKey, piRow, &irowKey, pBlockScanInfo, pReader, &pResRow->pTSRow);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
}
}
@ -3964,12 +3977,10 @@ static int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbRea
}
if (piter->hasVal && pRow != NULL) {
tRowGetKeyEx(pRow, &rowKey);
return doMergeMemTableMultiRows(pRow, &rowKey, uid, piter, pDelList, pResRow, pReader, freeTSRow);
}
if (piiter->hasVal && piRow != NULL) {
tRowGetKeyEx(piRow, &irowKey);
return doMergeMemTableMultiRows(piRow, &irowKey, uid, piiter, pDelList, pResRow, pReader, freeTSRow);
}
@ -4095,7 +4106,11 @@ int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t e
if (row.type == TSDBROW_ROW_FMT) {
code = doAppendRowFromTSRow(pBlock, pReader, row.pTSRow, pBlockScanInfo);
if (code == TSDB_CODE_SUCCESS) {
tRowGetKeyDeepCopy(row.pTSRow, &pBlockScanInfo->lastProcKey);
pBlockScanInfo->lastProcKey.ts = row.pTSRow->ts;
pBlockScanInfo->lastProcKey.numOfPKs = row.pTSRow->numOfPKs;
if (row.pTSRow->numOfPKs > 0) {
tRowGetPrimaryKeyDeepCopy(row.pTSRow, &pBlockScanInfo->lastProcKey);
}
}
if (freeTSRow) {

4
source/dnode/vnode/src/tsdb/tsdbReadUtil.c
View File

@ -212,7 +212,7 @@ static void initLastProcKey(STableBlockScanInfo *pScanInfo, STsdbReader* pReader
int32_t numOfPks = pReader->suppInfo.numOfPks;
bool asc = ASCENDING_TRAVERSE(pReader->info.order);
int8_t type = pReader->suppInfo.pk.type;
int8_t bytes = pReader->suppInfo.pk.bytes;
int32_t bytes = pReader->suppInfo.pk.bytes;
SRowKey* pRowKey = &pScanInfo->lastProcKey;
if (asc) {
@ -1056,7 +1056,7 @@ static int32_t sortUidComparFn(const void* p1, const void* p2) {
const SSttKeyRange* px1 = p1;
const SSttKeyRange* px2 = p2;
int32_t ret = tRowKeyCompare(&px1, px2);
int32_t ret = tRowKeyCompare(&px1->skey, &px2->skey);
return ret;
}

20
source/dnode/vnode/src/tsdb/tsdbReadUtil.h
View File

@ -38,14 +38,18 @@ extern "C" {
(_k)->ekey.ts = INT64_MIN; \
} while (0);
#define tRowGetKeyEx(_pRow, _pKey) \
do { \
if ((_pRow)->type == TSDBROW_ROW_FMT) { \
tRowGetKey((_pRow)->pTSRow, (_pKey)); \
} else { \
tColRowGetKey((_pRow)->pBlockData, (_pRow)->iRow, (_pKey)); \
} \
} while (0)
#define tRowGetKeyEx(_pRow, _pKey) \
{ \
if ((_pRow)->type == TSDBROW_ROW_FMT) { \
(_pKey)->ts = (_pRow)->pTSRow->ts; \
if ((_pRow)->pTSRow->numOfPKs > 0) { \
tRowGetPrimaryKey((_pRow)->pTSRow, (_pKey)); \
} \
} else { \
(_pKey)->ts = (_pRow)->pBlockData->aTSKEY[(_pRow)->iRow]; \
tColRowGetPrimaryKey((_pRow)->pBlockData, (_pRow)->iRow, (_pKey)); \
} \
}
typedef enum {
READER_STATUS_SUSPEND = 0x1,

10
source/dnode/vnode/src/tsdb/tsdbRetention.c
View File

@ -757,6 +757,16 @@ _exit:
int32_t tsdbS3Migrate(STsdb *tsdb, int64_t now, int32_t sync) {
int32_t code = 0;
extern int8_t tsS3EnabledCfg;
int32_t expired = grantCheck(TSDB_GRANT_OBJECT_STORAGE);
if (expired && tsS3Enabled) {
tsdbWarn("s3 grant expired: %d", expired);
tsS3Enabled = false;
} else if (!expired && tsS3EnabledCfg) {
tsS3Enabled = true;
}
if (!tsS3Enabled) {
return code;
}

5
source/dnode/vnode/src/tsdb/tsdbUtil.c
View File

@ -632,10 +632,7 @@ void tsdbRowGetKey(TSDBROW *row, STsdbRowKey *key) {
}
}
void tColRowGetKey(SBlockData *pBlock, int32_t irow, SRowKey *key) {
key->ts = pBlock->aTSKEY[irow];
key->numOfPKs = 0;
void tColRowGetPrimaryKey(SBlockData *pBlock, int32_t irow, SRowKey *key) {
for (int32_t i = 0; i < pBlock->nColData; i++) {
SColData *pColData = &pBlock->aColData[i];
if (pColData->cflag & COL_IS_KEY) {

12
source/libs/catalog/src/ctgCache.c
View File

@ -3226,14 +3226,24 @@ int32_t ctgGetTbTSMAFromCache(SCatalog* pCtg, SCtgTbTSMACtx* pCtx, int32_t dbIdx
// get tb cache
pName = taosArrayGet(pList, i);
pTbCache = taosHashAcquire(dbCache->tbCache, pName->tname, strlen(pName->tname));
if (!pTbCache || !pTbCache->pMeta) {
if (!pTbCache) {
ctgDebug("tb: %s.%s not in cache", dbFName, pName->tname);
ctgAddTSMAFetch(&pCtx->pFetches, dbIdx, i, fetchIdx, baseResIdx + i, flag, FETCH_TSMA_SOURCE_TB_META, NULL);
taosArrayPush(pCtx->pResList, &(SMetaRes){0});
continue;
}
CTG_LOCK(CTG_READ, &pTbCache->metaLock);
if (!pTbCache->pMeta) {
CTG_UNLOCK(CTG_READ, &pTbCache->metaLock);
ctgDebug("tb: %s.%s not in cache", dbFName, pName->tname);
ctgAddTSMAFetch(&pCtx->pFetches, dbIdx, i, fetchIdx, baseResIdx + i, flag, FETCH_TSMA_SOURCE_TB_META, NULL);
taosArrayPush(pCtx->pResList, &(SMetaRes){0});
taosHashRelease(dbCache->tbCache, pTbCache);
continue;
}
uint64_t suid = pTbCache->pMeta->suid;
int8_t tbType = pTbCache->pMeta->tableType;
CTG_UNLOCK(CTG_READ, &pTbCache->metaLock);
taosHashRelease(dbCache->tbCache, pTbCache);
SName tsmaSourceTbName = *pName;

30
source/libs/executor/inc/executorInt.h
View File

@ -446,17 +446,23 @@ typedef struct STimeWindowAggSupp {
SColumnInfoData timeWindowData; // query time window info for scalar function execution.
} STimeWindowAggSupp;
typedef struct SSteamOpBasicInfo {
int32_t primaryPkIndex;
bool updateOperatorInfo;
} SSteamOpBasicInfo;
typedef struct SStreamScanInfo {
SExprInfo* pPseudoExpr;
int32_t numOfPseudoExpr;
SExprSupp tbnameCalSup;
SExprSupp* pPartTbnameSup;
SExprSupp tagCalSup;
int32_t primaryTsIndex; // primary time stamp slot id
int32_t primaryKeyIndex;
SReadHandle readHandle;
SInterval interval; // if the upstream is an interval operator, the interval info is also kept here.
SColMatchInfo matchInfo;
SSteamOpBasicInfo basic;
SExprInfo* pPseudoExpr;
int32_t numOfPseudoExpr;
SExprSupp tbnameCalSup;
SExprSupp* pPartTbnameSup;
SExprSupp tagCalSup;
int32_t primaryTsIndex; // primary time stamp slot id
int32_t primaryKeyIndex;
SReadHandle readHandle;
SInterval interval; // if the upstream is an interval operator, the interval info is also kept here.
SColMatchInfo matchInfo;
SArray* pBlockLists; // multiple SSDatablock.
SSDataBlock* pRes; // result SSDataBlock
@ -568,10 +574,6 @@ typedef struct SOpCheckPointInfo {
SHashObj* children; // key:child id
} SOpCheckPointInfo;
typedef struct SSteamOpBasicInfo {
int32_t primaryPkIndex;
} SSteamOpBasicInfo;
typedef struct SStreamIntervalOperatorInfo {
SOptrBasicInfo binfo; // basic info
SSteamOpBasicInfo basic;

32
source/libs/executor/inc/streamexecutorInt.h Normal file
View File

@ -0,0 +1,32 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef STREAM_EXECUTORINT_H
#define STREAM_EXECUTORINT_H
#ifdef __cplusplus
extern "C" {
#endif
#include "executorInt.h"
void setStreamOperatorState(SSteamOpBasicInfo* pBasicInfo, EStreamType type);
bool needSaveStreamOperatorInfo(SSteamOpBasicInfo* pBasicInfo);
void saveStreamOperatorStateComplete(SSteamOpBasicInfo* pBasicInfo);
#ifdef __cplusplus
}
#endif
#endif // STREAM_EXECUTORINT_H

17
source/libs/executor/src/executor.c
View File

@ -824,14 +824,17 @@ int32_t qKillTask(qTaskInfo_t tinfo, int32_t rspCode) {
qDebug("%s sync killed execTask", GET_TASKID(pTaskInfo));
setTaskKilled(pTaskInfo, TSDB_CODE_TSC_QUERY_KILLED);
taosWLockLatch(&pTaskInfo->lock);
while (qTaskIsExecuting(pTaskInfo)) {
taosMsleep(10);
while(1) {
taosWLockLatch(&pTaskInfo->lock);
if (qTaskIsExecuting(pTaskInfo)) { // let's wait for 100 ms and try again
taosWUnLockLatch(&pTaskInfo->lock);
taosMsleep(100);
} else { // not running now
pTaskInfo->code = rspCode;
taosWUnLockLatch(&pTaskInfo->lock);
return TSDB_CODE_SUCCESS;
}
}
pTaskInfo->code = rspCode;
taosWUnLockLatch(&pTaskInfo->lock);
return TSDB_CODE_SUCCESS;
}
bool qTaskIsExecuting(qTaskInfo_t qinfo) {

18
source/libs/executor/src/scanoperator.c
View File

@ -20,6 +20,7 @@
#include "os.h"
#include "querynodes.h"
#include "systable.h"
#include "streamexecutorInt.h"
#include "tname.h"
#include "tdatablock.h"
@ -2426,10 +2427,13 @@ void streamScanOperatorSaveCheckpoint(SStreamScanInfo* pInfo) {
if (!pInfo->pState) {
return;
}
void* pBuf = NULL;
int32_t len = streamScanOperatorEncode(pInfo, &pBuf);
pInfo->stateStore.streamStateSaveInfo(pInfo->pState, STREAM_SCAN_OP_CHECKPOINT_NAME, strlen(STREAM_SCAN_OP_CHECKPOINT_NAME), pBuf, len);
taosMemoryFree(pBuf);
if (needSaveStreamOperatorInfo(&pInfo->basic)) {
void* pBuf = NULL;
int32_t len = streamScanOperatorEncode(pInfo, &pBuf);
pInfo->stateStore.streamStateSaveInfo(pInfo->pState, STREAM_SCAN_OP_CHECKPOINT_NAME, strlen(STREAM_SCAN_OP_CHECKPOINT_NAME), pBuf, len);
taosMemoryFree(pBuf);
saveStreamOperatorStateComplete(&pInfo->basic);
}
}
// other properties are recovered from the execution plan
@ -2582,6 +2586,7 @@ FETCH_NEXT_BLOCK:
case STREAM_NORMAL:
case STREAM_GET_ALL:
printDataBlock(pBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
setStreamOperatorState(&pInfo->basic, pBlock->info.type);
return pBlock;
case STREAM_RETRIEVE: {
pInfo->blockType = STREAM_INPUT__DATA_SUBMIT;
@ -2622,6 +2627,7 @@ FETCH_NEXT_BLOCK:
if (pInfo->pDeleteDataRes->info.rows > 0) {
printSpecDataBlock(pInfo->pDeleteDataRes, getStreamOpName(pOperator->operatorType), "delete result", GET_TASKID(pTaskInfo));
setStreamOperatorState(&pInfo->basic, pInfo->pDeleteDataRes->info.type);
return pInfo->pDeleteDataRes;
} else {
goto FETCH_NEXT_BLOCK;
@ -2639,6 +2645,7 @@ FETCH_NEXT_BLOCK:
if (pInfo->pDeleteDataRes->info.rows > 0) {
pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE;
printSpecDataBlock(pInfo->pDeleteDataRes, getStreamOpName(pOperator->operatorType), "delete result", GET_TASKID(pTaskInfo));
setStreamOperatorState(&pInfo->basic, pInfo->pDeleteDataRes->info.type);
return pInfo->pDeleteDataRes;
} else {
goto FETCH_NEXT_BLOCK;
@ -2652,6 +2659,7 @@ FETCH_NEXT_BLOCK:
break;
}
printDataBlock(pBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
setStreamOperatorState(&pInfo->basic, pBlock->info.type);
return pBlock;
} else if (pInfo->blockType == STREAM_INPUT__DATA_SUBMIT) {
qDebug("stream scan mode:%d, %s", pInfo->scanMode, id);
@ -2659,6 +2667,7 @@ FETCH_NEXT_BLOCK:
case STREAM_SCAN_FROM_RES: {
pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE;
doCheckUpdate(pInfo, pInfo->pRes->info.window.ekey, pInfo->pRes);
setStreamOperatorState(&pInfo->basic, pInfo->pRes->info.type);
doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL);
pInfo->pRes->info.dataLoad = 1;
blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex);
@ -2762,6 +2771,7 @@ FETCH_NEXT_BLOCK:
}
doCheckUpdate(pInfo, pBlockInfo->window.ekey, pInfo->pRes);
setStreamOperatorState(&pInfo->basic, pInfo->pRes->info.type);
doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL);
blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex);

19
source/libs/executor/src/streamcountwindowoperator.c
View File

@ -17,6 +17,7 @@
#include "functionMgt.h"
#include "operator.h"
#include "querytask.h"
#include "streamexecutorInt.h"
#include "tchecksum.h"
#include "tcommon.h"
#include "tdatablock.h"
@ -415,13 +416,16 @@ void* doStreamCountDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato
void doStreamCountSaveCheckpoint(SOperatorInfo* pOperator) {
SStreamCountAggOperatorInfo* pInfo = pOperator->info;
int32_t len = doStreamCountEncodeOpState(NULL, 0, pOperator, true);
void* buf = taosMemoryCalloc(1, len);
void* pBuf = buf;
len = doStreamCountEncodeOpState(&pBuf, len, pOperator, true);
pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_COUNT_OP_CHECKPOINT_NAME,
strlen(STREAM_COUNT_OP_CHECKPOINT_NAME), buf, len);
taosMemoryFree(buf);
if (needSaveStreamOperatorInfo(&pInfo->basic)) {
int32_t len = doStreamCountEncodeOpState(NULL, 0, pOperator, true);
void* buf = taosMemoryCalloc(1, len);
void* pBuf = buf;
len = doStreamCountEncodeOpState(&pBuf, len, pOperator, true);
pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_COUNT_OP_CHECKPOINT_NAME,
strlen(STREAM_COUNT_OP_CHECKPOINT_NAME), buf, len);
taosMemoryFree(buf);
saveStreamOperatorStateComplete(&pInfo->basic);
}
}
void doResetCountWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock) {
@ -550,6 +554,7 @@ static SSDataBlock* doStreamCountAgg(SOperatorInfo* pOperator) {
break;
}
printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo));
setStreamOperatorState(&pInfo->basic, pBlock->info.type);
if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) {
bool add = pInfo->destHasPrimaryKey && IS_NORMAL_COUNT_OP(pOperator);

19
source/libs/executor/src/streameventwindowoperator.c
View File

@ -18,6 +18,7 @@
#include "functionMgt.h"
#include "operator.h"
#include "querytask.h"
#include "streamexecutorInt.h"
#include "tchecksum.h"
#include "tcommon.h"
#include "tcompare.h"
@ -458,13 +459,16 @@ void* doStreamEventDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato
void doStreamEventSaveCheckpoint(SOperatorInfo* pOperator) {
SStreamEventAggOperatorInfo* pInfo = pOperator->info;
int32_t len = doStreamEventEncodeOpState(NULL, 0, pOperator);
void* buf = taosMemoryCalloc(1, len);
void* pBuf = buf;
len = doStreamEventEncodeOpState(&pBuf, len, pOperator);
pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_EVENT_OP_CHECKPOINT_NAME,
strlen(STREAM_EVENT_OP_CHECKPOINT_NAME), buf, len);
taosMemoryFree(buf);
if (needSaveStreamOperatorInfo(&pInfo->basic)) {
int32_t len = doStreamEventEncodeOpState(NULL, 0, pOperator);
void* buf = taosMemoryCalloc(1, len);
void* pBuf = buf;
len = doStreamEventEncodeOpState(&pBuf, len, pOperator);
pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_EVENT_OP_CHECKPOINT_NAME,
strlen(STREAM_EVENT_OP_CHECKPOINT_NAME), buf, len);
taosMemoryFree(buf);
saveStreamOperatorStateComplete(&pInfo->basic);
}
}
static SSDataBlock* buildEventResult(SOperatorInfo* pOperator) {
@ -531,6 +535,7 @@ static SSDataBlock* doStreamEventAgg(SOperatorInfo* pOperator) {
break;
}
printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo));
setStreamOperatorState(&pInfo->basic, pBlock->info.type);
if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||
pBlock->info.type == STREAM_CLEAR) {

30
source/libs/executor/src/streamexecutorInt.c Normal file
View File

@ -0,0 +1,30 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "executorInt.h"
void setStreamOperatorState(SSteamOpBasicInfo* pBasicInfo, EStreamType type) {
if (type != STREAM_GET_ALL && type != STREAM_CHECKPOINT) {
pBasicInfo->updateOperatorInfo = true;
}
}
bool needSaveStreamOperatorInfo(SSteamOpBasicInfo* pBasicInfo) {
return pBasicInfo->updateOperatorInfo;
}
void saveStreamOperatorStateComplete(SSteamOpBasicInfo* pBasicInfo) {
pBasicInfo->updateOperatorInfo = false;
}

58
source/libs/executor/src/streamtimewindowoperator.c
View File

@ -18,6 +18,7 @@
#include "functionMgt.h"
#include "operator.h"
#include "querytask.h"
#include "streamexecutorInt.h"
#include "tchecksum.h"
#include "tcommon.h"
#include "tcompare.h"
@ -1211,13 +1212,16 @@ void doStreamIntervalDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
void doStreamIntervalSaveCheckpoint(SOperatorInfo* pOperator) {
SStreamIntervalOperatorInfo* pInfo = pOperator->info;
int32_t len = doStreamIntervalEncodeOpState(NULL, 0, pOperator);
void* buf = taosMemoryCalloc(1, len);
void* pBuf = buf;
len = doStreamIntervalEncodeOpState(&pBuf, len, pOperator);
pInfo->stateStore.streamStateSaveInfo(pInfo->pState, STREAM_INTERVAL_OP_CHECKPOINT_NAME,
strlen(STREAM_INTERVAL_OP_CHECKPOINT_NAME), buf, len);
taosMemoryFree(buf);
if (needSaveStreamOperatorInfo(&pInfo->basic)) {
int32_t len = doStreamIntervalEncodeOpState(NULL, 0, pOperator);
void* buf = taosMemoryCalloc(1, len);
void* pBuf = buf;
len = doStreamIntervalEncodeOpState(&pBuf, len, pOperator);
pInfo->stateStore.streamStateSaveInfo(pInfo->pState, STREAM_INTERVAL_OP_CHECKPOINT_NAME,
strlen(STREAM_INTERVAL_OP_CHECKPOINT_NAME), buf, len);
taosMemoryFree(buf);
saveStreamOperatorStateComplete(&pInfo->basic);
}
}
static void copyIntervalDeleteKey(SSHashObj* pMap, SArray* pWins) {
@ -1347,6 +1351,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
}
pInfo->numOfDatapack++;
printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo));
setStreamOperatorState(&pInfo->basic, pBlock->info.type);
if (pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_PULL_DATA) {
pInfo->binfo.pRes->info.type = pBlock->info.type;
@ -2690,13 +2695,16 @@ void* doStreamSessionDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
void doStreamSessionSaveCheckpoint(SOperatorInfo* pOperator) {
SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
int32_t len = doStreamSessionEncodeOpState(NULL, 0, pOperator, true);
void* buf = taosMemoryCalloc(1, len);
void* pBuf = buf;
len = doStreamSessionEncodeOpState(&pBuf, len, pOperator, true);
pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_SESSION_OP_CHECKPOINT_NAME,
strlen(STREAM_SESSION_OP_CHECKPOINT_NAME), buf, len);
taosMemoryFree(buf);
if (needSaveStreamOperatorInfo(&pInfo->basic)) {
int32_t len = doStreamSessionEncodeOpState(NULL, 0, pOperator, true);
void* buf = taosMemoryCalloc(1, len);
void* pBuf = buf;
len = doStreamSessionEncodeOpState(&pBuf, len, pOperator, true);
pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_SESSION_OP_CHECKPOINT_NAME,
strlen(STREAM_SESSION_OP_CHECKPOINT_NAME), buf, len);
taosMemoryFree(buf);
saveStreamOperatorStateComplete(&pInfo->basic);
}
}
void resetUnCloseSessionWinInfo(SSHashObj* winMap) {
@ -2766,6 +2774,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
break;
}
printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo));
setStreamOperatorState(&pInfo->basic, pBlock->info.type);
if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||
pBlock->info.type == STREAM_CLEAR) {
@ -3176,6 +3185,7 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) {
break;
}
printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo));
setStreamOperatorState(&pInfo->basic, pBlock->info.type);
if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||
pBlock->info.type == STREAM_CLEAR) {
@ -3673,13 +3683,16 @@ void* doStreamStateDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato
void doStreamStateSaveCheckpoint(SOperatorInfo* pOperator) {
SStreamStateAggOperatorInfo* pInfo = pOperator->info;
int32_t len = doStreamStateEncodeOpState(NULL, 0, pOperator, true);
void* buf = taosMemoryCalloc(1, len);
void* pBuf = buf;
len = doStreamStateEncodeOpState(&pBuf, len, pOperator, true);
pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_STATE_OP_CHECKPOINT_NAME,
strlen(STREAM_STATE_OP_CHECKPOINT_NAME), buf, len);
taosMemoryFree(buf);
if (needSaveStreamOperatorInfo(&pInfo->basic)) {
int32_t len = doStreamStateEncodeOpState(NULL, 0, pOperator, true);
void* buf = taosMemoryCalloc(1, len);
void* pBuf = buf;
len = doStreamStateEncodeOpState(&pBuf, len, pOperator, true);
pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_STATE_OP_CHECKPOINT_NAME,
strlen(STREAM_STATE_OP_CHECKPOINT_NAME), buf, len);
taosMemoryFree(buf);
saveStreamOperatorStateComplete(&pInfo->basic);
}
}
static SSDataBlock* buildStateResult(SOperatorInfo* pOperator) {
@ -3746,6 +3759,7 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
break;
}
printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo));
setStreamOperatorState(&pInfo->basic, pBlock->info.type);
if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||
pBlock->info.type == STREAM_CLEAR) {
@ -4069,6 +4083,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
pInfo->numOfDatapack++;
printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo));
setStreamOperatorState(&pInfo->basic, pBlock->info.type);
if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||
pBlock->info.type == STREAM_CLEAR) {
@ -4465,6 +4480,7 @@ static SSDataBlock* doStreamMidIntervalAgg(SOperatorInfo* pOperator) {
}
pInfo->numOfDatapack++;
printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo));
setStreamOperatorState(&pInfo->basic, pBlock->info.type);
if (pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_PULL_DATA) {
pInfo->binfo.pRes->info.type = pBlock->info.type;

86
source/libs/executor/src/sysscanoperator.c
View File

@ -1651,50 +1651,56 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) {
SSysTableScanInfo* pInfo = pOperator->info;
char dbName[TSDB_DB_NAME_LEN] = {0};
if (isTaskKilled(pOperator->pTaskInfo)) {
setOperatorCompleted(pOperator);
return NULL;
}
while (1) {
blockDataCleanup(pInfo->pRes);
const char* name = tNameGetTableName(&pInfo->name);
if (pInfo->showRewrite) {
getDBNameFromCondition(pInfo->pCondition, dbName);
if (strncasecmp(name, TSDB_INS_TABLE_COMPACTS, TSDB_TABLE_FNAME_LEN) != 0 &&
strncasecmp(name, TSDB_INS_TABLE_COMPACT_DETAILS, TSDB_TABLE_FNAME_LEN) != 0) {
sprintf(pInfo->req.db, "%d.%s", pInfo->accountId, dbName);
}
} else if (strncasecmp(name, TSDB_INS_TABLE_COLS, TSDB_TABLE_FNAME_LEN) == 0) {
getDBNameFromCondition(pInfo->pCondition, dbName);
if (dbName[0]) sprintf(pInfo->req.db, "%d.%s", pInfo->accountId, dbName);
sysTableIsCondOnOneTable(pInfo->pCondition, pInfo->req.filterTb);
}
SSDataBlock* pBlock = NULL;
if (strncasecmp(name, TSDB_INS_TABLE_TABLES, TSDB_TABLE_FNAME_LEN) == 0) {
pBlock = sysTableScanUserTables(pOperator);
} else if (strncasecmp(name, TSDB_INS_TABLE_TAGS, TSDB_TABLE_FNAME_LEN) == 0) {
pBlock = sysTableScanUserTags(pOperator);
} else if (strncasecmp(name, TSDB_INS_TABLE_COLS, TSDB_TABLE_FNAME_LEN) == 0 && pInfo->readHandle.mnd == NULL) {
pBlock = sysTableScanUserCols(pOperator);
} else if (strncasecmp(name, TSDB_INS_TABLE_STABLES, TSDB_TABLE_FNAME_LEN) == 0 && pInfo->showRewrite &&
IS_SYS_DBNAME(dbName)) {
pBlock = sysTableScanUserSTables(pOperator);
} else { // load the meta from mnode of the given epset
pBlock = sysTableScanFromMNode(pOperator, pInfo, name, pTaskInfo);
}
sysTableScanFillTbName(pOperator, pInfo, name, pBlock);
if (pBlock != NULL) {
bool limitReached = applyLimitOffset(&pInfo->limitInfo, pBlock, pTaskInfo);
if (limitReached) {
if (isTaskKilled(pOperator->pTaskInfo)) {
setOperatorCompleted(pOperator);
return NULL;
}
return pBlock->info.rows > 0 ? pBlock : NULL;
} else {
return NULL;
blockDataCleanup(pInfo->pRes);
const char* name = tNameGetTableName(&pInfo->name);
if (pInfo->showRewrite) {
getDBNameFromCondition(pInfo->pCondition, dbName);
if (strncasecmp(name, TSDB_INS_TABLE_COMPACTS, TSDB_TABLE_FNAME_LEN) != 0 &&
strncasecmp(name, TSDB_INS_TABLE_COMPACT_DETAILS, TSDB_TABLE_FNAME_LEN) != 0) {
sprintf(pInfo->req.db, "%d.%s", pInfo->accountId, dbName);
}
} else if (strncasecmp(name, TSDB_INS_TABLE_COLS, TSDB_TABLE_FNAME_LEN) == 0) {
getDBNameFromCondition(pInfo->pCondition, dbName);
if (dbName[0]) sprintf(pInfo->req.db, "%d.%s", pInfo->accountId, dbName);
sysTableIsCondOnOneTable(pInfo->pCondition, pInfo->req.filterTb);
}
SSDataBlock* pBlock = NULL;
if (strncasecmp(name, TSDB_INS_TABLE_TABLES, TSDB_TABLE_FNAME_LEN) == 0) {
pBlock = sysTableScanUserTables(pOperator);
} else if (strncasecmp(name, TSDB_INS_TABLE_TAGS, TSDB_TABLE_FNAME_LEN) == 0) {
pBlock = sysTableScanUserTags(pOperator);
} else if (strncasecmp(name, TSDB_INS_TABLE_COLS, TSDB_TABLE_FNAME_LEN) == 0 && pInfo->readHandle.mnd == NULL) {
pBlock = sysTableScanUserCols(pOperator);
} else if (strncasecmp(name, TSDB_INS_TABLE_STABLES, TSDB_TABLE_FNAME_LEN) == 0 && pInfo->showRewrite &&
IS_SYS_DBNAME(dbName)) {
pBlock = sysTableScanUserSTables(pOperator);
} else { // load the meta from mnode of the given epset
pBlock = sysTableScanFromMNode(pOperator, pInfo, name, pTaskInfo);
}
sysTableScanFillTbName(pOperator, pInfo, name, pBlock);
if (pBlock != NULL) {
bool limitReached = applyLimitOffset(&pInfo->limitInfo, pBlock, pTaskInfo);
if (limitReached) {
setOperatorCompleted(pOperator);
}
if (pBlock->info.rows == 0) {
continue;
}
return pBlock;
} else {
return NULL;
}
}
}

2
source/libs/function/src/udfd.c
View File

@ -1424,7 +1424,7 @@ int main(int argc, char *argv[]) {
printf("failed to start since init log error\n");
}
if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 0) != 0) {
if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 0, true) != 0) {
fnError("failed to start since read config error");
return -2;
}

2
source/libs/function/test/runUdf.c
View File

@ -127,7 +127,7 @@ int aggregateFuncTest() {
int main(int argc, char *argv[]) {
parseArgs(argc, argv);
initLog();
if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 0) != 0) {
if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 0, true) != 0) {
fnError("failed to start since read config error");
return -1;
}

70
source/libs/geometry/src/geosWrapper.c
View File

@ -86,6 +86,67 @@ _exit:
return code;
}
static int initWktRegex(pcre2_code **ppRegex, pcre2_match_data **ppMatchData) {
int ret = 0;
char *wktPatternWithSpace = taosMemoryCalloc(4, 1024);
sprintf(
wktPatternWithSpace,
"^( *)point( *)z?m?( *)((empty)|(\\(( *)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}( *)\\)))|linestring( *)z?m?( "
"*)((empty)|(\\(( *)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}(( *)(,)( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}( *))*( *)\\)))|polygon( *)z?m?( "
"*)((empty)|(\\(( *)((empty)|(\\(( *)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}(( *)(,)( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}( *))*( *)\\)))(( *)(,)( "
"*)((empty)|(\\(( *)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}(( *)(,)( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}( *))*( *)\\)))( *))*( "
"*)\\)))|multipoint( *)z?m?( *)((empty)|(\\(( "
"*)((([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}|((empty)|(\\(( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}( *)\\))))(( *)(,)( "
"*)((([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}|((empty)|(\\(( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}( *)\\))))( *))*( "
"*)\\)))|multilinestring( *)z?m?( *)((empty)|(\\(( *)((empty)|(\\(( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}(( *)(,)( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}( *))*( *)\\)))(( *)(,)( "
"*)((empty)|(\\(( *)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}(( *)(,)( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}( *))*( *)\\)))( *))*( "
"*)\\)))|multipolygon( *)z?m?( *)((empty)|(\\(( *)((empty)|(\\(( *)((empty)|(\\(( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}(( *)(,)( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}( *))*( *)\\)))(( *)(,)( "
"*)((empty)|(\\(( *)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}(( *)(,)( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}( *))*( *)\\)))( *))*( *)\\)))(( *)(,)( "
"*)((empty)|(\\(( *)((empty)|(\\(( *)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}(( *)(,)( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}( *))*( *)\\)))(( *)(,)( "
"*)((empty)|(\\(( *)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}(( *)(,)( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}( *))*( *)\\)))( *))*( *)\\)))( *))*( "
"*)\\)))|(GEOCOLLECTION\\((?R)(( *)(,)( *)(?R))*( *)\\))( *)$");
ret = doRegComp(ppRegex, ppMatchData, wktPatternWithSpace);
taosMemoryFree(wktPatternWithSpace);
return ret;
}
int32_t initCtxGeomFromText() {
int32_t code = TSDB_CODE_FAILED;
SGeosContext* geosCtx = getThreadLocalGeosCtx();
@ -113,6 +174,10 @@ int32_t initCtxGeomFromText() {
}
}
if (geosCtx->WKTRegex == NULL) {
if (initWktRegex(&geosCtx->WKTRegex, &geosCtx->WKTMatchData) != 0) return code;
}
return TSDB_CODE_SUCCESS;
}
@ -125,6 +190,11 @@ int32_t doGeomFromText(const char *inputWKT, unsigned char **outputGeom, size_t
GEOSGeometry *geom = NULL;
unsigned char *wkb = NULL;
if (doRegExec(inputWKT, geosCtx->WKTRegex, geosCtx->WKTMatchData) != 0) {
code = TSDB_CODE_FUNC_FUNTION_PARA_VALUE;
goto _exit;
}
geom = GEOSWKTReader_read_r(geosCtx->handle, geosCtx->WKTReader, inputWKT);
if (geom == NULL) {
code = TSDB_CODE_FUNC_FUNTION_PARA_VALUE;

8
source/libs/nodes/src/nodesCodeFuncs.c
View File

@ -187,6 +187,8 @@ const char* nodesNodeName(ENodeType type) {
return "BalanceVgroupStmt";
case QUERY_NODE_BALANCE_VGROUP_LEADER_STMT:
return "BalanceVgroupLeaderStmt";
case QUERY_NODE_BALANCE_VGROUP_LEADER_DATABASE_STMT:
return "BalanceVgroupLeaderStmt";
case QUERY_NODE_MERGE_VGROUP_STMT:
return "MergeVgroupStmt";
case QUERY_NODE_SHOW_DB_ALIVE_STMT:
@ -7607,6 +7609,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
return TSDB_CODE_SUCCESS; // SBalanceVgroupStmt has no fields to serialize.
case QUERY_NODE_BALANCE_VGROUP_LEADER_STMT:
return TSDB_CODE_SUCCESS; // SBalanceVgroupLeaderStmt has no fields to serialize.
case QUERY_NODE_BALANCE_VGROUP_LEADER_DATABASE_STMT:
return TSDB_CODE_SUCCESS;
case QUERY_NODE_MERGE_VGROUP_STMT:
return mergeVgroupStmtToJson(pObj, pJson);
case QUERY_NODE_REDISTRIBUTE_VGROUP_STMT:
@ -7953,7 +7957,9 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
case QUERY_NODE_BALANCE_VGROUP_STMT:
return TSDB_CODE_SUCCESS; // SBalanceVgroupStmt has no fields to deserialize.
case QUERY_NODE_BALANCE_VGROUP_LEADER_STMT:
return TSDB_CODE_SUCCESS; // SBalanceVgroupLeaderStmt has no fields to deserialize.
return TSDB_CODE_SUCCESS;
case QUERY_NODE_BALANCE_VGROUP_LEADER_DATABASE_STMT:
return TSDB_CODE_SUCCESS; // SBalanceVgroupLeaderStmt has no fields to deserialize.
case QUERY_NODE_MERGE_VGROUP_STMT:
return jsonToMergeVgroupStmt(pJson, pObj);
case QUERY_NODE_REDISTRIBUTE_VGROUP_STMT:

3
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -473,6 +473,8 @@ SNode* nodesMakeNode(ENodeType type) {
return makeNode(type, sizeof(SBalanceVgroupStmt));
case QUERY_NODE_BALANCE_VGROUP_LEADER_STMT:
return makeNode(type, sizeof(SBalanceVgroupLeaderStmt));
case QUERY_NODE_BALANCE_VGROUP_LEADER_DATABASE_STMT:
return makeNode(type, sizeof(SBalanceVgroupLeaderStmt));
case QUERY_NODE_MERGE_VGROUP_STMT:
return makeNode(type, sizeof(SMergeVgroupStmt));
case QUERY_NODE_REDISTRIBUTE_VGROUP_STMT:
@ -1161,6 +1163,7 @@ void nodesDestroyNode(SNode* pNode) {
case QUERY_NODE_RESUME_STREAM_STMT: // no pointer field
case QUERY_NODE_BALANCE_VGROUP_STMT: // no pointer field
case QUERY_NODE_BALANCE_VGROUP_LEADER_STMT: // no pointer field
case QUERY_NODE_BALANCE_VGROUP_LEADER_DATABASE_STMT: // no pointer field
case QUERY_NODE_MERGE_VGROUP_STMT: // no pointer field
break;
case QUERY_NODE_REDISTRIBUTE_VGROUP_STMT:

1
source/libs/parser/inc/parAst.h
View File

@ -274,6 +274,7 @@ SNode* createKillStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pId
SNode* createKillQueryStmt(SAstCreateContext* pCxt, const SToken* pQueryId);
SNode* createBalanceVgroupStmt(SAstCreateContext* pCxt);
SNode* createBalanceVgroupLeaderStmt(SAstCreateContext* pCxt, const SToken* pVgId);
SNode* createBalanceVgroupLeaderDBNameStmt(SAstCreateContext* pCxt, const SToken* pDbName);
SNode* createMergeVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId1, const SToken* pVgId2);
SNode* createRedistributeVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId, SNodeList* pDnodes);
SNode* createSplitVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId);

1
source/libs/parser/inc/sql.y
View File

@ -773,6 +773,7 @@ cmd ::= KILL COMPACT NK_INTEGER(A).
/************************************************ merge/redistribute/ vgroup ******************************************/
cmd ::= BALANCE VGROUP. { pCxt->pRootNode = createBalanceVgroupStmt(pCxt); }
cmd ::= BALANCE VGROUP LEADER on_vgroup_id(A). { pCxt->pRootNode = createBalanceVgroupLeaderStmt(pCxt, &A); }
cmd ::= BALANCE VGROUP LEADER DATABASE db_name(A). { pCxt->pRootNode = createBalanceVgroupLeaderDBNameStmt(pCxt, &A); }
cmd ::= MERGE VGROUP NK_INTEGER(A) NK_INTEGER(B). { pCxt->pRootNode = createMergeVgroupStmt(pCxt, &A, &B); }
cmd ::= REDISTRIBUTE VGROUP NK_INTEGER(A) dnode_list(B). { pCxt->pRootNode = createRedistributeVgroupStmt(pCxt, &A, B); }
cmd ::= SPLIT VGROUP NK_INTEGER(A). { pCxt->pRootNode = createSplitVgroupStmt(pCxt, &A); }

10
source/libs/parser/src/parAstCreater.c
View File

@ -2809,6 +2809,16 @@ SNode* createBalanceVgroupLeaderStmt(SAstCreateContext* pCxt, const SToken* pVgI
return (SNode*)pStmt;
}
SNode* createBalanceVgroupLeaderDBNameStmt(SAstCreateContext* pCxt, const SToken* pDbName){
CHECK_PARSER_STATUS(pCxt);
SBalanceVgroupLeaderStmt* pStmt = (SBalanceVgroupLeaderStmt*)nodesMakeNode(QUERY_NODE_BALANCE_VGROUP_LEADER_DATABASE_STMT);
CHECK_OUT_OF_MEM(pStmt);
if (NULL != pDbName) {
COPY_STRING_FORM_ID_TOKEN(pStmt->dbName, pDbName);
}
return (SNode*)pStmt;
}
SNode* createMergeVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId1, const SToken* pVgId2) {
CHECK_PARSER_STATUS(pCxt);
SMergeVgroupStmt* pStmt = (SMergeVgroupStmt*)nodesMakeNode(QUERY_NODE_MERGE_VGROUP_STMT);

24
source/libs/parser/src/parAstParser.c
View File

@ -42,7 +42,6 @@ int32_t buildQueryAfterParse(SQuery** pQuery, SNode* pRootNode, int16_t placehol
return TSDB_CODE_SUCCESS;
}
int32_t parse(SParseContext* pParseCxt, SQuery** pQuery) {
SAstCreateContext cxt;
initAstCreateContext(pParseCxt, &cxt);
@ -635,8 +634,8 @@ static int32_t collectMetaKeyFromShowCompacts(SCollectMetaKeyCxt* pCxt, SShowStm
}
static int32_t collectMetaKeyFromShowCompactDetails(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) {
int32_t code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_COMPACT_DETAILS,
pCxt->pMetaCache);
int32_t code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB,
TSDB_INS_TABLE_COMPACT_DETAILS, pCxt->pMetaCache);
return code;
}
@ -691,7 +690,7 @@ static int32_t collectMetaKeyFromShowCreateView(SCollectMetaKeyCxt* pCxt, SShowC
int32_t code = catalogRemoveViewMeta(pCxt->pParseCxt->pCatalog, dbFName, 0, pStmt->viewName, 0);
if (TSDB_CODE_SUCCESS == code) {
code = reserveViewUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName, pStmt->viewName,
AUTH_TYPE_READ, pCxt->pMetaCache);
AUTH_TYPE_READ, pCxt->pMetaCache);
}
if (TSDB_CODE_SUCCESS == code) {
code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->viewName, pCxt->pMetaCache);
@ -700,7 +699,6 @@ static int32_t collectMetaKeyFromShowCreateView(SCollectMetaKeyCxt* pCxt, SShowC
return code;
}
static int32_t collectMetaKeyFromShowApps(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) {
return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_PERFORMANCE_SCHEMA_DB, TSDB_PERFS_TABLE_APPS,
pCxt->pMetaCache);
@ -759,24 +757,22 @@ static int32_t collectMetaKeyFromRevoke(SCollectMetaKeyCxt* pCxt, SRevokeStmt* p
return reserveTableMetaInCache(pCxt->pParseCxt->acctId, pStmt->objName, pStmt->tabName, pCxt->pMetaCache);
}
static int32_t collectMetaKeyFromCreateViewStmt(SCollectMetaKeyCxt* pCxt, SCreateViewStmt* pStmt) {
int32_t code =
reserveTableMetaInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->viewName, pCxt->pMetaCache);
int32_t code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->viewName, pCxt->pMetaCache);
if (TSDB_CODE_SUCCESS == code) {
code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName, NULL, AUTH_TYPE_WRITE,
pCxt->pMetaCache);
}
if (TSDB_CODE_SUCCESS == code) {
code = reserveViewUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName, pStmt->viewName, AUTH_TYPE_ALTER,
pCxt->pMetaCache);
code = reserveViewUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName, pStmt->viewName,
AUTH_TYPE_ALTER, pCxt->pMetaCache);
}
return code;
}
static int32_t collectMetaKeyFromDropViewStmt(SCollectMetaKeyCxt* pCxt, SDropViewStmt* pStmt) {
int32_t code = reserveViewUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName, pStmt->viewName, AUTH_TYPE_ALTER,
pCxt->pMetaCache);
int32_t code = reserveViewUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName,
pStmt->viewName, AUTH_TYPE_ALTER, pCxt->pMetaCache);
return code;
}
@ -787,7 +783,7 @@ static int32_t collectMetaKeyFromCreateTSMAStmt(SCollectMetaKeyCxt* pCxt, SCreat
code = reserveTSMAInfoInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, pCxt->pMetaCache);
if (TSDB_CODE_SUCCESS == code) {
char dstTbName[TSDB_TABLE_NAME_LEN] = {0};
snprintf(dstTbName, TSDB_TABLE_NAME_LEN, "%s"TSMA_RES_STB_POSTFIX, pStmt->tableName);
snprintf(dstTbName, TSDB_TABLE_NAME_LEN, "%s" TSMA_RES_STB_POSTFIX, pStmt->tableName);
code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, pStmt->dbName, dstTbName, pCxt->pMetaCache);
if (TSDB_CODE_SUCCESS == code) {
code = reserveTableVgroupInCache(pCxt->pParseCxt->acctId, pStmt->dbName, dstTbName, pCxt->pMetaCache);
@ -925,7 +921,7 @@ static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) {
case QUERY_NODE_SHOW_COMPACTS_STMT:
return collectMetaKeyFromShowCompacts(pCxt, (SShowStmt*)pStmt);
case QUERY_NODE_SHOW_COMPACT_DETAILS_STMT:
return collectMetaKeyFromShowCompactDetails(pCxt, (SShowStmt*)pStmt);
return collectMetaKeyFromShowCompactDetails(pCxt, (SShowStmt*)pStmt);
case QUERY_NODE_SHOW_GRANTS_FULL_STMT:
return collectMetaKeyFromShowGrantsFull(pCxt, (SShowStmt*)pStmt);
case QUERY_NODE_SHOW_GRANTS_LOGS_STMT:

17
source/libs/parser/src/parTranslater.c
View File

@ -4338,7 +4338,7 @@ int32_t translateTable(STranslateContext* pCxt, SNode** pTable, SNode* pJoinPare
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GET_META_ERROR, tstrerror(code));
}
#ifdef TD_ENTERPRISE
if (TSDB_VIEW_TABLE == pRealTable->pMeta->tableType) {
if (TSDB_VIEW_TABLE == pRealTable->pMeta->tableType && !pCurrSmt->tagScan) {
return translateView(pCxt, pTable, &name);
}
translateAudit(pCxt, pRealTable, &name);
@ -10578,6 +10578,7 @@ static int32_t translateBalanceVgroup(STranslateContext* pCxt, SBalanceVgroupStm
static int32_t translateBalanceVgroupLeader(STranslateContext* pCxt, SBalanceVgroupLeaderStmt* pStmt) {
SBalanceVgroupLeaderReq req = {0};
req.vgId = pStmt->vgId;
if(pStmt->dbName != NULL) strcpy(req.db, pStmt->dbName);
int32_t code =
buildCmdMsg(pCxt, TDMT_MND_BALANCE_VGROUP_LEADER, (FSerializeFunc)tSerializeSBalanceVgroupLeaderReq, &req);
tFreeSBalanceVgroupLeaderReq(&req);
@ -11263,6 +11264,9 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) {
case QUERY_NODE_BALANCE_VGROUP_LEADER_STMT:
code = translateBalanceVgroupLeader(pCxt, (SBalanceVgroupLeaderStmt*)pNode);
break;
case QUERY_NODE_BALANCE_VGROUP_LEADER_DATABASE_STMT:
code = translateBalanceVgroupLeader(pCxt, (SBalanceVgroupLeaderStmt*)pNode);
break;
case QUERY_NODE_MERGE_VGROUP_STMT:
code = translateMergeVgroup(pCxt, (SMergeVgroupStmt*)pNode);
break;
@ -11875,11 +11879,18 @@ static int32_t checkShowTags(STranslateContext* pCxt, const SShowStmt* pShow) {
toName(pCxt->pParseCxt->acctId, ((SValueNode*)pShow->pDbName)->literal,
((SValueNode*)pShow->pTbName)->literal, &name),
&pTableMeta, true);
taosMemoryFreeClear(pTableMeta);
if (TSDB_CODE_SUCCESS != code) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GET_META_ERROR, tstrerror(code));
code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GET_META_ERROR, tstrerror(code));
goto _exit;
}
if (TSDB_SUPER_TABLE != pTableMeta->tableType && TSDB_CHILD_TABLE != pTableMeta->tableType) {
code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAGS_PC,
"The _TAGS pseudo column can only be used for child table and super table queries");
goto _exit;
}
_exit:
taosMemoryFreeClear(pTableMeta);
return code;
}

4364
source/libs/parser/src/sql.c
View File

File diff suppressed because it is too large Load Diff

7
source/libs/planner/src/planOptimizer.c
View File

@ -246,11 +246,10 @@ static bool scanPathOptMayBeOptimized(SLogicNode* pNode) {
static bool scanPathOptShouldGetFuncs(SLogicNode* pNode) {
if (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode)) {
if (pNode->pParent && QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pNode->pParent)) {
if (WINDOW_TYPE_INTERVAL == ((SWindowLogicNode*)pNode->pParent)->winType) return true;
} else {
if (!pNode->pParent || QUERY_NODE_LOGIC_PLAN_WINDOW != nodeType(pNode->pParent) ||
WINDOW_TYPE_INTERVAL == ((SWindowLogicNode*)pNode->pParent)->winType)
return !scanPathOptHaveNormalCol(((SPartitionLogicNode*)pNode)->pPartitionKeys);
}
return false;
}
if ((QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pNode) &&

2
source/libs/stream/src/streamMeta.c
View File

@ -1631,6 +1631,8 @@ int32_t streamMetaAddFailedTask(SStreamMeta* pMeta, int64_t streamId, int32_t ta
streamMetaAddTaskLaunchResult(pMeta, hId.streamId, hId.taskId, startTs, now, false);
}
} else {
streamMetaRUnLock(pMeta);
stError("failed to locate the stream task:0x%" PRIx64 "-0x%x (vgId:%d), it may have been destroyed or stopped",
streamId, taskId, pMeta->vgId);
code = TSDB_CODE_STREAM_TASK_NOT_EXIST;

7
source/libs/stream/src/tstreamFileState.c
View File

@ -557,7 +557,6 @@ int32_t flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot,
const int32_t BATCH_LIMIT = 256;
int64_t st = taosGetTimestampMs();
int32_t numOfElems = listNEles(pSnapshot);
SListNode* pNode = NULL;
int idx = streamStateGetCfIdx(pFileState->pFileStore, pFileState->cfName);
@ -589,8 +588,11 @@ int32_t flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot,
}
taosMemoryFree(buf);
if (streamStateGetBatchSize(batch) > 0) {
int32_t numOfElems = streamStateGetBatchSize(batch);
if (numOfElems > 0) {
streamStatePutBatch_rocksdb(pFileState->pFileStore, batch);
} else {
goto _end;
}
streamStateClearBatch(batch);
@ -609,6 +611,7 @@ int32_t flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot,
streamStatePutBatch_rocksdb(pFileState->pFileStore, batch);
}
_end:
streamStateDestroyBatch(batch);
return code;
}

2
source/libs/transport/inc/transComm.h
View File

@ -123,7 +123,7 @@ typedef struct {
STransMsg* pRsp;
SEpSet epSet;
int8_t hasEpSet;
tsem_t* pSem;
tsem2_t* pSem;
int8_t inited;
SRWLatch latch;
} STransSyncMsg;

8
source/libs/transport/src/transCli.c
View File

@ -2450,7 +2450,7 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) {
pSyncMsg->hasEpSet = 1;
epsetAssign(&pSyncMsg->epSet, &pCtx->epSet);
}
tsem_post(pSyncMsg->pSem);
tsem2_post(pSyncMsg->pSem);
taosReleaseRef(transGetSyncMsgMgt(), pCtx->syncMsgRef);
} else {
rpcFreeCont(pResp->pCont);
@ -2679,8 +2679,8 @@ _RETURN:
return ret;
}
int64_t transCreateSyncMsg(STransMsg* pTransMsg) {
tsem_t* sem = taosMemoryCalloc(1, sizeof(tsem_t));
tsem_init(sem, 0, 0);
tsem2_t* sem = taosMemoryCalloc(1, sizeof(tsem2_t));
tsem2_init(sem, 0, 0);
STransSyncMsg* pSyncMsg = taosMemoryCalloc(1, sizeof(STransSyncMsg));
@ -2740,7 +2740,7 @@ int transSendRecvWithTimeout(void* shandle, SEpSet* pEpSet, STransMsg* pReq, STr
goto _RETURN;
}
ret = tsem_timewait(pSyncMsg->pSem, timeoutMs);
ret = tsem2_timewait(pSyncMsg->pSem, timeoutMs);
if (ret < 0) {
pRsp->code = TSDB_CODE_TIMEOUT_ERROR;
ret = TSDB_CODE_TIMEOUT_ERROR;

2
source/libs/transport/src/transComm.c
View File

@ -681,7 +681,7 @@ void transDestroySyncMsg(void* msg) {
if (msg == NULL) return;
STransSyncMsg* pSyncMsg = msg;
tsem_destroy(pSyncMsg->pSem);
tsem2_destroy(pSyncMsg->pSem);
taosMemoryFree(pSyncMsg->pSem);
transFreeMsg(pSyncMsg->pRsp->pCont);
taosMemoryFree(pSyncMsg->pRsp);

105
source/os/src/osSemaphore.c
View File

@ -215,14 +215,6 @@ int32_t taosGetAppName(char* name, int32_t* len) {
return 0;
}
int32_t tsem_wait(tsem_t* sem) {
int ret = 0;
do {
ret = sem_wait(sem);
} while (ret != 0 && errno == EINTR);
return ret;
}
int32_t tsem_timewait(tsem_t* sem, int64_t ms) {
int ret = 0;
@ -241,4 +233,101 @@ int32_t tsem_timewait(tsem_t* sem, int64_t ms) {
return ret;
}
int32_t tsem_wait(tsem_t* sem) {
int ret = 0;
do {
ret = sem_wait(sem);
} while (ret != 0 && errno == EINTR);
return ret;
}
int tsem2_init(tsem2_t* sem, int pshared, unsigned int value) {
int ret = taosThreadMutexInit(&sem->mutex, NULL);
if (ret != 0) return ret;
ret = taosThreadCondAttrInit(&sem->attr);
if (ret != 0)
{
taosThreadMutexDestroy(&sem->mutex);
return ret;
}
ret = taosThreadCondAttrSetclock(&sem->attr, CLOCK_MONOTONIC);
if (ret != 0)
{
taosThreadMutexDestroy(&sem->mutex);
taosThreadCondAttrDestroy(&sem->attr);
return ret;
}
ret = taosThreadCondInit(&sem->cond, &sem->attr);
if (ret != 0)
{
taosThreadMutexDestroy(&sem->mutex);
taosThreadCondAttrDestroy(&sem->attr);
return ret;
}
sem->count = value;
return 0;
}
int tsem2_post(tsem2_t *sem) {
taosThreadMutexLock(&sem->mutex);
sem->count++;
taosThreadCondSignal(&sem->cond);
taosThreadMutexUnlock(&sem->mutex);
return 0;
}
int tsem2_destroy(tsem2_t* sem) {
taosThreadMutexDestroy(&sem->mutex);
taosThreadCondDestroy(&sem->cond);
taosThreadCondAttrDestroy(&sem->attr);
return 0;
}
int32_t tsem2_wait(tsem2_t* sem) {
taosThreadMutexLock(&sem->mutex);
while (sem->count <= 0) {
int ret = taosThreadCondWait(&sem->cond, &sem->mutex);
if (0 == ret) {
continue;
} else {
taosThreadMutexUnlock(&sem->mutex);
return ret;
}
}
sem->count--;
taosThreadMutexUnlock(&sem->mutex);
return 0;
}
int32_t tsem2_timewait(tsem2_t* sem, int64_t ms) {
int ret = 0;
taosThreadMutexLock(&sem->mutex);
if (sem->count <= 0) {
struct timespec ts = {0};
if (clock_gettime(CLOCK_MONOTONIC, &ts) == -1) {
taosThreadMutexUnlock(&sem->mutex);
return -1;
}
ts.tv_sec += ms / 1000;
ts.tv_nsec += (ms % 1000) * 1000000;
ts.tv_sec += ts.tv_nsec / 1000000000;
ts.tv_nsec %= 1000000000;
while (sem->count <= 0) {
ret = taosThreadCondTimedWait(&sem->cond, &sem->mutex, &ts);
if (ret != 0) {
taosThreadMutexUnlock(&sem->mutex);
return ret;
}
}
}
sem->count--;
taosThreadMutexUnlock(&sem->mutex);
return ret;
}
#endif

10
source/os/src/osThread.c
View File

@ -170,6 +170,16 @@ int32_t taosThreadCondAttrInit(TdThreadCondAttr *attr) {
#endif
}
int32_t taosThreadCondAttrSetclock(TdThreadCondAttr *attr, int clockId) {
#ifdef __USE_WIN_THREAD
return 0;
#elif defined(__APPLE__)
return 0;
#else
return pthread_condattr_setclock(attr, clockId);
#endif
}
int32_t taosThreadCondAttrSetPshared(TdThreadCondAttr *attr, int32_t pshared) {
#ifdef __USE_WIN_THREAD
return 0;

133
source/os/test/osSemaphoreTests.cpp
View File

@ -71,7 +71,6 @@ TEST(osSemaphoreTests, WaitTime1) {
tsem_destroy(&sem);
}
TEST(osSemaphoreTests, WaitAndPost) {
tsem_t sem;
int result = tsem_init(&sem, 0, 0);
@ -106,3 +105,135 @@ TEST(osSemaphoreTests, TimedWait) {
result = tsem_destroy(&sem);
EXPECT_EQ(result, 0);
}
TEST(osSemaphoreTests, Performance1_1) {
tsem_t sem;
const int count = 100000;
tsem_init(&sem, 0, 0);
std::thread([&sem, count]() {
for (int i = 0; i < count; ++i) {
tsem_post(&sem);
}
}).detach();
for (int i = 0; i < count; ++i) {
tsem_wait(&sem);
}
tsem_destroy(&sem);
}
TEST(osSemaphoreTests, Performance1_2) {
tsem2_t sem;
const int count = 100000;
tsem2_init(&sem, 0, 0);
std::thread([&sem, count]() {
for (int i = 0; i < count; ++i) {
tsem2_post(&sem);
}
}).detach();
for (int i = 0; i < count; ++i) {
tsem2_wait(&sem);
}
tsem2_destroy(&sem);
}
TEST(osSemaphoreTests, Performance2_1) {
tsem_t sem;
const int count = 50000;
tsem_init(&sem, 0, 0);
std::thread([&sem, count]() {
for (int i = 0; i < count; ++i) {
tsem_post(&sem);
}
}).detach();
std::thread([&sem, count]() {
for (int i = 0; i < count; ++i) {
tsem_post(&sem);
}
}).detach();
for (int i = 0; i < count * 2; ++i) {
tsem_wait(&sem);
}
tsem_destroy(&sem);
}
TEST(osSemaphoreTests, Performance2_2) {
tsem2_t sem;
const int count = 50000;
tsem2_init(&sem, 0, 0);
std::thread([&sem, count]() {
for (int i = 0; i < count; ++i) {
tsem2_post(&sem);
}
}).detach();
std::thread([&sem, count]() {
for (int i = 0; i < count; ++i) {
tsem2_post(&sem);
}
}).detach();
for (int i = 0; i < count * 2; ++i) {
tsem2_wait(&sem);
}
tsem2_destroy(&sem);
}
TEST(osSemaphoreTests, Performance3_1) {
const int count = 100000;
for (int i = 0; i < count; ++i) {
tsem_t sem;
tsem_init(&sem, 0, 1);
EXPECT_EQ(tsem_timewait(&sem, 1000), 0);
tsem_destroy(&sem);
}
}
TEST(osSemaphoreTests, Performance3_2) {
const int count = 100000;
for (int i = 0; i < count; ++i) {
tsem2_t sem;
tsem2_init(&sem, 0, 1);
EXPECT_EQ(tsem2_timewait(&sem, 1000), 0);
tsem2_destroy(&sem);
}
}
TEST(osSemaphoreTests, Performance4_1) {
const int count = 1000;
for (int i = 0; i < count; ++i) {
tsem_t sem;
tsem_init(&sem, 0, 0);
std::thread([&sem, count]() {
tsem_post(&sem);
}).detach();
tsem_timewait(&sem, 1000);
tsem_destroy(&sem);
}
}
TEST(osSemaphoreTests, Performance4_2) {
const int count = 1000;
for (int i = 0; i < count; ++i) {
tsem2_t sem;
tsem2_init(&sem, 0, 0);
std::thread([&sem, count]() {
tsem2_post(&sem);
}).detach();
tsem2_timewait(&sem, 1000);
tsem2_destroy(&sem);
}
}

18
source/util/CMakeLists.txt
View File

@ -21,29 +21,29 @@ target_include_directories(
PRIVATE "${TD_SOURCE_DIR}/utils/TSZ/sz/inc"
PRIVATE "${TD_SOURCE_DIR}/utils/TSZ/zstd/"
PRIVATE "${TD_SOURCE_DIR}/contrib/lzma2/"
PRIVATE "${TD_SOURCE_DIR}/contrib/pcre2/"
)
target_link_directories(
util
PUBLIC "${TD_SOURCE_DIR}/contrib/lzma2"
PUBLIC "${TD_SOURCE_DIR}/contrib/pcre2"
)
if (TD_LINUX)
target_link_libraries(
util
PUBLIC os common
PUBLIC lz4_static fast-lzma2
PUBLIC api cjson geos_c TSZ
)
else()
PUBLIC lz4_static fast-lzma2 pcre2-8
PUBLIC api cjson geos_c TSZ
)
else()
target_link_libraries(
util
PUBLIC os common
PUBLIC lz4_static
PUBLIC api cjson geos_c TSZ
)
PUBLIC lz4_static pcre2-8
PUBLIC api cjson geos_c TSZ
)
endif()
if(${BUILD_TEST})

23
source/util/src/tcompression.c
View File

@ -61,9 +61,7 @@
#include "zstd.h"
#endif
#ifdef TD_TSZ
#include "td_sz.h"
#endif
int32_t tsCompressPlain2(const char *const input, const int32_t nelements, char *const output, const char type);
int32_t tsDecompressPlain2(const char *const input, const int32_t nelements, char *const output, const char type);
@ -322,7 +320,6 @@ static const int32_t TEST_NUMBER = 1;
#define safeInt64Add(a, b) (((a >= 0) && (b <= INT64_MAX - a)) || ((a < 0) && (b >= INT64_MIN - a)))
#ifdef TD_TSZ
bool lossyFloat = false;
bool lossyDouble = false;
@ -341,8 +338,6 @@ int32_t tsCompressInit(char *lossyColumns, float fPrecision, double dPrecision,
// exit call
void tsCompressExit() { tdszExit(); }
#endif
/*
* Compress Integer (Simple8B).
*/
@ -1214,7 +1209,6 @@ int32_t tsDecompressFloatImp(const char *const input, const int32_t nelements, c
return nelements * FLOAT_BYTES;
}
#ifdef TD_TSZ
//
// ---------- float double lossy -----------
//
@ -1283,7 +1277,6 @@ int32_t tsDecompressDoubleLossyImp(const char *input, int32_t compressedSize, co
// decompressed with sz
return tdszDecompress(SZ_DOUBLE, input + 1, compressedSize - 1, nelements, output);
}
#endif
#ifdef BUILD_NO_CALL
/*************************************************************************
@ -2463,13 +2456,11 @@ int32_t tsDecompressTimestamp(void *pIn, int32_t nIn, int32_t nEle, void *pOut,
// Float =====================================================
int32_t tsCompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
int32_t nBuf) {
#ifdef TD_TSZ
// lossy mode
if (lossyFloat) {
return tsCompressFloatLossyImp(pIn, nEle, pOut);
// lossless mode
} else {
#endif
if (cmprAlg == ONE_STAGE_COMP) {
return tsCompressFloatImp(pIn, nEle, pOut);
} else if (cmprAlg == TWO_STAGE_COMP) {
@ -2479,19 +2470,15 @@ int32_t tsCompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_
ASSERTS(0, "compress algo invalid");
return -1;
}
#ifdef TD_TSZ
}
#endif
}
int32_t tsDecompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
int32_t nBuf) {
#ifdef TD_TSZ
if (HEAD_ALGO(((uint8_t *)pIn)[0]) == ALGO_SZ_LOSSY) {
// decompress lossy
return tsDecompressFloatLossyImp(pIn, nIn, nEle, pOut);
} else {
#endif
// decompress lossless
if (cmprAlg == ONE_STAGE_COMP) {
return tsDecompressFloatImp(pIn, nEle, pOut);
@ -2502,20 +2489,16 @@ int32_t tsDecompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int3
ASSERTS(0, "compress algo invalid");
return -1;
}
#ifdef TD_TSZ
}
#endif
}
// Double =====================================================
int32_t tsCompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
int32_t nBuf) {
#ifdef TD_TSZ
if (lossyDouble) {
// lossy mode
return tsCompressDoubleLossyImp(pIn, nEle, pOut);
} else {
#endif
// lossless mode
if (cmprAlg == ONE_STAGE_COMP) {
return tsCompressDoubleImp(pIn, nEle, pOut);
@ -2526,19 +2509,15 @@ int32_t tsCompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32
ASSERTS(0, "compress algo invalid");
return -1;
}
#ifdef TD_TSZ
}
#endif
}
int32_t tsDecompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
int32_t nBuf) {
#ifdef TD_TSZ
if (HEAD_ALGO(((uint8_t *)pIn)[0]) == ALGO_SZ_LOSSY) {
// decompress lossy
return tsDecompressDoubleLossyImp(pIn, nIn, nEle, pOut);
} else {
#endif
// decompress lossless
if (cmprAlg == ONE_STAGE_COMP) {
return tsDecompressDoubleImp(pIn, nEle, pOut);
@ -2549,9 +2528,7 @@ int32_t tsDecompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int
ASSERTS(0, "compress algo invalid");
return -1;
}
#ifdef TD_TSZ
}
#endif
}
// Binary =====================================================

1
source/util/src/terror.c
View File

@ -259,6 +259,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_IN_CREATING, "Database in creating
TAOS_DEFINE_ERROR(TSDB_CODE_MND_ENCRYPT_NOT_ALLOW_CHANGE, "Encryption is not allowed to be changed after database is created")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INCONSIST_ENCRYPT_KEY, "Inconsistent encryption key")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ENCRYPT_KEY, "The cluster has not been set properly for database encryption")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_ENCRYPT_GRANT_EXPIRED, "The database encryption function grant expired")
// mnode-node
TAOS_DEFINE_ERROR(TSDB_CODE_MND_MNODE_ALREADY_EXIST, "Mnode already exists")

4
source/util/src/tgeosctx.c
View File

@ -43,6 +43,10 @@ void destroyThreadLocalGeosCtx() {
tlGeosCtx.WKBWriter = NULL;
}
if (tlGeosCtx.WKTRegex) {
destroyRegexes(tlGeosCtx.WKTRegex, tlGeosCtx.WKTMatchData);
}
if(tlGeosCtx.handle) {
GEOS_finish_r(tlGeosCtx.handle);
tlGeosCtx.handle = NULL;

35
source/util/src/tpcr2.c Normal file
View File

@ -0,0 +1,35 @@
#include "tpcre2.h"
int32_t doRegComp(pcre2_code** ppRegex, pcre2_match_data** ppMatchData, const char* pattern) {
uint32_t options = PCRE2_CASELESS;
int errorcode;
PCRE2_SIZE erroroffset;
*ppRegex = pcre2_compile((PCRE2_SPTR8)pattern, PCRE2_ZERO_TERMINATED, options, &errorcode, &erroroffset, NULL);
if (*ppRegex == NULL) {
PCRE2_UCHAR buffer[256];
pcre2_get_error_message(errorcode, buffer, sizeof(buffer));
return 1;
}
*ppMatchData = pcre2_match_data_create_from_pattern(*ppRegex, NULL);
return 0;
}
int32_t doRegExec(const char* pString, pcre2_code* pRegex, pcre2_match_data* pMatchData) {
int32_t ret = 0;
ret = pcre2_match(pRegex, (PCRE2_SPTR)pString, PCRE2_ZERO_TERMINATED, 0, 0, pMatchData, NULL);
if (ret < 0) {
PCRE2_UCHAR buffer[256];
pcre2_get_error_message(ret, buffer, sizeof(buffer));
return 1;
}
return (ret > 0) ? 0 : 1;
}
void destroyRegexes(pcre2_code* pWktRegex, pcre2_match_data* pWktMatchData) {
pcre2_code_free(pWktRegex);
pcre2_match_data_free(pWktMatchData);
}

86
tests/army/community/insert/insert_basic.py Normal file
View File

@ -0,0 +1,86 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import time
import random
import taos
import frame
import frame.etool
from frame.log import *
from frame.cases import *
from frame.sql import *
from frame.caseBase import *
from frame import *
class TDTestCase(TBase):
def checkGeometry(self):
tdLog.info(f"check geometry")
tdSql.execute("create database db_geometry;")
tdSql.execute("use db_geometry;")
tdSql.execute("create table t_ge (ts timestamp, id int, c1 GEOMETRY(512));")
tdSql.execute("insert into t_ge values(now, 1, 'MULTIPOINT ((0 0), (1 1))');")
tdSql.execute("insert into t_ge values(now, 1, 'MULTIPOINT (0 0, 1 1)');")
tdSql.execute("insert into t_ge values(now, 2, 'POINT (0 0)');")
tdSql.execute("insert into t_ge values(now, 2, 'POINT EMPTY');")
tdSql.execute("insert into t_ge values(now, 3, 'LINESTRING (0 0, 0 1, 1 2)');")
tdSql.execute("insert into t_ge values(now, 3, 'LINESTRING EMPTY');")
tdSql.execute("insert into t_ge values(now, 4, 'POLYGON ((0 0, 1 0, 1 1, 0 1, 0 0))');")
tdSql.execute("insert into t_ge values(now, 4, 'POLYGON ((0 0, 4 0, 4 4, 0 4, 0 0), (1 1, 1 2, 2 2, 2 1, 1 1))');")
tdSql.execute("insert into t_ge values(now, 4, 'POLYGON EMPTY');")
tdSql.execute("insert into t_ge values(now, 5, 'MULTILINESTRING ((0 0, 1 1), (2 2, 3 3))');")
tdSql.execute("insert into t_ge values(now, 6, 'MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1, 1 1)), ((4 3, 6 3, 6 1, 4 1, 4 3)))');")
tdSql.execute("insert into t_ge values(now, 7, 'GEOMETRYCOLLECTION (MULTIPOINT((0 0), (1 1)), POINT(3 4), LINESTRING(2 3, 3 4))');")
tdSql.query("select * from t_ge;")
tdSql.checkRows(12)
tdSql.query("select * from t_ge where id=1;")
tdSql.checkRows(2)
tdSql.query("select * from t_ge where id=2;")
tdSql.checkRows(2)
tdSql.query("select * from t_ge where id=3;")
tdSql.checkRows(2)
tdSql.query("select * from t_ge where id=4;")
tdSql.checkRows(3)
tdSql.query("select * from t_ge where id=5;")
tdSql.checkRows(1)
tdSql.query("select * from t_ge where id=6;")
tdSql.checkRows(1)
tdSql.query("select * from t_ge where id=7;")
tdSql.checkRows(1)
def checkDataType(self):
tdLog.info(f"check datatype")
self.checkGeometry()
# run
def run(self):
tdLog.debug(f"start to excute {__file__}")
# check insert datatype
self.checkDataType()
tdLog.success(f"{__file__} successfully executed")
tdCases.addLinux(__file__, TDTestCase())
tdCases.addWindows(__file__, TDTestCase())

181
tests/army/community/insert/test_column_tag_boundary.py Normal file
View File

@ -0,0 +1,181 @@
from frame.log import *
from frame.cases import *
from frame.sql import *
from frame.caseBase import *
from frame import *
from frame.eos import *
import random
import string
class TDTestCase(TBase):
"""Add test case to test column and tag boundary for task TD-28586
"""
def init(self, conn, logSql, replicaVar=1):
self.replicaVar = int(replicaVar)
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor())
# define the max properties of column and tag
self.super_table_max_column_num = 4096
self.max_tag_num = 128
self.max_tag_length = 16382
self.max_column_length = 65517
self.child_table_num = 1
self.insert_round_num = 700
self.row_num_per_round = 15
def prepare_data(self):
# database
tdSql.execute("create database db;")
tdSql.execute("use db;")
def test_binary_boundary(self):
# create tables
tdSql.execute(f"create table st_binary (ts timestamp, c1 binary({self.max_column_length})) tags (t1 binary({self.max_tag_length}));")
for i in range(self.child_table_num):
# create child table with max column and tag length
tag = ''.join(random.sample(string.ascii_lowercase, 1)) * self.max_tag_length
tdSql.execute(f"create table ct_binary{i+1} using st_binary tags('{tag}');")
# insert data
for j in range(self.insert_round_num):
sql = "insert into ct_binary%s values" % (i+1)
for k in range(self.row_num_per_round):
sql += "(now+%ss, '%s')," % (str(j * 10 + k + 1), 'a' * self.max_column_length)
tdSql.execute(sql)
tdLog.info(f"Insert {self.row_num_per_round} rows data into ct_binary{i+1} {j+1} times successfully")
tdSql.execute("flush database db;")
# check the data
for i in range(self.child_table_num):
tdSql.query(f"select * from ct_binary{i+1};")
tdSql.checkRows(10500)
row_num = random.randint(0, 9999)
tdSql.checkData(row_num, 1, 'a' * self.max_column_length)
tdSql.query(f"show tags from ct_binary{i+1};")
tdSql.checkData(0, 5, tag)
def test_varchar_boundary(self):
# create tables
tdSql.execute(f"create table st_varchar (ts timestamp, c1 varchar({self.max_column_length})) tags (t1 varchar({self.max_tag_length}));")
for i in range(self.child_table_num):
# create child table with max column and tag length
tag = ''.join(random.sample(string.ascii_lowercase, 1)) * self.max_tag_length
tdSql.execute(f"create table ct_varchar{i+1} using st_varchar tags('{tag}');")
# insert data
for j in range(self.insert_round_num):
sql = "insert into ct_varchar%s values" % (i+1)
for k in range(self.row_num_per_round):
sql += "(now+%ss, '%s')," % (str(j * 10 + k + 1), 'b' * self.max_column_length)
tdSql.execute(sql)
tdLog.info(f"Insert {self.row_num_per_round} rows data into ct_varchar{i+1} {j+1} times successfully")
tdSql.execute("flush database db;")
# check the data
for i in range(self.child_table_num):
tdSql.query(f"select * from ct_varchar{i+1};")
tdSql.checkRows(10500)
row_num = random.randint(0, 9999)
tdSql.checkData(row_num, 1, 'b' * self.max_column_length)
tdSql.query(f"show tags from ct_varchar{i+1};")
tdSql.checkData(0, 5, tag)
def gen_chinese_string(self, length):
start = 0x4e00
end = 0x9fa5
chinese_string = ''
for _ in range(length):
chinese_string += chr(random.randint(start, end))
return chinese_string
def test_nchar_boundary(self):
max_nchar_column_length = self.max_column_length // 4
max_nchar_tag_length = self.max_tag_length // 4
# create tables
tdSql.execute(f"create table st_nchar (ts timestamp, c1 nchar({max_nchar_column_length})) tags (t1 nchar({max_nchar_tag_length}));")
for i in range(self.child_table_num):
# create child table with max column and tag length
tag = self.gen_chinese_string(max_nchar_tag_length)
column = self.gen_chinese_string(max_nchar_column_length)
tdSql.execute(f"create table ct_nchar{i+1} using st_nchar tags('{tag}');")
# insert data
for j in range(self.insert_round_num):
sql = "insert into ct_nchar%s values" % (i+1)
for k in range(self.row_num_per_round):
sql += "(now+%ss, '%s')," % (str(j * 10 + k + 1), column)
tdSql.execute(sql)
tdLog.info(f"Insert {self.row_num_per_round} rows data into ct_nchar{i+1} {j+1} times successfully")
tdSql.execute("flush database db;")
# check the data
for i in range(self.child_table_num):
tdSql.query(f"select * from ct_nchar{i+1};")
tdSql.checkRows(10500)
row_num = random.randint(0, 9999)
tdSql.checkData(row_num, 1, column)
tdSql.query(f"show tags from ct_nchar{i+1};")
tdSql.checkData(0, 5, tag)
def test_varbinary_boundary(self):
row_num_per_round = 8
# create tables
tdSql.execute(f"create table st_varbinary (ts timestamp, c1 varbinary({self.max_column_length})) tags (t1 varbinary({self.max_tag_length}));")
for i in range(self.child_table_num):
# create child table with max column and tag length
tag = (''.join(random.sample(string.ascii_lowercase, 1)) * self.max_tag_length).encode().hex()
column = (''.join(random.sample(string.ascii_lowercase, 1)) * self.max_column_length).encode().hex()
tdSql.execute("create table ct_varbinary%s using st_varbinary tags('%s');" % (str(i+1), '\\x' + tag))
# insert data
for j in range(self.insert_round_num):
sql = "insert into ct_varbinary%s values" % (i+1)
for k in range(row_num_per_round):
sql += "(now+%ss, '%s')," % (str(j * 10 + k + 1), '\\x' + column)
tdSql.execute(sql)
tdLog.info(f"Insert {row_num_per_round} rows data into ct_varbinary{i+1} {j+1} times successfully")
tdSql.execute("flush database db;")
# check the data
for i in range(self.child_table_num):
tdSql.query(f"select * from ct_varbinary{i+1};")
tdSql.checkRows(5600)
row_num = random.randint(0, 5599)
tdSql.checkData(row_num, 1, bytes.fromhex(column))
tdSql.query(f"show tags from ct_varbinary{i+1};")
tdSql.checkData(0, 5, '\\x' + tag.upper())
def test_json_tag_boundary(self):
row_num_per_round = 8
max_json_tag_length = 4095
max_json_tag_key_length = 256
# create tables
tdSql.execute(f"create table st_json_tag (ts timestamp, c1 varbinary({self.max_column_length})) tags (t1 json);")
for i in range(self.child_table_num):
# create child table with max column and tag length
tag_key = ''.join(random.sample(string.ascii_lowercase, 1)) * max_json_tag_key_length
tag_value = ''.join(random.sample(string.ascii_lowercase, 1)) * (max_json_tag_length - max_json_tag_key_length - 7)
column = (''.join(random.sample(string.ascii_lowercase, 1)) * self.max_column_length).encode().hex()
tdSql.execute("create table ct_json_tag%s using st_json_tag tags('%s');" % (str(i+1), f'{{"{tag_key}":"{tag_value}"}}'))
# insert data
for j in range(self.insert_round_num):
sql = "insert into ct_json_tag%s values" % (i+1)
for k in range(row_num_per_round):
sql += "(now+%ss, '%s')," % (str(j * 10 + k + 1), '\\x' + column)
tdSql.execute(sql)
tdLog.info(f"Insert {row_num_per_round} rows data into ct_json_tag{i+1} {j+1} times successfully")
tdSql.execute("flush database db;")
# check the data
for i in range(self.child_table_num):
tdSql.query(f"select * from ct_json_tag{i+1} where t1->'{tag_key}' = '{tag_value}';")
tdSql.checkRows(5600)
row_num = random.randint(0, 5599)
tdSql.checkData(row_num, 1, bytes.fromhex(column))
def run(self):
self.prepare_data()
self.test_binary_boundary()
self.test_varchar_boundary()
self.test_nchar_boundary()
self.test_varbinary_boundary()
self.test_json_tag_boundary()
def stop(self):
tdSql.execute("drop database db;")
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

23
tests/army/community/query/show.py
View File

@ -102,6 +102,22 @@ class TDTestCase(TBase):
allRows = self.insert_rows * self.childtable_count
tdSql.checkFirstValue(sql, allRows)
def checkShowTags(self):
# verification for TD-29904
tdSql.error("show tags from t100000", expectErrInfo='Fail to get table info, error: Table does not exist')
sql = "show tags from child1"
tdSql.query(sql)
tdSql.checkRows(5)
sql = f"show tags from child1 from {self.db}"
tdSql.query(sql)
tdSql.checkRows(5)
sql = f"show tags from {self.db}.child1"
tdSql.query(sql)
tdSql.checkRows(5)
def checkShow(self):
# not support
sql = "show accounts;"
@ -125,10 +141,12 @@ class TDTestCase(TBase):
sqls = [
"show scores;",
"SHOW CLUSTER VARIABLES",
"SHOW BNODES;",
# "SHOW BNODES;",
]
tdSql.executes(sqls)
self.checkShowTags()
# run
def run(self):
@ -143,6 +161,9 @@ class TDTestCase(TBase):
# do action
self.doQuery()
# check show
self.checkShow()
tdLog.success(f"{__file__} successfully executed")

31
tests/army/community/storage/compressBasic.py
View File

@ -47,14 +47,20 @@ class TDTestCase(TBase):
dtypes = [ "tinyint","tinyint unsigned","smallint","smallint unsigned","int","int unsigned",
"bigint","bigint unsigned","timestamp","bool","float","double","binary(16)","nchar(16)",
"varchar(16)","varbinary(16)"]
def combineValid(self, datatype, encode, compress):
if datatype != "float" and datatype != "double":
if compress == "tsz":
return False
return True
def genAllSqls(self, stbName, max):
# encode
encodes = [
[["tinyint","tinyint unsigned","smallint","smallint unsigned","int","int unsigned","bigint","bigint unsigned"], ["simple8b"]],
[["timestamp","bigint","bigint unsigned"], ["delta-i"]],
[["bool"], ["bit-packing"]],
[["float","double"], ["delta-d"]]
[["tinyint","tinyint unsigned","smallint","smallint unsigned","int","int unsigned","bigint","bigint unsigned"], ["simple8B"]],
[["timestamp","bigint","bigint unsigned"], ["Delta-i"]],
[["bool"], ["Bit-packing"]],
[["float","double"], ["Delta-d"]]
]
c = 0 # column number
@ -65,7 +71,6 @@ class TDTestCase(TBase):
# loop append sqls
for lines in encodes:
print(lines)
for datatype in lines[0]:
for encode in lines[1]:
for compress in self.compresses:
@ -74,8 +79,9 @@ class TDTestCase(TBase):
# first
sql = f"create table {self.db}.st{t} (ts timestamp"
else:
sql += f", c{c} {datatype} ENCODE '{encode}' COMPRESS '{compress}' LEVEL '{level}'"
c += 1
if self.combineValid(datatype, encode, compress):
sql += f", c{c} {datatype} ENCODE '{encode}' COMPRESS '{compress}' LEVEL '{level}'"
c += 1
if c >= max:
# append sqls
@ -97,9 +103,11 @@ class TDTestCase(TBase):
# check error create
def errorCreate(self):
sqls = [
f"create table terr(ts timestamp, c0 int ENCODE 'abc') ",
f""
f"create table terr(ts timestamp, c0 int ENCODE 'simple8B' COMPRESS 'tsz' LEVEL 'high') ",
f"create table terr(ts timestamp, bi bigint encode 'bit-packing') tags (area int);"
f"create table terr(ts timestamp, ic int encode 'delta-d') tags (area int);"
]
tdSql.errors(sqls)
for dtype in self.dtypes:
# encode
@ -111,6 +119,11 @@ class TDTestCase(TBase):
# level
sql = f"create table terr(ts timestamp, c0 {dtype} LEVEL 'hig') "
tdSql.error(sql)
# tsz check
if dtype != "float" and dtype != "double":
sql = f"create table terr(ts timestamp, c0 {dtype} COMPRESS 'tsz') "
tdSql.error(sql)
# default value correct
def defaultCorrect(self):

4
tests/parallel_test/cases.task
View File

@ -11,14 +11,16 @@
# army-test
#
,,y,army,./pytest.sh python3 ./test.py -f enterprise/multi-level/mlevel_basic.py -N 3 -L 3 -D 2
,,y,army,./pytest.sh python3 ./test.py -f enterprise/s3/s3Basic.py -N 3
,,n,army,python3 ./test.py -f enterprise/s3/s3Basic.py -N 3
,,y,army,./pytest.sh python3 ./test.py -f community/cluster/snapshot.py -N 3 -L 3 -D 2
,,y,army,./pytest.sh python3 ./test.py -f community/query/function/test_func_elapsed.py
,,y,army,./pytest.sh python3 ./test.py -f community/query/test_join.py
,,y,army,./pytest.sh python3 ./test.py -f community/query/test_compare.py
,,y,army,./pytest.sh python3 ./test.py -f community/insert/test_column_tag_boundary.py
,,y,army,./pytest.sh python3 ./test.py -f community/query/fill/fill_desc.py -N 3 -L 3 -D 2
,,y,army,./pytest.sh python3 ./test.py -f community/cluster/incSnapshot.py -N 3
,,y,army,./pytest.sh python3 ./test.py -f community/query/query_basic.py -N 3
,,y,army,./pytest.sh python3 ./test.py -f community/insert/insert_basic.py -N 3
,,y,army,./pytest.sh python3 ./test.py -f community/cluster/splitVgroupByLearner.py -N 3
,,n,army,python3 ./test.py -f community/cmdline/fullopt.py
,,n,army,python3 ./test.py -f community/query/show.py -N 3

2
tests/parallel_test/run_case.sh
View File

@ -79,7 +79,7 @@ md5sum /home/TDinternal/debug/build/lib/libtaos.so
#define taospy 2.7.10
pip3 list|grep taospy
pip3 uninstall taospy -y
pip3 install --default-timeout=120 taospy==2.7.13
pip3 install --default-timeout=120 taospy==2.7.15
#define taos-ws-py 0.3.1
pip3 list|grep taos-ws-py

7
tests/script/sh/checkAsan.sh
View File

@ -72,7 +72,12 @@ python_error=$(cat ${LOG_DIR}/*.info | grep -w "stack" | wc -l)
#0 0x7f2d64f5a808 in __interceptor_malloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cc:144
#1 0x7f2d63fcf459 in strerror /build/glibc-SzIz7B/glibc-2.31/string/strerror.c:38
runtime_error=$(cat ${LOG_DIR}/*.asan | grep "runtime error" | grep -v "trees.c:873" | grep -v "sclfunc.c.*outside the range of representable values of type" | grep -v "signed integer overflow" | grep -v "strerror.c" | grep -v "asan_malloc_linux.cc" | grep -v "strerror.c" | grep -v "asan_malloc_linux.cpp" | grep -v "sclvector.c" | wc -l)
# TD-29953
#/home/TDinternal/community/utils/TSZ/sz/src/sz_double.c:388:59: runtime error: 2.64021e+25 is outside the range of representable values of type 'long unsigned int'
#/home/TDinternal/community/utils/TSZ/sz/src/sz_float.c:407:59: runtime error: 5.76041e+19 is outside the range of representable values of type 'long unsigned int'
#/home/TDinternal/community/source/libs/scalar/src/sclfunc.c:808:11: runtime error: -3.40401e+18 is outside the range of representable values of type 'int'
runtime_error=$(cat ${LOG_DIR}/*.asan | grep "runtime error" | grep -v "trees.c:873" | grep -v "sclfunc.c.*outside the range of representable values of type" | grep -v "signed integer overflow" | grep -v "strerror.c" | grep -v "asan_malloc_linux.cc" | grep -v "strerror.c" | grep -v "asan_malloc_linux.cpp" | grep -v "sclvector.c" | grep -v "sclfunc.c:808"| grep -v "sz_double.c:388" | grep -v "sz_float.c:407:59"| wc -l)
echo -e "\033[44;32;1m"asan error_num: $error_num"\033[0m"
echo -e "\033[44;32;1m"asan memory_leak: $memory_leak"\033[0m"

173
tests/script/tsim/query/sys_tbname.sim
View File

@ -135,4 +135,177 @@ endi
print =========================== td-24781
sql select DISTINCT (`precision`) from `information_schema`.`ins_databases` PARTITION BY `precision`
print =========================ins_stables
print create database test vgroups 4;
sql create database test vgroups 4;
sql use test;
sql create stable st1(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st2(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st3(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st4(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st5(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st6(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st7(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st8(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st9(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st10(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st11(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st12(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st13(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st14(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st15(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st16(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st17(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st18(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st19(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st20(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st21(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st22(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st23(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st24(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st25(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st26(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st27(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st28(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st29(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st30(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st31(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st32(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st33(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st34(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st35(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st36(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st37(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st38(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st39(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st40(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st41(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st42(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st43(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st44(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st45(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st46(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st47(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st48(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st49(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st50(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st51(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st52(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st53(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st54(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st55(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st56(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st57(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st58(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st59(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st60(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st61(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st62(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st63(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st64(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st65(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st66(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st67(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st68(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st69(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st70(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
print create database test1 vgroups 4;
sql create database test1 vgroups 4;
sql use test1;
sql create stable st1(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st2(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st3(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st4(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st5(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st6(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st7(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st8(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st9(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st10(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st11(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st12(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st13(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st14(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st15(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st16(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st17(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st18(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st19(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st20(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st21(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st22(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st23(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st24(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st25(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st26(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st27(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st28(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st29(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st30(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st31(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st32(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st33(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st34(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st35(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st36(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st37(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st38(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st39(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st40(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st41(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st42(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st43(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st44(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st45(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st46(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st47(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st48(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st49(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st50(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st51(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st52(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st53(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st54(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st55(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st56(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st57(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st58(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st59(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st60(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st61(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st62(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st63(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st64(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st65(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st66(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st67(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st68(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st69(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create stable st70(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sleep 1000
sql select * from information_schema.ins_stables where db_name = "test" limit 68,32;
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
print $data20 $data21 $data22 $data23
if $rows != 2 then
return -1
endi
sql select * from information_schema.ins_stables where db_name = "test1" limit 68,32;
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
print $data20 $data21 $data22 $data23
if $rows != 2 then
return -1
endi
#system sh/exec.sh -n dnode1 -s stop -x SIGINT

25
tests/system-test/0-others/information_schema.py
View File

@ -269,13 +269,17 @@ class TDTestCase:
def ins_grants_check(self):
grant_name_dict = {
'stream':'stream',
'subscription':'subscription',
'view':'view',
'audit':'audit',
'csv':'csv',
'storage':'multi_tier_storage',
'backup_restore':'backup_restore',
'stream':'Stream',
'subscription':'Subscription',
'view':'View',
'audit':'Audit',
'csv':'CSV',
'storage':'Multi-Tier Storage',
'backup_restore':'Data Backup & Restore',
'object_storage':'Object Storage',
'active_active':'Active-Active',
'dual_replica':'Dual-Replica HA',
'db_encryption':'Database Encryption',
'opc_da':'OPC_DA',
'opc_ua':'OPC_UA',
'pi':'Pi',
@ -285,7 +289,10 @@ class TDTestCase:
'avevahistorian':'avevaHistorian',
'opentsdb':'OpenTSDB',
'td2.6':'TDengine2.6',
'td3.0':'TDengine3.0'
'td3.0':'TDengine3.0',
'mysql':'MySQL',
'postgres':'PostgreSQL',
'oracle':'Oracle',
}
tdSql.execute('drop database if exists db2')
@ -297,7 +304,7 @@ class TDTestCase:
if result[i][0] in grant_name_dict:
tdSql.checkEqual(result[i][1], grant_name_dict[result[i][0]])
index += 1
tdSql.checkEqual(index, 17)
tdSql.checkEqual(index, 24)
tdSql.query(f'select * from information_schema.ins_grants_logs')
result = tdSql.queryResult
tdSql.checkEqual(True, len(result) >= 0)

27
tests/system-test/0-others/show_tag_index.py
View File

@ -78,6 +78,11 @@ class TDTestCase:
tdSql.execute(f'create table stb (ts timestamp, c0 int) tags (t0 int, t1 int)')
tdSql.execute(f'create table ctb1 using stb tags (1, 1)')
tdSql.execute(f'create table ctb2 using stb tags (2, 2)')
tdSql.execute(f'create table ntb (ts timestamp, c0 int)')
tdSql.execute(f'create view vtb as select * from stb')
tdSql.execute(f'create view vtb1 as select * from ctb1')
tdSql.execute(f'create view vtb2 as select * from ctb2')
tdSql.execute(f'create view vtbn as select * from ntb')
tdSql.execute(f'insert into ctb1 values (now, 1)')
tdSql.execute(f'insert into ctb2 values (now, 2)')
@ -113,9 +118,14 @@ class TDTestCase:
tdSql.error(f'show tags from `db`.`stb` from db')
tdSql.error(f'show tags from db.ctb1 from db')
tdSql.error(f'show tags from `db`.`ctb1` from db')
tdSql.error(f'show tags from tb_undef from db')
tdSql.error(f'show tags from db.tb_undef')
tdSql.error(f'show tags from tb_undef')
tdSql.error(f'show tags from tb_undef from db', expectErrInfo='Fail to get table info, error: Table does not exist')
tdSql.error(f'show tags from db.tb_undef', expectErrInfo='Fail to get table info, error: Table does not exist')
tdSql.error(f'show tags from tb_undef', expectErrInfo='Fail to get table info, error: Table does not exist')
tdSql.error(f'show tags from ntb', expectErrInfo='Tags can only applied to super table and child table')
tdSql.error(f'show tags from vtb', expectErrInfo='Tags can only applied to super table and child table')
tdSql.error(f'show tags from vtb1', expectErrInfo='Tags can only applied to super table and child table')
tdSql.error(f'show tags from vtb2', expectErrInfo='Tags can only applied to super table and child table')
tdSql.error(f'show tags from vtbn', expectErrInfo='Tags can only applied to super table and child table')
# show table tags
tdSql.query(f'show table tags from stb')
@ -148,9 +158,14 @@ class TDTestCase:
tdSql.error(f'show table tags from `db`.`stb` from db')
tdSql.error(f'show table tags from db.ctb1 from db')
tdSql.error(f'show table tags from `db`.`ctb1` from db')
tdSql.error(f'show table tags from tb_undef from db')
tdSql.error(f'show table tags from db.tb_undef')
tdSql.error(f'show table tags from tb_undef')
tdSql.error(f'show table tags from tb_undef from db', expectErrInfo='Fail to get table info, error: Table does not exist')
tdSql.error(f'show table tags from db.tb_undef', expectErrInfo='Fail to get table info, error: Table does not exist')
tdSql.error(f'show table tags from tb_undef', expectErrInfo='Fail to get table info, error: Table does not exist')
tdSql.error(f'show table tags from ntb', expectErrInfo='Tags can only applied to super table and child table')
tdSql.error(f'show table tags from vtb', expectErrInfo='Tags can only applied to super table and child table')
tdSql.error(f'show table tags from vtb1', expectErrInfo='Tags can only applied to super table and child table')
tdSql.error(f'show table tags from vtb2', expectErrInfo='Tags can only applied to super table and child table')
tdSql.error(f'show table tags from vtbn', expectErrInfo='Tags can only applied to super table and child table')
# show indexes
tdSql.execute(f'create index idx1 on stb (t1)')

4
tests/system-test/1-insert/composite_primary_key_insert.py
View File

@ -31,8 +31,8 @@ class LegalDataType(Enum):
UINT = 'INT UNSIGNED'
BIGINT = 'BIGINT'
UBIGINT = 'BIGINT UNSIGNED'
VARCHAR = 'VARCHAR(100)'
BINARY = 'BINARY(100)'
VARCHAR = 'VARCHAR(10000)'
BINARY = 'BINARY(10000)'
class TableType(Enum):

9
tests/system-test/2-query/partition_by_col.py
View File

@ -22,7 +22,7 @@ class TDTestCase:
self.vgroups = 4
self.ctbNum = 10
self.rowsPerTbl = 10000
self.duraion = '1h'
self.duraion = '1d'
def init(self, conn, logSql, replicaVar=1):
self.replicaVar = int(replicaVar)
@ -33,7 +33,7 @@ class TDTestCase:
if dropFlag == 1:
tsql.execute("drop database if exists %s"%(dbName))
tsql.execute("create database if not exists %s vgroups %d replica %d duration %s"%(dbName, vgroups, replica, duration))
tsql.execute("create database if not exists %s vgroups %d replica %d duration %s stt_trigger 1"%(dbName, vgroups, replica, duration))
tdLog.debug("complete to create database %s"%(dbName))
return
@ -266,11 +266,11 @@ class TDTestCase:
#'select _wstart as ts, count(*), t1 as a, %s from meters partition by t1, %s interval(30m)' % (col_name, col_name),
#'select _wstart as ts, count(*), t1 as a, %s from meters partition by t1, %s interval(1h)' % (col_name, col_name),
'select _wstart as ts, count(*), %s as a, %s from meters partition by %s interval(1s)' % (col_name, col_name, col_name),
'select _wstart as ts, count(*), %s as a, %s from meters partition by %s interval(30d)' % (col_name, col_name, col_name),
#'select _wstart as ts, count(*), %s as a, %s from meters partition by %s interval(30s)' % (col_name, col_name, col_name),
#'select _wstart as ts, count(*), %s as a, %s from meters partition by %s interval(1m)' % (col_name, col_name, col_name),
#'select _wstart as ts, count(*), %s as a, %s from meters partition by %s interval(30m)' % (col_name, col_name, col_name),
#'select _wstart as ts, count(*), %s as a, %s from meters partition by %s interval(1h)' % (col_name, col_name, col_name),
'select _wstart as ts, count(*), %s as a, %s from meters partition by %s interval(1h)' % (col_name, col_name, col_name),
'select _wstart as ts, count(*), tbname as a, %s from meters partition by %s, tbname interval(1s)' % (col_name, col_name),
'select _wstart as ts, count(*), t1 as a, %s from meters partition by %s, t1 interval(1s)' % (col_name, col_name),
@ -317,6 +317,7 @@ class TDTestCase:
def run(self):
self.prepareTestEnv()
tdSql.execute('flush database test')
#time.sleep(99999999)
self.test_sort_for_partition_hint()
self.test_sort_for_partition_res()

14
tests/system-test/8-stream/stream_basic.py
View File

@ -87,7 +87,7 @@ class TDTestCase:
tdSql.execute(f'insert into t2 using st tags(2) values(now, 1) (now+1s, 2)')
tdSql.execute(f'insert into t3 using st tags(3) values(now, 1) (now+1s, 2)')
tdSql.execute("create stream stream1 fill_history 1 into sta subtable(concat('new-', tname)) AS SELECT "
tdSql.execute("create stream stream1 fill_history 1 into sta subtable(concat('nee.w-', tname)) AS SELECT "
"_wstart, count(*), avg(i) FROM st PARTITION BY tbname tname INTERVAL(1m)", show=True)
tdSql.execute("create stream stream2 fill_history 1 into stb subtable(concat('new-', tname)) AS SELECT "
@ -97,25 +97,25 @@ class TDTestCase:
tdSql.query("select * from sta")
tdSql.checkRows(3)
tdSql.query("select tbname from sta order by tbname")
if not tdSql.getData(0, 0).startswith('new-t1_1.d1.sta_'):
if not tdSql.getData(0, 0).startswith('nee_w-t1_sta_'):
tdLog.exit("error1")
if not tdSql.getData(1, 0).startswith('new-t2_1.d1.sta_'):
if not tdSql.getData(1, 0).startswith('nee_w-t2_sta_'):
tdLog.exit("error2")
if not tdSql.getData(2, 0).startswith('new-t3_1.d1.sta_'):
if not tdSql.getData(2, 0).startswith('nee_w-t3_sta_'):
tdLog.exit("error3")
tdSql.query("select * from stb")
tdSql.checkRows(3)
tdSql.query("select tbname from stb order by tbname")
if not tdSql.getData(0, 0).startswith('new-t1_1.d1.stb_'):
if not tdSql.getData(0, 0).startswith('new-t1_stb_'):
tdLog.exit("error4")
if not tdSql.getData(1, 0).startswith('new-t2_1.d1.stb_'):
if not tdSql.getData(1, 0).startswith('new-t2_stb_'):
tdLog.exit("error5")
if not tdSql.getData(2, 0).startswith('new-t3_1.d1.stb_'):
if not tdSql.getData(2, 0).startswith('new-t3_stb_'):
tdLog.exit("error6")
# run

2666
utils/test/c/tsz_test.c
View File

File diff suppressed because it is too large Load Diff

46
utils/tsim/src/simExe.c
View File

@ -37,15 +37,15 @@ void simLogSql(char *sql, bool useSharp) {
char *simParseHostName(char *varName) {
static char hostName[140];
//#ifdef WINDOWS
// hostName[0] = '\"';
// taosGetFqdn(&hostName[1]);
// int strEndIndex = strlen(hostName);
// hostName[strEndIndex] = '\"';
// hostName[strEndIndex + 1] = '\0';
//#else
//#ifdef WINDOWS
// hostName[0] = '\"';
// taosGetFqdn(&hostName[1]);
// int strEndIndex = strlen(hostName);
// hostName[strEndIndex] = '\"';
// hostName[strEndIndex + 1] = '\0';
//#else
sprintf(hostName, "%s", "localhost");
//#endif
//#endif
return hostName;
}
@ -276,12 +276,16 @@ int32_t simExecuteExpression(SScript *script, char *exp) {
if (op1[0] == '=') {
strcpy(simGetVariable(script, var1 + 1, var1Len - 1), t3);
} else if (op1[0] == '<') {
val0 = atoi(t0);
val1 = atoi(t3);
int64_t val0 = atoll(t0);
int64_t val1 = atoll(t3);
// val0 = atoi(t0);
// val1 = atoi(t3);
if (val0 >= val1) result = -1;
} else if (op1[0] == '>') {
val0 = atoi(t0);
val1 = atoi(t3);
int64_t val0 = atoll(t0);
int64_t val1 = atoll(t3);
// val0 = atoi(t0);
// val1 = atoi(t3);
if (val0 <= val1) result = -1;
}
} else {
@ -378,16 +382,14 @@ bool simExecuteRunBackCmd(SScript *script, char *option) {
return true;
}
void simReplaceDirSep (char *buf){
void simReplaceDirSep(char *buf) {
#ifdef WINDOWS
int i=0;
while(buf[i] != '\0')
{
if(buf[i] == '/')
{
buf[i] = '\\';
}
i++;
int i = 0;
while (buf[i] != '\0') {
if (buf[i] == '/') {
buf[i] = '\\';
}
i++;
}
#endif
}
@ -505,7 +507,7 @@ bool simExecuteSystemContentCmd(SScript *script, char *option) {
}
bool simExecuteSetBIModeCmd(SScript *script, char *option) {
char buf[1024];
char buf[1024];
simVisuallizeOption(script, option, buf);
option = buf;

2
utils/tsim/src/simSystem.c
View File

@ -28,7 +28,7 @@ extern bool simExecSuccess;
int32_t simInitCfg() {
taosCreateLog("simlog", 1, configDir, NULL, NULL, NULL, NULL, 1);
taosInitCfg(configDir, NULL, NULL, NULL, NULL, 1);
taosInitCfg(configDir, NULL, NULL, NULL, NULL, 1, true);
SConfig *pCfg = taosGetCfg();
tstrncpy(simScriptDir, cfgGetItem(pCfg, "scriptDir")->str, PATH_MAX);