xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into enh/TD-22404-M

This commit is contained in:
kailixu 2023-04-12 18:30:43 +08:00
parent 7f6fdd4b6b d551075bba
commit 85e11d8cbf
97 changed files with 6072 additions and 4257 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmake/taostools_CMakeLists.txt.in
View File

@ -2,7 +2,7 @@
# taos-tools
ExternalProject_Add(taos-tools
GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
GIT_TAG 63635fc
GIT_TAG 0681d8b
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
BINARY_DIR ""
#BUILD_IN_SOURCE TRUE

1
include/common/tglobal.h
View File

@ -104,6 +104,7 @@ extern int32_t tsCacheLazyLoadThreshold; // cost threshold for last/last_row lo
// query client
extern int32_t tsQueryPolicy;
extern int32_t tsQueryRspPolicy;
extern int64_t tsQueryMaxConcurrentTables;
extern int32_t tsQuerySmaOptimize;
extern int32_t tsQueryRsmaTolerance;
extern bool tsQueryPlannerTrace;

32
include/common/tmsg.h
View File

@ -177,6 +177,12 @@ typedef enum _mgmt_table {
#define TSDB_ALTER_USER_SYSINFO 0xA
#define TSDB_ALTER_USER_ADD_SUBSCRIBE_TOPIC 0xB
#define TSDB_ALTER_USER_REMOVE_SUBSCRIBE_TOPIC 0xC
#define TSDB_ALTER_USER_ADD_READ_TABLE 0xD
#define TSDB_ALTER_USER_REMOVE_READ_TABLE 0xE
#define TSDB_ALTER_USER_ADD_WRITE_TABLE 0xF
#define TSDB_ALTER_USER_REMOVE_WRITE_TABLE 0x10
#define TSDB_ALTER_USER_ADD_ALL_TABLE 0x11
#define TSDB_ALTER_USER_REMOVE_ALL_TABLE 0x12
#define TSDB_ALTER_USER_PRIVILEGES 0x2
@ -669,13 +675,16 @@ int32_t tSerializeSCreateUserReq(void* buf, int32_t bufLen, SCreateUserReq* pReq
int32_t tDeserializeSCreateUserReq(void* buf, int32_t bufLen, SCreateUserReq* pReq);
typedef struct {
int8_t alterType;
int8_t superUser;
int8_t sysInfo;
int8_t enable;
char user[TSDB_USER_LEN];
char pass[TSDB_USET_PASSWORD_LEN];
char objname[TSDB_DB_FNAME_LEN]; // db or topic
int8_t alterType;
int8_t superUser;
int8_t sysInfo;
int8_t enable;
char user[TSDB_USER_LEN];
char pass[TSDB_USET_PASSWORD_LEN];
char objname[TSDB_DB_FNAME_LEN]; // db or topic
char tabName[TSDB_TABLE_NAME_LEN];
char* tagCond;
int32_t tagCondLen;
} SAlterUserReq;
int32_t tSerializeSAlterUserReq(void* buf, int32_t bufLen, SAlterUserReq* pReq);
@ -698,6 +707,9 @@ typedef struct {
SHashObj* createdDbs;
SHashObj* readDbs;
SHashObj* writeDbs;
SHashObj* readTbs;
SHashObj* writeTbs;
SHashObj* useDbs;
} SGetUserAuthRsp;
int32_t tSerializeSGetUserAuthRsp(void* buf, int32_t bufLen, SGetUserAuthRsp* pRsp);
@ -3187,9 +3199,9 @@ typedef struct {
SArray* blockTbName;
SArray* blockSchema;
// the following attributes are extended from SMqDataRsp
int32_t createTableNum;
SArray* createTableLen;
SArray* createTableReq;
int32_t createTableNum;
SArray* createTableLen;
SArray* createTableReq;
} STaosxRsp;
int32_t tEncodeSTaosxRsp(SEncoder* pEncoder, const STaosxRsp* pRsp);

98
include/common/ttokendef.h
View File

@ -63,55 +63,55 @@
#define TK_READ 45
#define TK_WRITE 46
#define TK_NK_DOT 47
#define TK_DNODE 48
#define TK_PORT 49
#define TK_DNODES 50
#define TK_NK_IPTOKEN 51
#define TK_FORCE 52
#define TK_LOCAL 53
#define TK_QNODE 54
#define TK_BNODE 55
#define TK_SNODE 56
#define TK_MNODE 57
#define TK_DATABASE 58
#define TK_USE 59
#define TK_FLUSH 60
#define TK_TRIM 61
#define TK_COMPACT 62
#define TK_IF 63
#define TK_NOT 64
#define TK_EXISTS 65
#define TK_BUFFER 66
#define TK_CACHEMODEL 67
#define TK_CACHESIZE 68
#define TK_COMP 69
#define TK_DURATION 70
#define TK_NK_VARIABLE 71
#define TK_MAXROWS 72
#define TK_MINROWS 73
#define TK_KEEP 74
#define TK_PAGES 75
#define TK_PAGESIZE 76
#define TK_TSDB_PAGESIZE 77
#define TK_PRECISION 78
#define TK_REPLICA 79
#define TK_VGROUPS 80
#define TK_SINGLE_STABLE 81
#define TK_RETENTIONS 82
#define TK_SCHEMALESS 83
#define TK_WAL_LEVEL 84
#define TK_WAL_FSYNC_PERIOD 85
#define TK_WAL_RETENTION_PERIOD 86
#define TK_WAL_RETENTION_SIZE 87
#define TK_WAL_ROLL_PERIOD 88
#define TK_WAL_SEGMENT_SIZE 89
#define TK_STT_TRIGGER 90
#define TK_TABLE_PREFIX 91
#define TK_TABLE_SUFFIX 92
#define TK_NK_COLON 93
#define TK_MAX_SPEED 94
#define TK_START 95
#define TK_WITH 96
#define TK_WITH 48
#define TK_DNODE 49
#define TK_PORT 50
#define TK_DNODES 51
#define TK_NK_IPTOKEN 52
#define TK_FORCE 53
#define TK_LOCAL 54
#define TK_QNODE 55
#define TK_BNODE 56
#define TK_SNODE 57
#define TK_MNODE 58
#define TK_DATABASE 59
#define TK_USE 60
#define TK_FLUSH 61
#define TK_TRIM 62
#define TK_COMPACT 63
#define TK_IF 64
#define TK_NOT 65
#define TK_EXISTS 66
#define TK_BUFFER 67
#define TK_CACHEMODEL 68
#define TK_CACHESIZE 69
#define TK_COMP 70
#define TK_DURATION 71
#define TK_NK_VARIABLE 72
#define TK_MAXROWS 73
#define TK_MINROWS 74
#define TK_KEEP 75
#define TK_PAGES 76
#define TK_PAGESIZE 77
#define TK_TSDB_PAGESIZE 78
#define TK_PRECISION 79
#define TK_REPLICA 80
#define TK_VGROUPS 81
#define TK_SINGLE_STABLE 82
#define TK_RETENTIONS 83
#define TK_SCHEMALESS 84
#define TK_WAL_LEVEL 85
#define TK_WAL_FSYNC_PERIOD 86
#define TK_WAL_RETENTION_PERIOD 87
#define TK_WAL_RETENTION_SIZE 88
#define TK_WAL_ROLL_PERIOD 89
#define TK_WAL_SEGMENT_SIZE 90
#define TK_STT_TRIGGER 91
#define TK_TABLE_PREFIX 92
#define TK_TABLE_SUFFIX 93
#define TK_NK_COLON 94
#define TK_MAX_SPEED 95
#define TK_START 96
#define TK_TIMESTAMP 97
#define TK_END 98
#define TK_TABLE 99

16
include/libs/catalog/catalog.h
View File

@ -29,6 +29,7 @@ extern "C" {
#include "tmsg.h"
#include "tname.h"
#include "transport.h"
#include "nodes.h"
typedef struct SCatalog SCatalog;
@ -49,10 +50,15 @@ typedef enum {
typedef struct SUserAuthInfo {
char user[TSDB_USER_LEN];
char dbFName[TSDB_DB_FNAME_LEN];
SName tbName;
AUTH_TYPE type;
} SUserAuthInfo;
typedef struct SUserAuthRes {
bool pass;
SNode* pCond;
} SUserAuthRes;
typedef struct SDbInfo {
int32_t vgVer;
int32_t tbNum;
@ -96,7 +102,7 @@ typedef struct SMetaData {
SArray* pTableIndex; // pRes = SArray<STableIndexInfo>*
SArray* pUdfList; // pRes = SFuncInfo*
SArray* pIndex; // pRes = SIndexInfo*
SArray* pUser; // pRes = bool*
SArray* pUser; // pRes = SUserAuthRes*
SArray* pQnodeList; // pRes = SArray<SQueryNodeLoad>*
SArray* pTableCfg; // pRes = STableCfg*
SArray* pDnodeList; // pRes = SArray<SEpSet>*
@ -312,11 +318,9 @@ int32_t catalogUpdateTableIndex(SCatalog* pCtg, STableIndexRsp* pRsp);
int32_t catalogGetUdfInfo(SCatalog* pCtg, SRequestConnInfo* pConn, const char* funcName, SFuncInfo* pInfo);
int32_t catalogChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, const char* user, const char* dbFName, AUTH_TYPE type,
bool* pass);
int32_t catalogChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, SUserAuthInfo *pAuth, SUserAuthRes* pRes);
int32_t catalogChkAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFName, AUTH_TYPE type, bool* pass,
bool* exists);
int32_t catalogChkAuthFromCache(SCatalog* pCtg, SUserAuthInfo *pAuth, SUserAuthRes* pRes, bool* exists);
int32_t catalogUpdateUserAuthInfo(SCatalog* pCtg, SGetUserAuthRsp* pAuth);

2
include/libs/nodes/cmdnodes.h
View File

@ -456,7 +456,9 @@ typedef struct SGrantStmt {
ENodeType type;
char userName[TSDB_USER_LEN];
char objName[TSDB_DB_NAME_LEN]; // db or topic
char tabName[TSDB_TABLE_NAME_LEN];
int64_t privileges;
SNode* pTagCond;
} SGrantStmt;
typedef SGrantStmt SRevokeStmt;

1
include/util/taoserror.h
View File

@ -241,6 +241,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_MND_INVALID_ALTER_OPER TAOS_DEF_ERROR_CODE(0, 0x0356)
#define TSDB_CODE_MND_AUTH_FAILURE TAOS_DEF_ERROR_CODE(0, 0x0357)
#define TSDB_CODE_MND_USER_NOT_AVAILABLE TAOS_DEF_ERROR_CODE(0, 0x0358)
#define TSDB_CODE_MND_PRIVILEDGE_EXIST TAOS_DEF_ERROR_CODE(0, 0x0359)
// mnode-stable-part1
#define TSDB_CODE_MND_STB_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0360)

1
include/util/tdef.h
View File

@ -198,6 +198,7 @@ typedef enum ELogicConditionType {
#define TSDB_STREAM_NAME_LEN 193 // it is a null-terminated string
#define TSDB_DB_NAME_LEN 65
#define TSDB_DB_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN + TSDB_NAME_DELIMITER_LEN)
#define TSDB_PRIVILEDGE_CONDITION_LEN 200
#define TSDB_FUNC_NAME_LEN 65
#define TSDB_FUNC_COMMENT_LEN 1024 * 1024

45
packaging/tools/install.sh
View File

@ -572,6 +572,21 @@ function install_config() {
done
}
function install_share_etc() {
[ ! -d ${script_dir}/share/etc ] && return
for c in `ls ${script_dir}/share/etc/`; do
if [ -e /etc/$c ]; then
out=/etc/$c.new.`date +%F`
${csudo}cp -f ${script_dir}/share/etc/$c $out ||:
else
${csudo}cp -f ${script_dir}/share/etc/$c /etc/$c ||:
fi
done
[ ! -d ${script_dir}/share/srv ] && return
${csudo} cp ${script_dir}/share/srv/* ${service_config_dir} ||:
}
function install_log() {
${csudo}rm -rf ${log_dir} || :
${csudo}mkdir -p ${log_dir} && ${csudo}chmod 777 ${log_dir}
@ -586,18 +601,20 @@ function install_data() {
}
function install_connector() {
[ -d "${script_dir}/connector/" ] && ${csudo}cp -rf ${script_dir}/connector/ ${install_main_dir}/
if [ -d "${script_dir}/connector/" ]; then
${csudo}cp -rf ${script_dir}/connector/ ${install_main_dir}/ || echo "failed to copy connector"
fi
}
function install_examples() {
if [ -d ${script_dir}/examples ]; then
${csudo}cp -rf ${script_dir}/examples/* ${install_main_dir}/examples
${csudo}cp -rf ${script_dir}/examples/* ${install_main_dir}/examples || echo "failed to copy examples"
fi
}
function install_web() {
if [ -d "${script_dir}/share" ]; then
${csudo}cp -rf ${script_dir}/share/* ${install_main_dir}/share
${csudo}cp -rf ${script_dir}/share/* ${install_main_dir}/share > /dev/null 2>&1 ||:
fi
}
@ -685,11 +702,33 @@ function clean_service_on_systemd() {
# if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then
# ${csudo}rm -f ${service_config_dir}/${serverName2}.service
# fi
x_service_config="${service_config_dir}/${xName2}.service"
if [ -e "$x_service_config" ]; then
if systemctl is-active --quiet ${xName2}; then
echo "${productName2} ${xName2} is running, stopping it..."
${csudo}systemctl stop ${xName2} &>/dev/null || echo &>/dev/null
fi
${csudo}systemctl disable ${xName2} &>/dev/null || echo &>/dev/null
${csudo}rm -f ${x_service_config}
fi
explorer_service_config="${service_config_dir}/${explorerName2}.service"
if [ -e "$explorer_service_config" ]; then
if systemctl is-active --quiet ${explorerName2}; then
echo "${productName2} ${explorerName2} is running, stopping it..."
${csudo}systemctl stop ${explorerName2} &>/dev/null || echo &>/dev/null
fi
${csudo}systemctl disable ${explorerName2} &>/dev/null || echo &>/dev/null
${csudo}rm -f ${explorer_service_config}
${csudo}rm -f /etc/${clientName2}/explorer.toml
fi
}
function install_service_on_systemd() {
clean_service_on_systemd
install_share_etc
[ -f ${script_dir}/cfg/${serverName2}.service ] &&
${csudo}cp ${script_dir}/cfg/${serverName2}.service \
${service_config_dir}/ || :

1
packaging/tools/makepkg.sh
View File

@ -150,6 +150,7 @@ fi
mkdir -p ${install_dir}/bin && cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || :
mkdir -p ${install_dir}/init.d && cp ${init_file_deb} ${install_dir}/init.d/${serverName}.deb
mkdir -p ${install_dir}/init.d && cp ${init_file_rpm} ${install_dir}/init.d/${serverName}.rpm
mkdir -p ${install_dir}/share && cp -rf ${build_dir}/share/{etc,srv} ${install_dir}/share ||:
if [ $adapterName != "taosadapter" ]; then
mv ${install_dir}/cfg/${clientName2}adapter.toml ${install_dir}/cfg/$adapterName.toml

22
packaging/tools/remove.sh
View File

@ -192,7 +192,27 @@ function clean_service_on_systemd() {
${csudo}systemctl stop ${tarbitrator_service_name} &>/dev/null || echo &>/dev/null
fi
${csudo}systemctl disable ${tarbitrator_service_name} &>/dev/null || echo &>/dev/null
${csudo}rm -f ${tarbitratord_service_config}
x_service_config="${service_config_dir}/${xName2}.service"
if [ -e "$x_service_config" ]; then
if systemctl is-active --quiet ${xName2}; then
echo "${productName2} ${xName2} is running, stopping it..."
${csudo}systemctl stop ${xName2} &>/dev/null || echo &>/dev/null
fi
${csudo}systemctl disable ${xName2} &>/dev/null || echo &>/dev/null
${csudo}rm -f ${x_service_config}
fi
explorer_service_config="${service_config_dir}/${explorerName2}.service"
if [ -e "$explorer_service_config" ]; then
if systemctl is-active --quiet ${explorerName2}; then
echo "${productName2} ${explorerName2} is running, stopping it..."
${csudo}systemctl stop ${explorerName2} &>/dev/null || echo &>/dev/null
fi
${csudo}systemctl disable ${explorerName2} &>/dev/null || echo &>/dev/null
${csudo}rm -f ${explorer_service_config}
${csudo}rm -f /etc/${clientName2}/explorer.toml
fi
}
function clean_service_on_sysvinit() {

9
source/client/inc/clientStmt.h
View File

@ -144,6 +144,15 @@ extern char *gStmtStatusStr[];
goto _return; \
} \
} while (0)
#define STMT_ERRI_JRET(c) \
do { \
code = c; \
if (code != TSDB_CODE_SUCCESS) { \
terrno = code; \
goto _return; \
} \
} while (0)
#define STMT_ELOG(param, ...) qError("stmt:%p " param, pStmt, __VA_ARGS__)
#define STMT_DLOG(param, ...) qDebug("stmt:%p " param, pStmt, __VA_ARGS__)

2
source/client/src/clientSml.c
View File

@ -699,7 +699,7 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns,
pReq.numOfTags = 1;
SField field = {0};
field.type = TSDB_DATA_TYPE_NCHAR;
field.bytes = 1;
field.bytes = TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE;
strcpy(field.name, tsSmlTagName);
taosArrayPush(pReq.pTags, &field);
}

36
source/client/src/clientStmt.c
View File

@ -975,15 +975,17 @@ int stmtIsInsert(TAOS_STMT* stmt, int* insert) {
}
int stmtGetTagFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) {
int32_t code = 0;
STscStmt* pStmt = (STscStmt*)stmt;
int32_t preCode = pStmt->errCode;
STMT_DLOG_E("start to get tag fields");
if (STMT_TYPE_QUERY == pStmt->sql.type) {
STMT_RET(TSDB_CODE_TSC_STMT_API_ERROR);
STMT_ERRI_JRET(TSDB_CODE_TSC_STMT_API_ERROR);
}
STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_FETCH_FIELDS));
STMT_ERRI_JRET(stmtSwitchStatus(pStmt, STMT_FETCH_FIELDS));
if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 &&
STMT_TYPE_MULTI_INSERT != pStmt->sql.type) {
@ -995,27 +997,33 @@ int stmtGetTagFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) {
pStmt->exec.pRequest = NULL;
}
STMT_ERR_RET(stmtCreateRequest(pStmt));
STMT_ERRI_JRET(stmtCreateRequest(pStmt));
if (pStmt->bInfo.needParse) {
STMT_ERR_RET(stmtParseSql(pStmt));
STMT_ERRI_JRET(stmtParseSql(pStmt));
}
STMT_ERR_RET(stmtFetchTagFields(stmt, nums, fields));
STMT_ERRI_JRET(stmtFetchTagFields(stmt, nums, fields));
return TSDB_CODE_SUCCESS;
_return:
pStmt->errCode = preCode;
return code;
}
int stmtGetColFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) {
int32_t code = 0;
STscStmt* pStmt = (STscStmt*)stmt;
int32_t preCode = pStmt->errCode;
STMT_DLOG_E("start to get col fields");
if (STMT_TYPE_QUERY == pStmt->sql.type) {
STMT_RET(TSDB_CODE_TSC_STMT_API_ERROR);
STMT_ERRI_JRET(TSDB_CODE_TSC_STMT_API_ERROR);
}
STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_FETCH_FIELDS));
STMT_ERRI_JRET(stmtSwitchStatus(pStmt, STMT_FETCH_FIELDS));
if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 &&
STMT_TYPE_MULTI_INSERT != pStmt->sql.type) {
@ -1027,15 +1035,19 @@ int stmtGetColFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) {
pStmt->exec.pRequest = NULL;
}
STMT_ERR_RET(stmtCreateRequest(pStmt));
STMT_ERRI_JRET(stmtCreateRequest(pStmt));
if (pStmt->bInfo.needParse) {
STMT_ERR_RET(stmtParseSql(pStmt));
STMT_ERRI_JRET(stmtParseSql(pStmt));
}
STMT_ERR_RET(stmtFetchColFields(stmt, nums, fields));
STMT_ERRI_JRET(stmtFetchColFields(stmt, nums, fields));
return TSDB_CODE_SUCCESS;
_return:
pStmt->errCode = preCode;
return code;
}
int stmtGetParamNum(TAOS_STMT* stmt, int* nums) {

3
source/client/src/clientTmq.c
View File

@ -1151,6 +1151,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
};
if (tsem_init(&param.rspSem, 0, 0) != 0) {
code = TSDB_CODE_TSC_INTERNAL_ERROR;
goto FAIL;
}
@ -1186,6 +1187,8 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
int32_t retryCnt = 0;
while (TSDB_CODE_MND_CONSUMER_NOT_READY == doAskEp(tmq)) {
if (retryCnt++ > MAX_RETRY_COUNT) {
tscError("consumer:0x%" PRIx64 ", mnd not ready for subscribe, retry:%d in 500ms", tmq->consumerId, retryCnt);
code = TSDB_CODE_TSC_INTERNAL_ERROR;
goto FAIL;
}

4
source/common/src/systable.c
View File

@ -296,7 +296,9 @@ static const SSysDbTableSchema vnodesSchema[] = {
static const SSysDbTableSchema userUserPrivilegesSchema[] = {
{.name = "user_name", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "privilege", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "object_name", .bytes = TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "db_name", .bytes = TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "table_name", .bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "condition", .bytes = TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
};
static const SSysTableMeta infosMeta[] = {

5
source/common/src/tdatablock.c
View File

@ -1590,12 +1590,13 @@ static void doShiftBitmap(char* nullBitmap, size_t n, size_t total) {
i += 1;
}
} else if (n > 8) {
int32_t gap = len - newLen;
int32_t remain = (total % 8 != 0 && total % 8 <= tail) ? 1 : 0;
int32_t gap = len - newLen - remain;
while (i < newLen) {
uint8_t v = p[i + gap];
p[i] = (v << tail);
if (i < newLen - 1) {
if (i < newLen - 1 + remain) {
uint8_t next = p[i + gap + 1];
p[i] |= (next >> (8 - tail));
}

9
source/common/src/tglobal.c
View File

@ -20,6 +20,10 @@
#include "tlog.h"
#include "tmisce.h"
#if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL)
#include "cus_name.h"
#endif
GRANT_CFG_DECLARE;
SConfig *tsCfg = NULL;
@ -99,6 +103,7 @@ char tsSmlChildTableName[TSDB_TABLE_NAME_LEN] = ""; // user defined child table
// query
int32_t tsQueryPolicy = 1;
int32_t tsQueryRspPolicy = 0;
int64_t tsQueryMaxConcurrentTables = 200; // unit is TSDB_TABLE_NUM_UNIT
bool tsEnableQueryHb = false;
int32_t tsQuerySmaOptimize = 0;
int32_t tsQueryRsmaTolerance = 1000; // the tolerance time (ms) to judge from which level to query rsma data.
@ -229,7 +234,7 @@ static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *input
taosExpandDir(inputCfgDir, cfgDir, PATH_MAX);
if (taosIsDir(cfgDir)) {
#ifdef CUS_PROMPT
snprintf(cfgFile, sizeof(cfgFile), "%s" TD_DIRSEP "%s.cfg", CUS_PROMPT, cfgDir);
snprintf(cfgFile, sizeof(cfgFile), "%s" TD_DIRSEP "%s.cfg", cfgDir, CUS_PROMPT);
#else
snprintf(cfgFile, sizeof(cfgFile), "%s" TD_DIRSEP "taos.cfg", cfgDir);
#endif
@ -336,6 +341,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
if (cfgAddInt32(pCfg, "maxRetryWaitTime", tsMaxRetryWaitTime, 0, 86400000, 0) != 0) return -1;
if (cfgAddBool(pCfg, "useAdapter", tsUseAdapter, true) != 0) return -1;
if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, true) != 0) return -1;
if (cfgAddInt64(pCfg, "queryMaxConcurrentTables", tsQueryMaxConcurrentTables, INT64_MIN, INT64_MAX, 1) != 0) return -1;
tsNumOfRpcThreads = tsNumOfCores / 2;
tsNumOfRpcThreads = TRANGE(tsNumOfRpcThreads, 2, TSDB_MAX_RPC_THREADS);
@ -731,6 +737,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
tsKeepColumnName = cfgGetItem(pCfg, "keepColumnName")->bval;
tsUseAdapter = cfgGetItem(pCfg, "useAdapter")->bval;
tsEnableCrashReport = cfgGetItem(pCfg, "crashReporting")->bval;
tsQueryMaxConcurrentTables = cfgGetItem(pCfg, "queryMaxConcurrentTables")->i64;
tsMaxRetryWaitTime = cfgGetItem(pCfg, "maxRetryWaitTime")->i32;

135
source/common/src/tmsg.c
View File

@ -1150,7 +1150,7 @@ int32_t tDeserializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) {
if (tDecodeI64(&decoder, &vload.compStorage) < 0) return -1;
if (tDecodeI64(&decoder, &vload.pointsWritten) < 0) return -1;
if (tDecodeI32(&decoder, &vload.numOfCachedTables) < 0) return -1;
if (tDecodeI32(&decoder, (int32_t*)&reserved) < 0) return -1;
if (tDecodeI32(&decoder, (int32_t *)&reserved) < 0) return -1;
if (tDecodeI64(&decoder, &reserved) < 0) return -1;
if (tDecodeI64(&decoder, &reserved) < 0) return -1;
if (taosArrayPush(pReq->pVloads, &vload) == NULL) {
@ -1368,6 +1368,12 @@ int32_t tSerializeSAlterUserReq(void *buf, int32_t bufLen, SAlterUserReq *pReq)
if (tEncodeCStr(&encoder, pReq->user) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->pass) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->objname) < 0) return -1;
int32_t len = strlen(pReq->tabName);
if (tEncodeI32(&encoder, len) < 0) return -1;
if (len > 0) {
if (tEncodeCStr(&encoder, pReq->tabName) < 0) return -1;
}
if (tEncodeBinary(&encoder, pReq->tagCond, pReq->tagCondLen) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
@ -1387,6 +1393,16 @@ int32_t tDeserializeSAlterUserReq(void *buf, int32_t bufLen, SAlterUserReq *pReq
if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->pass) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->objname) < 0) return -1;
if (!tDecodeIsEnd(&decoder)) {
int32_t len = 0;
if (tDecodeI32(&decoder, &len) < 0) return -1;
if (len > 0) {
if (tDecodeCStrTo(&decoder, pReq->tabName) < 0) return -1;
}
uint64_t tagCondLen = 0;
if (tDecodeBinaryAlloc(&decoder, (void **)&pReq->tagCond, &tagCondLen) < 0) return -1;
pReq->tagCondLen = tagCondLen;
}
tEndDecode(&decoder);
tDecoderClear(&decoder);
@ -1429,6 +1445,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
int32_t numOfCreatedDbs = taosHashGetSize(pRsp->createdDbs);
int32_t numOfReadDbs = taosHashGetSize(pRsp->readDbs);
int32_t numOfWriteDbs = taosHashGetSize(pRsp->writeDbs);
if (tEncodeI32(pEncoder, numOfCreatedDbs) < 0) return -1;
if (tEncodeI32(pEncoder, numOfReadDbs) < 0) return -1;
if (tEncodeI32(pEncoder, numOfWriteDbs) < 0) return -1;
@ -1451,6 +1468,54 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
db = taosHashIterate(pRsp->writeDbs, db);
}
int32_t numOfReadTbs = taosHashGetSize(pRsp->readTbs);
int32_t numOfWriteTbs = taosHashGetSize(pRsp->writeTbs);
int32_t numOfUseTbs = taosHashGetSize(pRsp->useDbs);
if (tEncodeI32(pEncoder, numOfReadTbs) < 0) return -1;
if (tEncodeI32(pEncoder, numOfWriteTbs) < 0) return -1;
if (tEncodeI32(pEncoder, numOfUseTbs) < 0) return -1;
char *tb = taosHashIterate(pRsp->readTbs, NULL);
while (tb != NULL) {
size_t keyLen = 0;
void *key = taosHashGetKey(tb, &keyLen);
if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
if (tEncodeCStr(pEncoder, key) < 0) return -1;
size_t valueLen = 0;
valueLen = strlen(tb);
if (tEncodeI32(pEncoder, valueLen) < 0) return -1;
if (tEncodeCStr(pEncoder, tb) < 0) return -1;
tb = taosHashIterate(pRsp->readTbs, tb);
}
tb = taosHashIterate(pRsp->writeTbs, NULL);
while (tb != NULL) {
size_t keyLen = 0;
void *key = taosHashGetKey(tb, &keyLen);
if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
if (tEncodeCStr(pEncoder, key) < 0) return -1;
size_t valueLen = 0;
valueLen = strlen(tb);
if (tEncodeI32(pEncoder, valueLen) < 0) return -1;
if (tEncodeCStr(pEncoder, tb) < 0) return -1;
tb = taosHashIterate(pRsp->writeTbs, tb);
}
int32_t *useDb = taosHashIterate(pRsp->useDbs, NULL);
while (useDb != NULL) {
size_t keyLen = 0;
void *key = taosHashGetKey(useDb, &keyLen);
if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
if (tEncodeCStr(pEncoder, key) < 0) return -1;
if (tEncodeI32(pEncoder, *useDb) < 0) return -1;
useDb = taosHashIterate(pRsp->useDbs, useDb);
}
return 0;
}
@ -1473,7 +1538,11 @@ int32_t tDeserializeSGetUserAuthRspImpl(SDecoder *pDecoder, SGetUserAuthRsp *pRs
pRsp->createdDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
pRsp->readDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
pRsp->writeDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
if (pRsp->readDbs == NULL || pRsp->writeDbs == NULL) {
pRsp->readTbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
pRsp->writeTbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
pRsp->useDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
if (pRsp->createdDbs == NULL || pRsp->readDbs == NULL || pRsp->writeDbs == NULL || pRsp->readTbs == NULL ||
pRsp->writeTbs == NULL || pRsp->useDbs == NULL) {
return -1;
}
@ -1512,6 +1581,63 @@ int32_t tDeserializeSGetUserAuthRspImpl(SDecoder *pDecoder, SGetUserAuthRsp *pRs
taosHashPut(pRsp->writeDbs, db, len, db, len);
}
if (!tDecodeIsEnd(pDecoder)) {
int32_t numOfReadTbs = 0;
int32_t numOfWriteTbs = 0;
int32_t numOfUseDbs = 0;
if (tDecodeI32(pDecoder, &numOfReadTbs) < 0) return -1;
if (tDecodeI32(pDecoder, &numOfWriteTbs) < 0) return -1;
if (tDecodeI32(pDecoder, &numOfUseDbs) < 0) return -1;
for (int32_t i = 0; i < numOfReadTbs; ++i) {
int32_t keyLen = 0;
if (tDecodeI32(pDecoder, &keyLen) < 0) return -1;
char *key = taosMemoryCalloc(keyLen + 1, sizeof(char));
if (tDecodeCStrTo(pDecoder, key) < 0) return -1;
int32_t valuelen = 0;
if (tDecodeI32(pDecoder, &valuelen) < 0) return -1;
char *value = taosMemoryCalloc(valuelen + 1, sizeof(char));
if (tDecodeCStrTo(pDecoder, value) < 0) return -1;
taosHashPut(pRsp->readTbs, key, strlen(key), value, valuelen + 1);
taosMemoryFree(key);
taosMemoryFree(value);
}
for (int32_t i = 0; i < numOfWriteTbs; ++i) {
int32_t keyLen = 0;
if (tDecodeI32(pDecoder, &keyLen) < 0) return -1;
char *key = taosMemoryCalloc(keyLen + 1, sizeof(char));
if (tDecodeCStrTo(pDecoder, key) < 0) return -1;
int32_t valuelen = 0;
if (tDecodeI32(pDecoder, &valuelen) < 0) return -1;
char *value = taosMemoryCalloc(valuelen + 1, sizeof(char));
if (tDecodeCStrTo(pDecoder, value) < 0) return -1;
taosHashPut(pRsp->writeTbs, key, strlen(key), value, valuelen + 1);
taosMemoryFree(key);
taosMemoryFree(value);
}
for (int32_t i = 0; i < numOfUseDbs; ++i) {
int32_t keyLen = 0;
if (tDecodeI32(pDecoder, &keyLen) < 0) return -1;
char *key = taosMemoryCalloc(keyLen + 1, sizeof(char));
if (tDecodeCStrTo(pDecoder, key) < 0) return -1;
int32_t ref = 0;
if (tDecodeI32(pDecoder, &ref) < 0) return -1;
taosHashPut(pRsp->useDbs, key, strlen(key), &ref, sizeof(ref));
}
}
return 0;
}
@ -1533,6 +1659,9 @@ void tFreeSGetUserAuthRsp(SGetUserAuthRsp *pRsp) {
taosHashCleanup(pRsp->createdDbs);
taosHashCleanup(pRsp->readDbs);
taosHashCleanup(pRsp->writeDbs);
taosHashCleanup(pRsp->writeTbs);
taosHashCleanup(pRsp->readTbs);
taosHashCleanup(pRsp->useDbs);
}
int32_t tSerializeSCreateDropMQSNodeReq(void *buf, int32_t bufLen, SMCreateQnodeReq *pReq) {
@ -6824,7 +6953,7 @@ int32_t tDecodeSMqDataRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) {
}
void tDeleteSMqDataRsp(SMqDataRsp *pRsp) {
pRsp->blockDataLen = taosArrayDestroy(pRsp->blockDataLen);;
pRsp->blockDataLen = taosArrayDestroy(pRsp->blockDataLen);
taosArrayDestroyP(pRsp->blockData, (FDelete)taosMemoryFree);
pRsp->blockData = NULL;
taosArrayDestroyP(pRsp->blockSchema, (FDelete)tDeleteSSchemaWrapper);

8
source/dnode/mgmt/mgmt_vnode/src/vmWorker.c
View File

@ -114,11 +114,11 @@ static void vmProcessFetchQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO
int32_t code = vnodeProcessFetchMsg(pVnode->pImpl, pMsg, pInfo);
if (code != 0) {
if (terrno != 0) code = terrno;
dGError("vgId:%d, msg:%p failed to fetch since %s", pVnode->vgId, pMsg, terrstr(code));
dGError("vnodeProcessFetchMsg vgId:%d, msg:%p failed to fetch since %s", pVnode->vgId, pMsg, terrstr());
vmSendRsp(pMsg, code);
}
dGTrace("vgId:%d, msg:%p is freed, code:0x%x", pVnode->vgId, pMsg, code);
dGTrace("vnodeProcessFetchMsg vgId:%d, msg:%p is freed, code:0x%x", pVnode->vgId, pMsg, code);
rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg);
}
@ -163,8 +163,8 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp
SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId);
if (pVnode == NULL) {
dGError("vgId:%d, msg:%p failed to put into vnode queue since %s, type:%s qtype:%d contLen:%d", pHead->vgId, pMsg,
terrstr(), TMSG_INFO(pMsg->msgType), qtype, pHead->contLen);
dGWarn("vgId:%d, msg:%p failed to put into vnode queue since %s, type:%s qtype:%d contLen:%d", pHead->vgId, pMsg,
terrstr(), TMSG_INFO(pMsg->msgType), qtype, pHead->contLen);
terrno = (terrno != 0) ? terrno : -1;
return terrno;
}

3
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -280,6 +280,9 @@ typedef struct {
SHashObj* readDbs;
SHashObj* writeDbs;
SHashObj* topics;
SHashObj* readTbs;
SHashObj* writeTbs;
SHashObj* useDbs;
SRWLatch lock;
} SUserObj;

1
source/dnode/mnode/impl/inc/mndStb.h
View File

@ -38,6 +38,7 @@ void mndFreeStb(SStbObj *pStb);
int32_t mndBuildSMCreateStbRsp(SMnode *pMnode, char *dbFName, char *stbFName, void **pCont, int32_t *pLen);
void mndExtractDbNameFromStbFullName(const char *stbFullName, char *dst);
void mndExtractShortDbNameFromStbFullName(const char *stbFullName, char *dst);
void mndExtractTbNameFromStbFullName(const char *stbFullName, char *dst, int32_t dstSize);
const char *mndGetStbStr(const char *src);

1
source/dnode/mnode/impl/inc/mndUser.h
View File

@ -31,6 +31,7 @@ void mndReleaseUser(SMnode *pMnode, SUserObj *pUser);
// for trans test
SSdbRaw *mndUserActionEncode(SUserObj *pUser);
SHashObj *mndDupDbHash(SHashObj *pOld);
SHashObj *mndDupTableHash(SHashObj *pOld);
SHashObj *mndDupTopicHash(SHashObj *pOld);
int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_t numOfUses, void **ppRsp,
int32_t *pRspLen);

1
source/dnode/mnode/impl/src/mndPrivilege.c
View File

@ -35,6 +35,7 @@ int32_t mndCheckTopicPrivilegeByName(SMnode *pMnode, const char *user, EOperType
int32_t mndSetUserAuthRsp(SMnode *pMnode, SUserObj *pUser, SGetUserAuthRsp *pRsp) {
memcpy(pRsp->user, pUser->user, TSDB_USER_LEN);
pRsp->superAuth = 1;
pRsp->enable = pUser->enable;
pRsp->version = pUser->authVersion;
return 0;
}

7
source/dnode/mnode/impl/src/mndStb.c
View File

@ -2613,6 +2613,13 @@ void mndExtractDbNameFromStbFullName(const char *stbFullName, char *dst) {
tNameGetFullDbName(&name, dst);
}
void mndExtractShortDbNameFromStbFullName(const char *stbFullName, char *dst) {
SName name = {0};
tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
tNameGetDbName(&name, dst);
}
void mndExtractTbNameFromStbFullName(const char *stbFullName, char *dst, int32_t dstSize) {
int32_t pos = -1;
int32_t num = 0;

411
source/dnode/mnode/impl/src/mndUser.c
View File

@ -18,11 +18,12 @@
#include "mndDb.h"
#include "mndPrivilege.h"
#include "mndShow.h"
#include "mndStb.h"
#include "mndTopic.h"
#include "mndTrans.h"
#include "tbase64.h"
#define USER_VER_NUMBER 2
#define USER_VER_NUMBER 3
#define USER_RESERVE_SIZE 64
static int32_t mndCreateDefaultUsers(SMnode *pMnode);
@ -124,9 +125,40 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) {
int32_t numOfReadDbs = taosHashGetSize(pUser->readDbs);
int32_t numOfWriteDbs = taosHashGetSize(pUser->writeDbs);
int32_t numOfReadStbs = taosHashGetSize(pUser->readTbs);
int32_t numOfWriteStbs = taosHashGetSize(pUser->writeTbs);
int32_t numOfTopics = taosHashGetSize(pUser->topics);
int32_t size = sizeof(SUserObj) + USER_RESERVE_SIZE + (numOfReadDbs + numOfWriteDbs) * TSDB_DB_FNAME_LEN +
numOfTopics * TSDB_TOPIC_FNAME_LEN;
int32_t numOfUseDbs = taosHashGetSize(pUser->useDbs);
int32_t size = sizeof(SUserObj) + USER_RESERVE_SIZE +
(numOfReadDbs + numOfWriteDbs + numOfUseDbs) * TSDB_DB_FNAME_LEN + numOfTopics * TSDB_TOPIC_FNAME_LEN;
char *stb = taosHashIterate(pUser->readTbs, NULL);
while (stb != NULL) {
size_t keyLen = 0;
void *key = taosHashGetKey(stb, &keyLen);
size += sizeof(int32_t);
size += keyLen;
size_t valueLen = 0;
valueLen = strlen(stb);
size += sizeof(int32_t);
size += valueLen;
stb = taosHashIterate(pUser->readTbs, stb);
}
stb = taosHashIterate(pUser->writeTbs, NULL);
while (stb != NULL) {
size_t keyLen = 0;
void *key = taosHashGetKey(stb, &keyLen);
size += sizeof(int32_t);
size += keyLen;
size_t valueLen = 0;
valueLen = strlen(stb);
size += sizeof(int32_t);
size += valueLen;
stb = taosHashIterate(pUser->writeTbs, stb);
}
SSdbRaw *pRaw = sdbAllocRaw(SDB_USER, USER_VER_NUMBER, size);
if (pRaw == NULL) goto _OVER;
@ -164,6 +196,49 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) {
topic = taosHashIterate(pUser->topics, topic);
}
SDB_SET_INT32(pRaw, dataPos, numOfReadStbs, _OVER)
SDB_SET_INT32(pRaw, dataPos, numOfWriteStbs, _OVER)
SDB_SET_INT32(pRaw, dataPos, numOfUseDbs, _OVER)
stb = taosHashIterate(pUser->readTbs, NULL);
while (stb != NULL) {
size_t keyLen = 0;
void *key = taosHashGetKey(stb, &keyLen);
SDB_SET_INT32(pRaw, dataPos, keyLen, _OVER)
SDB_SET_BINARY(pRaw, dataPos, key, keyLen, _OVER);
size_t valueLen = 0;
valueLen = strlen(stb) + 1;
SDB_SET_INT32(pRaw, dataPos, valueLen, _OVER)
SDB_SET_BINARY(pRaw, dataPos, stb, valueLen, _OVER);
stb = taosHashIterate(pUser->readTbs, stb);
}
stb = taosHashIterate(pUser->writeTbs, NULL);
while (stb != NULL) {
size_t keyLen = 0;
void *key = taosHashGetKey(stb, &keyLen);
SDB_SET_INT32(pRaw, dataPos, keyLen, _OVER)
SDB_SET_BINARY(pRaw, dataPos, key, keyLen, _OVER);
size_t valueLen = 0;
valueLen = strlen(stb) + 1;
SDB_SET_INT32(pRaw, dataPos, valueLen, _OVER)
SDB_SET_BINARY(pRaw, dataPos, stb, valueLen, _OVER);
stb = taosHashIterate(pUser->writeTbs, stb);
}
int32_t *useDb = taosHashIterate(pUser->useDbs, NULL);
while (useDb != NULL) {
size_t keyLen = 0;
void *key = taosHashGetKey(useDb, &keyLen);
SDB_SET_INT32(pRaw, dataPos, keyLen, _OVER)
SDB_SET_BINARY(pRaw, dataPos, key, keyLen, _OVER);
SDB_SET_INT32(pRaw, dataPos, *useDb, _OVER)
useDb = taosHashIterate(pUser->writeTbs, useDb);
}
SDB_SET_RESERVE(pRaw, dataPos, USER_RESERVE_SIZE, _OVER)
SDB_SET_DATALEN(pRaw, dataPos, _OVER)
@ -188,7 +263,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) {
int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER;
if (sver != 1 && sver != 2) {
if (sver != 1 && sver != 2 && sver != 3) {
terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
goto _OVER;
}
@ -249,6 +324,75 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) {
}
}
if (sver >= 3) {
int32_t numOfReadStbs = 0;
int32_t numOfWriteStbs = 0;
int32_t numOfUseDbs = 0;
SDB_GET_INT32(pRaw, dataPos, &numOfReadStbs, _OVER)
SDB_GET_INT32(pRaw, dataPos, &numOfWriteStbs, _OVER)
SDB_GET_INT32(pRaw, dataPos, &numOfUseDbs, _OVER)
pUser->readTbs =
taosHashInit(numOfReadStbs, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
pUser->writeTbs =
taosHashInit(numOfWriteStbs, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
pUser->useDbs = taosHashInit(numOfUseDbs, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
for (int32_t i = 0; i < numOfReadStbs; ++i) {
int32_t keyLen = 0;
SDB_GET_INT32(pRaw, dataPos, &keyLen, _OVER);
char *key = taosMemoryCalloc(keyLen, sizeof(char));
memset(key, 0, keyLen);
SDB_GET_BINARY(pRaw, dataPos, key, keyLen, _OVER);
int32_t valuelen = 0;
SDB_GET_INT32(pRaw, dataPos, &valuelen, _OVER);
char *value = taosMemoryCalloc(valuelen, sizeof(char));
memset(value, 0, valuelen);
SDB_GET_BINARY(pRaw, dataPos, value, valuelen, _OVER)
taosHashPut(pUser->readTbs, key, keyLen, value, valuelen);
taosMemoryFree(key);
taosMemoryFree(value);
}
for (int32_t i = 0; i < numOfWriteStbs; ++i) {
int32_t keyLen = 0;
SDB_GET_INT32(pRaw, dataPos, &keyLen, _OVER);
char *key = taosMemoryCalloc(keyLen, sizeof(char));
memset(key, 0, keyLen);
SDB_GET_BINARY(pRaw, dataPos, key, keyLen, _OVER);
int32_t valuelen = 0;
SDB_GET_INT32(pRaw, dataPos, &valuelen, _OVER);
char *value = taosMemoryCalloc(valuelen, sizeof(char));
memset(value, 0, valuelen);
SDB_GET_BINARY(pRaw, dataPos, value, valuelen, _OVER)
taosHashPut(pUser->writeTbs, key, keyLen, value, valuelen);
taosMemoryFree(key);
taosMemoryFree(value);
}
for (int32_t i = 0; i < numOfUseDbs; ++i) {
int32_t keyLen = 0;
SDB_GET_INT32(pRaw, dataPos, &keyLen, _OVER);
char *key = taosMemoryCalloc(keyLen, sizeof(char));
memset(key, 0, keyLen);
SDB_GET_BINARY(pRaw, dataPos, key, keyLen, _OVER);
int32_t ref = 0;
SDB_GET_INT32(pRaw, dataPos, &ref, _OVER);
taosHashPut(pUser->useDbs, key, keyLen, &ref, sizeof(ref));
}
}
SDB_GET_RESERVE(pRaw, dataPos, USER_RESERVE_SIZE, _OVER)
taosInitRWLatch(&pUser->lock);
@ -261,6 +405,9 @@ _OVER:
taosHashCleanup(pUser->readDbs);
taosHashCleanup(pUser->writeDbs);
taosHashCleanup(pUser->topics);
taosHashCleanup(pUser->readTbs);
taosHashCleanup(pUser->writeTbs);
taosHashCleanup(pUser->useDbs);
}
taosMemoryFreeClear(pRow);
return NULL;
@ -285,6 +432,57 @@ static int32_t mndUserActionInsert(SSdb *pSdb, SUserObj *pUser) {
return 0;
}
SHashObj *mndDupTableHash(SHashObj *pOld) {
SHashObj *pNew =
taosHashInit(taosHashGetSize(pOld), taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
if (pNew == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
char *tb = taosHashIterate(pOld, NULL);
while (tb != NULL) {
size_t keyLen = 0;
char *key = taosHashGetKey(tb, &keyLen);
int32_t valueLen = strlen(tb) + 1;
if (taosHashPut(pNew, key, keyLen, tb, valueLen) != 0) {
taosHashCancelIterate(pOld, tb);
taosHashCleanup(pNew);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
tb = taosHashIterate(pOld, tb);
}
return pNew;
}
SHashObj *mndDupUseDbHash(SHashObj *pOld) {
SHashObj *pNew =
taosHashInit(taosHashGetSize(pOld), taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
if (pNew == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
int32_t *db = taosHashIterate(pOld, NULL);
while (db != NULL) {
size_t keyLen = 0;
char *key = taosHashGetKey(db, &keyLen);
if (taosHashPut(pNew, key, keyLen, db, sizeof(*db)) != 0) {
taosHashCancelIterate(pOld, db);
taosHashCleanup(pNew);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
db = taosHashIterate(pOld, db);
}
return pNew;
}
static int32_t mndUserDupObj(SUserObj *pUser, SUserObj *pNew) {
memcpy(pNew, pUser, sizeof(SUserObj));
pNew->authVersion++;
@ -293,7 +491,10 @@ static int32_t mndUserDupObj(SUserObj *pUser, SUserObj *pNew) {
taosRLockLatch(&pUser->lock);
pNew->readDbs = mndDupDbHash(pUser->readDbs);
pNew->writeDbs = mndDupDbHash(pUser->writeDbs);
pNew->readTbs = mndDupTableHash(pUser->readTbs);
pNew->writeTbs = mndDupTableHash(pUser->writeTbs);
pNew->topics = mndDupTopicHash(pUser->topics);
pNew->useDbs = mndDupUseDbHash(pUser->useDbs);
taosRUnLockLatch(&pUser->lock);
if (pNew->readDbs == NULL || pNew->writeDbs == NULL || pNew->topics == NULL) {
@ -306,9 +507,15 @@ static void mndUserFreeObj(SUserObj *pUser) {
taosHashCleanup(pUser->readDbs);
taosHashCleanup(pUser->writeDbs);
taosHashCleanup(pUser->topics);
taosHashCleanup(pUser->readTbs);
taosHashCleanup(pUser->writeTbs);
taosHashCleanup(pUser->useDbs);
pUser->readDbs = NULL;
pUser->writeDbs = NULL;
pUser->topics = NULL;
pUser->readTbs = NULL;
pUser->writeTbs = NULL;
pUser->useDbs = NULL;
}
static int32_t mndUserActionDelete(SSdb *pSdb, SUserObj *pUser) {
@ -328,6 +535,9 @@ static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pOld, SUserObj *pNew) {
TSWAP(pOld->readDbs, pNew->readDbs);
TSWAP(pOld->writeDbs, pNew->writeDbs);
TSWAP(pOld->topics, pNew->topics);
TSWAP(pOld->readTbs, pNew->readTbs);
TSWAP(pOld->writeTbs, pNew->writeTbs);
TSWAP(pOld->useDbs, pNew->useDbs);
taosWUnLockLatch(&pOld->lock);
return 0;
@ -498,6 +708,71 @@ SHashObj *mndDupDbHash(SHashObj *pOld) { return mndDupObjHash(pOld, TSDB_DB_FNAM
SHashObj *mndDupTopicHash(SHashObj *pOld) { return mndDupObjHash(pOld, TSDB_TOPIC_FNAME_LEN); }
static int32_t mndTablePriviledge(SMnode *pMnode, SHashObj *hash, SHashObj *useDbHash, SAlterUserReq *alterReq,
SSdb *pSdb) {
void *pIter = NULL;
char tbFName[TSDB_TABLE_FNAME_LEN] = {0};
snprintf(tbFName, sizeof(tbFName), "%s.%s", alterReq->objname, alterReq->tabName);
int32_t len = strlen(tbFName) + 1;
if (alterReq->tagCond != NULL && alterReq->tagCondLen != 0) {
char *value = taosHashGet(hash, tbFName, len);
if (value != NULL) {
terrno = TSDB_CODE_MND_PRIVILEDGE_EXIST;
return -1;
}
int32_t condLen = alterReq->tagCondLen;
if (taosHashPut(hash, tbFName, len, alterReq->tagCond, condLen) != 0) {
return -1;
}
} else {
if (taosHashPut(hash, tbFName, len, "t", 2) != 0) {
return -1;
}
}
int32_t dbKeyLen = strlen(alterReq->objname) + 1;
int32_t ref = 1;
int32_t *currRef = taosHashGet(useDbHash, alterReq->objname, dbKeyLen);
if (NULL != currRef) {
ref = (*currRef) + 1;
}
if (taosHashPut(useDbHash, alterReq->objname, dbKeyLen, &ref, sizeof(ref)) != 0) {
return -1;
}
return 0;
}
static int32_t mndRemoveTablePriviledge(SMnode *pMnode, SHashObj *hash, SHashObj *useDbHash, SAlterUserReq *alterReq,
SSdb *pSdb) {
void *pIter = NULL;
char tbFName[TSDB_TABLE_FNAME_LEN] = {0};
snprintf(tbFName, sizeof(tbFName), "%s.%s", alterReq->objname, alterReq->tabName);
int32_t len = strlen(tbFName) + 1;
if (taosHashRemove(hash, tbFName, len) != 0) {
return -1;
}
int32_t dbKeyLen = strlen(alterReq->objname) + 1;
int32_t *currRef = taosHashGet(useDbHash, alterReq->objname, dbKeyLen);
if (NULL == currRef || 1 == *currRef) {
if (taosHashRemove(useDbHash, alterReq->objname, dbKeyLen) != 0) {
return -1;
}
return 0;
}
int32_t ref = (*currRef) - 1;
if (taosHashPut(useDbHash, alterReq->objname, dbKeyLen, &ref, sizeof(ref)) != 0) {
return -1;
}
return 0;
}
static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node;
SSdb *pSdb = pMnode->pSdb;
@ -637,6 +912,22 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) {
}
}
if (alterReq.alterType == TSDB_ALTER_USER_ADD_READ_TABLE) {
if (mndTablePriviledge(pMnode, newUser.readTbs, newUser.useDbs, &alterReq, pSdb) != 0) goto _OVER;
}
if (alterReq.alterType == TSDB_ALTER_USER_ADD_WRITE_TABLE) {
if (mndTablePriviledge(pMnode, newUser.writeTbs, newUser.useDbs, &alterReq, pSdb) != 0) goto _OVER;
}
if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_READ_TABLE) {
if (mndRemoveTablePriviledge(pMnode, newUser.readTbs, newUser.useDbs, &alterReq, pSdb) != 0) goto _OVER;
}
if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_WRITE_TABLE) {
if (mndRemoveTablePriviledge(pMnode, newUser.writeTbs, newUser.useDbs, &alterReq, pSdb) != 0) goto _OVER;
}
if (alterReq.alterType == TSDB_ALTER_USER_ADD_SUBSCRIBE_TOPIC) {
int32_t len = strlen(alterReq.objname) + 1;
SMqTopicObj *pTopic = mndAcquireTopic(pMnode, alterReq.objname);
@ -830,6 +1121,70 @@ static void mndCancelGetNextUser(SMnode *pMnode, void *pIter) {
sdbCancelFetch(pSdb, pIter);
}
static void mndLoopHash(SHashObj *hash, char *priType, SSDataBlock *pBlock, int32_t *numOfRows, char *user,
SShowObj *pShow) {
char *value = taosHashIterate(hash, NULL);
int32_t cols = 0;
while (value != NULL) {
cols = 0;
char userName[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(userName, user, pShow->pMeta->pSchemas[cols].bytes);
SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, *numOfRows, (const char *)userName, false);
char privilege[20] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(privilege, priType, pShow->pMeta->pSchemas[cols].bytes);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, *numOfRows, (const char *)privilege, false);
size_t keyLen = 0;
void *key = taosHashGetKey(value, &keyLen);
char dbName[TSDB_DB_NAME_LEN] = {0};
mndExtractShortDbNameFromStbFullName(key, dbName);
char dbNameContent[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(dbNameContent, dbName, pShow->pMeta->pSchemas[cols].bytes);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, *numOfRows, (const char *)dbNameContent, false);
char tableName[TSDB_TABLE_NAME_LEN] = {0};
mndExtractTbNameFromStbFullName(key, tableName, TSDB_TABLE_NAME_LEN);
char tableNameContent[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(tableNameContent, tableName, pShow->pMeta->pSchemas[cols].bytes);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, *numOfRows, (const char *)tableNameContent, false);
if (strcmp("t", value) != 0) {
SNode *pAst = NULL;
int32_t sqlLen = 0;
char sql[TSDB_EXPLAIN_RESULT_ROW_SIZE] = {0};
if (nodesStringToNode(value, &pAst) == 0) {
nodesNodeToSQL(pAst, sql, TSDB_EXPLAIN_RESULT_ROW_SIZE, &sqlLen);
nodesDestroyNode(pAst);
} else {
sqlLen = 5;
sprintf(sql, "error");
}
char obj[TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(obj, sql, pShow->pMeta->pSchemas[cols].bytes);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, *numOfRows, (const char *)obj, false);
} else {
char condition[TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(condition, "", pShow->pMeta->pSchemas[cols].bytes);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, *numOfRows, (const char *)condition, false);
}
(*numOfRows)++;
value = taosHashIterate(hash, value);
}
}
static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
SMnode *pMnode = pReq->info.node;
SSdb *pSdb = pMnode->pSdb;
@ -845,7 +1200,9 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
int32_t numOfReadDbs = taosHashGetSize(pUser->readDbs);
int32_t numOfWriteDbs = taosHashGetSize(pUser->writeDbs);
int32_t numOfTopics = taosHashGetSize(pUser->topics);
if (numOfRows + numOfReadDbs + numOfWriteDbs + numOfTopics >= rows) break;
int32_t numOfReadTbs = taosHashGetSize(pUser->readTbs);
int32_t numOfWriteTbs = taosHashGetSize(pUser->writeTbs);
if (numOfRows + numOfReadDbs + numOfWriteDbs + numOfTopics + numOfReadTbs + numOfWriteTbs >= rows) break;
if (pUser->superUser) {
cols = 0;
@ -864,6 +1221,16 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)objName, false);
char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(tableName, "", pShow->pMeta->pSchemas[cols].bytes);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)tableName, false);
char condition[TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(condition, "", pShow->pMeta->pSchemas[cols].bytes);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)condition, false);
numOfRows++;
}
@ -888,6 +1255,16 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)objName, false);
char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(tableName, "", pShow->pMeta->pSchemas[cols].bytes);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)tableName, false);
char condition[TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(condition, "", pShow->pMeta->pSchemas[cols].bytes);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)condition, false);
numOfRows++;
db = taosHashIterate(pUser->readDbs, db);
}
@ -913,10 +1290,24 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)objName, false);
char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(tableName, "", pShow->pMeta->pSchemas[cols].bytes);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)tableName, false);
char condition[TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(condition, "", pShow->pMeta->pSchemas[cols].bytes);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)condition, false);
numOfRows++;
db = taosHashIterate(pUser->writeDbs, db);
}
mndLoopHash(pUser->readTbs, "read", pBlock, &numOfRows, pUser->user, pShow);
mndLoopHash(pUser->writeTbs, "write", pBlock, &numOfRows, pUser->user, pShow);
char *topic = taosHashIterate(pUser->topics, NULL);
while (topic != NULL) {
cols = 0;
@ -936,6 +1327,16 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)topicName, false);
char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(tableName, "", pShow->pMeta->pSchemas[cols].bytes);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)tableName, false);
char condition[TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(condition, "", pShow->pMeta->pSchemas[cols].bytes);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)condition, false);
numOfRows++;
topic = taosHashIterate(pUser->topics, topic);
}

3
source/dnode/vnode/src/tq/tqRead.c
View File

@ -296,10 +296,9 @@ void tqCloseReader(STqReader* pReader) {
int32_t tqSeekVer(STqReader* pReader, int64_t ver, const char* id) {
if (walReadSeekVer(pReader->pWalReader, ver) < 0) {
tqDebug("tmq poll: wal reader failed to seek to ver:%"PRId64" code:%s, %s", ver, tstrerror(terrno), id);
return -1;
}
tqDebug("tmq poll: wal reader seek to ver:%"PRId64" %s", ver, id);
tqDebug("tmq poll: wal reader seek to ver success ver:%"PRId64" %s", ver, id);
return 0;
}

22
source/dnode/vnode/src/tsdb/tsdbCache.c
View File

@ -1468,11 +1468,14 @@ static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, SArray **ppCo
hasRow = true;
code = updateTSchema(TSDBROW_SVERSION(pRow), pr, uid);
if (TSDB_CODE_SUCCESS != code) {
goto _err;
int32_t sversion = TSDBROW_SVERSION(pRow);
if (sversion != -1) {
code = updateTSchema(sversion, pr, uid);
if (TSDB_CODE_SUCCESS != code) {
goto _err;
}
pTSchema = pr->pCurrSchema;
}
pTSchema = pr->pCurrSchema;
int16_t nCol = pTSchema->numOfCols;
TSKEY rowTs = TSDBROW_TS(pRow);
@ -1622,11 +1625,14 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach
hasRow = true;
code = updateTSchema(TSDBROW_SVERSION(pRow), pr, uid);
if (TSDB_CODE_SUCCESS != code) {
goto _err;
int32_t sversion = TSDBROW_SVERSION(pRow);
if (sversion != -1) {
code = updateTSchema(sversion, pr, uid);
if (TSDB_CODE_SUCCESS != code) {
goto _err;
}
pTSchema = pr->pCurrSchema;
}
pTSchema = pr->pCurrSchema;
int16_t nCol = pTSchema->numOfCols;
TSKEY rowTs = TSDBROW_TS(pRow);

28
source/dnode/vnode/src/tsdb/tsdbRead.c
View File

@ -2215,8 +2215,16 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo*
int64_t key = hasDataInFileBlock(pBlockData, pDumpInfo) ? pBlockData->aTSKEY[pDumpInfo->rowIndex] : INT64_MIN;
TSDBKEY k = TSDBROW_KEY(pRow);
TSDBKEY ik = TSDBROW_KEY(piRow);
TSDBKEY k = TSDBROW_KEY(pRow);
TSDBKEY ik = TSDBROW_KEY(piRow);
STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid);
if (pSchema == NULL) {
return code;
}
STSchema* piSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid);
if (piSchema == NULL) {
return code;
}
int64_t minKey = 0;
if (ASCENDING_TRAVERSE(pReader->order)) {
@ -2288,15 +2296,10 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo*
if (minKey == ik.ts) {
if (init) {
tsdbRowMergerAdd(&merge, piRow, NULL);
tsdbRowMergerAdd(&merge, piRow, piSchema);
} else {
init = true;
STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid);
if (pSchema == NULL) {
return code;
}
code = tsdbRowMergerInit(&merge, NULL, piRow, pSchema);
code = tsdbRowMergerInit(&merge, pSchema, piRow, piSchema);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
@ -2315,7 +2318,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo*
return code;
}
tsdbRowMergerAdd(&merge, pRow, NULL);
tsdbRowMergerAdd(&merge, pRow, pSchema);
} else {
STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid);
code = tsdbRowMergerInit(&merge, NULL, pRow, pSchema);
@ -2332,7 +2335,6 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo*
} else {
if (minKey == k.ts) {
init = true;
STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid);
code = tsdbRowMergerInit(&merge, NULL, pRow, pSchema);
if (code != TSDB_CODE_SUCCESS) {
return code;
@ -2347,11 +2349,11 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo*
if (minKey == ik.ts) {
if (init) {
tsdbRowMergerAdd(&merge, piRow, NULL);
tsdbRowMergerAdd(&merge, piRow, piSchema);
} else {
init = true;
STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid);
code = tsdbRowMergerInit(&merge, NULL, piRow, pSchema);
code = tsdbRowMergerInit(&merge, pSchema, piRow, piSchema);
if (code != TSDB_CODE_SUCCESS) {
return code;
}

4
source/libs/catalog/CMakeLists.txt
View File

@ -8,9 +8,9 @@ target_include_directories(
target_link_libraries(
catalog
PRIVATE os util transport qcom
PRIVATE os util transport qcom nodes
)
if(${BUILD_TEST})
ADD_SUBDIRECTORY(test)
endif(${BUILD_TEST})
endif(${BUILD_TEST})

25
source/libs/catalog/inc/catalogInt.h
View File

@ -99,6 +99,20 @@ typedef struct SCtgDebug {
uint32_t showCachePeriodSec;
} SCtgDebug;
typedef struct SCtgAuthReq {
SRequestConnInfo* pConn;
SUserAuthInfo* pRawReq;
SGetUserAuthRsp authInfo;
AUTH_TYPE singleType;
bool onlyCache;
} SCtgAuthReq;
typedef struct SCtgAuthRsp {
SUserAuthRes* pRawRes;
bool metaNotExists;
} SCtgAuthRsp;
typedef struct SCtgTbCacheInfo {
bool inCache;
uint64_t dbId;
@ -214,12 +228,8 @@ typedef struct SCtgRentMgmt {
} SCtgRentMgmt;
typedef struct SCtgUserAuth {
int32_t version;
SRWLatch lock;
bool superUser;
SHashObj* createdDbs;
SHashObj* readDbs;
SHashObj* writeDbs;
SRWLatch lock;
SGetUserAuthRsp userAuth;
} SCtgUserAuth;
typedef struct SCatalog {
@ -703,7 +713,7 @@ int32_t ctgTbMetaExistInCache(SCatalog* pCtg, char* dbFName, char* tbName, int32
int32_t ctgReadTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta);
int32_t ctgReadTbVerFromCache(SCatalog* pCtg, SName* pTableName, int32_t* sver, int32_t* tver, int32_t* tbType,
uint64_t* suid, char* stbName);
int32_t ctgChkAuthFromCache(SCatalog* pCtg, char* user, char* dbFName, AUTH_TYPE type, bool* inCache, bool* pass);
int32_t ctgChkAuthFromCache(SCatalog *pCtg, SUserAuthInfo *pReq, bool *inCache, SCtgAuthRsp* pRes);
int32_t ctgDropDbCacheEnqueue(SCatalog* pCtg, const char* dbFName, int64_t dbId);
int32_t ctgDropDbVgroupEnqueue(SCatalog* pCtg, const char* dbFName, bool syncReq);
int32_t ctgDropStbMetaEnqueue(SCatalog* pCtg, const char* dbFName, int64_t dbId, const char* stbName, uint64_t suid,
@ -806,6 +816,7 @@ int32_t ctgAcquireVgMetaFromCache(SCatalog *pCtg, const char *dbFName, const cha
int32_t ctgCopyTbMeta(SCatalog *pCtg, SCtgTbMetaCtx *ctx, SCtgDBCache **pDb, SCtgTbCache **pTb, STableMeta **pTableMeta, char* dbFName);
void ctgReleaseVgMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, SCtgTbCache *pCache);
void ctgReleaseTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, SCtgTbCache *pCache);
int32_t ctgChkSetAuthRes(SCatalog *pCtg, SCtgAuthReq *req, SCtgAuthRsp* res);
extern SCatalogMgmt gCtgMgmt;
extern SCtgDebug gCTGDebug;

53
source/libs/catalog/src/catalog.c
View File

@ -319,14 +319,13 @@ _return:
CTG_RET(code);
}
int32_t ctgChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, const char* user, const char* dbFName, AUTH_TYPE type,
bool* pass, bool* exists) {
int32_t ctgChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, SUserAuthInfo *pReq, SUserAuthRes* pRes, bool* exists) {
bool inCache = false;
int32_t code = 0;
SCtgAuthRsp rsp = {0};
rsp.pRawRes = pRes;
*pass = false;
CTG_ERR_RET(ctgChkAuthFromCache(pCtg, (char*)user, (char*)dbFName, type, &inCache, pass));
CTG_ERR_RET(ctgChkAuthFromCache(pCtg, pReq, &inCache, &rsp));
if (inCache) {
if (exists) {
@ -339,30 +338,22 @@ int32_t ctgChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, const char* user, co
return TSDB_CODE_SUCCESS;
}
SGetUserAuthRsp authRsp = {0};
CTG_ERR_RET(ctgGetUserDbAuthFromMnode(pCtg, pConn, user, &authRsp, NULL));
SCtgAuthReq req = {0};
req.pRawReq = pReq;
req.pConn = pConn;
req.onlyCache = exists ? true : false;
CTG_ERR_RET(ctgGetUserDbAuthFromMnode(pCtg, pConn, pReq->user, &req.authInfo, NULL));
if (authRsp.superAuth) {
*pass = true;
goto _return;
}
if (authRsp.createdDbs && taosHashGet(authRsp.createdDbs, dbFName, strlen(dbFName))) {
*pass = true;
goto _return;
}
if (CTG_AUTH_READ(type) && authRsp.readDbs && taosHashGet(authRsp.readDbs, dbFName, strlen(dbFName))) {
*pass = true;
} else if (CTG_AUTH_WRITE(type) && authRsp.writeDbs && taosHashGet(authRsp.writeDbs, dbFName, strlen(dbFName))) {
*pass = true;
CTG_ERR_JRET(ctgChkSetAuthRes(pCtg, &req, &rsp));
if (rsp.metaNotExists && exists) {
*exists = false;
}
_return:
ctgUpdateUserEnqueue(pCtg, &authRsp, false);
ctgUpdateUserEnqueue(pCtg, &req.authInfo, false);
return TSDB_CODE_SUCCESS;
CTG_RET(code);
}
int32_t ctgGetTbType(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName, int32_t* tbType) {
@ -1368,7 +1359,7 @@ int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion** users, uint32_
void* key = taosHashGetKey(pAuth, &len);
strncpy((*users)[i].user, key, len);
(*users)[i].user[len] = 0;
(*users)[i].version = pAuth->version;
(*users)[i].version = pAuth->userAuth.version;
++i;
if (i >= *num) {
taosHashCancelIterate(pCtg->userCache, pAuth);
@ -1448,32 +1439,30 @@ _return:
CTG_API_LEAVE(code);
}
int32_t catalogChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, const char* user, const char* dbFName, AUTH_TYPE type,
bool* pass) {
int32_t catalogChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, SUserAuthInfo *pAuth, SUserAuthRes* pRes) {
CTG_API_ENTER();
if (NULL == pCtg || NULL == pConn || NULL == user || NULL == dbFName || NULL == pass) {
if (NULL == pCtg || NULL == pConn || NULL == pAuth || NULL == pRes) {
CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
}
int32_t code = 0;
CTG_ERR_JRET(ctgChkAuth(pCtg, pConn, user, dbFName, type, pass, NULL));
CTG_ERR_JRET(ctgChkAuth(pCtg, pConn, pAuth, pRes, NULL));
_return:
CTG_API_LEAVE(code);
}
int32_t catalogChkAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFName, AUTH_TYPE type,
bool* pass, bool* exists) {
int32_t catalogChkAuthFromCache(SCatalog* pCtg, SUserAuthInfo *pAuth, SUserAuthRes* pRes, bool* exists) {
CTG_API_ENTER();
if (NULL == pCtg || NULL == user || NULL == dbFName || NULL == pass || NULL == exists) {
if (NULL == pCtg || NULL == pAuth || NULL == pRes || NULL == exists) {
CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
}
int32_t code = 0;
CTG_ERR_JRET(ctgChkAuth(pCtg, NULL, user, dbFName, type, pass, exists));
CTG_ERR_JRET(ctgChkAuth(pCtg, NULL, pAuth, pRes, exists));
_return:

71
source/libs/catalog/src/ctgAsync.c
View File

@ -1550,45 +1550,20 @@ _return:
int32_t ctgHandleGetUserRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) {
int32_t code = 0;
SCtgTask* pTask = tReq->pTask;
SCtgUserCtx* ctx = (SCtgUserCtx*)pTask->taskCtx;
SCatalog* pCtg = pTask->pJob->pCtg;
bool pass = false;
SGetUserAuthRsp* pOut = (SGetUserAuthRsp*)pTask->msgCtx.out;
CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target));
if (pOut->superAuth) {
pass = true;
goto _return;
}
ctgUpdateUserEnqueue(pCtg, pOut, true);
taosMemoryFreeClear(pTask->msgCtx.out);
if (pOut->createdDbs && taosHashGet(pOut->createdDbs, ctx->user.dbFName, strlen(ctx->user.dbFName))) {
pass = true;
goto _return;
}
CTG_ERR_JRET((*gCtgAsyncFps[pTask->type].launchFp)(pTask));
if (CTG_AUTH_READ(ctx->user.type) && pOut->readDbs &&
taosHashGet(pOut->readDbs, ctx->user.dbFName, strlen(ctx->user.dbFName))) {
pass = true;
} else if (CTG_AUTH_WRITE(ctx->user.type) && pOut->writeDbs &&
taosHashGet(pOut->writeDbs, ctx->user.dbFName, strlen(ctx->user.dbFName))) {
pass = true;
}
return TSDB_CODE_SUCCESS;
_return:
if (TSDB_CODE_SUCCESS == code) {
pTask->res = taosMemoryCalloc(1, sizeof(bool));
if (NULL == pTask->res) {
code = TSDB_CODE_OUT_OF_MEMORY;
} else {
*(bool*)pTask->res = pass;
}
}
ctgUpdateUserEnqueue(pCtg, pOut, false);
taosMemoryFreeClear(pTask->msgCtx.out);
ctgHandleTaskEnd(pTask, code);
CTG_RET(code);
@ -2067,31 +2042,39 @@ int32_t ctgLaunchGetUdfTask(SCtgTask* pTask) {
}
int32_t ctgLaunchGetUserTask(SCtgTask* pTask) {
int32_t code = 0;
SCatalog* pCtg = pTask->pJob->pCtg;
SRequestConnInfo* pConn = &pTask->pJob->conn;
SCtgUserCtx* pCtx = (SCtgUserCtx*)pTask->taskCtx;
bool inCache = false;
bool pass = false;
SCtgAuthRsp rsp = {0};
SCtgJob* pJob = pTask->pJob;
SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1);
if (NULL == pMsgCtx->pBatchs) {
pMsgCtx->pBatchs = pJob->pBatchs;
}
CTG_ERR_RET(ctgChkAuthFromCache(pCtg, pCtx->user.user, pCtx->user.dbFName, pCtx->user.type, &inCache, &pass));
rsp.pRawRes = taosMemoryCalloc(1, sizeof(SUserAuthRes));
if (NULL == rsp.pRawRes) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
CTG_ERR_RET(ctgChkAuthFromCache(pCtg, &pCtx->user, &inCache, &rsp));
if (inCache) {
pTask->res = taosMemoryCalloc(1, sizeof(bool));
if (NULL == pTask->res) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
*(bool*)pTask->res = pass;
pTask->res = rsp.pRawRes;
CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0));
return TSDB_CODE_SUCCESS;
}
CTG_ERR_RET(ctgGetUserDbAuthFromMnode(pCtg, pConn, pCtx->user.user, NULL, pTask));
taosMemoryFreeClear(rsp.pRawRes);
if (rsp.metaNotExists) {
CTG_ERR_RET(ctgLaunchSubTask(pTask, CTG_TASK_GET_TB_META, ctgGetTbCfgCb, &pCtx->user.tbName));
} else {
CTG_ERR_RET(ctgGetUserDbAuthFromMnode(pCtg, pConn, pCtx->user.user, NULL, pTask));
}
return TSDB_CODE_SUCCESS;
}
@ -2139,6 +2122,20 @@ _return:
CTG_RET(ctgHandleTaskEnd(pTask, pTask->subRes.code));
}
int32_t ctgGetUserCb(SCtgTask* pTask) {
int32_t code = 0;
CTG_ERR_JRET(pTask->subRes.code);
CTG_RET(ctgLaunchGetUserTask(pTask));
_return:
CTG_RET(ctgHandleTaskEnd(pTask, pTask->subRes.code));
}
int32_t ctgCompDbVgTasks(SCtgTask* pTask, void* param, bool* equal) {
SCtgDbVgCtx* ctx = pTask->taskCtx;

131
source/libs/catalog/src/ctgCache.c
View File

@ -238,10 +238,11 @@ _return:
return TSDB_CODE_SUCCESS;
}
int32_t ctgAcquireVgMetaFromCache(SCatalog *pCtg, const char *dbFName, const char *tbName, SCtgDBCache **pDb, SCtgTbCache **pTb) {
int32_t ctgAcquireVgMetaFromCache(SCatalog *pCtg, const char *dbFName, const char *tbName, SCtgDBCache **pDb,
SCtgTbCache **pTb) {
SCtgDBCache *dbCache = NULL;
SCtgTbCache *tbCache = NULL;
bool vgInCache = false;
bool vgInCache = false;
ctgAcquireDBCache(pCtg, dbFName, &dbCache);
if (NULL == dbCache) {
@ -306,7 +307,6 @@ _return:
return TSDB_CODE_SUCCESS;
}
/*
int32_t ctgAcquireStbMetaFromCache(SCatalog *pCtg, char *dbFName, uint64_t suid, SCtgDBCache **pDb, SCtgTbCache **pTb) {
SCtgDBCache *dbCache = NULL;
@ -360,9 +360,10 @@ _return:
}
*/
int32_t ctgAcquireStbMetaFromCache(SCtgDBCache *dbCache, SCatalog *pCtg, char *dbFName, uint64_t suid, SCtgTbCache **pTb) {
int32_t ctgAcquireStbMetaFromCache(SCtgDBCache *dbCache, SCatalog *pCtg, char *dbFName, uint64_t suid,
SCtgTbCache **pTb) {
SCtgTbCache *pCache = NULL;
char *stName = taosHashAcquire(dbCache->stbCache, &suid, sizeof(suid));
char *stName = taosHashAcquire(dbCache->stbCache, &suid, sizeof(suid));
if (NULL == stName) {
ctgDebug("stb 0x%" PRIx64 " not in cache, dbFName:%s", suid, dbFName);
goto _return;
@ -459,10 +460,11 @@ int32_t ctgTbMetaExistInCache(SCatalog *pCtg, char *dbFName, char *tbName, int32
return TSDB_CODE_SUCCESS;
}
int32_t ctgCopyTbMeta(SCatalog *pCtg, SCtgTbMetaCtx *ctx, SCtgDBCache **pDb, SCtgTbCache **pTb, STableMeta **pTableMeta, char* dbFName) {
int32_t ctgCopyTbMeta(SCatalog *pCtg, SCtgTbMetaCtx *ctx, SCtgDBCache **pDb, SCtgTbCache **pTb, STableMeta **pTableMeta,
char *dbFName) {
SCtgDBCache *dbCache = *pDb;
SCtgTbCache *tbCache = *pTb;
STableMeta *tbMeta = tbCache->pMeta;
STableMeta *tbMeta = tbCache->pMeta;
ctx->tbInfo.inCache = true;
ctx->tbInfo.dbId = dbCache->dbId;
ctx->tbInfo.suid = tbMeta->suid;
@ -491,12 +493,12 @@ int32_t ctgCopyTbMeta(SCatalog *pCtg, SCtgTbMetaCtx *ctx, SCtgDBCache **pDb, SCt
memcpy(*pTableMeta, tbMeta, metaSize);
//ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache);
// ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache);
CTG_UNLOCK(CTG_READ, &tbCache->metaLock);
taosHashRelease(dbCache->tbCache, tbCache);
*pTb = NULL;
ctgDebug("Got ctb %s meta from cache, will continue to get its stb meta, type:%d, dbFName:%s", ctx->pName->tname,
ctx->tbInfo.tbType, dbFName);
@ -528,7 +530,6 @@ int32_t ctgCopyTbMeta(SCatalog *pCtg, SCtgTbMetaCtx *ctx, SCtgDBCache **pDb, SCt
return TSDB_CODE_SUCCESS;
}
int32_t ctgReadTbMetaFromCache(SCatalog *pCtg, SCtgTbMetaCtx *ctx, STableMeta **pTableMeta) {
int32_t code = 0;
SCtgDBCache *dbCache = NULL;
@ -598,17 +599,17 @@ int32_t ctgReadTbVerFromCache(SCatalog *pCtg, SName *pTableName, int32_t *sver,
// PROCESS FOR CHILD TABLE
//ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache);
// ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache);
if (tbCache) {
CTG_UNLOCK(CTG_READ, &tbCache->metaLock);
taosHashRelease(dbCache->tbCache, tbCache);
}
ctgDebug("Got ctb %s ver from cache, will continue to get its stb ver, dbFName:%s", pTableName->tname, dbFName);
ctgAcquireStbMetaFromCache(dbCache, pCtg, dbFName, *suid, &tbCache);
if (NULL == tbCache) {
//ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache);
// ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache);
ctgDebug("stb 0x%" PRIx64 " meta not in cache", *suid);
return TSDB_CODE_SUCCESS;
}
@ -678,55 +679,40 @@ _return:
CTG_RET(code);
}
int32_t ctgChkAuthFromCache(SCatalog *pCtg, char *user, char *dbFName, AUTH_TYPE type, bool *inCache, bool *pass) {
char *p = strchr(dbFName, '.');
if (p) {
++p;
} else {
p = dbFName;
}
if (IS_SYS_DBNAME(p)) {
int32_t ctgChkAuthFromCache(SCatalog *pCtg, SUserAuthInfo *pReq, bool *inCache, SCtgAuthRsp *pRes) {
if (IS_SYS_DBNAME(pReq->tbName.dbname)) {
*inCache = true;
*pass = true;
ctgDebug("sysdb %s, pass", dbFName);
pRes->pRawRes->pass = true;
ctgDebug("sysdb %s, pass", pReq->tbName.dbname);
return TSDB_CODE_SUCCESS;
}
SCtgUserAuth *pUser = (SCtgUserAuth *)taosHashGet(pCtg->userCache, user, strlen(user));
SCtgUserAuth *pUser = (SCtgUserAuth *)taosHashGet(pCtg->userCache, pReq->user, strlen(pReq->user));
if (NULL == pUser) {
ctgDebug("user not in cache, user:%s", user);
ctgDebug("user not in cache, user:%s", pReq->user);
goto _return;
}
*inCache = true;
ctgDebug("Got user from cache, user:%s", user);
ctgDebug("Got user from cache, user:%s", pReq->user);
CTG_CACHE_STAT_INC(numOfUserHit, 1);
if (pUser->superUser) {
*pass = true;
return TSDB_CODE_SUCCESS;
}
SCtgAuthReq req = {0};
req.pRawReq = pReq;
req.onlyCache = true;
CTG_LOCK(CTG_READ, &pUser->lock);
if (pUser->createdDbs && taosHashGet(pUser->createdDbs, dbFName, strlen(dbFName))) {
*pass = true;
CTG_UNLOCK(CTG_READ, &pUser->lock);
return TSDB_CODE_SUCCESS;
}
if (pUser->readDbs && taosHashGet(pUser->readDbs, dbFName, strlen(dbFName)) && CTG_AUTH_READ(type)) {
*pass = true;
}
if (pUser->writeDbs && taosHashGet(pUser->writeDbs, dbFName, strlen(dbFName)) && CTG_AUTH_WRITE(type)) {
*pass = true;
}
memcpy(&req.authInfo, &pUser->userAuth, sizeof(pUser->userAuth));
int32_t code = ctgChkSetAuthRes(pCtg, &req, pRes);
CTG_UNLOCK(CTG_READ, &pUser->lock);
CTG_ERR_JRET(code);
return TSDB_CODE_SUCCESS;
if (pRes->metaNotExists) {
goto _return;
}
CTG_RET(code);
_return:
@ -1700,9 +1686,9 @@ void ctgFreeAllInstance(void) {
taosHashClear(gCtgMgmt.pCluster);
}
int32_t ctgVgInfoIdComp(void const* lp, void const* rp) {
int32_t* key = (int32_t*)lp;
SVgroupInfo* pVg = (SVgroupInfo*)rp;
int32_t ctgVgInfoIdComp(void const *lp, void const *rp) {
int32_t *key = (int32_t *)lp;
SVgroupInfo *pVg = (SVgroupInfo *)rp;
if (*key < pVg->vgId) {
return -1;
@ -1713,7 +1699,6 @@ int32_t ctgVgInfoIdComp(void const* lp, void const* rp) {
return 0;
}
int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) {
int32_t code = 0;
SCtgUpdateVgMsg *msg = operation->data;
@ -1778,10 +1763,10 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) {
dbCache = NULL;
//if (!IS_SYS_DBNAME(dbFName)) {
tstrncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName));
CTG_ERR_JRET(ctgMetaRentUpdate(&msg->pCtg->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion),
ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare));
// if (!IS_SYS_DBNAME(dbFName)) {
tstrncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName));
CTG_ERR_JRET(ctgMetaRentUpdate(&msg->pCtg->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion),
ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare));
//}
_return:
@ -2024,11 +2009,7 @@ int32_t ctgOpUpdateUser(SCtgCacheOperation *operation) {
if (NULL == pUser) {
SCtgUserAuth userAuth = {0};
userAuth.version = msg->userAuth.version;
userAuth.superUser = msg->userAuth.superAuth;
userAuth.createdDbs = msg->userAuth.createdDbs;
userAuth.readDbs = msg->userAuth.readDbs;
userAuth.writeDbs = msg->userAuth.writeDbs;
memcpy(&userAuth.userAuth, &msg->userAuth, sizeof(msg->userAuth));
if (taosHashPut(pCtg->userCache, msg->userAuth.user, strlen(msg->userAuth.user), &userAuth, sizeof(userAuth))) {
ctgError("taosHashPut user %s to cache failed", msg->userAuth.user);
@ -2040,22 +2021,32 @@ int32_t ctgOpUpdateUser(SCtgCacheOperation *operation) {
return TSDB_CODE_SUCCESS;
}
pUser->version = msg->userAuth.version;
CTG_LOCK(CTG_WRITE, &pUser->lock);
taosHashCleanup(pUser->createdDbs);
pUser->createdDbs = msg->userAuth.createdDbs;
taosHashCleanup(pUser->userAuth.createdDbs);
pUser->userAuth.createdDbs = msg->userAuth.createdDbs;
msg->userAuth.createdDbs = NULL;
taosHashCleanup(pUser->readDbs);
pUser->readDbs = msg->userAuth.readDbs;
taosHashCleanup(pUser->userAuth.readDbs);
pUser->userAuth.readDbs = msg->userAuth.readDbs;
msg->userAuth.readDbs = NULL;
taosHashCleanup(pUser->writeDbs);
pUser->writeDbs = msg->userAuth.writeDbs;
taosHashCleanup(pUser->userAuth.writeDbs);
pUser->userAuth.writeDbs = msg->userAuth.writeDbs;
msg->userAuth.writeDbs = NULL;
taosHashCleanup(pUser->userAuth.readTbs);
pUser->userAuth.readTbs = msg->userAuth.readTbs;
msg->userAuth.readTbs = NULL;
taosHashCleanup(pUser->userAuth.writeTbs);
pUser->userAuth.writeTbs = msg->userAuth.writeTbs;
msg->userAuth.writeTbs = NULL;
taosHashCleanup(pUser->userAuth.useDbs);
pUser->userAuth.useDbs = msg->userAuth.useDbs;
msg->userAuth.useDbs = NULL;
CTG_UNLOCK(CTG_WRITE, &pUser->lock);
_return:
@ -2063,6 +2054,9 @@ _return:
taosHashCleanup(msg->userAuth.createdDbs);
taosHashCleanup(msg->userAuth.readDbs);
taosHashCleanup(msg->userAuth.writeDbs);
taosHashCleanup(msg->userAuth.readTbs);
taosHashCleanup(msg->userAuth.writeTbs);
taosHashCleanup(msg->userAuth.useDbs);
taosMemoryFreeClear(msg);
@ -2255,6 +2249,9 @@ void ctgFreeCacheOperationData(SCtgCacheOperation *op) {
taosHashCleanup(msg->userAuth.createdDbs);
taosHashCleanup(msg->userAuth.readDbs);
taosHashCleanup(msg->userAuth.writeDbs);
taosHashCleanup(msg->userAuth.readTbs);
taosHashCleanup(msg->userAuth.writeTbs);
taosHashCleanup(msg->userAuth.useDbs);
taosMemoryFreeClear(op->data);
break;
}

241
source/libs/catalog/src/ctgUtil.c
View File

@ -174,9 +174,12 @@ void ctgFreeSMetaData(SMetaData* pData) {
}
void ctgFreeSCtgUserAuth(SCtgUserAuth* userCache) {
taosHashCleanup(userCache->createdDbs);
taosHashCleanup(userCache->readDbs);
taosHashCleanup(userCache->writeDbs);
taosHashCleanup(userCache->userAuth.createdDbs);
taosHashCleanup(userCache->userAuth.readDbs);
taosHashCleanup(userCache->userAuth.writeDbs);
taosHashCleanup(userCache->userAuth.readTbs);
taosHashCleanup(userCache->userAuth.writeTbs);
taosHashCleanup(userCache->userAuth.useDbs);
}
void ctgFreeMetaRent(SCtgRentMgmt* mgmt) {
@ -423,6 +426,9 @@ void ctgFreeMsgCtx(SCtgMsgCtx* pCtx) {
taosHashCleanup(pOut->createdDbs);
taosHashCleanup(pOut->readDbs);
taosHashCleanup(pOut->writeDbs);
taosHashCleanup(pOut->readTbs);
taosHashCleanup(pOut->writeTbs);
taosHashCleanup(pOut->useDbs);
taosMemoryFreeClear(pCtx->out);
break;
}
@ -869,19 +875,19 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, const SName
vgInfo = taosArraySearch(dbInfo->vgArray, &hashValue, ctgHashValueComp, TD_EQ);
/*
void* pIter = taosHashIterate(dbInfo->vgHash, NULL);
while (pIter) {
vgInfo = pIter;
if (hashValue >= vgInfo->hashBegin && hashValue <= vgInfo->hashEnd) {
taosHashCancelIterate(dbInfo->vgHash, pIter);
break;
}
/*
void* pIter = taosHashIterate(dbInfo->vgHash, NULL);
while (pIter) {
vgInfo = pIter;
if (hashValue >= vgInfo->hashBegin && hashValue <= vgInfo->hashEnd) {
taosHashCancelIterate(dbInfo->vgHash, pIter);
break;
}
pIter = taosHashIterate(dbInfo->vgHash, pIter);
vgInfo = NULL;
}
*/
pIter = taosHashIterate(dbInfo->vgHash, pIter);
vgInfo = NULL;
}
*/
if (NULL == vgInfo) {
ctgError("no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, db,
@ -906,7 +912,7 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SCtgTaskReq* tReq, SDBVgInfo*
CTG_ERR_RET(ctgMakeVgArray(dbInfo));
int32_t vgNum = taosArrayGetSize(dbInfo->vgArray);
int32_t vgNum = taosArrayGetSize(dbInfo->vgArray);
if (vgNum <= 0) {
ctgError("db vgroup cache invalid, db:%s, vgroup number:%d", dbFName, vgNum);
CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
@ -986,43 +992,43 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SCtgTaskReq* tReq, SDBVgInfo*
CTG_RET(code);
}
int32_t ctgGetVgIdsFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, char* dbFName, const char* pTbs[], int32_t tbNum, int32_t* vgId) {
int32_t code = 0;
CTG_ERR_RET(ctgMakeVgArray(dbInfo));
int32_t ctgGetVgIdsFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, char* dbFName, const char* pTbs[], int32_t tbNum,
int32_t* vgId) {
int32_t code = 0;
CTG_ERR_RET(ctgMakeVgArray(dbInfo));
int32_t vgNum = taosArrayGetSize(dbInfo->vgArray);
int32_t vgNum = taosArrayGetSize(dbInfo->vgArray);
if (vgNum <= 0) {
ctgError("db vgroup cache invalid, db:%s, vgroup number:%d", dbFName, vgNum);
CTG_ERR_RET(TSDB_CODE_TSC_DB_NOT_SELECTED);
}
if (vgNum <= 0) {
ctgError("db vgroup cache invalid, db:%s, vgroup number:%d", dbFName, vgNum);
CTG_ERR_RET(TSDB_CODE_TSC_DB_NOT_SELECTED);
}
SVgroupInfo* vgInfo = NULL;
char tbFullName[TSDB_TABLE_FNAME_LEN];
snprintf(tbFullName, sizeof(tbFullName), "%s.", dbFName);
int32_t offset = strlen(tbFullName);
for (int32_t i = 0; i < tbNum; ++i) {
snprintf(tbFullName + offset, sizeof(tbFullName) - offset, "%s", pTbs[i]);
uint32_t hashValue = taosGetTbHashVal(tbFullName, (uint32_t)strlen(tbFullName), dbInfo->hashMethod,
dbInfo->hashPrefix, dbInfo->hashSuffix);
SVgroupInfo* vgInfo = NULL;
char tbFullName[TSDB_TABLE_FNAME_LEN];
snprintf(tbFullName, sizeof(tbFullName), "%s.", dbFName);
int32_t offset = strlen(tbFullName);
vgInfo = taosArraySearch(dbInfo->vgArray, &hashValue, ctgHashValueComp, TD_EQ);
if (NULL == vgInfo) {
ctgError("no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, dbFName,
(int32_t)taosArrayGetSize(dbInfo->vgArray));
CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
for (int32_t i = 0; i < tbNum; ++i) {
snprintf(tbFullName + offset, sizeof(tbFullName) - offset, "%s", pTbs[i]);
uint32_t hashValue = taosGetTbHashVal(tbFullName, (uint32_t)strlen(tbFullName), dbInfo->hashMethod,
dbInfo->hashPrefix, dbInfo->hashSuffix);
vgId[i] = vgInfo->vgId;
vgInfo = taosArraySearch(dbInfo->vgArray, &hashValue, ctgHashValueComp, TD_EQ);
if (NULL == vgInfo) {
ctgError("no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, dbFName,
(int32_t)taosArrayGetSize(dbInfo->vgArray));
CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
ctgDebug("Got tb %s vgId:%d", tbFullName, vgInfo->vgId);
}
vgId[i] = vgInfo->vgId;
CTG_RET(code);
ctgDebug("Got tb %s vgId:%d", tbFullName, vgInfo->vgId);
}
CTG_RET(code);
}
int32_t ctgStbVersionSearchCompare(const void* key1, const void* key2) {
if (*(uint64_t*)key1 < ((SSTableVersion*)key2)->suid) {
return -1;
@ -1067,26 +1073,25 @@ int32_t ctgMakeVgArray(SDBVgInfo* dbInfo) {
if (NULL == dbInfo) {
return TSDB_CODE_SUCCESS;
}
if (dbInfo->vgHash && NULL == dbInfo->vgArray) {
dbInfo->vgArray = taosArrayInit(100, sizeof(SVgroupInfo));
if (NULL == dbInfo->vgArray) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
void* pIter = taosHashIterate(dbInfo->vgHash, NULL);
void* pIter = taosHashIterate(dbInfo->vgHash, NULL);
while (pIter) {
taosArrayPush(dbInfo->vgArray, pIter);
pIter = taosHashIterate(dbInfo->vgHash, pIter);
}
taosArraySort(dbInfo->vgArray, ctgVgInfoComp);
}
return TSDB_CODE_SUCCESS;
}
int32_t ctgCloneVgInfo(SDBVgInfo* src, SDBVgInfo** dst) {
CTG_ERR_RET(ctgMakeVgArray(src));
@ -1330,6 +1335,143 @@ static void* ctgCloneDnodeList(void* pSrc) { return taosArrayDup((const SArray*)
static void ctgFreeDnodeList(void* p) { taosArrayDestroy((SArray*)((SMetaRes*)p)->pRes); }
int32_t ctgChkSetTbAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) {
int32_t code = 0;
STableMeta* pMeta = NULL;
SGetUserAuthRsp* pInfo = &req->authInfo;
SHashObj* pTbs = (AUTH_TYPE_READ == req->singleType) ? pInfo->readTbs : pInfo->writeTbs;
char tbFullName[TSDB_TABLE_FNAME_LEN];
tNameExtractFullName(&req->pRawReq->tbName, tbFullName);
char* pCond = taosHashGet(pTbs, tbFullName, strlen(tbFullName));
if (pCond) {
if (strlen(pCond) > 1) {
CTG_ERR_RET(nodesStringToNode(pCond, &res->pRawRes->pCond));
}
res->pRawRes->pass = true;
return TSDB_CODE_SUCCESS;
}
res->pRawRes->pass = false;
// CTG_ERR_RET(catalogGetCachedTableMeta(pCtg, &req->pRawReq->tbName, &pMeta));
// if (NULL == pMeta) {
// if (req->onlyCache) {
// res->metaNotExists = true;
// ctgDebug("db %s tb %s meta not in cache for auth", req->pRawReq->tbName.dbname, req->pRawReq->tbName.tname);
// return TSDB_CODE_SUCCESS;
// }
// CTG_ERR_RET(catalogGetTableMeta(pCtg, req->pConn, &req->pRawReq->tbName, &pMeta));
// }
// if (TSDB_SUPER_TABLE == pMeta->tableType || TSDB_NORMAL_TABLE == pMeta->tableType) {
// res->pRawRes->pass = false;
// goto _return;
// }
// if (TSDB_CHILD_TABLE == pMeta->tableType) {
// res->pRawRes->pass = true;
// /*
// char stbName[TSDB_TABLE_NAME_LEN] = {0};
// CTG_ERR_JRET(ctgGetCachedStbNameFromSuid(pCtg, pMeta->suid, stbName));
// if (0 == stbName[0]) {
// if (req->onlyCache) {
// res->notExists = true;
// return TSDB_CODE_SUCCESS;
// }
// CTG_ERR_RET(catalogRefreshTableMeta(pCtg, req->pConn, &req->pRawReq->tbName, 0));
// }
// */
// }
_return:
taosMemoryFree(pMeta);
CTG_RET(code);
}
int32_t ctgChkSetAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) {
int32_t code = 0;
SUserAuthInfo* pReq = req->pRawReq;
SUserAuthRes* pRes = res->pRawRes;
SGetUserAuthRsp* pInfo = &req->authInfo;
pRes->pass = false;
pRes->pCond = NULL;
if (!pInfo->enable) {
pRes->pass = false;
return TSDB_CODE_SUCCESS;
}
if (pInfo->superAuth) {
pRes->pass = true;
return TSDB_CODE_SUCCESS;
}
char dbFName[TSDB_DB_FNAME_LEN];
tNameGetFullDbName(&pReq->tbName, dbFName);
if (pInfo->createdDbs && taosHashGet(pInfo->createdDbs, dbFName, strlen(dbFName))) {
pRes->pass = true;
return TSDB_CODE_SUCCESS;
}
switch (pReq->type) {
case AUTH_TYPE_READ: {
if (pInfo->readTbs && taosHashGetSize(pInfo->readTbs) > 0) {
req->singleType = AUTH_TYPE_READ;
CTG_ERR_RET(ctgChkSetTbAuthRes(pCtg, req, res));
if (pRes->pass) {
return TSDB_CODE_SUCCESS;
}
}
if (pInfo->readDbs && taosHashGet(pInfo->readDbs, dbFName, strlen(dbFName))) {
pRes->pass = true;
return TSDB_CODE_SUCCESS;
}
break;
}
case AUTH_TYPE_WRITE: {
if (pInfo->writeTbs && taosHashGetSize(pInfo->writeTbs) > 0) {
req->singleType = AUTH_TYPE_WRITE;
CTG_ERR_RET(ctgChkSetTbAuthRes(pCtg, req, res));
if (pRes->pass) {
return TSDB_CODE_SUCCESS;
}
}
if (pInfo->writeDbs && taosHashGet(pInfo->writeDbs, dbFName, strlen(dbFName))) {
pRes->pass = true;
return TSDB_CODE_SUCCESS;
}
break;
}
case AUTH_TYPE_READ_OR_WRITE: {
if ((pInfo->readDbs && taosHashGet(pInfo->readDbs, dbFName, strlen(dbFName))) ||
(pInfo->writeDbs && taosHashGet(pInfo->writeDbs, dbFName, strlen(dbFName))) ||
(pInfo->useDbs && taosHashGet(pInfo->useDbs, dbFName, strlen(dbFName)))) {
pRes->pass = true;
return TSDB_CODE_SUCCESS;
}
break;
}
default:
break;
}
return TSDB_CODE_SUCCESS;
}
#if 0
static int32_t ctgCloneMetaDataArray(SArray* pSrc, __array_item_dup_fn_t copyFunc, SArray** pDst) {
if (NULL == pSrc) {
@ -1423,4 +1565,3 @@ void catalogFreeMetaData(SMetaData* pData) {
taosMemoryFree(pData);
}
#endif

2
source/libs/catalog/test/CMakeLists.txt
View File

@ -9,7 +9,7 @@ IF(NOT TD_DARWIN)
ADD_EXECUTABLE(catalogTest ${SOURCE_LIST})
TARGET_LINK_LIBRARIES(
catalogTest
PUBLIC os util common catalog transport gtest qcom taos_static
PUBLIC os util common nodes catalog transport gtest qcom taos_static
)
TARGET_INCLUDE_DIRECTORIES(

165
source/libs/catalog/test/catalogTests.cpp
View File

@ -27,8 +27,8 @@
#ifdef WINDOWS
#define TD_USE_WINSOCK
#endif
#include "catalogInt.h"
#include "catalog.h"
#include "catalogInt.h"
#include "os.h"
#include "stub.h"
#include "taos.h"
@ -48,7 +48,7 @@ void ctgTestSetRspCTableMeta();
void ctgTestSetRspSTableMeta();
void ctgTestSetRspMultiSTableMeta();
extern int32_t clientConnRefPool;
extern int32_t clientConnRefPool;
enum {
CTGT_RSP_VGINFO = 1,
@ -298,7 +298,7 @@ void ctgTestBuildSTableMetaRsp(STableMetaRsp *rspMsg) {
void ctgTestRspDbVgroups(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
rpcFreeCont(pMsg->pCont);
SUseDbRsp usedbRsp = {0};
strcpy(usedbRsp.db, ctgTestDbname);
usedbRsp.vgVersion = ctgTestVgVersion;
@ -343,7 +343,7 @@ void ctgTestRspDbVgroups(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *
void ctgTestRspTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
rpcFreeCont(pMsg->pCont);
STableMetaRsp metaRsp = {0};
strcpy(metaRsp.dbFName, ctgTestDbname);
strcpy(metaRsp.tbName, ctgTestTablename);
@ -384,13 +384,13 @@ void ctgTestRspTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *
void ctgTestRspTableMetaNotExist(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
rpcFreeCont(pMsg->pCont);
pRsp->code = CTG_ERR_CODE_TABLE_NOT_EXIST;
}
void ctgTestRspCTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
rpcFreeCont(pMsg->pCont);
STableMetaRsp metaRsp = {0};
strcpy(metaRsp.dbFName, ctgTestDbname);
strcpy(metaRsp.tbName, ctgTestCurrentCTableName ? ctgTestCurrentCTableName : ctgTestCTablename);
@ -438,7 +438,7 @@ void ctgTestRspCTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg
void ctgTestRspSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
rpcFreeCont(pMsg->pCont);
STableMetaRsp metaRsp = {0};
strcpy(metaRsp.dbFName, ctgTestDbname);
strcpy(metaRsp.tbName, ctgTestCurrentSTableName ? ctgTestCurrentSTableName : ctgTestSTablename);
@ -450,7 +450,7 @@ void ctgTestRspSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg
metaRsp.sversion = ctgTestSVersion;
metaRsp.tversion = ctgTestTVersion;
metaRsp.suid = ctgTestSuid;
metaRsp.tuid = ctgTestSuid+1;
metaRsp.tuid = ctgTestSuid + 1;
metaRsp.vgId = 0;
metaRsp.pSchemas = (SSchema *)taosMemoryMalloc((metaRsp.numOfTags + metaRsp.numOfColumns) * sizeof(SSchema));
@ -486,7 +486,7 @@ void ctgTestRspSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg
void ctgTestRspMultiSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
rpcFreeCont(pMsg->pCont);
static int32_t idx = 1;
STableMetaRsp metaRsp = {0};
@ -536,16 +536,14 @@ void ctgTestRspMultiSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRp
tFreeSTableMetaRsp(&metaRsp);
}
void ctgTestRspErrIndexInfo(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
rpcFreeCont(pMsg->pCont);
pRsp->code = TSDB_CODE_MND_DB_INDEX_NOT_EXIST;
pRsp->contLen = 0;
pRsp->pCont = NULL;
}
void ctgTestRspUserAuth(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
rpcFreeCont(pMsg->pCont);
@ -553,6 +551,7 @@ void ctgTestRspUserAuth(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *p
strcpy(userRsp.user, ctgTestUsername);
userRsp.version = 1;
userRsp.superAuth = 1;
userRsp.enable = 1;
int32_t contLen = tSerializeSGetUserAuthRsp(NULL, 0, &userRsp);
void *pReq = rpcMallocCont(contLen);
@ -565,7 +564,7 @@ void ctgTestRspUserAuth(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *p
void ctgTestRspTableCfg(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
rpcFreeCont(pMsg->pCont);
static int32_t idx = 1;
STableCfgRsp tblRsp = {0};
@ -600,7 +599,7 @@ void ctgTestRspTableCfg(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *p
void ctgTestRspTableIndex(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
rpcFreeCont(pMsg->pCont);
static int32_t idx = 1;
STableIndexRsp tblRsp = {0};
@ -611,10 +610,10 @@ void ctgTestRspTableIndex(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg
STableIndexInfo info = {0};
for (int32_t i = 0; i < ctgTestIndexNum; ++i) {
info.interval = 1 + i;
info.expr = (char*)taosMemoryCalloc(1, 10);
info.expr = (char *)taosMemoryCalloc(1, 10);
taosArrayPush(tblRsp.pIndex, &info);
}
int32_t contLen = tSerializeSTableIndexRsp(NULL, 0, &tblRsp);
void *pReq = rpcMallocCont(contLen);
tSerializeSTableIndexRsp(pReq, contLen, &tblRsp);
@ -628,12 +627,12 @@ void ctgTestRspTableIndex(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg
void ctgTestRspDBCfg(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
rpcFreeCont(pMsg->pCont);
static int32_t idx = 1;
SDbCfgRsp dbRsp = {0};
dbRsp.numOfVgroups = ctgTestVgNum;
int32_t contLen = tSerializeSDbCfgRsp(NULL, 0, &dbRsp);
void *pReq = rpcMallocCont(contLen);
tSerializeSDbCfgRsp(pReq, contLen, &dbRsp);
@ -651,7 +650,7 @@ void ctgTestRspQnodeList(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *
for (int32_t i = 0; i < ctgTestQnodeNum; ++i) {
SQueryNodeLoad nodeLoad = {0};
nodeLoad.addr.nodeId = i;
(void)taosArrayPush(qlistRsp.qnodeList, &nodeLoad);
}
@ -675,7 +674,7 @@ void ctgTestRspUdfInfo(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pR
SFuncInfo funcInfo = {0};
strcpy(funcInfo.name, "func1");
funcInfo.funcType = ctgTestFuncType;
(void)taosArrayPush(funcRsp.pFuncInfos, &funcInfo);
int32_t rspLen = tSerializeSRetrieveFuncRsp(NULL, 0, &funcRsp);
@ -694,7 +693,7 @@ void ctgTestRspSvrVer(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRs
SServerVerRsp verRsp = {0};
strcpy(verRsp.ver, "1.0");
int32_t rspLen = tSerializeSServerVerRsp(NULL, 0, &verRsp);
void *pReq = rpcMallocCont(rspLen);
tSerializeSServerVerRsp(pReq, rspLen, &verRsp);
@ -713,9 +712,9 @@ void ctgTestRspDndeList(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *p
epSet.numOfEps = 1;
tstrncpy(epSet.eps[0].fqdn, "localhost", TSDB_FQDN_LEN);
epSet.eps[0].port = 6030;
(void)taosArrayPush(dRsp.dnodeList, &epSet);
int32_t rspLen = tSerializeSDnodeListRsp(NULL, 0, &dRsp);
void *pReq = rpcMallocCont(rspLen);
tSerializeSDnodeListRsp(pReq, rspLen, &dRsp);
@ -727,8 +726,6 @@ void ctgTestRspDndeList(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *p
tFreeSDnodeListRsp(&dRsp);
}
void ctgTestRspAuto(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
switch (pMsg->msgType) {
case TDMT_MND_USE_DB:
@ -745,7 +742,7 @@ void ctgTestRspAuto(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp)
ctgTestRspDBCfg(shandle, pEpSet, pMsg, pRsp);
break;
case TDMT_MND_QNODE_LIST:
ctgTestRspQnodeList(shandle, pEpSet, pMsg, pRsp);
ctgTestRspQnodeList(shandle, pEpSet, pMsg, pRsp);
break;
case TDMT_MND_RETRIEVE_FUNC:
ctgTestRspUdfInfo(shandle, pEpSet, pMsg, pRsp);
@ -763,7 +760,6 @@ void ctgTestRspAuto(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp)
return;
}
void ctgTestRspByIdx(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
switch (ctgTestRspFunc[ctgTestRspIdx]) {
case CTGT_RSP_VGINFO:
@ -1058,7 +1054,7 @@ void ctgTestSetRspDbVgroupsAndMultiSuperMeta() {
void *ctgTestGetDbVgroupThread(void *param) {
struct SCatalog *pCtg = (struct SCatalog *)param;
int32_t code = 0;
SRequestConnInfo connInfo = {0};
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
SArray *vgList = NULL;
int32_t n = 0;
@ -1204,7 +1200,6 @@ void *ctgTestSetCtableMetaThread(void *param) {
return NULL;
}
void ctgTestFetchRows(TAOS_RES *result, int32_t *rows) {
TAOS_ROW row;
int num_fields = taos_num_fields(result);
@ -1220,23 +1215,22 @@ void ctgTestFetchRows(TAOS_RES *result, int32_t *rows) {
}
}
void ctgTestExecQuery(TAOS * taos, char* sql, bool fetch, int32_t *rows) {
void ctgTestExecQuery(TAOS *taos, char *sql, bool fetch, int32_t *rows) {
TAOS_RES *result = taos_query(taos, sql);
int code = taos_errno(result);
int code = taos_errno(result);
ASSERT_EQ(code, 0);
if (fetch) {
ctgTestFetchRows(result, rows);
}
taos_free_result(result);
}
TEST(tableMeta, normalTable) {
struct SCatalog *pCtg = NULL;
SVgroupInfo vgInfo = {0};
SRequestConnInfo connInfo = {0};
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
ctgTestInitLogFile();
@ -1372,7 +1366,7 @@ TEST(tableMeta, normalTable) {
TEST(tableMeta, childTableCase) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo connInfo = {0};
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
SVgroupInfo vgInfo = {0};
@ -1484,7 +1478,7 @@ TEST(tableMeta, childTableCase) {
TEST(tableMeta, superTableCase) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo connInfo = {0};
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
SVgroupInfo vgInfo = {0};
@ -1626,7 +1620,7 @@ TEST(tableMeta, superTableCase) {
TEST(tableMeta, rmStbMeta) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo connInfo = {0};
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
SVgroupInfo vgInfo = {0};
@ -1696,7 +1690,7 @@ TEST(tableMeta, rmStbMeta) {
TEST(tableMeta, updateStbMeta) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo connInfo = {0};
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
SVgroupInfo vgInfo = {0};
@ -1785,7 +1779,7 @@ TEST(tableMeta, updateStbMeta) {
TEST(getIndexInfo, notExists) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo connInfo = {0};
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
SVgroupInfo vgInfo = {0};
SArray *vgList = NULL;
@ -1809,13 +1803,13 @@ TEST(getIndexInfo, notExists) {
SIndexInfo info;
code = catalogGetIndexMeta(pCtg, mockPointer, "index1", &info);
ASSERT_TRUE(code != 0);
catalogDestroy();
}
TEST(refreshGetMeta, normal2normal) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo connInfo = {0};
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
SVgroupInfo vgInfo = {0};
SArray *vgList = NULL;
@ -1894,7 +1888,7 @@ TEST(refreshGetMeta, normal2normal) {
TEST(refreshGetMeta, normal2notexist) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo connInfo = {0};
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
SVgroupInfo vgInfo = {0};
SArray *vgList = NULL;
@ -1964,7 +1958,7 @@ TEST(refreshGetMeta, normal2notexist) {
TEST(refreshGetMeta, normal2child) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo connInfo = {0};
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
SVgroupInfo vgInfo = {0};
SArray *vgList = NULL;
@ -2045,10 +2039,9 @@ TEST(refreshGetMeta, normal2child) {
ctgTestCurrentSTableName = NULL;
}
TEST(refreshGetMeta, stable2child) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo connInfo = {0};
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
SVgroupInfo vgInfo = {0};
SArray *vgList = NULL;
@ -2134,7 +2127,7 @@ TEST(refreshGetMeta, stable2child) {
TEST(refreshGetMeta, stable2stable) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo connInfo = {0};
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
SVgroupInfo vgInfo = {0};
SArray *vgList = NULL;
@ -2220,7 +2213,7 @@ TEST(refreshGetMeta, stable2stable) {
TEST(refreshGetMeta, child2stable) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo connInfo = {0};
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
SVgroupInfo vgInfo = {0};
SArray *vgList = NULL;
@ -2306,7 +2299,7 @@ TEST(refreshGetMeta, child2stable) {
TEST(tableDistVgroup, normalTable) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo connInfo = {0};
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
SVgroupInfo *vgInfo = NULL;
SArray *vgList = NULL;
@ -2343,7 +2336,7 @@ TEST(tableDistVgroup, normalTable) {
TEST(tableDistVgroup, childTableCase) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo connInfo = {0};
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
SVgroupInfo *vgInfo = NULL;
SArray *vgList = NULL;
@ -2381,7 +2374,7 @@ TEST(tableDistVgroup, childTableCase) {
TEST(tableDistVgroup, superTableCase) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo connInfo = {0};
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
SVgroupInfo *vgInfo = NULL;
SArray *vgList = NULL;
@ -2430,7 +2423,7 @@ TEST(tableDistVgroup, superTableCase) {
TEST(dbVgroup, getSetDbVgroupCase) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo connInfo = {0};
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
SVgroupInfo vgInfo = {0};
SVgroupInfo *pvgInfo = NULL;
@ -2520,7 +2513,7 @@ TEST(dbVgroup, getSetDbVgroupCase) {
TEST(multiThread, getSetRmSameDbVgroup) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo connInfo = {0};
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
SVgroupInfo vgInfo = {0};
SVgroupInfo *pvgInfo = NULL;
@ -2572,7 +2565,7 @@ TEST(multiThread, getSetRmSameDbVgroup) {
TEST(multiThread, getSetRmDiffDbVgroup) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo connInfo = {0};
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
SVgroupInfo vgInfo = {0};
SVgroupInfo *pvgInfo = NULL;
@ -2624,7 +2617,7 @@ TEST(multiThread, getSetRmDiffDbVgroup) {
TEST(multiThread, ctableMeta) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo connInfo = {0};
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
SVgroupInfo vgInfo = {0};
SVgroupInfo *pvgInfo = NULL;
@ -2675,7 +2668,7 @@ TEST(multiThread, ctableMeta) {
TEST(rentTest, allRent) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo connInfo = {0};
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
SVgroupInfo vgInfo = {0};
SVgroupInfo *pvgInfo = NULL;
@ -2754,7 +2747,7 @@ TEST(rentTest, allRent) {
TEST(apiTest, catalogRefreshDBVgInfo_test) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo connInfo = {0};
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
ctgTestInitLogFile();
@ -2781,7 +2774,7 @@ TEST(apiTest, catalogRefreshDBVgInfo_test) {
TEST(apiTest, catalogChkAuth_test) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo connInfo = {0};
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
ctgTestInitLogFile();
@ -2800,15 +2793,19 @@ TEST(apiTest, catalogChkAuth_test) {
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
bool pass = false;
SUserAuthInfo authInfo = {0};
SUserAuthRes authRes = {0};
strcpy(authInfo.user, ctgTestUsername);
toName(1, ctgTestDbname, ctgTestSTablename, &authInfo.tbName);
authInfo.type = AUTH_TYPE_READ;
bool exists = false;
code = catalogChkAuthFromCache(pCtg, ctgTestUsername, ctgTestDbname, AUTH_TYPE_READ, &pass, &exists);
code = catalogChkAuthFromCache(pCtg, &authInfo, &authRes, &exists);
ASSERT_EQ(code, 0);
ASSERT_EQ(exists, false);
code = catalogChkAuth(pCtg, mockPointer, ctgTestUsername, ctgTestDbname, AUTH_TYPE_READ, &pass);
code = catalogChkAuth(pCtg, mockPointer, &authInfo, &authRes);
ASSERT_EQ(code, 0);
ASSERT_EQ(pass, true);
ASSERT_EQ(authRes.pass, true);
while (true) {
uint64_t n = 0;
@ -2820,9 +2817,9 @@ TEST(apiTest, catalogChkAuth_test) {
}
}
code = catalogChkAuthFromCache(pCtg, ctgTestUsername, ctgTestDbname, AUTH_TYPE_READ, &pass, &exists);
code = catalogChkAuthFromCache(pCtg, &authInfo, &authRes, &exists);
ASSERT_EQ(code, 0);
ASSERT_EQ(pass, true);
ASSERT_EQ(authRes.pass, true);
ASSERT_EQ(exists, true);
catalogDestroy();
@ -2830,7 +2827,7 @@ TEST(apiTest, catalogChkAuth_test) {
TEST(apiTest, catalogRefreshGetTableCfg_test) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo connInfo = {0};
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
ctgTestInitLogFile();
@ -2868,7 +2865,7 @@ TEST(apiTest, catalogRefreshGetTableCfg_test) {
TEST(apiTest, catalogGetTableIndex_test) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo connInfo = {0};
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
ctgTestInitLogFile();
@ -2904,7 +2901,7 @@ TEST(apiTest, catalogGetTableIndex_test) {
TEST(apiTest, catalogGetDBCfg_test) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo connInfo = {0};
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
ctgTestInitLogFile();
@ -2937,7 +2934,7 @@ TEST(apiTest, catalogGetDBCfg_test) {
TEST(apiTest, catalogGetQnodeList_test) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo connInfo = {0};
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
ctgTestInitLogFile();
@ -2962,17 +2959,16 @@ TEST(apiTest, catalogGetQnodeList_test) {
ASSERT_EQ(taosArrayGetSize(qnodeList), ctgTestQnodeNum);
for (int32_t i = 0; i < ctgTestQnodeNum; ++i) {
SQueryNodeLoad * pLoad = (SQueryNodeLoad *)taosArrayGet(qnodeList, i);
SQueryNodeLoad *pLoad = (SQueryNodeLoad *)taosArrayGet(qnodeList, i);
ASSERT_EQ(pLoad->addr.nodeId, i);
}
catalogDestroy();
}
TEST(apiTest, catalogGetUdfInfo_test) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo connInfo = {0};
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
ctgTestInitLogFile();
@ -2999,10 +2995,9 @@ TEST(apiTest, catalogGetUdfInfo_test) {
catalogDestroy();
}
TEST(apiTest, catalogGetServerVersion_test) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo connInfo = {0};
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
ctgTestInitLogFile();
@ -3021,7 +3016,7 @@ TEST(apiTest, catalogGetServerVersion_test) {
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
char* ver = NULL;
char *ver = NULL;
code = catalogGetServerVersion(pCtg, mockPointer, &ver);
ASSERT_EQ(code, 0);
ASSERT_TRUE(0 == strcmp(ver, "1.0"));
@ -3031,7 +3026,7 @@ TEST(apiTest, catalogGetServerVersion_test) {
TEST(apiTest, catalogUpdateTableIndex_test) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo connInfo = {0};
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
ctgTestInitLogFile();
@ -3061,10 +3056,9 @@ TEST(apiTest, catalogUpdateTableIndex_test) {
catalogDestroy();
}
TEST(apiTest, catalogGetDnodeList_test) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo connInfo = {0};
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
ctgTestInitLogFile();
@ -3083,7 +3077,7 @@ TEST(apiTest, catalogGetDnodeList_test) {
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
SArray* pList = NULL;
SArray *pList = NULL;
code = catalogGetDnodeList(pCtg, mockPointer, &pList);
ASSERT_EQ(code, 0);
ASSERT_EQ(taosArrayGetSize(pList), 1);
@ -3095,23 +3089,23 @@ TEST(apiTest, catalogGetDnodeList_test) {
#ifdef INTEGRATION_TEST
TEST(intTest, autoCreateTableTest) {
struct SCatalog *pCtg = NULL;
struct SCatalog *pCtg = NULL;
TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0);
ASSERT_TRUE(NULL != taos);
TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0);
ASSERT_TRUE(NULL != taos);
ctgdEnableDebug("api", true);
ctgdEnableDebug("meta", true);
ctgdEnableDebug("cache", true);
ctgdEnableDebug("lock", true);
ctgTestExecQuery(taos, "drop database if exists db1", false, NULL);
ctgTestExecQuery(taos, "create database db1", false, NULL);
ctgTestExecQuery(taos, "create stable db1.st1 (ts timestamp, f1 int) tags(tg1 int)", false, NULL);
ctgTestExecQuery(taos, "insert into db1.tb1 using db1.st1 tags(1) values(now, 1)", false, NULL);
ctgdGetOneHandle(&pCtg);
while (true) {
uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
if (2 != n) {
@ -3137,15 +3131,14 @@ TEST(intTest, autoCreateTableTest) {
ctgTestExecQuery(taos, "alter table db1.st1 add column f2 double", false, NULL);
ctgdEnableDebug("stopUpdate", false);
ctgTestExecQuery(taos, "insert into db1.tb1 (ts, f1) values(now, 4)", false, NULL);
taos_close(taos);
}
#endif
int main(int argc, char **argv) {
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();

1
source/libs/executor/inc/dataSinkInt.h
View File

@ -29,7 +29,6 @@ struct SDataSinkHandle;
typedef struct SDataSinkManager {
SDataSinkMgtCfg cfg;
TdThreadMutex mutex;
} SDataSinkManager;
typedef int32_t (*FPutDataBlock)(struct SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue);

4
source/libs/executor/src/aggregateoperator.c
View File

@ -172,9 +172,9 @@ int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) {
int32_t scanFlag = MAIN_SCAN;
bool hasValidBlock = false;
bool blockAllocated = false;
while (1) {
bool blockAllocated = false;
SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
if (pBlock == NULL) {
if (!hasValidBlock) {
@ -570,4 +570,4 @@ void functionCtxRestore(SqlFunctionCtx* pCtx, SFunctionCtxStatus* pStatus) {
pCtx->input.colDataSMAIsSet = pStatus->hasAgg;
pCtx->input.numOfRows = pStatus->numOfRows;
pCtx->input.startRowIndex = pStatus->startOffset;
}
}

1
source/libs/executor/src/dataSinkMgt.c
View File

@ -23,7 +23,6 @@ SDataSinkStat gDataSinkStat = {0};
int32_t dsDataSinkMgtInit(SDataSinkMgtCfg* cfg) {
gDataSinkManager.cfg = *cfg;
taosThreadMutexInit(&gDataSinkManager.mutex, NULL);
return 0; // to avoid compiler eror
}

7
source/libs/executor/src/executor.c
View File

@ -1062,6 +1062,7 @@ int32_t qStreamSetScanMemData(qTaskInfo_t tinfo, SPackedData submit) {
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
if((pTaskInfo->execModel != OPTR_EXEC_MODEL_QUEUE) || (pTaskInfo->streamInfo.submit.msgStr != NULL)){
qError("qStreamSetScanMemData err:%d,%p", pTaskInfo->execModel, pTaskInfo->streamInfo.submit.msgStr);
terrno = TSDB_CODE_PAR_INTERNAL_ERROR;
return -1;
}
qDebug("set the submit block for future scan");
@ -1102,7 +1103,6 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
// let's seek to the next version in wal file
if (tqSeekVer(pInfo->tqReader, pOffset->version + 1, id) < 0) {
qError("tqSeekVer failed ver:%"PRId64", %s", pOffset->version + 1, id);
return -1;
}
} else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) {
@ -1125,6 +1125,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
} else {
taosRUnLockLatch(&pTaskInfo->lock);
qError("no table in table list, %s", id);
terrno = TSDB_CODE_PAR_INTERNAL_ERROR;
return -1;
}
}
@ -1143,6 +1144,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
} else {
qError("vgId:%d uid:%" PRIu64 " not found in table list, total:%d, index:%d %s", pTaskInfo->id.vgId, uid,
numOfTables, pScanInfo->currentTable, id);
terrno = TSDB_CODE_PAR_INTERNAL_ERROR;
return -1;
}
@ -1175,6 +1177,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
pScanBaseInfo->cond.twindows.skey = oldSkey;
} else {
qError("invalid pOffset->type:%d, %s", pOffset->type, id);
terrno = TSDB_CODE_PAR_INTERNAL_ERROR;
return -1;
}
@ -1189,6 +1192,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
if (setForSnapShot(sContext, pOffset->uid) != 0) {
qError("setDataForSnapShot error. uid:%" PRId64" , %s", pOffset->uid, id);
terrno = TSDB_CODE_PAR_INTERNAL_ERROR;
return -1;
}
@ -1224,6 +1228,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
SSnapContext* sContext = pInfo->sContext;
if (setForSnapShot(sContext, pOffset->uid) != 0) {
qError("setForSnapShot error. uid:%" PRIu64 " ,version:%" PRId64, pOffset->uid, pOffset->version);
terrno = TSDB_CODE_PAR_INTERNAL_ERROR;
return -1;
}
qDebug("tmqsnap qStreamPrepareScan snapshot meta uid:%" PRId64 " ts %" PRId64 " %s", pOffset->uid, pOffset->ts, id);

2
source/libs/executor/src/executorimpl.c
View File

@ -999,6 +999,7 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scan
SOperatorInfo* extractOperatorInTree(SOperatorInfo* pOperator, int32_t type, const char* id) {
if (pOperator == NULL) {
qError("invalid operator, failed to find tableScanOperator %s", id);
terrno = TSDB_CODE_PAR_INTERNAL_ERROR;
return NULL;
}
@ -1007,6 +1008,7 @@ SOperatorInfo* extractOperatorInTree(SOperatorInfo* pOperator, int32_t type, con
} else {
if (pOperator->pDownstream == NULL || pOperator->pDownstream[0] == NULL) {
qError("invalid operator, failed to find tableScanOperator %s", id);
terrno = TSDB_CODE_PAR_INTERNAL_ERROR;
return NULL;
}

10
source/libs/function/src/udfd.c
View File

@ -571,13 +571,13 @@ int32_t udfdLoadUdf(char *udfName, SUdf *udf) {
char initFuncName[TSDB_FUNC_NAME_LEN + 5] = {0};
char *initSuffix = "_init";
strcpy(initFuncName, udfName);
strncat(initFuncName, initSuffix, strlen(initSuffix));
strncat(initFuncName, initSuffix, TSDB_FUNC_NAME_LEN + 5 - strlen(initFuncName) - 1);
uv_dlsym(&udf->lib, initFuncName, (void **)(&udf->initFunc));
char destroyFuncName[TSDB_FUNC_NAME_LEN + 5] = {0};
char *destroySuffix = "_destroy";
strcpy(destroyFuncName, udfName);
strncat(destroyFuncName, destroySuffix, strlen(destroySuffix));
strncat(destroyFuncName, destroySuffix, TSDB_FUNC_NAME_LEN + 5 - strlen(destroyFuncName) - 1);
uv_dlsym(&udf->lib, destroyFuncName, (void **)(&udf->destroyFunc));
if (udf->funcType == TSDB_FUNC_TYPE_SCALAR) {
@ -591,17 +591,17 @@ int32_t udfdLoadUdf(char *udfName, SUdf *udf) {
char startFuncName[TSDB_FUNC_NAME_LEN + 6] = {0};
char *startSuffix = "_start";
strncpy(startFuncName, processFuncName, sizeof(startFuncName));
strncat(startFuncName, startSuffix, strlen(startSuffix));
strncat(startFuncName, startSuffix, TSDB_FUNC_NAME_LEN + 6 - strlen(startSuffix) - 1);
uv_dlsym(&udf->lib, startFuncName, (void **)(&udf->aggStartFunc));
char finishFuncName[TSDB_FUNC_NAME_LEN + 7] = {0};
char *finishSuffix = "_finish";
strncpy(finishFuncName, processFuncName, sizeof(finishFuncName));
strncat(finishFuncName, finishSuffix, strlen(finishSuffix));
strncat(finishFuncName, finishSuffix, TSDB_FUNC_NAME_LEN + 7 - strlen(finishFuncName) - 1);
uv_dlsym(&udf->lib, finishFuncName, (void **)(&udf->aggFinishFunc));
char mergeFuncName[TSDB_FUNC_NAME_LEN + 6] = {0};
char *mergeSuffix = "_merge";
strncpy(mergeFuncName, processFuncName, sizeof(mergeFuncName));
strncat(mergeFuncName, mergeSuffix, strlen(mergeSuffix));
strncat(mergeFuncName, mergeSuffix, TSDB_FUNC_NAME_LEN + 6 - strlen(mergeFuncName) - 1);
uv_dlsym(&udf->lib, mergeFuncName, (void **)(&udf->aggMergeFunc));
}
return 0;

4
source/libs/function/test/udf1.c
View File

@ -31,7 +31,7 @@ DLL_EXPORT int32_t udf1(SUdfDataBlock *block, SUdfColumn *resultCol) {
}
}
if (j == block->numOfCols) {
int32_t luckyNum = 88;
int32_t luckyNum = 1;
udfColDataSet(resultCol, i, (char *)&luckyNum, false);
}
}
@ -43,4 +43,4 @@ DLL_EXPORT int32_t udf1(SUdfDataBlock *block, SUdfColumn *resultCol) {
Sleep(1);
#endif
return 0;
}
}

12
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -923,9 +923,15 @@ void nodesDestroyNode(SNode* pNode) {
taosMemoryFree(((SDescribeStmt*)pNode)->pMeta);
break;
case QUERY_NODE_RESET_QUERY_CACHE_STMT: // no pointer field
case QUERY_NODE_COMPACT_DATABASE_STMT: // no pointer field
case QUERY_NODE_CREATE_FUNCTION_STMT: // no pointer field
case QUERY_NODE_DROP_FUNCTION_STMT: // no pointer field
break;
case QUERY_NODE_COMPACT_DATABASE_STMT: {
SCompactDatabaseStmt* pStmt = (SCompactDatabaseStmt*)pNode;
nodesDestroyNode(pStmt->pStart);
nodesDestroyNode(pStmt->pEnd);
break;
}
case QUERY_NODE_CREATE_FUNCTION_STMT: // no pointer field
case QUERY_NODE_DROP_FUNCTION_STMT: // no pointer field
break;
case QUERY_NODE_CREATE_STREAM_STMT: {
SCreateStreamStmt* pStmt = (SCreateStreamStmt*)pNode;

11
source/libs/parser/inc/parAst.h
View File

@ -82,6 +82,11 @@ typedef struct SAlterOption {
SNodeList* pList;
} SAlterOption;
typedef struct STokenPair {
SToken first;
SToken second;
} STokenPair;
extern SToken nil_token;
void initAstCreateContext(SParseContext* pParseCxt, SAstCreateContext* pCxt);
@ -227,8 +232,10 @@ SNode* createMergeVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId1, cons
SNode* createRedistributeVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId, SNodeList* pDnodes);
SNode* createSplitVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId);
SNode* createSyncdbStmt(SAstCreateContext* pCxt, const SToken* pDbName);
SNode* createGrantStmt(SAstCreateContext* pCxt, int64_t privileges, SToken* pDbName, SToken* pUserName);
SNode* createRevokeStmt(SAstCreateContext* pCxt, int64_t privileges, SToken* pDbName, SToken* pUserName);
SNode* createGrantStmt(SAstCreateContext* pCxt, int64_t privileges, STokenPair* pPrivLevel, SToken* pUserName,
SNode* pTagCond);
SNode* createRevokeStmt(SAstCreateContext* pCxt, int64_t privileges, STokenPair* pPrivLevel, SToken* pUserName,
SNode* pTagCond);
SNode* createDeleteStmt(SAstCreateContext* pCxt, SNode* pTable, SNode* pWhere);
SNode* createInsertStmt(SAstCreateContext* pCxt, SNode* pTable, SNodeList* pCols, SNode* pQuery);

6
source/libs/parser/inc/parUtil.h
View File

@ -97,9 +97,8 @@ int32_t reserveTableVgroupInCache(int32_t acctId, const char* pDb, const char* p
int32_t reserveTableVgroupInCacheExt(const SName* pName, SParseMetaCache* pMetaCache);
int32_t reserveDbVgVersionInCache(int32_t acctId, const char* pDb, SParseMetaCache* pMetaCache);
int32_t reserveDbCfgInCache(int32_t acctId, const char* pDb, SParseMetaCache* pMetaCache);
int32_t reserveUserAuthInCache(int32_t acctId, const char* pUser, const char* pDb, AUTH_TYPE type,
int32_t reserveUserAuthInCache(int32_t acctId, const char* pUser, const char* pDb, const char* pTable, AUTH_TYPE type,
SParseMetaCache* pMetaCache);
int32_t reserveUserAuthInCacheExt(const char* pUser, const SName* pName, AUTH_TYPE type, SParseMetaCache* pMetaCache);
int32_t reserveUdfInCache(const char* pFunc, SParseMetaCache* pMetaCache);
int32_t reserveTableIndexInCache(int32_t acctId, const char* pDb, const char* pTable, SParseMetaCache* pMetaCache);
int32_t reserveTableCfgInCache(int32_t acctId, const char* pDb, const char* pTable, SParseMetaCache* pMetaCache);
@ -110,8 +109,7 @@ int32_t getTableVgroupFromCache(SParseMetaCache* pMetaCache, const SName* pName,
int32_t getDbVgVersionFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, int32_t* pVersion, int64_t* pDbId,
int32_t* pTableNum, int64_t* pStateTs);
int32_t getDbCfgFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, SDbCfgInfo* pInfo);
int32_t getUserAuthFromCache(SParseMetaCache* pMetaCache, const char* pUser, const char* pDbFName, AUTH_TYPE type,
bool* pPass);
int32_t getUserAuthFromCache(SParseMetaCache* pMetaCache, SUserAuthInfo* pAuthReq, SUserAuthRes* pAuthRes);
int32_t getUdfInfoFromCache(SParseMetaCache* pMetaCache, const char* pFunc, SFuncInfo* pInfo);
int32_t getTableIndexFromCache(SParseMetaCache* pMetaCache, const SName* pName, SArray** pIndexes);
int32_t getTableCfgFromCache(SParseMetaCache* pMetaCache, const SName* pName, STableCfg** pOutput);

16
source/libs/parser/inc/sql.y
View File

@ -94,8 +94,8 @@ sysinfo_opt(A) ::= .
sysinfo_opt(A) ::= SYSINFO NK_INTEGER(B). { A = taosStr2Int8(B.z, NULL, 10); }
/************************************************ grant/revoke ********************************************************/
cmd ::= GRANT privileges(A) ON priv_level(B) TO user_name(C). { pCxt->pRootNode = createGrantStmt(pCxt, A, &B, &C); }
cmd ::= REVOKE privileges(A) ON priv_level(B) FROM user_name(C). { pCxt->pRootNode = createRevokeStmt(pCxt, A, &B, &C); }
cmd ::= GRANT privileges(A) ON priv_level(B) with_opt(D) TO user_name(C). { pCxt->pRootNode = createGrantStmt(pCxt, A, &B, &C, D); }
cmd ::= REVOKE privileges(A) ON priv_level(B) with_opt(D) FROM user_name(C). { pCxt->pRootNode = createRevokeStmt(pCxt, A, &B, &C, D); }
%type privileges { int64_t }
%destructor privileges { }
@ -113,11 +113,15 @@ priv_type_list(A) ::= priv_type_list(B) NK_COMMA priv_type(C).
priv_type(A) ::= READ. { A = PRIVILEGE_TYPE_READ; }
priv_type(A) ::= WRITE. { A = PRIVILEGE_TYPE_WRITE; }
%type priv_level { SToken }
%type priv_level { STokenPair }
%destructor priv_level { }
priv_level(A) ::= NK_STAR(B) NK_DOT NK_STAR. { A = B; }
priv_level(A) ::= db_name(B) NK_DOT NK_STAR. { A = B; }
priv_level(A) ::= topic_name(B). { A = B; }
priv_level(A) ::= NK_STAR(B) NK_DOT NK_STAR(C). { A.first = B; A.second = C; }
priv_level(A) ::= db_name(B) NK_DOT NK_STAR(C). { A.first = B; A.second = C; }
priv_level(A) ::= db_name(B) NK_DOT table_name(C). { A.first = B; A.second = C; }
priv_level(A) ::= topic_name(B). { A.first = B; A.second = nil_token; }
with_opt(A) ::= . { A = NULL; }
with_opt(A) ::= WITH search_condition(B). { A = B; }
/************************************************ create/drop/alter dnode *********************************************/
cmd ::= CREATE DNODE dnode_endpoint(A). { pCxt->pRootNode = createCreateDnodeStmt(pCxt, &A, NULL); }

22
source/libs/parser/src/parAstCreater.c
View File

@ -1961,29 +1961,39 @@ SNode* createSyncdbStmt(SAstCreateContext* pCxt, const SToken* pDbName) {
return pStmt;
}
SNode* createGrantStmt(SAstCreateContext* pCxt, int64_t privileges, SToken* pDbName, SToken* pUserName) {
SNode* createGrantStmt(SAstCreateContext* pCxt, int64_t privileges, STokenPair* pPrivLevel, SToken* pUserName,
SNode* pTagCond) {
CHECK_PARSER_STATUS(pCxt);
if (!checkDbName(pCxt, pDbName, false) || !checkUserName(pCxt, pUserName)) {
if (!checkDbName(pCxt, &pPrivLevel->first, false) || !checkUserName(pCxt, pUserName)) {
return NULL;
}
SGrantStmt* pStmt = (SGrantStmt*)nodesMakeNode(QUERY_NODE_GRANT_STMT);
CHECK_OUT_OF_MEM(pStmt);
pStmt->privileges = privileges;
COPY_STRING_FORM_ID_TOKEN(pStmt->objName, pDbName);
COPY_STRING_FORM_ID_TOKEN(pStmt->objName, &pPrivLevel->first);
if (TK_NK_NIL != pPrivLevel->second.type && TK_NK_STAR != pPrivLevel->second.type) {
COPY_STRING_FORM_ID_TOKEN(pStmt->tabName, &pPrivLevel->second);
}
COPY_STRING_FORM_ID_TOKEN(pStmt->userName, pUserName);
pStmt->pTagCond = pTagCond;
return (SNode*)pStmt;
}
SNode* createRevokeStmt(SAstCreateContext* pCxt, int64_t privileges, SToken* pDbName, SToken* pUserName) {
SNode* createRevokeStmt(SAstCreateContext* pCxt, int64_t privileges, STokenPair* pPrivLevel, SToken* pUserName,
SNode* pTagCond) {
CHECK_PARSER_STATUS(pCxt);
if (!checkDbName(pCxt, pDbName, false) || !checkUserName(pCxt, pUserName)) {
if (!checkDbName(pCxt, &pPrivLevel->first, false) || !checkUserName(pCxt, pUserName)) {
return NULL;
}
SRevokeStmt* pStmt = (SRevokeStmt*)nodesMakeNode(QUERY_NODE_REVOKE_STMT);
CHECK_OUT_OF_MEM(pStmt);
pStmt->privileges = privileges;
COPY_STRING_FORM_ID_TOKEN(pStmt->objName, pDbName);
COPY_STRING_FORM_ID_TOKEN(pStmt->objName, &pPrivLevel->first);
if (TK_NK_NIL != pPrivLevel->second.type && TK_NK_STAR != pPrivLevel->second.type) {
COPY_STRING_FORM_ID_TOKEN(pStmt->tabName, &pPrivLevel->second);
}
COPY_STRING_FORM_ID_TOKEN(pStmt->userName, pUserName);
pStmt->pTagCond = pTagCond;
return (SNode*)pStmt;
}

50
source/libs/parser/src/parAstParser.c
View File

@ -154,7 +154,8 @@ static int32_t collectMetaKeyFromRealTableImpl(SCollectMetaKeyCxt* pCxt, const c
code = reserveTableVgroupInCache(pCxt->pParseCxt->acctId, pDb, pTable, pCxt->pMetaCache);
}
if (TSDB_CODE_SUCCESS == code) {
code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pDb, authType, pCxt->pMetaCache);
code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pDb, pTable, authType,
pCxt->pMetaCache);
}
if (TSDB_CODE_SUCCESS == code) {
code = reserveDbVgInfoInCache(pCxt->pParseCxt->acctId, pDb, pCxt->pMetaCache);
@ -247,7 +248,7 @@ static int32_t collectMetaKeyFromCreateTable(SCollectMetaKeyCxt* pCxt, SCreateTa
code = reserveTableVgroupInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, pCxt->pMetaCache);
}
if (TSDB_CODE_SUCCESS == code) {
code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName, AUTH_TYPE_WRITE,
code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName, NULL, AUTH_TYPE_WRITE,
pCxt->pMetaCache);
}
return code;
@ -267,8 +268,8 @@ static int32_t collectMetaKeyFromCreateMultiTable(SCollectMetaKeyCxt* pCxt, SCre
code = reserveTableVgroupInCache(pCxt->pParseCxt->acctId, pClause->dbName, pClause->tableName, pCxt->pMetaCache);
}
if (TSDB_CODE_SUCCESS == code) {
code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pClause->dbName, AUTH_TYPE_WRITE,
pCxt->pMetaCache);
code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pClause->dbName, NULL,
AUTH_TYPE_WRITE, pCxt->pMetaCache);
}
if (TSDB_CODE_SUCCESS != code) {
break;
@ -286,6 +287,10 @@ static int32_t collectMetaKeyFromDropTable(SCollectMetaKeyCxt* pCxt, SDropTableS
if (TSDB_CODE_SUCCESS == code) {
code = reserveTableVgroupInCache(pCxt->pParseCxt->acctId, pClause->dbName, pClause->tableName, pCxt->pMetaCache);
}
if (TSDB_CODE_SUCCESS == code) {
code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pClause->dbName,
pClause->tableName, AUTH_TYPE_WRITE, pCxt->pMetaCache);
}
if (TSDB_CODE_SUCCESS != code) {
break;
}
@ -293,6 +298,11 @@ static int32_t collectMetaKeyFromDropTable(SCollectMetaKeyCxt* pCxt, SDropTableS
return code;
}
static int32_t collectMetaKeyFromDropStable(SCollectMetaKeyCxt* pCxt, SDropSuperTableStmt* pStmt) {
return reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName, pStmt->tableName,
AUTH_TYPE_WRITE, pCxt->pMetaCache);
}
static int32_t collectMetaKeyFromAlterTable(SCollectMetaKeyCxt* pCxt, SAlterTableStmt* pStmt) {
int32_t code = reserveDbCfgInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pCxt->pMetaCache);
if (TSDB_CODE_SUCCESS == code) {
@ -301,6 +311,10 @@ static int32_t collectMetaKeyFromAlterTable(SCollectMetaKeyCxt* pCxt, SAlterTabl
if (TSDB_CODE_SUCCESS == code) {
code = reserveTableVgroupInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, pCxt->pMetaCache);
}
if (TSDB_CODE_SUCCESS == code) {
code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName, pStmt->tableName,
AUTH_TYPE_WRITE, pCxt->pMetaCache);
}
return code;
}
@ -309,6 +323,10 @@ static int32_t collectMetaKeyFromAlterStable(SCollectMetaKeyCxt* pCxt, SAlterTab
if (TSDB_CODE_SUCCESS == code) {
code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, pCxt->pMetaCache);
}
if (TSDB_CODE_SUCCESS == code) {
code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName, pStmt->tableName,
AUTH_TYPE_WRITE, pCxt->pMetaCache);
}
return code;
}
@ -439,8 +457,9 @@ static int32_t collectMetaKeyFromShowStables(SCollectMetaKeyCxt* pCxt, SShowStmt
int32_t code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_STABLES,
pCxt->pMetaCache);
if (TSDB_CODE_SUCCESS == code) {
code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser,
((SValueNode*)pStmt->pDbName)->literal, AUTH_TYPE_READ_OR_WRITE, pCxt->pMetaCache);
code =
reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, ((SValueNode*)pStmt->pDbName)->literal,
NULL, AUTH_TYPE_READ_OR_WRITE, pCxt->pMetaCache);
}
return code;
}
@ -457,8 +476,9 @@ static int32_t collectMetaKeyFromShowTables(SCollectMetaKeyCxt* pCxt, SShowStmt*
code = reserveDbVgInfoInCache(pCxt->pParseCxt->acctId, ((SValueNode*)pStmt->pDbName)->literal, pCxt->pMetaCache);
}
if (TSDB_CODE_SUCCESS == code) {
code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser,
((SValueNode*)pStmt->pDbName)->literal, AUTH_TYPE_READ_OR_WRITE, pCxt->pMetaCache);
code =
reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, ((SValueNode*)pStmt->pDbName)->literal,
NULL, AUTH_TYPE_READ_OR_WRITE, pCxt->pMetaCache);
}
return code;
}
@ -561,7 +581,8 @@ static int32_t collectMetaKeyFromShowCreateTable(SCollectMetaKeyCxt* pCxt, SShow
code = reserveDbCfgInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pCxt->pMetaCache);
}
if (TSDB_CODE_SUCCESS == code) {
code = reserveUserAuthInCacheExt(pCxt->pParseCxt->pUser, &name, AUTH_TYPE_READ, pCxt->pMetaCache);
code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName, NULL, AUTH_TYPE_READ,
pCxt->pMetaCache);
}
return code;
}
@ -610,6 +631,13 @@ static int32_t collectMetaKeyFromCompactDatabase(SCollectMetaKeyCxt* pCxt, SComp
return reserveDbCfgInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pCxt->pMetaCache);
}
static int32_t collectMetaKeyFromGrant(SCollectMetaKeyCxt* pCxt, SGrantStmt* pStmt) {
if ('\0' == pStmt->tabName[0]) {
return TSDB_CODE_SUCCESS;
}
return reserveTableMetaInCache(pCxt->pParseCxt->acctId, pStmt->objName, pStmt->tabName, pCxt->pMetaCache);
}
static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) {
pCxt->pStmt = pStmt;
switch (nodeType(pStmt)) {
@ -627,6 +655,8 @@ static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) {
return collectMetaKeyFromCreateMultiTable(pCxt, (SCreateMultiTablesStmt*)pStmt);
case QUERY_NODE_DROP_TABLE_STMT:
return collectMetaKeyFromDropTable(pCxt, (SDropTableStmt*)pStmt);
case QUERY_NODE_DROP_SUPER_TABLE_STMT:
return collectMetaKeyFromDropStable(pCxt, (SDropSuperTableStmt*)pStmt);
case QUERY_NODE_ALTER_TABLE_STMT:
return collectMetaKeyFromAlterTable(pCxt, (SAlterTableStmt*)pStmt);
case QUERY_NODE_ALTER_SUPER_TABLE_STMT:
@ -645,6 +675,8 @@ static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) {
return collectMetaKeyFromCompactDatabase(pCxt, (SCompactDatabaseStmt*)pStmt);
case QUERY_NODE_CREATE_STREAM_STMT:
return collectMetaKeyFromCreateStream(pCxt, (SCreateStreamStmt*)pStmt);
case QUERY_NODE_GRANT_STMT:
return collectMetaKeyFromGrant(pCxt, (SGrantStmt*)pStmt);
case QUERY_NODE_SHOW_DNODES_STMT:
return collectMetaKeyFromShowDnodes(pCxt, (SShowStmt*)pStmt);
case QUERY_NODE_SHOW_MNODES_STMT:

153
source/libs/parser/src/parAuthenticator.c
View File

@ -23,49 +23,112 @@ typedef struct SAuthCxt {
int32_t errCode;
} SAuthCxt;
typedef struct SSelectAuthCxt {
SAuthCxt* pAuthCxt;
SSelectStmt* pSelect;
} SSelectAuthCxt;
static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt);
static int32_t checkAuth(SAuthCxt* pCxt, const char* pDbName, AUTH_TYPE type) {
static void setUserAuthInfo(SParseContext* pCxt, const char* pDbName, const char* pTabName, AUTH_TYPE type,
SUserAuthInfo* pAuth) {
snprintf(pAuth->user, sizeof(pAuth->user), "%s", pCxt->pUser);
if (NULL == pTabName) {
tNameSetDbName(&pAuth->tbName, pCxt->acctId, pDbName, strlen(pDbName));
} else {
toName(pCxt->acctId, pDbName, pTabName, &pAuth->tbName);
}
pAuth->type = type;
}
static int32_t checkAuth(SAuthCxt* pCxt, const char* pDbName, const char* pTabName, AUTH_TYPE type, SNode** pCond) {
SParseContext* pParseCxt = pCxt->pParseCxt;
if (pParseCxt->isSuperUser) {
return TSDB_CODE_SUCCESS;
}
SName name;
tNameSetDbName(&name, pParseCxt->acctId, pDbName, strlen(pDbName));
char dbFname[TSDB_DB_FNAME_LEN] = {0};
tNameGetFullDbName(&name, dbFname);
int32_t code = TSDB_CODE_SUCCESS;
bool pass = false;
SUserAuthInfo authInfo = {0};
setUserAuthInfo(pCxt->pParseCxt, pDbName, pTabName, type, &authInfo);
int32_t code = TSDB_CODE_SUCCESS;
SUserAuthRes authRes = {0};
if (NULL != pCxt->pMetaCache) {
code = getUserAuthFromCache(pCxt->pMetaCache, pParseCxt->pUser, dbFname, type, &pass);
code = getUserAuthFromCache(pCxt->pMetaCache, &authInfo, &authRes);
} else {
SRequestConnInfo conn = {.pTrans = pParseCxt->pTransporter,
.requestId = pParseCxt->requestId,
.requestObjRefId = pParseCxt->requestRid,
.mgmtEps = pParseCxt->mgmtEpSet};
code = catalogChkAuth(pParseCxt->pCatalog, &conn, pParseCxt->pUser, dbFname, type, &pass);
code = catalogChkAuth(pParseCxt->pCatalog, &conn, &authInfo, &authRes);
}
return TSDB_CODE_SUCCESS == code ? (pass ? TSDB_CODE_SUCCESS : TSDB_CODE_PAR_PERMISSION_DENIED) : code;
if (TSDB_CODE_SUCCESS == code && NULL != pCond) {
*pCond = authRes.pCond;
}
return TSDB_CODE_SUCCESS == code ? (authRes.pass ? TSDB_CODE_SUCCESS : TSDB_CODE_PAR_PERMISSION_DENIED) : code;
}
static EDealRes authSubquery(SAuthCxt* pCxt, SNode* pStmt) {
return TSDB_CODE_SUCCESS == authQuery(pCxt, pStmt) ? DEAL_RES_CONTINUE : DEAL_RES_ERROR;
}
static int32_t mergeStableTagCond(SNode** pWhere, SNode** pTagCond) {
SLogicConditionNode* pLogicCond = (SLogicConditionNode*)nodesMakeNode(QUERY_NODE_LOGIC_CONDITION);
if (NULL == pLogicCond) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pLogicCond->node.resType.type = TSDB_DATA_TYPE_BOOL;
pLogicCond->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes;
pLogicCond->condType = LOGIC_COND_TYPE_AND;
int32_t code = nodesListMakeStrictAppend(&pLogicCond->pParameterList, *pTagCond);
if (TSDB_CODE_SUCCESS == code) {
code = nodesListMakeAppend(&pLogicCond->pParameterList, *pWhere);
}
if (TSDB_CODE_SUCCESS == code) {
*pWhere = (SNode*)pLogicCond;
} else {
nodesDestroyNode((SNode*)pLogicCond);
}
return code;
}
static int32_t appendStableTagCond(SNode** pWhere, SNode* pTagCond) {
SNode* pTagCondCopy = nodesCloneNode(pTagCond);
if (NULL == pTagCondCopy) {
return TSDB_CODE_OUT_OF_MEMORY;
}
if (NULL == *pWhere) {
*pWhere = pTagCondCopy;
return TSDB_CODE_SUCCESS;
}
if (QUERY_NODE_LOGIC_CONDITION == nodeType(*pWhere) &&
LOGIC_COND_TYPE_AND == ((SLogicConditionNode*)*pWhere)->condType) {
return nodesListStrictAppend(((SLogicConditionNode*)*pWhere)->pParameterList, pTagCondCopy);
}
return mergeStableTagCond(pWhere, &pTagCondCopy);
}
static EDealRes authSelectImpl(SNode* pNode, void* pContext) {
SAuthCxt* pCxt = pContext;
SSelectAuthCxt* pCxt = pContext;
SAuthCxt* pAuthCxt = pCxt->pAuthCxt;
if (QUERY_NODE_REAL_TABLE == nodeType(pNode)) {
pCxt->errCode = checkAuth(pCxt, ((SRealTableNode*)pNode)->table.dbName, AUTH_TYPE_READ);
return TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_CONTINUE : DEAL_RES_ERROR;
SNode* pTagCond = NULL;
STableNode* pTable = (STableNode*)pNode;
pAuthCxt->errCode = checkAuth(pAuthCxt, pTable->dbName, pTable->tableName, AUTH_TYPE_READ, &pTagCond);
if (TSDB_CODE_SUCCESS == pAuthCxt->errCode && NULL != pTagCond) {
pAuthCxt->errCode = appendStableTagCond(&pCxt->pSelect->pWhere, pTagCond);
}
return TSDB_CODE_SUCCESS == pAuthCxt->errCode ? DEAL_RES_CONTINUE : DEAL_RES_ERROR;
} else if (QUERY_NODE_TEMP_TABLE == nodeType(pNode)) {
return authSubquery(pCxt, ((STempTableNode*)pNode)->pSubquery);
return authSubquery(pAuthCxt, ((STempTableNode*)pNode)->pSubquery);
}
return DEAL_RES_CONTINUE;
}
static int32_t authSelect(SAuthCxt* pCxt, SSelectStmt* pSelect) {
nodesWalkSelectStmt(pSelect, SQL_CLAUSE_FROM, authSelectImpl, pCxt);
SSelectAuthCxt cxt = {.pAuthCxt = pCxt, .pSelect = pSelect};
nodesWalkSelectStmt(pSelect, SQL_CLAUSE_FROM, authSelectImpl, &cxt);
return pCxt->errCode;
}
@ -85,11 +148,20 @@ static int32_t authDropUser(SAuthCxt* pCxt, SDropUserStmt* pStmt) {
}
static int32_t authDelete(SAuthCxt* pCxt, SDeleteStmt* pDelete) {
return checkAuth(pCxt, ((SRealTableNode*)pDelete->pFromTable)->table.dbName, AUTH_TYPE_WRITE);
SNode* pTagCond = NULL;
STableNode* pTable = (STableNode*)pDelete->pFromTable;
int32_t code = checkAuth(pCxt, pTable->dbName, pTable->tableName, AUTH_TYPE_WRITE, &pTagCond);
if (TSDB_CODE_SUCCESS == code && NULL != pTagCond) {
code = appendStableTagCond(&pDelete->pWhere, pTagCond);
}
return code;
}
static int32_t authInsert(SAuthCxt* pCxt, SInsertStmt* pInsert) {
int32_t code = checkAuth(pCxt, ((SRealTableNode*)pInsert->pTable)->table.dbName, AUTH_TYPE_WRITE);
SNode* pTagCond = NULL;
STableNode* pTable = (STableNode*)pInsert->pTable;
// todo check tag condition for subtable
int32_t code = checkAuth(pCxt, pTable->dbName, pTable->tableName, AUTH_TYPE_WRITE, &pTagCond);
if (TSDB_CODE_SUCCESS == code) {
code = authQuery(pCxt, pInsert->pQuery);
}
@ -97,22 +169,27 @@ static int32_t authInsert(SAuthCxt* pCxt, SInsertStmt* pInsert) {
}
static int32_t authShowTables(SAuthCxt* pCxt, SShowStmt* pStmt) {
return checkAuth(pCxt, ((SValueNode*)pStmt->pDbName)->literal, AUTH_TYPE_READ_OR_WRITE);
return checkAuth(pCxt, ((SValueNode*)pStmt->pDbName)->literal, NULL, AUTH_TYPE_READ_OR_WRITE, NULL);
}
static int32_t authShowCreateTable(SAuthCxt* pCxt, SShowCreateTableStmt* pStmt) {
return checkAuth(pCxt, pStmt->dbName, AUTH_TYPE_READ);
SNode* pTagCond = NULL;
// todo check tag condition for subtable
return checkAuth(pCxt, pStmt->dbName, NULL, AUTH_TYPE_READ, &pTagCond);
}
static int32_t authCreateTable(SAuthCxt* pCxt, SCreateTableStmt* pStmt) {
return checkAuth(pCxt, pStmt->dbName, AUTH_TYPE_WRITE);
SNode* pTagCond = NULL;
// todo check tag condition for subtable
return checkAuth(pCxt, pStmt->dbName, NULL, AUTH_TYPE_WRITE, &pTagCond);
}
static int32_t authCreateMultiTable(SAuthCxt* pCxt, SCreateMultiTablesStmt* pStmt) {
int32_t code = TSDB_CODE_SUCCESS;
SNode* pNode = NULL;
FOREACH(pNode, pStmt->pSubTables) {
code = checkAuth(pCxt, ((SCreateSubTableClause*)pNode)->dbName, AUTH_TYPE_WRITE);
SCreateSubTableClause* pClause = (SCreateSubTableClause*)pNode;
code = checkAuth(pCxt, pClause->dbName, NULL, AUTH_TYPE_WRITE, NULL);
if (TSDB_CODE_SUCCESS != code) {
break;
}
@ -120,6 +197,29 @@ static int32_t authCreateMultiTable(SAuthCxt* pCxt, SCreateMultiTablesStmt* pStm
return code;
}
static int32_t authDropTable(SAuthCxt* pCxt, SDropTableStmt* pStmt) {
int32_t code = TSDB_CODE_SUCCESS;
SNode* pNode = NULL;
FOREACH(pNode, pStmt->pTables) {
SDropTableClause* pClause = (SDropTableClause*)pNode;
code = checkAuth(pCxt, pClause->dbName, pClause->tableName, AUTH_TYPE_WRITE, NULL);
if (TSDB_CODE_SUCCESS != code) {
break;
}
}
return code;
}
static int32_t authDropStable(SAuthCxt* pCxt, SDropSuperTableStmt* pStmt) {
return checkAuth(pCxt, pStmt->dbName, pStmt->tableName, AUTH_TYPE_WRITE, NULL);
}
static int32_t authAlterTable(SAuthCxt* pCxt, SAlterTableStmt* pStmt) {
SNode* pTagCond = NULL;
// todo check tag condition for subtable
return checkAuth(pCxt, pStmt->dbName, pStmt->tableName, AUTH_TYPE_WRITE, NULL);
}
static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt) {
switch (nodeType(pStmt)) {
case QUERY_NODE_SET_OPERATOR:
@ -136,6 +236,13 @@ static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt) {
return authCreateTable(pCxt, (SCreateTableStmt*)pStmt);
case QUERY_NODE_CREATE_MULTI_TABLES_STMT:
return authCreateMultiTable(pCxt, (SCreateMultiTablesStmt*)pStmt);
case QUERY_NODE_DROP_TABLE_STMT:
return authDropTable(pCxt, (SDropTableStmt*)pStmt);
case QUERY_NODE_DROP_SUPER_TABLE_STMT:
return authDropStable(pCxt, (SDropSuperTableStmt*)pStmt);
case QUERY_NODE_ALTER_TABLE_STMT:
case QUERY_NODE_ALTER_SUPER_TABLE_STMT:
return authAlterTable(pCxt, (SAlterTableStmt*)pStmt);
case QUERY_NODE_SHOW_DNODES_STMT:
case QUERY_NODE_SHOW_MNODES_STMT:
case QUERY_NODE_SHOW_MODULES_STMT:
@ -146,7 +253,7 @@ static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt) {
case QUERY_NODE_SHOW_LICENCES_STMT:
case QUERY_NODE_SHOW_VGROUPS_STMT:
case QUERY_NODE_SHOW_DB_ALIVE_STMT:
case QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT:
case QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT:
case QUERY_NODE_SHOW_CREATE_DATABASE_STMT:
case QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT:
case QUERY_NODE_SHOW_VNODES_STMT:

139
source/libs/parser/src/parInsertSql.c
View File

@ -15,6 +15,7 @@
#include "parInsertUtil.h"
#include "parToken.h"
#include "scalar.h"
#include "tglobal.h"
#include "ttime.h"
@ -565,6 +566,120 @@ static int32_t checkAndTrimValue(SToken* pToken, char* tmpTokenBuf, SMsgBuf* pMs
return TSDB_CODE_SUCCESS;
}
typedef struct SRewriteTagCondCxt {
SArray* pTagVals;
SArray* pTagName;
int32_t code;
} SRewriteTagCondCxt;
static int32_t rewriteTagCondColumnImpl(STagVal* pVal, SNode** pNode) {
SValueNode* pValue = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
if (NULL == pValue) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pValue->node.resType.type = pVal->type;
switch (pVal->type) {
case TSDB_DATA_TYPE_BOOL:
pValue->datum.b = *(int8_t*)(&pVal->i64);
break;
case TSDB_DATA_TYPE_TINYINT:
pValue->datum.i = *(int8_t*)(&pVal->i64);
break;
case TSDB_DATA_TYPE_SMALLINT:
pValue->datum.i = *(int16_t*)(&pVal->i64);
break;
case TSDB_DATA_TYPE_INT:
pValue->datum.i = *(int32_t*)(&pVal->i64);
break;
case TSDB_DATA_TYPE_BIGINT:
pValue->datum.i = pVal->i64;
break;
case TSDB_DATA_TYPE_FLOAT:
pValue->datum.d = *(float*)(&pVal->i64);
break;
case TSDB_DATA_TYPE_DOUBLE:
pValue->datum.d = *(double*)(&pVal->i64);
break;
case TSDB_DATA_TYPE_VARCHAR:
case TSDB_DATA_TYPE_NCHAR:
pValue->datum.p = taosMemoryCalloc(1, pVal->nData + VARSTR_HEADER_SIZE);
if (NULL == pValue->datum.p) {
return TSDB_CODE_OUT_OF_MEMORY;
}
varDataSetLen(pValue->datum.p, pVal->nData);
memcpy(varDataVal(pValue->datum.p), pVal->pData, pVal->nData);
break;
case TSDB_DATA_TYPE_TIMESTAMP:
pValue->datum.i = pVal->i64;
break;
case TSDB_DATA_TYPE_UTINYINT:
pValue->datum.i = *(uint8_t*)(&pVal->i64);
break;
case TSDB_DATA_TYPE_USMALLINT:
pValue->datum.i = *(uint16_t*)(&pVal->i64);
break;
case TSDB_DATA_TYPE_UINT:
pValue->datum.i = *(uint32_t*)(&pVal->i64);
break;
case TSDB_DATA_TYPE_UBIGINT:
pValue->datum.i = *(uint64_t*)(&pVal->i64);
break;
case TSDB_DATA_TYPE_JSON:
case TSDB_DATA_TYPE_VARBINARY:
case TSDB_DATA_TYPE_DECIMAL:
case TSDB_DATA_TYPE_BLOB:
case TSDB_DATA_TYPE_MEDIUMBLOB:
default:
return TSDB_CODE_FAILED;
}
return TSDB_CODE_SUCCESS;
}
static int32_t rewriteTagCondColumn(SArray* pTagVals, SArray* pTagName, SNode** pNode) {
SColumnNode* pCol = (SColumnNode*)*pNode;
int32_t ntags = taosArrayGetSize(pTagName);
for (int32_t i = 0; i < ntags; ++i) {
char* pTagColName = taosArrayGet(pTagName, i);
if (0 == strcmp(pTagColName, pCol->colName)) {
return rewriteTagCondColumnImpl(taosArrayGet(pTagVals, i), pNode);
}
}
return TSDB_CODE_PAR_PERMISSION_DENIED;
}
static EDealRes rewriteTagCond(SNode** pNode, void* pContext) {
if (QUERY_NODE_COLUMN == nodeType(*pNode)) {
SRewriteTagCondCxt* pCxt = pContext;
pCxt->code = rewriteTagCondColumn(pCxt->pTagVals, pCxt->pTagName, pNode);
return (TSDB_CODE_SUCCESS == pCxt->code ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR);
}
return DEAL_RES_CONTINUE;
}
static int32_t setTagVal(SArray* pTagVals, SArray* pTagName, SNode* pCond) {
SRewriteTagCondCxt cxt = {.code = TSDB_CODE_SUCCESS, .pTagVals = pTagVals, .pTagName = pTagName};
nodesRewriteExpr(&pCond, rewriteTagCond, &cxt);
return cxt.code;
}
static int32_t checkTagCondResult(SNode* pResult) {
return (QUERY_NODE_VALUE == nodeType(pResult) && ((SValueNode*)pResult)->datum.b) ? TSDB_CODE_SUCCESS
: TSDB_CODE_PAR_PERMISSION_DENIED;
}
int32_t checkSubtablePrivilege(SArray* pTagVals, SArray* pTagName, SNode* pCond) {
int32_t code = setTagVal(pTagVals, pTagName, pCond);
SNode* pNew = NULL;
if (TSDB_CODE_SUCCESS == code) {
code = scalarCalculateConstants(pCond, &pNew);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkTagCondResult(pNew);
}
nodesDestroyNode(pNew);
return code;
}
// pSql -> tag1_value, ...)
static int32_t parseTagsClauseImpl(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) {
int32_t code = TSDB_CODE_SUCCESS;
@ -722,25 +837,31 @@ static int32_t parseUsingClauseBottom(SInsertParseContext* pCxt, SVnodeModifyOpS
return code;
}
static void setUserAuthInfo(SParseContext* pCxt, SName* pTbName, SUserAuthInfo* pInfo) {
snprintf(pInfo->user, sizeof(pInfo->user), "%s", pCxt->pUser);
memcpy(&pInfo->tbName, pTbName, sizeof(SName));
pInfo->type = AUTH_TYPE_WRITE;
}
static int32_t checkAuth(SParseContext* pCxt, SName* pTbName, bool* pMissCache) {
char dbFName[TSDB_DB_FNAME_LEN];
tNameGetFullDbName(pTbName, dbFName);
int32_t code = TSDB_CODE_SUCCESS;
bool pass = true;
bool exists = true;
int32_t code = TSDB_CODE_SUCCESS;
SUserAuthInfo authInfo = {0};
setUserAuthInfo(pCxt, pTbName, &authInfo);
SUserAuthRes authRes = {0};
bool exists = true;
if (pCxt->async) {
code = catalogChkAuthFromCache(pCxt->pCatalog, pCxt->pUser, dbFName, AUTH_TYPE_WRITE, &pass, &exists);
code = catalogChkAuthFromCache(pCxt->pCatalog, &authInfo, &authRes, &exists);
} else {
SRequestConnInfo conn = {.pTrans = pCxt->pTransporter,
.requestId = pCxt->requestId,
.requestObjRefId = pCxt->requestRid,
.mgmtEps = pCxt->mgmtEpSet};
code = catalogChkAuth(pCxt->pCatalog, &conn, pCxt->pUser, dbFName, AUTH_TYPE_WRITE, &pass);
code = catalogChkAuth(pCxt->pCatalog, &conn, &authInfo, &authRes);
}
if (TSDB_CODE_SUCCESS == code) {
if (!exists) {
*pMissCache = true;
} else if (!pass) {
} else if (!authRes.pass) {
code = TSDB_CODE_PAR_PERMISSION_DENIED;
}
}
@ -1901,7 +2022,7 @@ static int32_t buildInsertUserAuthReq(const char* pUser, SName* pName, SArray**
SUserAuthInfo userAuth = {.type = AUTH_TYPE_WRITE};
snprintf(userAuth.user, sizeof(userAuth.user), "%s", pUser);
tNameGetFullDbName(pName, userAuth.dbFName);
memcpy(&userAuth.tbName, pName, sizeof(SName));
taosArrayPush(*pUserAuth, &userAuth);
return TSDB_CODE_SUCCESS;

65
source/libs/parser/src/parTranslater.c
View File

@ -978,7 +978,8 @@ static EDealRes translateColumnUseAlias(STranslateContext* pCxt, SColumnNode** p
}
static EDealRes translateColumn(STranslateContext* pCxt, SColumnNode** pCol) {
if (NULL == pCxt->pCurrStmt || isSelectStmt(pCxt->pCurrStmt) && NULL == ((SSelectStmt*)pCxt->pCurrStmt)->pFromTable) {
if (NULL == pCxt->pCurrStmt ||
(isSelectStmt(pCxt->pCurrStmt) && NULL == ((SSelectStmt*)pCxt->pCurrStmt)->pFromTable)) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_COLUMN, (*pCol)->colName);
}
@ -6474,22 +6475,69 @@ static int32_t translateDropFunction(STranslateContext* pCxt, SDropFunctionStmt*
return buildCmdMsg(pCxt, TDMT_MND_DROP_FUNC, (FSerializeFunc)tSerializeSDropFuncReq, &req);
}
static int32_t createRealTableForGrantTable(SGrantStmt* pStmt, SRealTableNode** pTable) {
SRealTableNode* pRealTable = (SRealTableNode*)nodesMakeNode(QUERY_NODE_REAL_TABLE);
if (NULL == pRealTable) {
return TSDB_CODE_OUT_OF_MEMORY;
}
strcpy(pRealTable->table.dbName, pStmt->objName);
strcpy(pRealTable->table.tableName, pStmt->tabName);
strcpy(pRealTable->table.tableAlias, pStmt->tabName);
*pTable = pRealTable;
return TSDB_CODE_SUCCESS;
}
static int32_t translateGrantTagCond(STranslateContext* pCxt, SGrantStmt* pStmt, SAlterUserReq* pReq) {
if (NULL == pStmt->pTagCond) {
return TSDB_CODE_SUCCESS;
}
if ('\0' == pStmt->tabName[0] || '*' == pStmt->tabName[0]) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR,
"The With clause can only be used for table level privilege");
}
pCxt->pCurrStmt = (SNode*)pStmt;
SRealTableNode* pTable = NULL;
int32_t code = createRealTableForGrantTable(pStmt, &pTable);
if (TSDB_CODE_SUCCESS == code) {
SName name;
code = getTableMetaImpl(pCxt, toName(pCxt->pParseCxt->acctId, pTable->table.dbName, pTable->table.tableName, &name),
&(pTable->pMeta));
}
if (TSDB_CODE_SUCCESS == code) {
code = addNamespace(pCxt, pTable);
}
if (TSDB_CODE_SUCCESS == code) {
code = translateExpr(pCxt, &pStmt->pTagCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = nodesNodeToString(pStmt->pTagCond, false, &pReq->tagCond, &pReq->tagCondLen);
}
nodesDestroyNode((SNode*)pTable);
return code;
}
static int32_t translateGrant(STranslateContext* pCxt, SGrantStmt* pStmt) {
SAlterUserReq req = {0};
if (BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_ALL) ||
(BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_READ) &&
BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_WRITE))) {
req.alterType = TSDB_ALTER_USER_ADD_ALL_DB;
req.alterType = ('\0' == pStmt->tabName[0] ? TSDB_ALTER_USER_ADD_ALL_DB : TSDB_ALTER_USER_ADD_ALL_TABLE);
} else if (BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_READ)) {
req.alterType = TSDB_ALTER_USER_ADD_READ_DB;
req.alterType = ('\0' == pStmt->tabName[0] ? TSDB_ALTER_USER_ADD_READ_DB : TSDB_ALTER_USER_ADD_READ_TABLE);
} else if (BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_WRITE)) {
req.alterType = TSDB_ALTER_USER_ADD_WRITE_DB;
req.alterType = ('\0' == pStmt->tabName[0] ? TSDB_ALTER_USER_ADD_WRITE_DB : TSDB_ALTER_USER_ADD_WRITE_TABLE);
} else if (BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_SUBSCRIBE)) {
req.alterType = TSDB_ALTER_USER_ADD_SUBSCRIBE_TOPIC;
}
strcpy(req.user, pStmt->userName);
sprintf(req.objname, "%d.%s", pCxt->pParseCxt->acctId, pStmt->objName);
return buildCmdMsg(pCxt, TDMT_MND_ALTER_USER, (FSerializeFunc)tSerializeSAlterUserReq, &req);
sprintf(req.tabName, "%s", pStmt->tabName);
int32_t code = translateGrantTagCond(pCxt, pStmt, &req);
if (TSDB_CODE_SUCCESS == code) {
code = buildCmdMsg(pCxt, TDMT_MND_ALTER_USER, (FSerializeFunc)tSerializeSAlterUserReq, &req);
}
return code;
}
static int32_t translateRevoke(STranslateContext* pCxt, SRevokeStmt* pStmt) {
@ -6497,16 +6545,17 @@ static int32_t translateRevoke(STranslateContext* pCxt, SRevokeStmt* pStmt) {
if (BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_ALL) ||
(BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_READ) &&
BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_WRITE))) {
req.alterType = TSDB_ALTER_USER_REMOVE_ALL_DB;
req.alterType = ('\0' == pStmt->tabName[0] ? TSDB_ALTER_USER_REMOVE_ALL_DB : TSDB_ALTER_USER_REMOVE_ALL_TABLE);
} else if (BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_READ)) {
req.alterType = TSDB_ALTER_USER_REMOVE_READ_DB;
req.alterType = ('\0' == pStmt->tabName[0] ? TSDB_ALTER_USER_REMOVE_READ_DB : TSDB_ALTER_USER_REMOVE_READ_TABLE);
} else if (BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_WRITE)) {
req.alterType = TSDB_ALTER_USER_REMOVE_WRITE_DB;
req.alterType = ('\0' == pStmt->tabName[0] ? TSDB_ALTER_USER_REMOVE_WRITE_DB : TSDB_ALTER_USER_REMOVE_WRITE_TABLE);
} else if (BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_SUBSCRIBE)) {
req.alterType = TSDB_ALTER_USER_REMOVE_SUBSCRIBE_TOPIC;
}
strcpy(req.user, pStmt->userName);
sprintf(req.objname, "%d.%s", pCxt->pParseCxt->acctId, pStmt->objName);
sprintf(req.tabName, "%s", pStmt->tabName);
return buildCmdMsg(pCxt, TDMT_MND_ALTER_USER, (FSerializeFunc)tSerializeSAlterUserReq, &req);
}

88
source/libs/parser/src/parUtil.c
View File

@ -17,7 +17,7 @@
#include "cJSON.h"
#include "querynodes.h"
#define USER_AUTH_KEY_MAX_LEN TSDB_USER_LEN + TSDB_DB_FNAME_LEN + 2
#define USER_AUTH_KEY_MAX_LEN TSDB_USER_LEN + TSDB_TABLE_FNAME_LEN + 2
const void* nullPointer = NULL;
@ -496,24 +496,44 @@ int32_t getVnodeSysTableTargetName(int32_t acctId, SNode* pWhere, SName* pName)
return TSDB_CODE_SUCCESS;
}
static int32_t userAuthToString(int32_t acctId, const char* pUser, const char* pDb, AUTH_TYPE type, char* pStr) {
return sprintf(pStr, "%s*%d.%s*%d", pUser, acctId, pDb, type);
static int32_t userAuthToString(int32_t acctId, const char* pUser, const char* pDb, const char* pTable, AUTH_TYPE type,
char* pStr) {
return sprintf(pStr, "%s*%d*%s*%s*%d", pUser, acctId, pDb, (NULL != pTable && '\0' == pTable[0]) ? NULL : pTable,
type);
}
static int32_t userAuthToStringExt(const char* pUser, const char* pDbFName, AUTH_TYPE type, char* pStr) {
return sprintf(pStr, "%s*%s*%d", pUser, pDbFName, type);
static int32_t getIntegerFromAuthStr(const char* pStart, char** pNext) {
char* p = strchr(pStart, '*');
char buf[10] = {0};
if (NULL == p) {
strcpy(buf, pStart);
*pNext = NULL;
} else {
strncpy(buf, pStart, p - pStart);
*pNext = ++p;
}
return taosStr2Int32(buf, NULL, 10);
}
static void getStringFromAuthStr(const char* pStart, char* pStr, char** pNext) {
char* p = strchr(pStart, '*');
if (NULL == p) {
strcpy(pStr, pStart);
*pNext = NULL;
} else {
strncpy(pStr, pStart, p - pStart);
*pNext = ++p;
}
}
static void stringToUserAuth(const char* pStr, int32_t len, SUserAuthInfo* pUserAuth) {
char* p1 = strchr(pStr, '*');
strncpy(pUserAuth->user, pStr, p1 - pStr);
++p1;
char* p2 = strchr(p1, '*');
strncpy(pUserAuth->dbFName, p1, p2 - p1);
++p2;
char buf[10] = {0};
strncpy(buf, p2, len - (p2 - pStr));
pUserAuth->type = taosStr2Int32(buf, NULL, 10);
char* p = NULL;
getStringFromAuthStr(pStr, pUserAuth->user, &p);
pUserAuth->tbName.acctId = getIntegerFromAuthStr(p, &p);
getStringFromAuthStr(p, pUserAuth->tbName.dbname, &p);
getStringFromAuthStr(p, pUserAuth->tbName.tname, &p);
pUserAuth->tbName.type = TSDB_TABLE_NAME_T;
pUserAuth->type = getIntegerFromAuthStr(p, &p);
}
static int32_t buildTableReq(SHashObj* pTablesHash, SArray** pTables) {
@ -584,10 +604,12 @@ static int32_t buildUserAuthReq(SHashObj* pUserAuthHash, SArray** pUserAuth) {
}
void* p = taosHashIterate(pUserAuthHash, NULL);
while (NULL != p) {
size_t len = 0;
char* pKey = taosHashGetKey(p, &len);
size_t len = 0;
char* pKey = taosHashGetKey(p, &len);
char key[USER_AUTH_KEY_MAX_LEN] = {0};
strncpy(key, pKey, len);
SUserAuthInfo userAuth = {0};
stringToUserAuth(pKey, len, &userAuth);
stringToUserAuth(key, len, &userAuth);
taosArrayPush(*pUserAuth, &userAuth);
p = taosHashIterate(pUserAuthHash, p);
}
@ -712,7 +734,8 @@ static int32_t putUserAuthToCache(const SArray* pUserAuthReq, const SArray* pUse
for (int32_t i = 0; i < nvgs; ++i) {
SUserAuthInfo* pUser = taosArrayGet(pUserAuthReq, i);
char key[USER_AUTH_KEY_MAX_LEN] = {0};
int32_t len = userAuthToStringExt(pUser->user, pUser->dbFName, pUser->type, key);
int32_t len = userAuthToString(pUser->tbName.acctId, pUser->user, pUser->tbName.dbname, pUser->tbName.tname,
pUser->type, key);
if (TSDB_CODE_SUCCESS != putMetaDataToHash(key, len, pUserAuthData, i, pUserAuth)) {
return TSDB_CODE_OUT_OF_MEMORY;
}
@ -910,33 +933,24 @@ static int32_t reserveUserAuthInCacheImpl(const char* pKey, int32_t len, SParseM
return TSDB_CODE_OUT_OF_MEMORY;
}
}
bool pass = false;
return taosHashPut(pMetaCache->pUserAuth, pKey, len, &pass, sizeof(pass));
return taosHashPut(pMetaCache->pUserAuth, pKey, len, &nullPointer, POINTER_BYTES);
}
int32_t reserveUserAuthInCache(int32_t acctId, const char* pUser, const char* pDb, AUTH_TYPE type,
int32_t reserveUserAuthInCache(int32_t acctId, const char* pUser, const char* pDb, const char* pTable, AUTH_TYPE type,
SParseMetaCache* pMetaCache) {
char key[USER_AUTH_KEY_MAX_LEN] = {0};
int32_t len = userAuthToString(acctId, pUser, pDb, type, key);
int32_t len = userAuthToString(acctId, pUser, pDb, pTable, type, key);
return reserveUserAuthInCacheImpl(key, len, pMetaCache);
}
int32_t reserveUserAuthInCacheExt(const char* pUser, const SName* pName, AUTH_TYPE type, SParseMetaCache* pMetaCache) {
char dbFName[TSDB_DB_FNAME_LEN] = {0};
tNameGetFullDbName(pName, dbFName);
char key[USER_AUTH_KEY_MAX_LEN] = {0};
int32_t len = userAuthToStringExt(pUser, dbFName, type, key);
return reserveUserAuthInCacheImpl(key, len, pMetaCache);
}
int32_t getUserAuthFromCache(SParseMetaCache* pMetaCache, const char* pUser, const char* pDbFName, AUTH_TYPE type,
bool* pPass) {
char key[USER_AUTH_KEY_MAX_LEN] = {0};
int32_t len = userAuthToStringExt(pUser, pDbFName, type, key);
bool* pRes = NULL;
int32_t code = getMetaDataFromHash(key, len, pMetaCache->pUserAuth, (void**)&pRes);
int32_t getUserAuthFromCache(SParseMetaCache* pMetaCache, SUserAuthInfo* pAuthReq, SUserAuthRes* pAuthRes) {
char key[USER_AUTH_KEY_MAX_LEN] = {0};
int32_t len = userAuthToString(pAuthReq->tbName.acctId, pAuthReq->user, pAuthReq->tbName.dbname,
pAuthReq->tbName.tname, pAuthReq->type, key);
SUserAuthRes* pAuth = NULL;
int32_t code = getMetaDataFromHash(key, len, pMetaCache->pUserAuth, (void**)&pAuth);
if (TSDB_CODE_SUCCESS == code) {
*pPass = *pRes;
memcpy(pAuthRes, pAuth, sizeof(SUserAuthRes));
}
return code;
}

6503
source/libs/parser/src/sql.c
View File

File diff suppressed because it is too large Load Diff

10
source/libs/parser/test/mockCatalog.cpp
View File

@ -279,15 +279,13 @@ int32_t __catalogGetDBCfg(SCatalog* pCtg, SRequestConnInfo* pConn, const char* d
return g_mockCatalogService->catalogGetDBCfg(dbFName, pDbCfg);
}
int32_t __catalogChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, const char* user, const char* dbFName, AUTH_TYPE type,
bool* pass) {
*pass = true;
int32_t __catalogChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, SUserAuthInfo *pAuth, SUserAuthRes* pRes) {
pRes->pass = true;
return 0;
}
int32_t __catalogChkAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFName, AUTH_TYPE type, bool* pass,
bool* exists) {
*pass = true;
int32_t __catalogChkAuthFromCache(SCatalog* pCtg, SUserAuthInfo *pAuth, SUserAuthRes* pRes, bool* exists) {
pRes->pass = true;
*exists = true;
return 0;
}

4
source/libs/parser/test/mockCatalogService.cpp
View File

@ -589,8 +589,8 @@ class MockCatalogServiceImpl {
*pUserAuthData = taosArrayInit(num, sizeof(SMetaRes));
for (int32_t i = 0; i < num; ++i) {
SMetaRes res = {0};
res.pRes = taosMemoryCalloc(1, sizeof(bool));
*(bool*)(res.pRes) = true;
res.pRes = taosMemoryCalloc(1, sizeof(SUserAuthRes));
((SUserAuthRes*)res.pRes)->pass = true;
taosArrayPush(*pUserAuthData, &res);
}
}

12
source/libs/planner/src/planOptimizer.c
View File

@ -1079,11 +1079,23 @@ static bool sortPriKeyOptMayBeOptimized(SLogicNode* pNode) {
if (!sortPriKeyOptIsPriKeyOrderBy(pSort->pSortKeys) || 1 != LIST_LENGTH(pSort->node.pChildren)) {
return false;
}
SNode* pChild;
FOREACH(pChild, pSort->node.pChildren) {
SLogicNode* pSortDescendent = optFindPossibleNode((SLogicNode*)pChild, sortPriKeyOptMayBeOptimized);
if (pSortDescendent != NULL) {
return false;
}
}
return true;
}
static int32_t sortPriKeyOptGetSequencingNodesImpl(SLogicNode* pNode, bool groupSort, bool* pNotOptimize,
SNodeList** pSequencingNodes) {
if (NULL != pNode->pLimit || NULL != pNode->pSlimit) {
*pNotOptimize = false;
return TSDB_CODE_SUCCESS;
}
switch (nodeType(pNode)) {
case QUERY_NODE_LOGIC_PLAN_SCAN: {
SScanLogicNode* pScan = (SScanLogicNode*)pNode;

5
source/libs/scheduler/inc/schInt.h
View File

@ -54,7 +54,6 @@ typedef enum {
#define SCHEDULE_DEFAULT_MAX_JOB_NUM 1000
#define SCHEDULE_DEFAULT_MAX_TASK_NUM 1000
#define SCHEDULE_DEFAULT_MAX_NODE_TABLE_NUM 200 // unit is TSDB_TABLE_NUM_UNIT
#define SCHEDULE_DEFAULT_POLICY SCH_LOAD_SEQ
#define SCHEDULE_DEFAULT_MAX_NODE_NUM 20
@ -134,7 +133,7 @@ typedef struct SSchStatusFps {
typedef struct SSchedulerCfg {
uint32_t maxJobNum;
int32_t maxNodeTableNum;
int64_t maxNodeTableNum;
SCH_POLICY schPolicy;
bool enableReSchedule;
} SSchedulerCfg;
@ -175,7 +174,7 @@ typedef struct SSchHbCallbackParam {
typedef struct SSchFlowControl {
SRWLatch lock;
bool sorted;
int32_t tableNumSum;
int64_t tableNumSum;
uint32_t execTaskNum;
SArray *taskList; // Element is SSchTask*
} SSchFlowControl;

24
source/libs/scheduler/src/schFlowCtrl.c
View File

@ -46,7 +46,7 @@ int32_t schChkJobNeedFlowCtrl(SSchJob *pJob, SSchLevel *pLevel) {
return TSDB_CODE_SUCCESS;
}
int32_t sum = 0;
int64_t sum = 0;
int32_t taskNum = taosArrayGetSize(pJob->dataSrcTasks);
for (int32_t i = 0; i < taskNum; ++i) {
SSchTask *pTask = *(SSchTask **)taosArrayGet(pJob->dataSrcTasks, i);
@ -55,7 +55,7 @@ int32_t schChkJobNeedFlowCtrl(SSchJob *pJob, SSchLevel *pLevel) {
}
if (schMgmt.cfg.maxNodeTableNum <= 0 || sum < schMgmt.cfg.maxNodeTableNum) {
SCH_JOB_DLOG("job no need flow ctrl, totalTableNum:%d", sum);
SCH_JOB_DLOG("job no need flow ctrl, totalTableNum:%" PRId64, sum);
return TSDB_CODE_SUCCESS;
}
@ -68,7 +68,7 @@ int32_t schChkJobNeedFlowCtrl(SSchJob *pJob, SSchLevel *pLevel) {
SCH_SET_JOB_NEED_FLOW_CTRL(pJob);
SCH_JOB_DLOG("job NEED flow ctrl, totalTableNum:%d", sum);
SCH_JOB_DLOG("job NEED flow ctrl, totalTableNum:%" PRId64, sum);
return TSDB_CODE_SUCCESS;
}
@ -94,7 +94,7 @@ int32_t schDecTaskFlowQuota(SSchJob *pJob, SSchTask *pTask) {
--ctrl->execTaskNum;
ctrl->tableNumSum -= pTask->plan->execNodeStat.tableNum;
SCH_TASK_DLOG("task quota removed, fqdn:%s, port:%d, tableNum:%d, remainNum:%d, remainExecTaskNum:%d", ep->fqdn,
SCH_TASK_DLOG("task quota removed, fqdn:%s, port:%d, tableNum:%d, remainNum:%" PRId64 ", remainExecTaskNum:%d", ep->fqdn,
ep->port, pTask->plan->execNodeStat.tableNum, ctrl->tableNumSum, ctrl->execTaskNum);
_return:
@ -125,7 +125,7 @@ int32_t schCheckIncTaskFlowQuota(SSchJob *pJob, SSchTask *pTask, bool *enough) {
SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
SCH_TASK_DLOG("task quota added, fqdn:%s, port:%d, tableNum:%d, remainNum:%d, remainExecTaskNum:%d", ep->fqdn,
SCH_TASK_DLOG("task quota added, fqdn:%s, port:%d, tableNum:%d, remainNum:%" PRId64 ", remainExecTaskNum:%d", ep->fqdn,
ep->port, pTask->plan->execNodeStat.tableNum, nctrl.tableNumSum, nctrl.execTaskNum);
*enough = true;
@ -142,7 +142,7 @@ int32_t schCheckIncTaskFlowQuota(SSchJob *pJob, SSchTask *pTask, bool *enough) {
break;
}
int32_t sum = pTask->plan->execNodeStat.tableNum + ctrl->tableNumSum;
int64_t sum = pTask->plan->execNodeStat.tableNum + ctrl->tableNumSum;
if (sum <= schMgmt.cfg.maxNodeTableNum) {
ctrl->tableNumSum = sum;
@ -173,7 +173,7 @@ int32_t schCheckIncTaskFlowQuota(SSchJob *pJob, SSchTask *pTask, bool *enough) {
_return:
SCH_TASK_DLOG("task quota %s added, fqdn:%s, port:%d, tableNum:%d, remainNum:%d, remainExecTaskNum:%d",
SCH_TASK_DLOG("task quota %s added, fqdn:%s, port:%d, tableNum:%d, remainNum:%" PRId64 ", remainExecTaskNum:%d",
((*enough) ? "" : "NOT"), ep->fqdn, ep->port, pTask->plan->execNodeStat.tableNum, ctrl->tableNumSum,
ctrl->execTaskNum);
@ -203,7 +203,7 @@ int32_t schLaunchTasksInFlowCtrlListImpl(SSchJob *pJob, SSchFlowControl *ctrl) {
return TSDB_CODE_SUCCESS;
}
int32_t remainNum = schMgmt.cfg.maxNodeTableNum - ctrl->tableNumSum;
int64_t remainNum = schMgmt.cfg.maxNodeTableNum - ctrl->tableNumSum;
int32_t taskNum = taosArrayGetSize(ctrl->taskList);
int32_t code = 0;
SSchTask *pTask = NULL;
@ -217,7 +217,7 @@ int32_t schLaunchTasksInFlowCtrlListImpl(SSchJob *pJob, SSchFlowControl *ctrl) {
SEp *ep = SCH_GET_CUR_EP(&pTask->plan->execNode);
if (pTask->plan->execNodeStat.tableNum > remainNum && ctrl->execTaskNum > 0) {
SCH_TASK_DLOG("task NOT to launch, fqdn:%s, port:%d, tableNum:%d, remainNum:%d, remainExecTaskNum:%d", ep->fqdn,
SCH_TASK_DLOG("task NOT to launch, fqdn:%s, port:%d, tableNum:%d, remainNum:%" PRId64 ", remainExecTaskNum:%d", ep->fqdn,
ep->port, pTask->plan->execNodeStat.tableNum, ctrl->tableNumSum, ctrl->execTaskNum);
continue;
@ -228,14 +228,14 @@ int32_t schLaunchTasksInFlowCtrlListImpl(SSchJob *pJob, SSchFlowControl *ctrl) {
taosArrayRemove(ctrl->taskList, i);
SCH_TASK_DLOG("task to launch, fqdn:%s, port:%d, tableNum:%d, remainNum:%d, remainExecTaskNum:%d", ep->fqdn,
SCH_TASK_DLOG("task to launch, fqdn:%s, port:%d, tableNum:%d, remainNum:%" PRId64 ", remainExecTaskNum:%d", ep->fqdn,
ep->port, pTask->plan->execNodeStat.tableNum, ctrl->tableNumSum, ctrl->execTaskNum);
SCH_ERR_JRET(schAsyncLaunchTaskImpl(pJob, pTask));
remainNum -= pTask->plan->execNodeStat.tableNum;
if (remainNum <= 0) {
SCH_TASK_DLOG("no more task to launch, fqdn:%s, port:%d, remainNum:%d, remainExecTaskNum:%d", ep->fqdn, ep->port,
SCH_TASK_DLOG("no more task to launch, fqdn:%s, port:%d, remainNum:%" PRId64 ", remainExecTaskNum:%d", ep->fqdn, ep->port,
ctrl->tableNumSum, ctrl->execTaskNum);
break;
@ -244,7 +244,7 @@ int32_t schLaunchTasksInFlowCtrlListImpl(SSchJob *pJob, SSchFlowControl *ctrl) {
if (i < (taskNum - 1)) {
SSchTask *pLastTask = *(SSchTask **)taosArrayGetLast(ctrl->taskList);
if (remainNum < pLastTask->plan->execNodeStat.tableNum) {
SCH_TASK_DLOG("no more task to launch, fqdn:%s, port:%d, remainNum:%d, remainExecTaskNum:%d, smallestInList:%d",
SCH_TASK_DLOG("no more task to launch, fqdn:%s, port:%d, remainNum:%" PRId64 ", remainExecTaskNum:%d, smallestInList:%d",
ep->fqdn, ep->port, ctrl->tableNumSum, ctrl->execTaskNum, pLastTask->plan->execNodeStat.tableNum);
break;

6
source/libs/scheduler/src/scheduler.c
View File

@ -18,6 +18,7 @@
#include "schInt.h"
#include "tmsg.h"
#include "tref.h"
#include "tglobal.h"
SSchedulerMgmt schMgmt = {
.jobRef = -1,
@ -30,11 +31,12 @@ int32_t schedulerInit() {
}
schMgmt.cfg.maxJobNum = SCHEDULE_DEFAULT_MAX_JOB_NUM;
schMgmt.cfg.maxNodeTableNum = SCHEDULE_DEFAULT_MAX_NODE_TABLE_NUM;
schMgmt.cfg.maxNodeTableNum = tsQueryMaxConcurrentTables;
schMgmt.cfg.schPolicy = SCHEDULE_DEFAULT_POLICY;
schMgmt.cfg.enableReSchedule = true;
qDebug("schedule policy init to %d", schMgmt.cfg.schPolicy);
qDebug("schedule init, policy: %d, maxNodeTableNum: %" PRId64", reSchedule:%d",
schMgmt.cfg.schPolicy, schMgmt.cfg.maxNodeTableNum, schMgmt.cfg.enableReSchedule);
schMgmt.jobRef = taosOpenRef(schMgmt.cfg.maxJobNum, schFreeJobImpl);
if (schMgmt.jobRef < 0) {

14
source/libs/sync/inc/syncPipeline.h
View File

@ -77,18 +77,19 @@ static FORCE_INLINE int32_t syncLogReplGetNextRetryBackoff(SSyncLogReplMgr* pMgr
SyncTerm syncLogReplGetPrevLogTerm(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index);
int32_t syncLogReplReplicateOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode);
int32_t syncLogReplReplicateOneTo(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index, SyncTerm* pTerm,
SRaftId* pDestId, bool* pBarrier);
int32_t syncLogReplReplicateAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode);
int32_t syncLogReplReplicateProbe(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index);
int32_t syncLogReplDoOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode);
int32_t syncLogReplAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode);
int32_t syncLogReplProbe(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index);
int32_t syncLogReplRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode);
int32_t syncLogReplSendTo(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index, SyncTerm* pTerm, SRaftId* pDestId,
bool* pBarrier);
int32_t syncLogReplProcessReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg);
int32_t syncLogReplProcessReplyAsRecovery(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg);
int32_t syncLogReplProcessReplyAsNormal(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg);
int32_t syncLogReplProcessHeartbeatReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncHeartbeatReply* pMsg);
int32_t syncLogReplRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode);
// SSyncLogBuffer
SSyncLogBuffer* syncLogBufferCreate();
@ -100,6 +101,7 @@ int32_t syncLogBufferReInit(SSyncLogBuffer* pBuf, SSyncNode* pNode);
int64_t syncLogBufferGetEndIndex(SSyncLogBuffer* pBuf);
SyncTerm syncLogBufferGetLastMatchTerm(SSyncLogBuffer* pBuf);
bool syncLogBufferIsEmpty(SSyncLogBuffer* pBuf);
int32_t syncLogBufferAppend(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEntry* pEntry);
int32_t syncLogBufferAccept(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEntry* pEntry, SyncTerm prevTerm);
int64_t syncLogBufferProceed(SSyncLogBuffer* pBuf, SSyncNode* pNode, SyncTerm* pMatchTerm);

2
source/libs/sync/inc/syncRaftEntry.h
View File

@ -45,7 +45,7 @@ SSyncRaftEntry* syncEntryBuildNoop(SyncTerm term, SyncIndex index, int32_t vgId)
void syncEntryDestroy(SSyncRaftEntry* pEntry);
void syncEntry2OriginalRpc(const SSyncRaftEntry* pEntry, SRpcMsg* pRpcMsg); // step 7
static FORCE_INLINE bool syncLogIsReplicationBarrier(SSyncRaftEntry* pEntry) {
static FORCE_INLINE bool syncLogReplBarrier(SSyncRaftEntry* pEntry) {
return pEntry->originalRpcType == TDMT_SYNC_NOOP;
}

2
source/libs/sync/src/syncMain.c
View File

@ -599,7 +599,7 @@ int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak, int64_
sNTrace(pSyncNode, "propose msg, type:%s", TMSG_INFO(pMsg->msgType));
code = (*pSyncNode->syncEqMsg)(pSyncNode->msgcb, &rpcMsg);
if (code != 0) {
sError("vgId:%d, failed to propose msg while enqueue since %s", pSyncNode->vgId, terrstr());
sWarn("vgId:%d, failed to propose msg while enqueue since %s", pSyncNode->vgId, terrstr());
(void)syncRespMgrDel(pSyncNode->pSyncRespMgr, seqNum);
}

45
source/libs/sync/src/syncPipeline.c
View File

@ -633,7 +633,7 @@ int32_t syncLogReplRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode) {
SRaftId* pDestId = &pNode->replicasId[pMgr->peerId];
if (pMgr->retryBackoff == SYNC_MAX_RETRY_BACKOFF) {
syncLogReplReset(pMgr);
sWarn("vgId:%d, reset sync log repl mgr since retry backoff exceeding limit. peer:%" PRIx64, pNode->vgId,
sWarn("vgId:%d, reset sync log repl since retry backoff exceeding limit. peer:%" PRIx64, pNode->vgId,
pDestId->addr);
return -1;
}
@ -658,15 +658,15 @@ int32_t syncLogReplRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode) {
if (pMgr->states[pos].acked) {
if (pMgr->matchIndex < index && pMgr->states[pos].timeMs + (syncGetRetryMaxWaitMs() << 3) < nowMs) {
syncLogReplReset(pMgr);
sWarn("vgId:%d, reset sync log repl mgr since stagnation. index:%" PRId64 ", peer:%" PRIx64, pNode->vgId,
index, pDestId->addr);
sWarn("vgId:%d, reset sync log repl since stagnation. index:%" PRId64 ", peer:%" PRIx64, pNode->vgId, index,
pDestId->addr);
goto _out;
}
continue;
}
bool barrier = false;
if (syncLogReplReplicateOneTo(pMgr, pNode, index, &term, pDestId, &barrier) < 0) {
if (syncLogReplSendTo(pMgr, pNode, index, &term, pDestId, &barrier) < 0) {
sError("vgId:%d, failed to replicate sync log entry since %s. index:%" PRId64 ", dest:%" PRIx64 "", pNode->vgId,
terrstr(), index, pDestId->addr);
goto _out;
@ -708,7 +708,7 @@ int32_t syncLogReplProcessReplyAsRecovery(SSyncLogReplMgr* pMgr, SSyncNode* pNod
ASSERT(pMgr->matchIndex == 0);
if (pMsg->matchIndex < 0) {
pMgr->restored = true;
sInfo("vgId:%d, sync log repl mgr restored. peer: dnode:%d (%" PRIx64 "), mgr: rs(%d) [%" PRId64 " %" PRId64
sInfo("vgId:%d, sync log repl restored. peer: dnode:%d (%" PRIx64 "), mgr: rs(%d) [%" PRId64 " %" PRId64
", %" PRId64 "), buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")",
pNode->vgId, DID(&destId), destId.addr, pMgr->restored, pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex,
pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex);
@ -725,7 +725,7 @@ int32_t syncLogReplProcessReplyAsRecovery(SSyncLogReplMgr* pMgr, SSyncNode* pNod
if (pMsg->success && pMsg->matchIndex == pMsg->lastSendIndex) {
pMgr->matchIndex = pMsg->matchIndex;
pMgr->restored = true;
sInfo("vgId:%d, sync log repl mgr restored. peer: dnode:%d (%" PRIx64 "), mgr: rs(%d) [%" PRId64 " %" PRId64
sInfo("vgId:%d, sync log repl restored. peer: dnode:%d (%" PRIx64 "), mgr: rs(%d) [%" PRId64 " %" PRId64
", %" PRId64 "), buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")",
pNode->vgId, DID(&destId), destId.addr, pMgr->restored, pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex,
pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex);
@ -774,14 +774,14 @@ int32_t syncLogReplProcessReplyAsRecovery(SSyncLogReplMgr* pMgr, SSyncNode* pNod
// attempt to replicate the raft log at index
(void)syncLogReplReset(pMgr);
return syncLogReplReplicateProbe(pMgr, pNode, index);
return syncLogReplProbe(pMgr, pNode, index);
}
int32_t syncLogReplProcessHeartbeatReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncHeartbeatReply* pMsg) {
SSyncLogBuffer* pBuf = pNode->pLogBuf;
taosThreadMutexLock(&pBuf->mutex);
if (pMsg->startTime != 0 && pMsg->startTime != pMgr->peerStartTime) {
sInfo("vgId:%d, reset sync log repl mgr in heartbeat. peer:%" PRIx64 ", start time:%" PRId64 ", old:%" PRId64 "",
sInfo("vgId:%d, reset sync log repl in heartbeat. peer:%" PRIx64 ", start time:%" PRId64 ", old:%" PRId64 "",
pNode->vgId, pMsg->srcId.addr, pMsg->startTime, pMgr->peerStartTime);
syncLogReplReset(pMgr);
pMgr->peerStartTime = pMsg->startTime;
@ -794,8 +794,7 @@ int32_t syncLogReplProcessReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncApp
SSyncLogBuffer* pBuf = pNode->pLogBuf;
taosThreadMutexLock(&pBuf->mutex);
if (pMsg->startTime != pMgr->peerStartTime) {
sInfo("vgId:%d, reset sync log repl mgr in appendlog reply. peer:%" PRIx64 ", start time:%" PRId64
", old:%" PRId64,
sInfo("vgId:%d, reset sync log repl in appendlog reply. peer:%" PRIx64 ", start time:%" PRId64 ", old:%" PRId64,
pNode->vgId, pMsg->srcId.addr, pMsg->startTime, pMgr->peerStartTime);
syncLogReplReset(pMgr);
pMgr->peerStartTime = pMsg->startTime;
@ -810,16 +809,16 @@ int32_t syncLogReplProcessReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncApp
return 0;
}
int32_t syncLogReplReplicateOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode) {
int32_t syncLogReplDoOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode) {
if (pMgr->restored) {
(void)syncLogReplReplicateAttempt(pMgr, pNode);
(void)syncLogReplAttempt(pMgr, pNode);
} else {
(void)syncLogReplReplicateProbe(pMgr, pNode, pNode->pLogBuf->matchIndex);
(void)syncLogReplProbe(pMgr, pNode, pNode->pLogBuf->matchIndex);
}
return 0;
}
int32_t syncLogReplReplicateProbe(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index) {
int32_t syncLogReplProbe(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index) {
ASSERT(!pMgr->restored);
ASSERT(pMgr->startIndex >= 0);
int64_t retryMaxWaitMs = syncGetRetryMaxWaitMs();
@ -834,7 +833,7 @@ int32_t syncLogReplReplicateProbe(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncI
SRaftId* pDestId = &pNode->replicasId[pMgr->peerId];
bool barrier = false;
SyncTerm term = -1;
if (syncLogReplReplicateOneTo(pMgr, pNode, index, &term, pDestId, &barrier) < 0) {
if (syncLogReplSendTo(pMgr, pNode, index, &term, pDestId, &barrier) < 0) {
sError("vgId:%d, failed to replicate log entry since %s. index:%" PRId64 ", dest: 0x%016" PRIx64 "", pNode->vgId,
terrstr(), index, pDestId->addr);
return -1;
@ -857,7 +856,7 @@ int32_t syncLogReplReplicateProbe(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncI
return 0;
}
int32_t syncLogReplReplicateAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode) {
int32_t syncLogReplAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode) {
ASSERT(pMgr->restored);
SRaftId* pDestId = &pNode->replicasId[pMgr->peerId];
@ -879,7 +878,7 @@ int32_t syncLogReplReplicateAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode) {
SRaftId* pDestId = &pNode->replicasId[pMgr->peerId];
bool barrier = false;
SyncTerm term = -1;
if (syncLogReplReplicateOneTo(pMgr, pNode, index, &term, pDestId, &barrier) < 0) {
if (syncLogReplSendTo(pMgr, pNode, index, &term, pDestId, &barrier) < 0) {
sError("vgId:%d, failed to replicate log entry since %s. index:%" PRId64 ", dest: 0x%016" PRIx64 "", pNode->vgId,
terrstr(), index, pDestId->addr);
return -1;
@ -932,7 +931,7 @@ int32_t syncLogReplProcessReplyAsNormal(SSyncLogReplMgr* pMgr, SSyncNode* pNode,
pMgr->startIndex = pMgr->matchIndex;
}
return syncLogReplReplicateAttempt(pMgr, pNode);
return syncLogReplAttempt(pMgr, pNode);
}
SSyncLogReplMgr* syncLogReplCreate() {
@ -1127,8 +1126,8 @@ SSyncRaftEntry* syncLogBufferGetOneEntry(SSyncLogBuffer* pBuf, SSyncNode* pNode,
return pEntry;
}
int32_t syncLogReplReplicateOneTo(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index, SyncTerm* pTerm,
SRaftId* pDestId, bool* pBarrier) {
int32_t syncLogReplSendTo(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index, SyncTerm* pTerm, SRaftId* pDestId,
bool* pBarrier) {
SSyncRaftEntry* pEntry = NULL;
SRpcMsg msgOut = {0};
bool inBuf = false;
@ -1141,14 +1140,14 @@ int32_t syncLogReplReplicateOneTo(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncI
if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST) {
SSyncLogReplMgr* pMgr = syncNodeGetLogReplMgr(pNode, pDestId);
if (pMgr) {
sInfo("vgId:%d, reset sync log repl mgr of peer:%" PRIx64 " since %s. index:%" PRId64, pNode->vgId,
pDestId->addr, terrstr(), index);
sInfo("vgId:%d, reset sync log repl of peer:%" PRIx64 " since %s. index:%" PRId64, pNode->vgId, pDestId->addr,
terrstr(), index);
(void)syncLogReplReset(pMgr);
}
}
goto _err;
}
*pBarrier = syncLogIsReplicationBarrier(pEntry);
*pBarrier = syncLogReplBarrier(pEntry);
prevLogTerm = syncLogReplGetPrevLogTerm(pMgr, pNode, index);
if (prevLogTerm < 0) {

2
source/libs/sync/src/syncReplication.c
View File

@ -74,7 +74,7 @@ int32_t syncNodeReplicateWithoutLock(SSyncNode* pNode) {
continue;
}
SSyncLogReplMgr* pMgr = pNode->logReplMgrs[i];
(void)syncLogReplReplicateOnce(pMgr, pNode);
(void)syncLogReplDoOnce(pMgr, pNode);
}
return 0;
}

10
source/libs/wal/src/walRead.c
View File

@ -207,17 +207,12 @@ int32_t walReadSeekVer(SWalReader *pReader, int64_t ver) {
return 0;
}
// pReader->curInvalid = 1;
// pReader->curVersion = ver;
if (ver > pWal->vers.lastVer || ver < pWal->vers.firstVer) {
wDebug("vgId:%d, invalid index:%" PRId64 ", first index:%" PRId64 ", last index:%" PRId64, pReader->pWal->cfg.vgId,
wInfo("vgId:%d, invalid index:%" PRId64 ", first index:%" PRId64 ", last index:%" PRId64, pReader->pWal->cfg.vgId,
ver, pWal->vers.firstVer, pWal->vers.lastVer);
terrno = TSDB_CODE_WAL_LOG_NOT_EXIST;
return -1;
}
// if (ver < pWal->vers.snapshotVer) {
// }
if (walReadSeekVerImpl(pReader, ver) < 0) {
return -1;
@ -236,8 +231,6 @@ static int32_t walFetchHeadNew(SWalReader *pRead, int64_t fetchVer) {
if (pRead->curVersion != fetchVer) {
if (walReadSeekVer(pRead, fetchVer) < 0) {
// pRead->curVersion = fetchVer;
// pRead->curInvalid = 1;
return -1;
}
seeked = true;
@ -256,7 +249,6 @@ static int32_t walFetchHeadNew(SWalReader *pRead, int64_t fetchVer) {
} else {
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
}
// pRead->curInvalid = 1;
return -1;
}
}

3
source/util/src/terror.c
View File

@ -190,6 +190,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_USER_FROM_CONN, "Can not get user from
TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_USERS, "Too many users")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ALTER_OPER, "Invalid alter operation")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_AUTH_FAILURE, "Authentication failure")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_PRIVILEDGE_EXIST, "User already have this priviledge")
//mnode-stable-part1
TAOS_DEFINE_ERROR(TSDB_CODE_MND_STB_ALREADY_EXIST, "STable already exists")
@ -319,7 +320,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SNODE_ALREADY_DEPLOYED, "Snode already deploye
TAOS_DEFINE_ERROR(TSDB_CODE_SNODE_NOT_DEPLOYED, "Snode not deployed")
// vnode
TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VGROUP_ID, "Vnode moved to another dnode or was deleted")
TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VGROUP_ID, "Vnode is closed or removed")
TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_WRITE_AUTH, "Database write operation denied")
TAOS_DEFINE_ERROR(TSDB_CODE_VND_NOT_EXIST, "Vnode not exist")
TAOS_DEFINE_ERROR(TSDB_CODE_VND_ALREADY_EXIST, "Vnode already exist")

4
tests/develop-test/5-taos-tools/taosbenchmark/default_json.py
View File

@ -61,9 +61,9 @@ class TDTestCase:
os.system("%s" % cmd)
tdSql.execute("reset query cache")
tdSql.query("show db.tables")
tdSql.checkRows(10)
tdSql.checkRows(8)
tdSql.query("select count(*) from db.stb")
tdSql.checkData(0, 0, 100)
tdSql.checkData(0, 0, 80)
def stop(self):
tdSql.close()

7
tests/develop-test/5-taos-tools/taosbenchmark/json/default.json
View File

@ -15,13 +15,18 @@
"num_of_records_per_req": 10,
"databases": [{
"dbinfo": {
"name": "db"
"name": "db",
"drop": "yes"
},
"super_tables": [{
"child_table_exists":"no",
"name": "stb",
"childtable_prefix": "stb_",
"childtable_count": 10,
"insert_rows": 10,
"childtable_from": 1,
"childtable_to": 9,
"columns": [{"type": "INT"}],
"tags": [{"type": "INT"}]
}]

1
tests/parallel_test/cases.task
View File

@ -875,6 +875,7 @@
,,y,script,./test.sh -f tsim/query/udf_with_const.sim
,,y,script,./test.sh -f tsim/query/join_interval.sim
,,y,script,./test.sh -f tsim/query/unionall_as_table.sim
,,y,script,./test.sh -f tsim/query/multi_order_by.sim
,,y,script,./test.sh -f tsim/query/sys_tbname.sim
,,y,script,./test.sh -f tsim/query/groupby.sim
,,y,script,./test.sh -f tsim/query/event.sim

5
tests/pytest/util/autogen.py
View File

@ -51,10 +51,7 @@ class AutoGen:
metas = []
for i in range(cnt):
colname = f"{pre}{i}"
if i < len(types):
sel = i
else:
sel = random.randint(0, len(types)-1)
sel = i % len(types)
coltype = types[sel]
sql = f"{colname} {coltype}"
if sqls != "":

57
tests/script/tsim/query/multi_order_by.sim Normal file
View File

@ -0,0 +1,57 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/exec.sh -n dnode1 -s start
sql connect
sql create database test;
sql use test;
sql create table t(ts timestamp, f int);
sql insert into t values(now,0)(now+1s, 1)(now+2s, 2)(now+3s,3)(now+4s,4)(now+5s,5)(now+6s,6)(now+7s,7)(now+8s,8)(now+9s,9)
sql select * from (select * from t order by ts desc limit 3 offset 2) order by ts;
print $data01 $data11 $data21
if $data01 != 5 then
return -1
endi
if $data11 != 6 then
return -1
endi
if $data21 != 7 then
return -1
endi
sql select * from (select * from t order by ts limit 3 offset 2) order by ts desc;
print $data01 $data11 $data21
if $data01 != 4 then
return -1
endi
if $data11 != 3 then
return -1
endi
if $data21 != 2 then
return -1
endi
sql select * from (select * from t order by ts desc limit 3 offset 2) order by ts desc;
print $data01 $data11 $data21
if $data01 != 7 then
return -1
endi
if $data11 != 6 then
return -1
endi
if $data21 != 5 then
return -1
endi
sql select * from (select * from t order by ts limit 3 offset 2) order by ts;
print $data01 $data11 $data21
if $data01 != 2 then
return -1
endi
if $data11 != 3 then
return -1
endi
if $data21 != 4 then
return -1
endi
system sh/exec.sh -n dnode1 -s stop -x SIGINT

16
tests/script/tsim/valgrind/checkUdf.sim
View File

@ -29,10 +29,10 @@ sql select udf1(f) from t;
if $rows != 2 then
return -1
endi
if $data00 != 88 then
if $data00 != 1 then
return -1
endi
if $data10 != 88 then
if $data10 != 1 then
return -1
endi
@ -51,10 +51,10 @@ sql select udf1(f1, f2) from t2;
if $rows != 2 then
return -1
endi
if $data00 != 88 then
if $data00 != 1 then
return -1
endi
if $data10 != 88 then
if $data10 != 1 then
return -1
endi
@ -72,10 +72,10 @@ print $rows , $data00 , $data10 , $data20 , $data30
if $rows != 4 then
return -1
endi
if $data00 != 88 then
if $data00 != 1 then
return -1
endi
if $data10 != 88 then
if $data10 != 1 then
return -1
endi
@ -114,10 +114,10 @@ print $rows , $data00 , $data01
if $rows != 1 then
return -1
endi
if $data00 != 176.000000000 then
if $data00 != 2.000000000 then
return -1
endi
if $data01 != 152.420471066 then
if $data01 != 1.732050808 then
return -1
endi

50
tests/system-test/0-others/udfTest.py
View File

@ -191,20 +191,20 @@ class TDTestCase:
tdSql.checkData(0,0,None)
tdSql.checkData(0,1,None)
tdSql.checkData(0,2,1)
tdSql.checkData(0,3,88)
tdSql.checkData(0,3,1)
tdSql.checkData(0,4,1.000000000)
tdSql.checkData(0,5,88)
tdSql.checkData(0,5,1)
tdSql.checkData(0,6,"binary1")
tdSql.checkData(0,7,88)
tdSql.checkData(0,7,1)
tdSql.checkData(3,0,3)
tdSql.checkData(3,1,88)
tdSql.checkData(3,1,1)
tdSql.checkData(3,2,33333)
tdSql.checkData(3,3,88)
tdSql.checkData(3,3,1)
tdSql.checkData(3,4,33.000000000)
tdSql.checkData(3,5,88)
tdSql.checkData(3,5,1)
tdSql.checkData(3,6,"binary1")
tdSql.checkData(3,7,88)
tdSql.checkData(3,7,1)
tdSql.checkData(11,0,None)
tdSql.checkData(11,1,None)
@ -213,7 +213,7 @@ class TDTestCase:
tdSql.checkData(11,4,None)
tdSql.checkData(11,5,None)
tdSql.checkData(11,6,"binary1")
tdSql.checkData(11,7,88)
tdSql.checkData(11,7,1)
tdSql.query("select c1 , udf1(c1) ,c2 ,udf1(c2), c3 ,udf1(c3), c4 ,udf1(c4) from stb1 order by c1")
tdSql.checkData(0,0,None)
@ -226,13 +226,13 @@ class TDTestCase:
tdSql.checkData(0,7,None)
tdSql.checkData(20,0,8)
tdSql.checkData(20,1,88)
tdSql.checkData(20,1,1)
tdSql.checkData(20,2,88888)
tdSql.checkData(20,3,88)
tdSql.checkData(20,3,1)
tdSql.checkData(20,4,888)
tdSql.checkData(20,5,88)
tdSql.checkData(20,5,1)
tdSql.checkData(20,6,88)
tdSql.checkData(20,7,88)
tdSql.checkData(20,7,1)
# aggregate functions
@ -375,14 +375,14 @@ class TDTestCase:
tdSql.checkRows(25)
tdSql.checkData(0,0,None)
tdSql.checkData(0,1,None)
tdSql.checkData(1,0,88)
tdSql.checkData(1,0,1)
tdSql.checkData(1,1,8)
tdSql.query("select abs(udf1(c1)) , abs(ceil(c1)) from ct1 order by ts;")
tdSql.checkRows(13)
tdSql.checkData(0,0,88)
tdSql.checkData(0,0,1)
tdSql.checkData(0,1,8)
tdSql.checkData(1,0,88)
tdSql.checkData(1,0,1)
tdSql.checkData(1,1,7)
# bug fix for crash
@ -401,9 +401,9 @@ class TDTestCase:
tdSql.query("select c1 ,udf1(c1) , c6 ,udf1(c6) from stb1 where c1 > 8 order by ts")
tdSql.checkRows(3)
tdSql.checkData(0,0,9)
tdSql.checkData(0,1,88)
tdSql.checkData(0,1,1)
tdSql.checkData(0,2,-99.990000000)
tdSql.checkData(0,3,88)
tdSql.checkData(0,3,1)
tdSql.query("select sub1.c1, sub2.c2 from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null")
tdSql.checkData(0,0,0)
@ -412,20 +412,20 @@ class TDTestCase:
tdSql.checkData(1,1,10)
tdSql.query("select udf1(sub1.c1), udf1(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null")
tdSql.checkData(0,0,88)
tdSql.checkData(0,1,88)
tdSql.checkData(1,0,88)
tdSql.checkData(1,1,88)
tdSql.checkData(0,0,1)
tdSql.checkData(0,1,1)
tdSql.checkData(1,0,1)
tdSql.checkData(1,1,1)
tdSql.query("select sub1.c1 , udf1(sub1.c1), sub2.c2 ,udf1(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null")
tdSql.checkData(0,0,0)
tdSql.checkData(0,1,88)
tdSql.checkData(0,1,1)
tdSql.checkData(0,2,0)
tdSql.checkData(0,3,88)
tdSql.checkData(0,3,1)
tdSql.checkData(1,0,1)
tdSql.checkData(1,1,88)
tdSql.checkData(1,1,1)
tdSql.checkData(1,2,10)
tdSql.checkData(1,3,88)
tdSql.checkData(1,3,1)
tdSql.query("select udf2(sub1.c1), udf2(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null")
tdSql.checkData(0,0,16.881943016)

50
tests/system-test/0-others/udf_cfg2.py
View File

@ -193,20 +193,20 @@ class TDTestCase:
tdSql.checkData(0,0,None)
tdSql.checkData(0,1,None)
tdSql.checkData(0,2,1)
tdSql.checkData(0,3,88)
tdSql.checkData(0,3,1)
tdSql.checkData(0,4,1.000000000)
tdSql.checkData(0,5,88)
tdSql.checkData(0,5,1)
tdSql.checkData(0,6,"binary1")
tdSql.checkData(0,7,88)
tdSql.checkData(0,7,1)
tdSql.checkData(3,0,3)
tdSql.checkData(3,1,88)
tdSql.checkData(3,1,1)
tdSql.checkData(3,2,33333)
tdSql.checkData(3,3,88)
tdSql.checkData(3,3,1)
tdSql.checkData(3,4,33.000000000)
tdSql.checkData(3,5,88)
tdSql.checkData(3,5,1)
tdSql.checkData(3,6,"binary1")
tdSql.checkData(3,7,88)
tdSql.checkData(3,7,1)
tdSql.checkData(11,0,None)
tdSql.checkData(11,1,None)
@ -215,7 +215,7 @@ class TDTestCase:
tdSql.checkData(11,4,None)
tdSql.checkData(11,5,None)
tdSql.checkData(11,6,"binary1")
tdSql.checkData(11,7,88)
tdSql.checkData(11,7,1)
tdSql.query("select c1 , udf1(c1) ,c2 ,udf1(c2), c3 ,udf1(c3), c4 ,udf1(c4) from stb1 order by c1")
tdSql.checkData(0,0,None)
@ -228,13 +228,13 @@ class TDTestCase:
tdSql.checkData(0,7,None)
tdSql.checkData(20,0,8)
tdSql.checkData(20,1,88)
tdSql.checkData(20,1,1)
tdSql.checkData(20,2,88888)
tdSql.checkData(20,3,88)
tdSql.checkData(20,3,1)
tdSql.checkData(20,4,888)
tdSql.checkData(20,5,88)
tdSql.checkData(20,5,1)
tdSql.checkData(20,6,88)
tdSql.checkData(20,7,88)
tdSql.checkData(20,7,1)
# aggregate functions
@ -377,14 +377,14 @@ class TDTestCase:
tdSql.checkRows(25)
tdSql.checkData(0,0,None)
tdSql.checkData(0,1,None)
tdSql.checkData(1,0,88)
tdSql.checkData(1,0,1)
tdSql.checkData(1,1,8)
tdSql.query("select abs(udf1(c1)) , abs(ceil(c1)) from ct1 order by ts;")
tdSql.checkRows(13)
tdSql.checkData(0,0,88)
tdSql.checkData(0,0,1)
tdSql.checkData(0,1,8)
tdSql.checkData(1,0,88)
tdSql.checkData(1,0,1)
tdSql.checkData(1,1,7)
# bug fix for crash
@ -403,9 +403,9 @@ class TDTestCase:
tdSql.query("select c1 ,udf1(c1) , c6 ,udf1(c6) from stb1 where c1 > 8 order by ts")
tdSql.checkRows(3)
tdSql.checkData(0,0,9)
tdSql.checkData(0,1,88)
tdSql.checkData(0,1,1)
tdSql.checkData(0,2,-99.990000000)
tdSql.checkData(0,3,88)
tdSql.checkData(0,3,1)
tdSql.query("select sub1.c1, sub2.c2 from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null")
tdSql.checkData(0,0,0)
@ -414,20 +414,20 @@ class TDTestCase:
tdSql.checkData(1,1,10)
tdSql.query("select udf1(sub1.c1), udf1(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null")
tdSql.checkData(0,0,88)
tdSql.checkData(0,1,88)
tdSql.checkData(1,0,88)
tdSql.checkData(1,1,88)
tdSql.checkData(0,0,1)
tdSql.checkData(0,1,1)
tdSql.checkData(1,0,1)
tdSql.checkData(1,1,1)
tdSql.query("select sub1.c1 , udf1(sub1.c1), sub2.c2 ,udf1(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null")
tdSql.checkData(0,0,0)
tdSql.checkData(0,1,88)
tdSql.checkData(0,1,1)
tdSql.checkData(0,2,0)
tdSql.checkData(0,3,88)
tdSql.checkData(0,3,1)
tdSql.checkData(1,0,1)
tdSql.checkData(1,1,88)
tdSql.checkData(1,1,1)
tdSql.checkData(1,2,10)
tdSql.checkData(1,3,88)
tdSql.checkData(1,3,1)
tdSql.query("select udf2(sub1.c1), udf2(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null")
tdSql.checkData(0,0,16.881943016)

50
tests/system-test/0-others/udf_create.py
View File

@ -193,20 +193,20 @@ class TDTestCase:
tdSql.checkData(0,0,None)
tdSql.checkData(0,1,None)
tdSql.checkData(0,2,1)
tdSql.checkData(0,3,88)
tdSql.checkData(0,3,1)
tdSql.checkData(0,4,1.000000000)
tdSql.checkData(0,5,88)
tdSql.checkData(0,5,1)
tdSql.checkData(0,6,"binary1")
tdSql.checkData(0,7,88)
tdSql.checkData(0,7,1)
tdSql.checkData(3,0,3)
tdSql.checkData(3,1,88)
tdSql.checkData(3,1,1)
tdSql.checkData(3,2,33333)
tdSql.checkData(3,3,88)
tdSql.checkData(3,3,1)
tdSql.checkData(3,4,33.000000000)
tdSql.checkData(3,5,88)
tdSql.checkData(3,5,1)
tdSql.checkData(3,6,"binary1")
tdSql.checkData(3,7,88)
tdSql.checkData(3,7,1)
tdSql.checkData(11,0,None)
tdSql.checkData(11,1,None)
@ -215,7 +215,7 @@ class TDTestCase:
tdSql.checkData(11,4,None)
tdSql.checkData(11,5,None)
tdSql.checkData(11,6,"binary1")
tdSql.checkData(11,7,88)
tdSql.checkData(11,7,1)
tdSql.query("select c1 , udf1(c1) ,c2 ,udf1(c2), c3 ,udf1(c3), c4 ,udf1(c4) from stb1 order by c1")
tdSql.checkData(0,0,None)
@ -228,13 +228,13 @@ class TDTestCase:
tdSql.checkData(0,7,None)
tdSql.checkData(20,0,8)
tdSql.checkData(20,1,88)
tdSql.checkData(20,1,1)
tdSql.checkData(20,2,88888)
tdSql.checkData(20,3,88)
tdSql.checkData(20,3,1)
tdSql.checkData(20,4,888)
tdSql.checkData(20,5,88)
tdSql.checkData(20,5,1)
tdSql.checkData(20,6,88)
tdSql.checkData(20,7,88)
tdSql.checkData(20,7,1)
# aggregate functions
@ -377,14 +377,14 @@ class TDTestCase:
tdSql.checkRows(25)
tdSql.checkData(0,0,None)
tdSql.checkData(0,1,None)
tdSql.checkData(1,0,88)
tdSql.checkData(1,0,1)
tdSql.checkData(1,1,8)
tdSql.query("select abs(udf1(c1)) , abs(ceil(c1)) from ct1 order by ts;")
tdSql.checkRows(13)
tdSql.checkData(0,0,88)
tdSql.checkData(0,0,1)
tdSql.checkData(0,1,8)
tdSql.checkData(1,0,88)
tdSql.checkData(1,0,1)
tdSql.checkData(1,1,7)
# bug fix for crash
@ -403,9 +403,9 @@ class TDTestCase:
tdSql.query("select c1 ,udf1(c1) , c6 ,udf1(c6) from stb1 where c1 > 8 order by ts")
tdSql.checkRows(3)
tdSql.checkData(0,0,9)
tdSql.checkData(0,1,88)
tdSql.checkData(0,1,1)
tdSql.checkData(0,2,-99.990000000)
tdSql.checkData(0,3,88)
tdSql.checkData(0,3,1)
tdSql.query("select sub1.c1, sub2.c2 from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null")
tdSql.checkData(0,0,0)
@ -414,20 +414,20 @@ class TDTestCase:
tdSql.checkData(1,1,10)
tdSql.query("select udf1(sub1.c1), udf1(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null")
tdSql.checkData(0,0,88)
tdSql.checkData(0,1,88)
tdSql.checkData(1,0,88)
tdSql.checkData(1,1,88)
tdSql.checkData(0,0,1)
tdSql.checkData(0,1,1)
tdSql.checkData(1,0,1)
tdSql.checkData(1,1,1)
tdSql.query("select sub1.c1 , udf1(sub1.c1), sub2.c2 ,udf1(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null")
tdSql.checkData(0,0,0)
tdSql.checkData(0,1,88)
tdSql.checkData(0,1,1)
tdSql.checkData(0,2,0)
tdSql.checkData(0,3,88)
tdSql.checkData(0,3,1)
tdSql.checkData(1,0,1)
tdSql.checkData(1,1,88)
tdSql.checkData(1,1,1)
tdSql.checkData(1,2,10)
tdSql.checkData(1,3,88)
tdSql.checkData(1,3,1)
tdSql.query("select udf2(sub1.c1), udf2(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null")
tdSql.checkData(0,0,16.881943016)

54
tests/system-test/0-others/udf_restart_taosd.py
View File

@ -190,20 +190,20 @@ class TDTestCase:
tdSql.checkData(0,0,None)
tdSql.checkData(0,1,None)
tdSql.checkData(0,2,1)
tdSql.checkData(0,3,88)
tdSql.checkData(0,3,1)
tdSql.checkData(0,4,1.000000000)
tdSql.checkData(0,5,88)
tdSql.checkData(0,5,1)
tdSql.checkData(0,6,"binary1")
tdSql.checkData(0,7,88)
tdSql.checkData(0,7,1)
tdSql.checkData(3,0,3)
tdSql.checkData(3,1,88)
tdSql.checkData(3,1,1)
tdSql.checkData(3,2,33333)
tdSql.checkData(3,3,88)
tdSql.checkData(3,3,1)
tdSql.checkData(3,4,33.000000000)
tdSql.checkData(3,5,88)
tdSql.checkData(3,5,1)
tdSql.checkData(3,6,"binary1")
tdSql.checkData(3,7,88)
tdSql.checkData(3,7,1)
tdSql.checkData(11,0,None)
tdSql.checkData(11,1,None)
@ -212,7 +212,7 @@ class TDTestCase:
tdSql.checkData(11,4,None)
tdSql.checkData(11,5,None)
tdSql.checkData(11,6,"binary1")
tdSql.checkData(11,7,88)
tdSql.checkData(11,7,1)
tdSql.query("select c1 , udf1(c1) ,c2 ,udf1(c2), c3 ,udf1(c3), c4 ,udf1(c4) from stb1 order by c1")
tdSql.checkData(0,0,None)
@ -225,13 +225,13 @@ class TDTestCase:
tdSql.checkData(0,7,None)
tdSql.checkData(20,0,8)
tdSql.checkData(20,1,88)
tdSql.checkData(20,1,1)
tdSql.checkData(20,2,88888)
tdSql.checkData(20,3,88)
tdSql.checkData(20,3,1)
tdSql.checkData(20,4,888)
tdSql.checkData(20,5,88)
tdSql.checkData(20,5,1)
tdSql.checkData(20,6,88)
tdSql.checkData(20,7,88)
tdSql.checkData(20,7,1)
# aggregate functions
@ -374,14 +374,14 @@ class TDTestCase:
tdSql.checkRows(25)
tdSql.checkData(0,0,None)
tdSql.checkData(0,1,None)
tdSql.checkData(1,0,88)
tdSql.checkData(1,0,1)
tdSql.checkData(1,1,8)
tdSql.query("select abs(udf1(c1)) , abs(ceil(c1)) from ct1 order by ts;")
tdSql.checkRows(13)
tdSql.checkData(0,0,88)
tdSql.checkData(0,0,1)
tdSql.checkData(0,1,8)
tdSql.checkData(1,0,88)
tdSql.checkData(1,0,1)
tdSql.checkData(1,1,7)
# bug fix for crash
@ -400,9 +400,9 @@ class TDTestCase:
tdSql.query("select c1 ,udf1(c1) , c6 ,udf1(c6) from stb1 where c1 > 8 order by ts")
tdSql.checkRows(3)
tdSql.checkData(0,0,9)
tdSql.checkData(0,1,88)
tdSql.checkData(0,1,1)
tdSql.checkData(0,2,-99.990000000)
tdSql.checkData(0,3,88)
tdSql.checkData(0,3,1)
tdSql.query("select sub1.c1, sub2.c2 from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null")
tdSql.checkData(0,0,0)
@ -411,20 +411,20 @@ class TDTestCase:
tdSql.checkData(1,1,10)
tdSql.query("select udf1(sub1.c1), udf1(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null")
tdSql.checkData(0,0,88)
tdSql.checkData(0,1,88)
tdSql.checkData(1,0,88)
tdSql.checkData(1,1,88)
tdSql.checkData(0,0,1)
tdSql.checkData(0,1,1)
tdSql.checkData(1,0,1)
tdSql.checkData(1,1,1)
tdSql.query("select sub1.c1 , udf1(sub1.c1), sub2.c2 ,udf1(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null")
tdSql.checkData(0,0,0)
tdSql.checkData(0,1,88)
tdSql.checkData(0,1,1)
tdSql.checkData(0,2,0)
tdSql.checkData(0,3,88)
tdSql.checkData(0,3,1)
tdSql.checkData(1,0,1)
tdSql.checkData(1,1,88)
tdSql.checkData(1,1,1)
tdSql.checkData(1,2,10)
tdSql.checkData(1,3,88)
tdSql.checkData(1,3,1)
tdSql.query("select udf2(sub1.c1), udf2(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null")
tdSql.checkData(0,0,16.881943016)
@ -468,12 +468,12 @@ class TDTestCase:
tdSql.checkData(1,0,1)
tdSql.checkData(1,1,1)
tdSql.checkData(1,2,1.110000000)
tdSql.checkData(1,3,88)
tdSql.checkData(1,3,1)
tdSql.query("select c1,c6,udf1(c1,c6) from stb1 order by ts")
tdSql.checkData(1,0,8)
tdSql.checkData(1,1,88.880000000)
tdSql.checkData(1,2,88)
tdSql.checkData(1,2,1)
tdSql.query("select abs(udf1(c1,c6,c1,c6)) , abs(ceil(c1)) from stb1 where c1 is not null order by ts;")
tdSql.checkRows(22)

4
tests/system-test/0-others/user_control.py
View File

@ -444,7 +444,7 @@ class TDTestCase:
self.__grant_user_privileges(privilege="", dbname="db", user_name=self.__user_list[0]) ,
self.__grant_user_privileges(privilege=" ".join(self.__privilege), user_name=self.__user_list[0]) ,
f"GRANT {self.__privilege[0]} ON * TO {self.__user_list[0]}" ,
f"GRANT {self.__privilege[0]} ON {DBNAME}.{NTBNAME} TO {self.__user_list[0]}" ,
# f"GRANT {self.__privilege[0]} ON {DBNAME}.{NTBNAME} TO {self.__user_list[0]}" ,
]
def __revoke_err(self):
@ -456,7 +456,7 @@ class TDTestCase:
self.__revoke_user_privileges(privilege="", dbname="db", user_name=self.__user_list[0]) ,
self.__revoke_user_privileges(privilege=" ".join(self.__privilege), user_name=self.__user_list[0]) ,
f"REVOKE {self.__privilege[0]} ON * FROM {self.__user_list[0]}" ,
f"REVOKE {self.__privilege[0]} ON {DBNAME}.{NTBNAME} FROM {self.__user_list[0]}" ,
# f"REVOKE {self.__privilege[0]} ON {DBNAME}.{NTBNAME} FROM {self.__user_list[0]}" ,
]
def test_grant_err(self):

2
tests/system-test/2-query/odbc.py
View File

@ -22,7 +22,7 @@ class TDTestCase:
tdSql.execute("insert into db.ctb using db.stb tags(1) (ts, c1) values (now, 1)")
tdSql.query("select count(*) from information_schema.ins_columns")
tdSql.checkData(0, 0, 272)
tdSql.checkData(0, 0, 274)
tdSql.query("select * from information_schema.ins_columns where table_name = 'ntb'")
tdSql.checkRows(14)

4
tests/system-test/2-query/sml.py
View File

@ -24,7 +24,7 @@ class TDTestCase:
tdSql.init(conn.cursor(), True)
#tdSql.init(conn.cursor(), logSql) # output sql.txt file
def checkFileContent(self, dbname="sml_db"):
def checkContent(self, dbname="sml_db"):
simClientCfg="%s/taos.cfg"%tdDnodes.getSimCfgPath()
buildPath = tdCom.getBuildPath()
cmdStr = '%s/build/bin/sml_test %s'%(buildPath, simClientCfg)
@ -102,7 +102,7 @@ class TDTestCase:
def run(self):
tdSql.prepare()
self.checkFileContent()
self.checkContent()
def stop(self):
tdSql.close()

28
tests/system-test/7-tmq/dropDbR3ConflictTransaction.py
View File

@ -32,34 +32,6 @@ class TDTestCase:
tdSql.init(conn.cursor())
#tdSql.init(conn.cursor(), logSql) # output sql.txt file
def checkFileContent(self, consumerId, queryString):
buildPath = tdCom.getBuildPath()
cfgPath = tdCom.getClientCfgPath()
dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId)
cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile)
tdLog.info(cmdStr)
os.system(cmdStr)
consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId)
tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile))
consumeFile = open(consumeRowsFile, mode='r')
queryFile = open(dstFile, mode='r')
# skip first line for it is schema
queryFile.readline()
while True:
dst = queryFile.readline()
src = consumeFile.readline()
if dst:
if dst != src:
tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId)
else:
break
return
def prepareTestEnv(self):
tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ")
paraDict = {'dbName': 'dbt',

30
tests/system-test/7-tmq/tmq3mnodeSwitch.py
View File

@ -138,34 +138,6 @@ class TDTestCase:
else:
tdLog.exit("three mnodes is not ready in 10s ")
def checkFileContent(self, consumerId, queryString):
buildPath = tdCom.getBuildPath()
cfgPath = tdCom.getClientCfgPath()
dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId)
cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile)
tdLog.info(cmdStr)
os.system(cmdStr)
consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId)
tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile))
consumeFile = open(consumeRowsFile, mode='r')
queryFile = open(dstFile, mode='r')
# skip first line for it is schema
queryFile.readline()
while True:
dst = queryFile.readline()
src = consumeFile.readline()
if dst:
if dst != src:
tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId)
else:
break
return
def tmqCase1(self):
tdLog.printNoPrefix("======== test case 1: ")
paraDict = {'dbName': 'db1',
@ -256,7 +228,7 @@ class TDTestCase:
tdLog.exit("0 tmq consume rows error!")
if expectRowsList[0] == resultList[0]:
self.checkFileContent(consumerId, queryString)
tmqCom.checkFileContent(consumerId, queryString)
time.sleep(10)
for i in range(len(topicNameList)):

35
tests/system-test/7-tmq/tmqCheckData.py
View File

@ -5,6 +5,7 @@ import time
import socket
import os
import threading
import math
from util.log import *
from util.sql import *
@ -21,34 +22,6 @@ class TDTestCase:
tdSql.init(conn.cursor())
#tdSql.init(conn.cursor(), logSql) # output sql.txt file
def checkFileContent(self, consumerId, queryString):
buildPath = tdCom.getBuildPath()
cfgPath = tdCom.getClientCfgPath()
dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId)
cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile)
tdLog.info(cmdStr)
os.system(cmdStr)
consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId)
tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile))
consumeFile = open(consumeRowsFile, mode='r')
queryFile = open(dstFile, mode='r')
# skip first line for it is schema
queryFile.readline()
while True:
dst = queryFile.readline()
src = consumeFile.readline()
if dst:
if dst != src:
tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId)
else:
break
return
def tmqCase1(self):
tdLog.printNoPrefix("======== test case 1: ")
paraDict = {'dbName': 'db1',
@ -109,7 +82,7 @@ class TDTestCase:
tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0]))
tdLog.exit("0 tmq consume rows error!")
self.checkFileContent(consumerId, queryString)
tmqCom.checkFileContent(consumerId, queryString)
# reinit consume info, and start tmq_sim, then check consume result
tmqCom.initConsumerTable()
@ -135,7 +108,7 @@ class TDTestCase:
tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[1], resultList[0]))
tdLog.exit("1 tmq consume rows error!")
self.checkFileContent(consumerId, queryString)
tmqCom.checkFileContent(consumerId, queryString)
# reinit consume info, and start tmq_sim, then check consume result
tmqCom.initConsumerTable()
@ -161,7 +134,7 @@ class TDTestCase:
# tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[2], resultList[0]))
# tdLog.exit("2 tmq consume rows error!")
# self.checkFileContent(consumerId, queryString)
# tmqCom.checkFileContent(consumerId, queryString)
time.sleep(10)
for i in range(len(topicNameList)):

34
tests/system-test/7-tmq/tmqCheckData1.py
View File

@ -21,34 +21,6 @@ class TDTestCase:
tdSql.init(conn.cursor())
#tdSql.init(conn.cursor(), logSql) # output sql.txt file
def checkFileContent(self, consumerId, queryString):
buildPath = tdCom.getBuildPath()
cfgPath = tdCom.getClientCfgPath()
dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId)
cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile)
tdLog.info(cmdStr)
os.system(cmdStr)
consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId)
tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile))
consumeFile = open(consumeRowsFile, mode='r')
queryFile = open(dstFile, mode='r')
# skip first line for it is schema
queryFile.readline()
while True:
dst = queryFile.readline()
src = consumeFile.readline()
if dst:
if dst != src:
tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId)
else:
break
return
def tmqCase1(self):
tdLog.printNoPrefix("======== test case 1: ")
paraDict = {'dbName': 'db1',
@ -109,7 +81,7 @@ class TDTestCase:
tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0]))
tdLog.exit("0 tmq consume rows error!")
self.checkFileContent(consumerId, queryString)
tmqCom.checkFileContent(consumerId, queryString)
# reinit consume info, and start tmq_sim, then check consume result
tmqCom.initConsumerTable()
@ -134,7 +106,7 @@ class TDTestCase:
tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[1], resultList[0]))
tdLog.exit("1 tmq consume rows error!")
self.checkFileContent(consumerId, queryString)
tmqCom.checkFileContent(consumerId, queryString)
# reinit consume info, and start tmq_sim, then check consume result
tmqCom.initConsumerTable()
@ -159,7 +131,7 @@ class TDTestCase:
tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[2], resultList[0]))
tdLog.exit("2 tmq consume rows error!")
self.checkFileContent(consumerId, queryString)
tmqCom.checkFileContent(consumerId, queryString)
time.sleep(10)
for i in range(len(topicNameList)):

24
tests/system-test/7-tmq/tmqCommon.py
View File

@ -10,7 +10,7 @@
###################################################################
# -*- coding: utf-8 -*-
import math
from asyncore import loop
from collections import defaultdict
import subprocess
@ -462,18 +462,24 @@ class TMQCom:
for i in range(0,skipRowsOfCons):
consumeFile.readline()
lines = 0
while True:
dst = queryFile.readline()
src = consumeFile.readline()
lines += 1
if dst:
if dst != src:
tdLog.info("src row: %s"%src)
tdLog.info("dst row: %s"%dst)
tdLog.exit("consumerId %d consume rows[%d] is not match the rows by direct query"%(consumerId, lines))
else:
dstSplit = dst.split(',')
srcSplit = src.split(',')
if not dst or not src:
break
if len(dstSplit) != len(srcSplit):
tdLog.exit("consumerId %d consume rows len is not match the rows by direct query,len(dstSplit):%d != len(srcSplit):%d, dst:%s, src:%s"
%(consumerId, len(dstSplit), len(srcSplit), dst, src))
for i in range(len(dstSplit)):
if srcSplit[i] != dstSplit[i]:
srcFloat = float(srcSplit[i])
dstFloat = float(dstSplit[i])
if not math.isclose(srcFloat, dstFloat, abs_tol=1e-9):
tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId)
return
def getResultFileByTaosShell(self, consumerId, queryString):

28
tests/system-test/7-tmq/tmqConsumerGroup.py
View File

@ -21,34 +21,6 @@ class TDTestCase:
tdSql.init(conn.cursor())
#tdSql.init(conn.cursor(), logSql) # output sql.txt file
def checkFileContent(self, consumerId, queryString):
buildPath = tdCom.getBuildPath()
cfgPath = tdCom.getClientCfgPath()
dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId)
cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile)
tdLog.info(cmdStr)
os.system(cmdStr)
consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId)
tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile))
consumeFile = open(consumeRowsFile, mode='r')
queryFile = open(dstFile, mode='r')
# skip first line for it is schema
queryFile.readline()
while True:
dst = queryFile.readline()
src = consumeFile.readline()
if dst:
if dst != src:
tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId)
else:
break
return
def tmqCase1(self):
tdLog.printNoPrefix("======== test case 1: ")
paraDict = {'dbName': 'db1',

36
tests/system-test/7-tmq/tmqUdf-multCtb-snapshot0.py
View File

@ -60,34 +60,6 @@ class TDTestCase:
tdLog.exit("create udf functions fail")
return
def checkFileContent(self, consumerId, queryString):
buildPath = tdCom.getBuildPath()
cfgPath = tdCom.getClientCfgPath()
dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId)
cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile)
tdLog.info(cmdStr)
os.system(cmdStr)
consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId)
tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile))
consumeFile = open(consumeRowsFile, mode='r')
queryFile = open(dstFile, mode='r')
# skip first line for it is schema
queryFile.readline()
while True:
dst = queryFile.readline()
src = consumeFile.readline()
if dst:
if dst != src:
tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId)
else:
break
return
def prepareTestEnv(self):
tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ")
paraDict = {'dbName': 'dbt',
@ -199,7 +171,7 @@ class TDTestCase:
tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0]))
tdLog.exit("0 tmq consume rows error!")
# self.checkFileContent(consumerId, queryString)
# tmqCom.checkFileContent(consumerId, queryString)
# tdLog.printNoPrefix("consumerId %d check data ok!"%(consumerId))
# reinit consume info, and start tmq_sim, then check consume result
@ -226,7 +198,7 @@ class TDTestCase:
tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[1], resultList[0]))
tdLog.exit("1 tmq consume rows error!")
# self.checkFileContent(consumerId, queryString)
# tmqCom.checkFileContent(consumerId, queryString)
# tdLog.printNoPrefix("consumerId %d check data ok!"%(consumerId))
time.sleep(10)
@ -309,7 +281,7 @@ class TDTestCase:
tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0]))
tdLog.exit("2 tmq consume rows error!")
# self.checkFileContent(consumerId, queryString)
# tmqCom.checkFileContent(consumerId, queryString)
# tdLog.printNoPrefix("consumerId %d check data ok!"%(consumerId))
# reinit consume info, and start tmq_sim, then check consume result
@ -336,7 +308,7 @@ class TDTestCase:
tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[1], resultList[0]))
tdLog.exit("3 tmq consume rows error!")
# self.checkFileContent(consumerId, queryString)
# tmqCom.checkFileContent(consumerId, queryString)
# tdLog.printNoPrefix("consumerId %d check data ok!"%(consumerId))
time.sleep(10)

36
tests/system-test/7-tmq/tmqUdf-multCtb-snapshot1.py
View File

@ -60,34 +60,6 @@ class TDTestCase:
tdLog.exit("create udf functions fail")
return
def checkFileContent(self, consumerId, queryString):
buildPath = tdCom.getBuildPath()
cfgPath = tdCom.getClientCfgPath()
dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId)
cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile)
tdLog.info(cmdStr)
os.system(cmdStr)
consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId)
tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile))
consumeFile = open(consumeRowsFile, mode='r')
queryFile = open(dstFile, mode='r')
# skip first line for it is schema
queryFile.readline()
while True:
dst = queryFile.readline()
src = consumeFile.readline()
if dst:
if dst != src:
tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId)
else:
break
return
def prepareTestEnv(self):
tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ")
paraDict = {'dbName': 'dbt',
@ -199,7 +171,7 @@ class TDTestCase:
tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0]))
tdLog.exit("0 tmq consume rows error!")
# self.checkFileContent(consumerId, queryString)
# tmqCom.checkFileContent(consumerId, queryString)
# tdLog.printNoPrefix("consumerId %d check data ok!"%(consumerId))
# reinit consume info, and start tmq_sim, then check consume result
@ -226,7 +198,7 @@ class TDTestCase:
tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[1], resultList[0]))
tdLog.exit("1 tmq consume rows error!")
# self.checkFileContent(consumerId, queryString)
# tmqCom.checkFileContent(consumerId, queryString)
# tdLog.printNoPrefix("consumerId %d check data ok!"%(consumerId))
time.sleep(10)
@ -309,7 +281,7 @@ class TDTestCase:
tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0]))
tdLog.exit("2 tmq consume rows error!")
# self.checkFileContent(consumerId, queryString)
# tmqCom.checkFileContent(consumerId, queryString)
# tdLog.printNoPrefix("consumerId %d check data ok!"%(consumerId))
# reinit consume info, and start tmq_sim, then check consume result
@ -336,7 +308,7 @@ class TDTestCase:
tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[1], resultList[0]))
tdLog.exit("3 tmq consume rows error!")
# self.checkFileContent(consumerId, queryString)
# tmqCom.checkFileContent(consumerId, queryString)
# tdLog.printNoPrefix("consumerId %d check data ok!"%(consumerId))
time.sleep(10)

36
tests/system-test/7-tmq/tmqUdf.py
View File

@ -60,34 +60,6 @@ class TDTestCase:
tdLog.exit("create udf functions fail")
return
def checkFileContent(self, consumerId, queryString):
buildPath = tdCom.getBuildPath()
cfgPath = tdCom.getClientCfgPath()
dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId)
cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile)
tdLog.info(cmdStr)
os.system(cmdStr)
consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId)
tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile))
consumeFile = open(consumeRowsFile, mode='r')
queryFile = open(dstFile, mode='r')
# skip first line for it is schema
queryFile.readline()
while True:
dst = queryFile.readline()
src = consumeFile.readline()
if dst:
if dst != src:
tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId)
else:
break
return
def prepareTestEnv(self):
tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ")
paraDict = {'dbName': 'dbt',
@ -199,7 +171,7 @@ class TDTestCase:
tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0]))
tdLog.exit("0 tmq consume rows error!")
self.checkFileContent(consumerId, queryString)
tmqCom.checkFileContent(consumerId, queryString)
tdLog.printNoPrefix("consumerId %d check data ok!"%(consumerId))
@ -227,7 +199,7 @@ class TDTestCase:
tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[1], resultList[0]))
tdLog.exit("1 tmq consume rows error!")
self.checkFileContent(consumerId, queryString)
tmqCom.checkFileContent(consumerId, queryString)
tdLog.printNoPrefix("consumerId %d check data ok!"%(consumerId))
time.sleep(10)
@ -310,7 +282,7 @@ class TDTestCase:
tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0]))
tdLog.exit("2 tmq consume rows error!")
self.checkFileContent(consumerId, queryString)
tmqCom.checkFileContent(consumerId, queryString)
tdLog.printNoPrefix("consumerId %d check data ok!"%(consumerId))
# reinit consume info, and start tmq_sim, then check consume result
@ -337,7 +309,7 @@ class TDTestCase:
tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[1], resultList[0]))
tdLog.exit("3 tmq consume rows error!")
self.checkFileContent(consumerId, queryString)
tmqCom.checkFileContent(consumerId, queryString)
tdLog.printNoPrefix("consumerId %d check data ok!"%(consumerId))
time.sleep(10)

34
tests/system-test/99-TDcase/TD-16821.py
View File

@ -21,34 +21,6 @@ class TDTestCase:
tdSql.init(conn.cursor())
#tdSql.init(conn.cursor(), logSql) # output sql.txt file
def checkFileContent(self, consumerId, queryString):
buildPath = tdCom.getBuildPath()
cfgPath = tdCom.getClientCfgPath()
dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId)
cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile)
tdLog.info(cmdStr)
os.system(cmdStr)
consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId)
tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile))
consumeFile = open(consumeRowsFile, mode='r')
queryFile = open(dstFile, mode='r')
# skip first line for it is schema
queryFile.readline()
while True:
dst = queryFile.readline()
src = consumeFile.readline()
if dst:
if dst != src:
tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId)
else:
break
return
def tmqCase1(self):
tdLog.printNoPrefix("======== test case 1: ")
paraDict = {'dbName': 'db1',
@ -113,7 +85,7 @@ class TDTestCase:
tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0]))
tdLog.exit("0 tmq consume rows error!")
self.checkFileContent(consumerId, queryString)
tmqCom.checkFileContent(consumerId, queryString)
# reinit consume info, and start tmq_sim, then check consume result
tmqCom.initConsumerTable()
@ -139,7 +111,7 @@ class TDTestCase:
tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[1], resultList[0]))
tdLog.exit("1 tmq consume rows error!")
self.checkFileContent(consumerId, queryString)
tmqCom.checkFileContent(consumerId, queryString)
# reinit consume info, and start tmq_sim, then check consume result
tmqCom.initConsumerTable()
@ -165,7 +137,7 @@ class TDTestCase:
# tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[2], resultList[0]))
# tdLog.exit("2 tmq consume rows error!")
# self.checkFileContent(consumerId, queryString)
# tmqCom.checkFileContent(consumerId, queryString)
time.sleep(10)
for i in range(len(topicNameList)):

688
tests/system-test/runAllOne.sh Normal file
View File

@ -0,0 +1,688 @@
# start -N3
echo " ********** -N 3 *************"
python3 ./test.py -f 7-tmq/tmqDelete-multiCtb.py -N 3 -n 3
python3 ./test.py -f 1-insert/alter_database.py -P
python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py -P
python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py -P
python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py -P
python3 ./test.py -f 1-insert/test_stmt_muti_insert_query.py -P
python3 ./test.py -f 1-insert/test_stmt_set_tbname_tag.py -P
python3 ./test.py -f 1-insert/alter_stable.py -P
python3 ./test.py -f 1-insert/alter_table.py -P
python3 ./test.py -f 1-insert/boundary.py -P
python3 ./test.py -f 1-insert/insertWithMoreVgroup.py -P
python3 ./test.py -f 1-insert/table_comment.py -P
python3 ./test.py -f 1-insert/time_range_wise.py -P
python3 ./test.py -f 1-insert/block_wise.py -P
python3 ./test.py -f 1-insert/create_retentions.py -P
python3 ./test.py -f 1-insert/mutil_stage.py -P
python3 ./test.py -f 1-insert/table_param_ttl.py -P
python3 ./test.py -f 1-insert/table_param_ttl.py -P -R
python3 ./test.py -f 1-insert/update_data_muti_rows.py -P
python3 ./test.py -f 1-insert/db_tb_name_check.py -P
python3 ./test.py -f 1-insert/InsertFuturets.py -P
python3 ./test.py -f 1-insert/insert_wide_column.py -P
python3 ./test.py -f 2-query/nestedQuery.py -P
python3 ./test.py -f 2-query/nestedQuery_str.py -P
python3 ./test.py -f 2-query/nestedQuery_math.py -P
python3 ./test.py -f 2-query/nestedQuery_time.py -P
python3 ./test.py -f 2-query/nestedQuery_26.py -P
python3 ./test.py -f 2-query/nestedQuery_str.py -P -Q 2
python3 ./test.py -f 2-query/nestedQuery_math.py -P -Q 2
python3 ./test.py -f 2-query/nestedQuery_time.py -P -Q 2
python3 ./test.py -f 2-query/nestedQuery.py -P -Q 2
python3 ./test.py -f 2-query/nestedQuery_26.py -P -Q 2
python3 ./test.py -f 2-query/columnLenUpdated.py -P
python3 ./test.py -f 2-query/columnLenUpdated.py -P -Q 2
python3 ./test.py -f 2-query/columnLenUpdated.py -P -Q 3
python3 ./test.py -f 2-query/columnLenUpdated.py -P -Q 4
python3 ./test.py -f 2-query/nestedQuery.py -P -Q 4
python3 ./test.py -f 2-query/nestedQuery_str.py -P -Q 4
python3 ./test.py -f 2-query/nestedQuery_math.py -P -Q 4
python3 ./test.py -f 2-query/nestedQuery_time.py -P -Q 4
python3 ./test.py -f 2-query/nestedQuery_26.py -P -Q 4
python3 ./test.py -f 7-tmq/tmqShow.py -P
python3 ./test.py -f 7-tmq/tmqDropStb.py -P
python3 ./test.py -f 7-tmq/subscribeStb0.py -P
python3 ./test.py -f 7-tmq/subscribeStb1.py -P
python3 ./test.py -f 7-tmq/subscribeStb2.py -P
python3 ./test.py -f 7-tmq/subscribeStb3.py -P
python3 ./test.py -f 7-tmq/subscribeDb0.py -P -N 3 -n 3
python3 ./test.py -f 1-insert/delete_stable.py -P
python3 ./test.py -f 2-query/out_of_order.py -P -Q 3
python3 ./test.py -f 2-query/out_of_order.py -P
python3 ./test.py -f 2-query/insert_null_none.py -P
python3 ./test.py -f 2-query/insert_null_none.py -P -R
python3 ./test.py -f 2-query/insert_null_none.py -P -Q 2
python3 ./test.py -f 2-query/insert_null_none.py -P -Q 3
python3 ./test.py -f 2-query/insert_null_none.py -P -Q 4
python3 ./test.py -f 1-insert/database_pre_suf.py -P
python3 ./test.py -f 2-query/concat.py -P -Q 3
python3 ./test.py -f 2-query/out_of_order.py -P -Q 2
python3 ./test.py -f 2-query/out_of_order.py -P -Q 4
python3 ./test.py -f 2-query/nestedQuery.py -P -Q 3
python3 ./test.py -f 2-query/nestedQuery_str.py -P -Q 3
python3 ./test.py -f 2-query/nestedQuery_math.py -P -Q 3
python3 ./test.py -f 2-query/nestedQuery_time.py -P -Q 3
python3 ./test.py -f 2-query/nestedQuery_26.py -P -Q 3
python3 ./test.py -f 7-tmq/create_wrong_topic.py -P
python3 ./test.py -f 7-tmq/dropDbR3ConflictTransaction.py -P -N 3
python3 ./test.py -f 7-tmq/basic5.py -P
python3 ./test.py -f 7-tmq/subscribeDb.py -P -N 3 -n 3
python3 ./test.py -f 7-tmq/subscribeDb1.py -P
python3 ./test.py -f 7-tmq/subscribeDb2.py -P
python3 ./test.py -f 7-tmq/subscribeDb3.py -P
python3 ./test.py -f 7-tmq/subscribeDb4.py -P
python3 ./test.py -f 7-tmq/subscribeStb.py -P
python3 ./test.py -f 7-tmq/subscribeStb4.py -P
python3 ./test.py -f 7-tmq/db.py -P
python3 ./test.py -f 7-tmq/tmqError.py -P
python3 ./test.py -f 7-tmq/schema.py -P
python3 ./test.py -f 7-tmq/stbFilter.py -P
python3 ./test.py -f 7-tmq/tmqCheckData.py -P
python3 ./test.py -f 7-tmq/tmqCheckData1.py -P
python3 ./test.py -f 7-tmq/tmqConsumerGroup.py -P
python3 ./test.py -f 7-tmq/tmqAlterSchema.py -P
python3 ./test.py -f 7-tmq/tmqConsFromTsdb.py -P -N 3 -n 3
python3 ./test.py -f 7-tmq/tmqConsFromTsdb1.py -P -N 3 -n 3
python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg.py -P
python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg.py -P
python3 ./test.py -f 7-tmq/tmqConsFromTsdb-1ctb.py -P
python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-1ctb.py -P
python3 ./test.py -f 7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py -P
python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py -P
python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py -P
python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py -P
python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py -P
python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py -P
python3 ./test.py -f 7-tmq/tmqAutoCreateTbl.py -P
python3 ./test.py -f 7-tmq/tmqDnodeRestart.py -P
python3 ./test.py -f 7-tmq/tmqDnodeRestart1.py -P
python3 ./test.py -f 7-tmq/tmqUpdate-1ctb.py -P
python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot0.py -P
python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot1.py -P
python3 ./test.py -f 7-tmq/tmqDropStbCtb.py -P
python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot0.py -P
python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot1.py -P
python3 ./test.py -f 7-tmq/tmqUdf-multCtb-snapshot0.py -P
python3 ./test.py -f 7-tmq/tmqUdf-multCtb-snapshot1.py -P
python3 ./test.py -f 7-tmq/stbTagFilter-1ctb.py -P
python3 ./test.py -f 7-tmq/dataFromTsdbNWal.py -P
python3 ./test.py -f 7-tmq/dataFromTsdbNWal-multiCtb.py -P
python3 ./test.py -f 7-tmq/tmq_taosx.py -P
python3 ./test.py -f 7-tmq/raw_block_interface_test.py -P
python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py -P
python3 ./test.py -f 99-TDcase/TD-19201.py -P
python3 ./test.py -f 99-TDcase/TD-21561.py -P
python3 ./test.py -f 0-others/taosShell.py -P
python3 ./test.py -f 0-others/taosShellError.py -P
python3 ./test.py -f 0-others/taosShellNetChk.py -P
python3 ./test.py -f 0-others/telemetry.py -P
python3 ./test.py -f 0-others/backquote_check.py -P
python3 ./test.py -f 0-others/taosdMonitor.py -P
python3 ./test.py -f 0-others/udfTest.py -P
python3 ./test.py -f 0-others/udf_create.py -P
python3 ./test.py -f 0-others/udf_restart_taosd.py -P
python3 ./test.py -f 0-others/udf_cfg1.py -P
python3 ./test.py -f 0-others/udf_cfg2.py -P
python3 ./test.py -f 0-others/cachemodel.py -P
python3 ./test.py -f 0-others/sysinfo.py -P
python3 ./test.py -f 0-others/user_control.py -P
python3 ./test.py -f 0-others/user_manage.py -P
python3 ./test.py -f 0-others/fsync.py -P
python3 ./test.py -f 0-others/tag_index_basic.py -P
python3 ./test.py -f 0-others/show.py -P
python3 ./test.py -f 0-others/information_schema.py -P
python3 ./test.py -f 2-query/abs.py -P
python3 ./test.py -f 2-query/abs.py -P -R
python3 ./test.py -f 2-query/and_or_for_byte.py -P
python3 ./test.py -f 2-query/and_or_for_byte.py -P -R
python3 ./test.py -f 2-query/apercentile.py -P
python3 ./test.py -f 2-query/apercentile.py -P -R
python3 ./test.py -f 2-query/arccos.py -P
python3 ./test.py -f 2-query/arccos.py -P -R
python3 ./test.py -f 2-query/arcsin.py -P
python3 ./test.py -f 2-query/arcsin.py -P -R
python3 ./test.py -f 2-query/arctan.py -P
python3 ./test.py -f 2-query/arctan.py -P -R
python3 ./test.py -f 2-query/avg.py -P
python3 ./test.py -f 2-query/avg.py -P -R
python3 ./test.py -f 2-query/between.py -P
python3 ./test.py -f 2-query/between.py -P -R
python3 ./test.py -f 2-query/bottom.py -P
python3 ./test.py -f 2-query/bottom.py -P -R
python3 ./test.py -f 2-query/cast.py -P
python3 ./test.py -f 2-query/cast.py -P -R
python3 ./test.py -f 2-query/ceil.py -P
python3 ./test.py -f 2-query/ceil.py -P -R
python3 ./test.py -f 2-query/char_length.py -P
python3 ./test.py -f 2-query/char_length.py -P -R
python3 ./test.py -f 2-query/check_tsdb.py -P
python3 ./test.py -f 2-query/check_tsdb.py -P -R
python3 ./test.py -f 2-query/concat.py -P
python3 ./test.py -f 2-query/concat.py -P -R
python3 ./test.py -f 2-query/concat_ws.py -P
python3 ./test.py -f 2-query/concat_ws.py -P -R
python3 ./test.py -f 2-query/concat_ws2.py -P
python3 ./test.py -f 2-query/concat_ws2.py -P -R
python3 ./test.py -f 2-query/cos.py -P
python3 ./test.py -f 2-query/cos.py -P -R
python3 ./test.py -f 2-query/count_partition.py -P
python3 ./test.py -f 2-query/count_partition.py -P -R
python3 ./test.py -f 2-query/count.py -P
python3 ./test.py -f 2-query/count.py -P -R
python3 ./test.py -f 2-query/countAlwaysReturnValue.py -P
python3 ./test.py -f 2-query/countAlwaysReturnValue.py -P -R
python3 ./test.py -f 2-query/db.py -P
python3 ./test.py -f 2-query/diff.py -P
python3 ./test.py -f 2-query/diff.py -P -R
python3 ./test.py -f 2-query/distinct.py -P
python3 ./test.py -f 2-query/distinct.py -P -R
python3 ./test.py -f 2-query/distribute_agg_apercentile.py -P
python3 ./test.py -f 2-query/distribute_agg_apercentile.py -P -R
python3 ./test.py -f 2-query/distribute_agg_avg.py -P
python3 ./test.py -f 2-query/distribute_agg_avg.py -P -R
python3 ./test.py -f 2-query/distribute_agg_count.py -P
python3 ./test.py -f 2-query/distribute_agg_count.py -P -R
python3 ./test.py -f 2-query/distribute_agg_max.py -P
python3 ./test.py -f 2-query/distribute_agg_max.py -P -R
python3 ./test.py -f 2-query/distribute_agg_min.py -P
python3 ./test.py -f 2-query/distribute_agg_min.py -P -R
python3 ./test.py -f 2-query/distribute_agg_spread.py -P
python3 ./test.py -f 2-query/distribute_agg_spread.py -P -R
python3 ./test.py -f 2-query/distribute_agg_stddev.py -P
python3 ./test.py -f 2-query/distribute_agg_stddev.py -P -R
python3 ./test.py -f 2-query/distribute_agg_sum.py -P
python3 ./test.py -f 2-query/distribute_agg_sum.py -P -R
python3 ./test.py -f 2-query/explain.py -P
python3 ./test.py -f 2-query/explain.py -P -R
python3 ./test.py -f 2-query/first.py -P
python3 ./test.py -f 2-query/first.py -P -R
python3 ./test.py -f 2-query/floor.py -P
python3 ./test.py -f 2-query/floor.py -P -R
python3 ./test.py -f 2-query/function_null.py -P
python3 ./test.py -f 2-query/function_null.py -P -R
python3 ./test.py -f 2-query/function_stateduration.py -P
python3 ./test.py -f 2-query/function_stateduration.py -P -R
python3 ./test.py -f 2-query/histogram.py -P
python3 ./test.py -f 2-query/histogram.py -P -R
python3 ./test.py -f 2-query/hyperloglog.py -P
python3 ./test.py -f 2-query/hyperloglog.py -P -R
python3 ./test.py -f 2-query/interp.py -P
python3 ./test.py -f 2-query/interp.py -P -R
python3 ./test.py -f 2-query/irate.py -P
python3 ./test.py -f 2-query/irate.py -P -R
python3 ./test.py -f 2-query/join.py -P
python3 ./test.py -f 2-query/join.py -P -R
python3 ./test.py -f 2-query/last_row.py -P
python3 ./test.py -f 2-query/last_row.py -P -R
python3 ./test.py -f 2-query/last.py -P
python3 ./test.py -f 2-query/last.py -P -R
python3 ./test.py -f 2-query/leastsquares.py -P
python3 ./test.py -f 2-query/leastsquares.py -P -R
python3 ./test.py -f 2-query/length.py -P
python3 ./test.py -f 2-query/length.py -P -R
python3 ./test.py -f 2-query/limit.py -P
python3 ./test.py -f 2-query/log.py -P
python3 ./test.py -f 2-query/log.py -P -R
python3 ./test.py -f 2-query/lower.py -P
python3 ./test.py -f 2-query/lower.py -P -R
python3 ./test.py -f 2-query/ltrim.py -P
python3 ./test.py -f 2-query/ltrim.py -P -R
python3 ./test.py -f 2-query/mavg.py -P
python3 ./test.py -f 2-query/mavg.py -P -R
python3 ./test.py -f 2-query/max_partition.py -P
python3 ./test.py -f 2-query/max_partition.py -P -R
python3 ./test.py -f 2-query/max_min_last_interval.py -P
python3 ./test.py -f 2-query/last_row_interval.py -P
python3 ./test.py -f 2-query/max.py -P
python3 ./test.py -f 2-query/max.py -P -R
python3 ./test.py -f 2-query/min.py -P
python3 ./test.py -f 2-query/min.py -P -R
python3 ./test.py -f 2-query/mode.py -P
python3 ./test.py -f 2-query/mode.py -P -R
python3 ./test.py -f 2-query/Now.py -P
python3 ./test.py -f 2-query/Now.py -P -R
python3 ./test.py -f 2-query/percentile.py -P
python3 ./test.py -f 2-query/percentile.py -P -R
python3 ./test.py -f 2-query/pow.py -P
python3 ./test.py -f 2-query/pow.py -P -R
python3 ./test.py -f 2-query/query_cols_tags_and_or.py -P
python3 ./test.py -f 2-query/query_cols_tags_and_or.py -P -R
python3 ./test.py -f 2-query/round.py -P
python3 ./test.py -f 2-query/round.py -P -R
python3 ./test.py -f 2-query/rtrim.py -P
python3 ./test.py -f 2-query/rtrim.py -P -R
python3 ./test.py -f 1-insert/drop.py -P -N 3 -M 3 -i False -n 3
python3 ./test.py -f 7-tmq/tmqUpdateWithConsume.py -P -N 3 -n 3
python3 ./test.py -f 2-query/db.py -P -N 3 -n 3 -R
python3 ./test.py -f 2-query/stablity.py -P
python3 ./test.py -f 2-query/stablity_1.py -P
python3 ./test.py -f 2-query/elapsed.py -P
python3 ./test.py -f 2-query/csum.py -P
python3 ./test.py -f 2-query/function_diff.py -P
python3 ./test.py -f 2-query/tagFilter.py -P
python3 ./test.py -f 2-query/projectionDesc.py -P
python3 ./test.py -f 2-query/queryQnode.py -P
python3 ./test.py -f 6-cluster/5dnode1mnode.py -P
# -N 4
echo " ********** -N 4 *************"
python3 ./test.py -f 6-cluster/vnode/4dnode1mnode_basic_createDb_replica1.py -N 4 -M 1
python3 ./test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas.py -P -N 4 -M 1
python3 ./test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas_querys.py -P -N 4 -M 1
python3 ./test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas.py -P -N 4 -M 1
python3 ./test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys.py -P -N 4 -M 1
python3 ./test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py -P -N 4 -M 1
python3 ./test.py -f 2-query/varchar.py -P -R
python3 ./test.py -f 2-query/case_when.py -P
python3 ./test.py -f 2-query/case_when.py -P -R
python3 ./test.py -f 2-query/blockSMA.py -P
python3 ./test.py -f 2-query/blockSMA.py -P -R
python3 ./test.py -f 2-query/projectionDesc.py -P
python3 ./test.py -f 2-query/projectionDesc.py -P -R
python3 ./test.py -f 1-insert/update_data.py -P
python3 ./test.py -f 1-insert/tb_100w_data_order.py -P
python3 ./test.py -f 1-insert/delete_childtable.py -P
python3 ./test.py -f 1-insert/delete_normaltable.py -P
python3 ./test.py -f 1-insert/keep_expired.py -P
python3 ./test.py -f 1-insert/drop.py -P
python3 ./test.py -f 2-query/join2.py -P
python3 ./test.py -f 2-query/union1.py -P
python3 ./test.py -f 2-query/concat2.py -P
python3 ./test.py -f 2-query/json_tag.py -P
python3 ./test.py -f 2-query/nestedQueryInterval.py -P
# -N 5
echo " ********** -N 5 *************"
python3 ./test.py -f 6-cluster/5dnode2mnode.py -P -N 5
python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -P -N 5 -M 3 -i False
python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -P -N 5 -M 3
python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -P -N 5 -M 3 -i False
python3 ./test.py -f 6-cluster/5dnode3mnodeStopLoop.py -P -N 5 -M 3
python3 ./test.py -f 6-cluster/5dnode3mnodeRecreateMnode.py -P -N 5 -M 3
python3 ./test.py -f 6-cluster/5dnode3mnodeStopFollowerLeader.py -P -N 5 -M 3
python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -P -N 5 -M 3
python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -P -N 5 -M 3
python3 ./test.py -f 0-others/taosdShell.py -P -N 5 -M 3 -Q 3
python3 ./test.py -f 7-tmq/tmqSubscribeStb-r3.py -P -N 5
python3 ./test.py -f 2-query/timezone.py -P
python3 ./test.py -f 2-query/timezone.py -P -R
python3 ./test.py -f 2-query/To_iso8601.py -P
python3 ./test.py -f 2-query/To_iso8601.py -P -R
python3 ./test.py -f 2-query/To_unixtimestamp.py -P
python3 ./test.py -f 2-query/To_unixtimestamp.py -P -R
python3 ./test.py -f 2-query/Today.py -P
python3 ./test.py -f 2-query/Today.py -P -R
python3 ./test.py -f 2-query/top.py -P
python3 ./test.py -f 2-query/top.py -P -R
python3 ./test.py -f 2-query/tsbsQuery.py -P
python3 ./test.py -f 2-query/tsbsQuery.py -P -R
python3 ./test.py -f 2-query/ttl_comment.py -P
python3 ./test.py -f 2-query/ttl_comment.py -P -R
python3 ./test.py -f 2-query/twa.py -P
python3 ./test.py -f 2-query/twa.py -P -R
python3 ./test.py -f 2-query/union.py -P
python3 ./test.py -f 2-query/union.py -P -R
python3 ./test.py -f 2-query/unique.py -P
python3 ./test.py -f 2-query/unique.py -P -R
python3 ./test.py -f 2-query/upper.py -P
python3 ./test.py -f 2-query/upper.py -P -R
python3 ./test.py -f 2-query/varchar.py -P
# -N6
echo " ********** -N 6 *************"
python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 6 -M 3
python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -P -N 6 -M 3 -n 3
python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -P -N 6 -M 3
python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -P -N 6 -M 3 -n 3
python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -P -N 6 -M 3
python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -P -N 6 -M 3 -n 3
python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeModifyMeta.py -P -N 6 -M 3
python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeModifyMeta.py -P -N 6 -M 3
python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py -P -N 6 -M 3
python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py -P -N 6 -M 3 -n 3
python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py -P -N 6 -M 3
python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py -P -N 6 -M 3 -n 3
python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py -P -N 6 -M 3
python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py -P -N 6 -M 3 -n 3
python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertData.py -P -N 6 -M 3
python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertData.py -P -N 6 -M 3 -n 3
python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -P -N 6 -M 3
python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -P -N 6 -M 3 -n 3
python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -P -N 6 -M 3
python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -P -N 6 -M 3 -n 3
python3 ./test.py -f 0-others/balance_vgroups_r1.py -P -N 6
python3 ./test.py -f 2-query/sample.py -P
python3 ./test.py -f 2-query/sample.py -P -R
python3 ./test.py -f 2-query/sin.py -P
python3 ./test.py -f 2-query/sin.py -P -R
python3 ./test.py -f 2-query/smaTest.py -P
python3 ./test.py -f 2-query/smaTest.py -P -R
python3 ./test.py -f 2-query/sml.py -P
python3 ./test.py -f 2-query/sml.py -P -R
python3 ./test.py -f 2-query/spread.py -P
python3 ./test.py -f 2-query/spread.py -P -R
python3 ./test.py -f 2-query/sqrt.py -P
python3 ./test.py -f 2-query/sqrt.py -P -R
python3 ./test.py -f 2-query/statecount.py -P
python3 ./test.py -f 2-query/statecount.py -P -R
python3 ./test.py -f 2-query/stateduration.py -P
python3 ./test.py -f 2-query/stateduration.py -P -R
python3 ./test.py -f 2-query/substr.py -P
python3 ./test.py -f 2-query/substr.py -P -R
python3 ./test.py -f 2-query/sum.py -P
python3 ./test.py -f 2-query/sum.py -P -R
python3 ./test.py -f 2-query/tail.py -P
python3 ./test.py -f 2-query/tail.py -P -R
python3 ./test.py -f 2-query/tan.py -P
python3 ./test.py -f 2-query/tan.py -P -R
python3 ./test.py -f 2-query/Timediff.py -P
python3 ./test.py -f 2-query/Timediff.py -P -R
python3 ./test.py -f 2-query/timetruncate.py -P
python3 ./test.py -f 2-query/timetruncate.py -P -R
# N-7
echo " ********** -N 7 *************"
python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6
python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -P -N 7 -M 3 -C 6 -n 3
python3 ./test.py -f 2-query/between.py -P -Q 2
python3 ./test.py -f 2-query/distinct.py -P -Q 2
python3 ./test.py -f 2-query/varchar.py -P -Q 2
python3 ./test.py -f 2-query/ltrim.py -P -Q 2
python3 ./test.py -f 2-query/rtrim.py -P -Q 2
python3 ./test.py -f 2-query/length.py -P -Q 2
python3 ./test.py -f 2-query/char_length.py -P -Q 2
python3 ./test.py -f 2-query/upper.py -P -Q 2
python3 ./test.py -f 2-query/lower.py -P -Q 2
python3 ./test.py -f 2-query/join.py -P -Q 2
python3 ./test.py -f 2-query/join2.py -P -Q 2
python3 ./test.py -f 2-query/cast.py -P -Q 2
python3 ./test.py -f 2-query/substr.py -P -Q 2
python3 ./test.py -f 2-query/union.py -P -Q 2
python3 ./test.py -f 2-query/union1.py -P -Q 2
python3 ./test.py -f 2-query/concat.py -P -Q 2
python3 ./test.py -f 2-query/concat2.py -P -Q 2
python3 ./test.py -f 2-query/concat_ws.py -P -Q 2
python3 ./test.py -f 2-query/concat_ws2.py -P -Q 2
python3 ./test.py -f 2-query/check_tsdb.py -P -Q 2
python3 ./test.py -f 2-query/spread.py -P -Q 2
python3 ./test.py -f 2-query/hyperloglog.py -P -Q 2
python3 ./test.py -f 2-query/explain.py -P -Q 2
python3 ./test.py -f 2-query/leastsquares.py -P -Q 2
python3 ./test.py -f 2-query/timezone.py -P -Q 2
python3 ./test.py -f 2-query/Now.py -P -Q 2
python3 ./test.py -f 2-query/Today.py -P -Q 2
python3 ./test.py -f 2-query/max.py -P -Q 2
python3 ./test.py -f 2-query/min.py -P -Q 2
python3 ./test.py -f 2-query/mode.py -P -Q 2
python3 ./test.py -f 2-query/count.py -P -Q 2
python3 ./test.py -f 2-query/countAlwaysReturnValue.py -P -Q 2
python3 ./test.py -f 2-query/last.py -P -Q 2
python3 ./test.py -f 2-query/first.py -P -Q 2
python3 ./test.py -f 2-query/To_iso8601.py -P -Q 2
python3 ./test.py -f 2-query/To_unixtimestamp.py -P -Q 2
python3 ./test.py -f 2-query/timetruncate.py -P -Q 2
python3 ./test.py -f 2-query/diff.py -P -Q 2
python3 ./test.py -f 2-query/Timediff.py -P -Q 2
python3 ./test.py -f 2-query/json_tag.py -P -Q 2
python3 ./test.py -f 2-query/top.py -P -Q 2
python3 ./test.py -f 2-query/bottom.py -P -Q 2
python3 ./test.py -f 2-query/percentile.py -P -Q 2
python3 ./test.py -f 2-query/apercentile.py -P -Q 2
python3 ./test.py -f 2-query/abs.py -P -Q 2
python3 ./test.py -f 2-query/ceil.py -P -Q 2
python3 ./test.py -f 2-query/floor.py -P -Q 2
python3 ./test.py -f 2-query/round.py -P -Q 2
python3 ./test.py -f 2-query/log.py -P -Q 2
python3 ./test.py -f 2-query/pow.py -P -Q 2
python3 ./test.py -f 2-query/sqrt.py -P -Q 2
python3 ./test.py -f 2-query/sin.py -P -Q 2
python3 ./test.py -f 2-query/cos.py -P -Q 2
python3 ./test.py -f 2-query/tan.py -P -Q 2
python3 ./test.py -f 2-query/arcsin.py -P -Q 2
python3 ./test.py -f 2-query/arccos.py -P -Q 2
python3 ./test.py -f 2-query/arctan.py -P -Q 2
python3 ./test.py -f 2-query/query_cols_tags_and_or.py -P -Q 2
python3 ./test.py -f 2-query/interp.py -P -Q 2
python3 ./test.py -f 2-query/nestedQueryInterval.py -P -Q 2
python3 ./test.py -f 2-query/stablity.py -P -Q 2
python3 ./test.py -f 2-query/stablity_1.py -P -Q 2
python3 ./test.py -f 2-query/avg.py -P -Q 2
python3 ./test.py -f 2-query/elapsed.py -P -Q 2
python3 ./test.py -f 2-query/csum.py -P -Q 2
python3 ./test.py -f 2-query/mavg.py -P -Q 2
python3 ./test.py -f 2-query/sample.py -P -Q 2
python3 ./test.py -f 2-query/function_diff.py -P -Q 2
python3 ./test.py -f 2-query/unique.py -P -Q 2
python3 ./test.py -f 2-query/stateduration.py -P -Q 2
python3 ./test.py -f 2-query/function_stateduration.py -P -Q 2
python3 ./test.py -f 2-query/statecount.py -P -Q 2
python3 ./test.py -f 2-query/tail.py -P -Q 2
python3 ./test.py -f 2-query/ttl_comment.py -P -Q 2
python3 ./test.py -f 2-query/distribute_agg_count.py -P -Q 2
python3 ./test.py -f 2-query/distribute_agg_max.py -P -Q 2
python3 ./test.py -f 2-query/distribute_agg_min.py -P -Q 2
python3 ./test.py -f 2-query/distribute_agg_sum.py -P -Q 2
python3 ./test.py -f 2-query/distribute_agg_spread.py -P -Q 2
python3 ./test.py -f 2-query/distribute_agg_apercentile.py -P -Q 2
python3 ./test.py -f 2-query/distribute_agg_avg.py -P -Q 2
python3 ./test.py -f 2-query/distribute_agg_stddev.py -P -Q 2
python3 ./test.py -f 2-query/twa.py -P -Q 2
python3 ./test.py -f 2-query/irate.py -P -Q 2
python3 ./test.py -f 2-query/function_null.py -P -Q 2
python3 ./test.py -f 2-query/count_partition.py -P -Q 2
python3 ./test.py -f 2-query/max_partition.py -P -Q 2
python3 ./test.py -f 2-query/max_min_last_interval.py -P -Q 2
python3 ./test.py -f 2-query/last_row_interval.py -P -Q 2
python3 ./test.py -f 2-query/last_row.py -P -Q 2
python3 ./test.py -f 2-query/tsbsQuery.py -P -Q 2
python3 ./test.py -f 2-query/sml.py -P -Q 2
python3 ./test.py -f 2-query/case_when.py -P -Q 2
python3 ./test.py -f 2-query/blockSMA.py -P -Q 2
python3 ./test.py -f 2-query/projectionDesc.py -P -Q 2
python3 ./test.py -f 99-TDcase/TD-21561.py -P -Q 2
python3 ./test.py -f 2-query/between.py -P -Q 3
python3 ./test.py -f 2-query/distinct.py -P -Q 3
python3 ./test.py -f 2-query/varchar.py -P -Q 3
python3 ./test.py -f 2-query/ltrim.py -P -Q 3
python3 ./test.py -f 2-query/rtrim.py -P -Q 3
python3 ./test.py -f 2-query/length.py -P -Q 3
python3 ./test.py -f 2-query/char_length.py -P -Q 3
python3 ./test.py -f 2-query/upper.py -P -Q 3
python3 ./test.py -f 2-query/lower.py -P -Q 3
python3 ./test.py -f 2-query/join.py -P -Q 3
python3 ./test.py -f 2-query/join2.py -P -Q 3
python3 ./test.py -f 2-query/cast.py -P -Q 3
python3 ./test.py -f 2-query/substr.py -P -Q 3
python3 ./test.py -f 2-query/union.py -P -Q 3
python3 ./test.py -f 2-query/union1.py -P -Q 3
python3 ./test.py -f 2-query/concat2.py -P -Q 3
python3 ./test.py -f 2-query/concat_ws.py -P -Q 3
python3 ./test.py -f 2-query/concat_ws2.py -P -Q 3
python3 ./test.py -f 2-query/check_tsdb.py -P -Q 3
python3 ./test.py -f 2-query/spread.py -P -Q 3
python3 ./test.py -f 2-query/hyperloglog.py -P -Q 3
python3 ./test.py -f 2-query/explain.py -P -Q 3
python3 ./test.py -f 2-query/leastsquares.py -P -Q 3
python3 ./test.py -f 2-query/timezone.py -P -Q 3
python3 ./test.py -f 2-query/Now.py -P -Q 3
python3 ./test.py -f 2-query/Today.py -P -Q 3
python3 ./test.py -f 2-query/max.py -P -Q 3
python3 ./test.py -f 2-query/min.py -P -Q 3
python3 ./test.py -f 2-query/mode.py -P -Q 3
python3 ./test.py -f 2-query/count.py -P -Q 3
python3 ./test.py -f 2-query/countAlwaysReturnValue.py -P -Q 3
python3 ./test.py -f 2-query/last.py -P -Q 3
python3 ./test.py -f 2-query/first.py -P -Q 3
python3 ./test.py -f 2-query/To_iso8601.py -P -Q 3
python3 ./test.py -f 2-query/To_unixtimestamp.py -P -Q 3
python3 ./test.py -f 2-query/timetruncate.py -P -Q 3
python3 ./test.py -f 2-query/diff.py -P -Q 3
python3 ./test.py -f 2-query/Timediff.py -P -Q 3
python3 ./test.py -f 2-query/json_tag.py -P -Q 3
python3 ./test.py -f 2-query/top.py -P -Q 3
python3 ./test.py -f 2-query/bottom.py -P -Q 3
python3 ./test.py -f 2-query/percentile.py -P -Q 3
python3 ./test.py -f 2-query/apercentile.py -P -Q 3
python3 ./test.py -f 2-query/abs.py -P -Q 3
python3 ./test.py -f 2-query/ceil.py -P -Q 3
python3 ./test.py -f 2-query/floor.py -P -Q 3
python3 ./test.py -f 2-query/round.py -P -Q 3
python3 ./test.py -f 2-query/log.py -P -Q 3
python3 ./test.py -f 2-query/pow.py -P -Q 3
python3 ./test.py -f 2-query/sqrt.py -P -Q 3
python3 ./test.py -f 2-query/sin.py -P -Q 3
python3 ./test.py -f 2-query/cos.py -P -Q 3
python3 ./test.py -f 2-query/tan.py -P -Q 3
python3 ./test.py -f 2-query/arcsin.py -P -Q 3
python3 ./test.py -f 2-query/arccos.py -P -Q 3
python3 ./test.py -f 2-query/arctan.py -P -Q 3
python3 ./test.py -f 2-query/query_cols_tags_and_or.py -P -Q 3
python3 ./test.py -f 2-query/nestedQueryInterval.py -P -Q 3
python3 ./test.py -f 2-query/stablity.py -P -Q 3
python3 ./test.py -f 2-query/stablity_1.py -P -Q 3
python3 ./test.py -f 2-query/avg.py -P -Q 3
python3 ./test.py -f 2-query/elapsed.py -P -Q 3
python3 ./test.py -f 2-query/csum.py -P -Q 3
python3 ./test.py -f 2-query/mavg.py -P -Q 3
python3 ./test.py -f 2-query/sample.py -P -Q 3
python3 ./test.py -f 2-query/function_diff.py -P -Q 3
python3 ./test.py -f 2-query/unique.py -P -Q 3
python3 ./test.py -f 2-query/stateduration.py -P -Q 3
python3 ./test.py -f 2-query/function_stateduration.py -P -Q 3
python3 ./test.py -f 2-query/statecount.py -P -Q 3
python3 ./test.py -f 2-query/tail.py -P -Q 3
python3 ./test.py -f 2-query/ttl_comment.py -P -Q 3
python3 ./test.py -f 2-query/distribute_agg_count.py -P -Q 3
python3 ./test.py -f 2-query/distribute_agg_max.py -P -Q 3
python3 ./test.py -f 2-query/distribute_agg_min.py -P -Q 3
python3 ./test.py -f 2-query/distribute_agg_sum.py -P -Q 3
python3 ./test.py -f 2-query/distribute_agg_spread.py -P -Q 3
python3 ./test.py -f 2-query/distribute_agg_apercentile.py -P -Q 3
python3 ./test.py -f 2-query/distribute_agg_avg.py -P -Q 3
python3 ./test.py -f 2-query/distribute_agg_stddev.py -P -Q 3
python3 ./test.py -f 2-query/twa.py -P -Q 3
python3 ./test.py -f 2-query/irate.py -P -Q 3
python3 ./test.py -f 2-query/function_null.py -P -Q 3
python3 ./test.py -f 2-query/count_partition.py -P -Q 3
python3 ./test.py -f 2-query/max_partition.py -P -Q 3
python3 ./test.py -f 2-query/max_min_last_interval.py -P -Q 3
python3 ./test.py -f 2-query/last_row_interval.py -P -Q 3
python3 ./test.py -f 2-query/last_row.py -P -Q 3
python3 ./test.py -f 2-query/tsbsQuery.py -P -Q 3
python3 ./test.py -f 2-query/sml.py -P -Q 3
python3 ./test.py -f 2-query/interp.py -P -Q 3
python3 ./test.py -f 2-query/case_when.py -P -Q 3
python3 ./test.py -f 2-query/blockSMA.py -P -Q 3
python3 ./test.py -f 2-query/projectionDesc.py -P -Q 3
python3 ./test.py -f 99-TDcase/TD-21561.py -P -Q 3
python3 ./test.py -f 2-query/between.py -P -Q 4
python3 ./test.py -f 2-query/distinct.py -P -Q 4
python3 ./test.py -f 2-query/varchar.py -P -Q 4
python3 ./test.py -f 2-query/ltrim.py -P -Q 4
python3 ./test.py -f 2-query/rtrim.py -P -Q 4
python3 ./test.py -f 2-query/length.py -P -Q 4
python3 ./test.py -f 2-query/char_length.py -P -Q 4
python3 ./test.py -f 2-query/upper.py -P -Q 4
python3 ./test.py -f 2-query/lower.py -P -Q 4
python3 ./test.py -f 2-query/join.py -P -Q 4
python3 ./test.py -f 2-query/join2.py -P -Q 4
python3 ./test.py -f 2-query/substr.py -P -Q 4
python3 ./test.py -f 2-query/union.py -P -Q 4
python3 ./test.py -f 2-query/union1.py -P -Q 4
python3 ./test.py -f 2-query/concat.py -P -Q 4
python3 ./test.py -f 2-query/concat2.py -P -Q 4
python3 ./test.py -f 2-query/concat_ws.py -P -Q 4
python3 ./test.py -f 2-query/concat_ws2.py -P -Q 4
python3 ./test.py -f 2-query/check_tsdb.py -P -Q 4
python3 ./test.py -f 2-query/spread.py -P -Q 4
python3 ./test.py -f 2-query/hyperloglog.py -P -Q 4
python3 ./test.py -f 2-query/explain.py -P -Q 4
python3 ./test.py -f 2-query/leastsquares.py -P -Q 4
python3 ./test.py -f 2-query/timezone.py -P -Q 4
python3 ./test.py -f 2-query/Now.py -P -Q 4
python3 ./test.py -f 2-query/Today.py -P -Q 4
python3 ./test.py -f 2-query/max.py -P -Q 4
python3 ./test.py -f 2-query/min.py -P -Q 4
python3 ./test.py -f 2-query/mode.py -P -Q 4
python3 ./test.py -f 2-query/count.py -P -Q 4
python3 ./test.py -f 2-query/countAlwaysReturnValue.py -P -Q 4
python3 ./test.py -f 2-query/last.py -P -Q 4
python3 ./test.py -f 2-query/first.py -P -Q 4
python3 ./test.py -f 2-query/To_iso8601.py -P -Q 4
python3 ./test.py -f 2-query/To_unixtimestamp.py -P -Q 4
python3 ./test.py -f 2-query/timetruncate.py -P -Q 4
python3 ./test.py -f 2-query/diff.py -P -Q 4
python3 ./test.py -f 2-query/Timediff.py -P -Q 4
python3 ./test.py -f 2-query/json_tag.py -P -Q 4
python3 ./test.py -f 2-query/top.py -P -Q 4
python3 ./test.py -f 2-query/bottom.py -P -Q 4
python3 ./test.py -f 2-query/percentile.py -P -Q 4
python3 ./test.py -f 2-query/apercentile.py -P -Q 4
python3 ./test.py -f 2-query/abs.py -P -Q 4
python3 ./test.py -f 2-query/ceil.py -P -Q 4
python3 ./test.py -f 2-query/floor.py -P -Q 4
python3 ./test.py -f 2-query/round.py -P -Q 4
python3 ./test.py -f 2-query/log.py -P -Q 4
python3 ./test.py -f 2-query/pow.py -P -Q 4
python3 ./test.py -f 2-query/sqrt.py -P -Q 4
python3 ./test.py -f 2-query/sin.py -P -Q 4
python3 ./test.py -f 2-query/cos.py -P -Q 4
python3 ./test.py -f 2-query/tan.py -P -Q 4
python3 ./test.py -f 2-query/arcsin.py -P -Q 4
python3 ./test.py -f 2-query/arccos.py -P -Q 4
python3 ./test.py -f 2-query/arctan.py -P -Q 4
python3 ./test.py -f 2-query/query_cols_tags_and_or.py -P -Q 4
python3 ./test.py -f 2-query/nestedQueryInterval.py -P -Q 4
python3 ./test.py -f 2-query/stablity.py -P -Q 4
python3 ./test.py -f 2-query/stablity_1.py -P -Q 4
python3 ./test.py -f 2-query/avg.py -P -Q 4
python3 ./test.py -f 2-query/elapsed.py -P -Q 4
python3 ./test.py -f 2-query/csum.py -P -Q 4
python3 ./test.py -f 2-query/mavg.py -P -Q 4
python3 ./test.py -f 2-query/sample.py -P -Q 4
python3 ./test.py -f 2-query/cast.py -P -Q 4
python3 ./test.py -f 2-query/function_diff.py -P -Q 4
python3 ./test.py -f 2-query/unique.py -P -Q 4
python3 ./test.py -f 2-query/stateduration.py -P -Q 4
python3 ./test.py -f 2-query/function_stateduration.py -P -Q 4
python3 ./test.py -f 2-query/statecount.py -P -Q 4
python3 ./test.py -f 2-query/tail.py -P -Q 4
python3 ./test.py -f 2-query/ttl_comment.py -P -Q 4
python3 ./test.py -f 2-query/distribute_agg_count.py -P -Q 4
python3 ./test.py -f 2-query/distribute_agg_max.py -P -Q 4
python3 ./test.py -f 2-query/distribute_agg_min.py -P -Q 4
python3 ./test.py -f 2-query/distribute_agg_sum.py -P -Q 4
python3 ./test.py -f 2-query/distribute_agg_spread.py -P -Q 4
python3 ./test.py -f 2-query/distribute_agg_apercentile.py -P -Q 4
python3 ./test.py -f 2-query/distribute_agg_avg.py -P -Q 4
python3 ./test.py -f 2-query/distribute_agg_stddev.py -P -Q 4
python3 ./test.py -f 2-query/twa.py -P -Q 4
python3 ./test.py -f 2-query/irate.py -P -Q 4
python3 ./test.py -f 2-query/function_null.py -P -Q 4
python3 ./test.py -f 2-query/count_partition.py -P -Q 4
python3 ./test.py -f 2-query/max_partition.py -P -Q 4
python3 ./test.py -f 2-query/max_min_last_interval.py -P -Q 4
python3 ./test.py -f 2-query/last_row_interval.py -P -Q 4
python3 ./test.py -f 2-query/last_row.py -P -Q 4
python3 ./test.py -f 2-query/tsbsQuery.py -P -Q 4
python3 ./test.py -f 2-query/sml.py -P -Q 4
python3 ./test.py -f 2-query/interp.py -P -Q 4
python3 ./test.py -f 2-query/case_when.py -P -Q 4
python3 ./test.py -f 2-query/insert_select.py -P
python3 ./test.py -f 2-query/insert_select.py -P -R
python3 ./test.py -f 2-query/insert_select.py -P -Q 2
python3 ./test.py -f 2-query/insert_select.py -P -Q 3
python3 ./test.py -f 2-query/insert_select.py -P -Q 4
python3 ./test.py -f 2-query/out_of_order.py -P -R
python3 ./test.py -f 2-query/blockSMA.py -P -Q 4
python3 ./test.py -f 2-query/projectionDesc.py -P -Q 4
python3 ./test.py -f 2-query/odbc.py -P
python3 ./test.py -f 99-TDcase/TD-21561.py -P -Q 4
python3 ./test.py -f 99-TDcase/TD-20582.py -P

76
tests/system-test/test.py
View File

@ -13,6 +13,7 @@
# pip install src/connector/python/
# -*- coding: utf-8 -*-
import os
import sys
import getopt
import subprocess
@ -22,6 +23,7 @@ import json
import platform
import socket
import threading
import importlib
import toml
sys.path.append("../pytest")
@ -53,8 +55,39 @@ def checkRunTimeError():
if hwnd:
os.system("TASKKILL /F /IM taosd.exe")
#
# run case on previous cluster
#
def runOnPreviousCluster(host, config, fileName):
print("enter run on previeous")
# load case module
sep = "/"
if platform.system().lower() == 'windows':
sep = os.sep
moduleName = fileName.replace(".py", "").replace(sep, ".")
uModule = importlib.import_module(moduleName)
case = uModule.TDTestCase()
# create conn
conn = taos.connect(host, config)
# run case
case.init(conn, False)
try:
case.run()
except Exception as e:
tdLog.notice(repr(e))
tdLog.exit("%s failed" % (fileName))
# stop
case.stop()
if __name__ == "__main__":
#
# analysis paramaters
#
fileName = "all"
deployPath = ""
masterIp = ""
@ -75,8 +108,9 @@ if __name__ == "__main__":
replicaVar = 1
asan = False
independentMnode = True
opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:N:M:Q:C:RD:n:i:a', [
'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd','dnodeNums','mnodeNums','queryPolicy','createDnodeNums','restful','adaptercfgupdate','replicaVar','independentMnode'])
previousCluster = False
opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:N:M:Q:C:RD:n:i:aP', [
'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd','dnodeNums','mnodeNums','queryPolicy','createDnodeNums','restful','adaptercfgupdate','replicaVar','independentMnode','previous'])
for key, value in opts:
if key in ['-h', '--help']:
tdLog.printNoPrefix(
@ -101,6 +135,7 @@ if __name__ == "__main__":
tdLog.printNoPrefix('-n the number of replicas')
tdLog.printNoPrefix('-i independentMnode Mnode')
tdLog.printNoPrefix('-a address sanitizer mode')
tdLog.printNoPrefix('-P run case with [P]revious cluster, do not create new cluster to run case.')
sys.exit(0)
@ -182,6 +217,12 @@ if __name__ == "__main__":
if key in ['-n', '--replicaVar']:
replicaVar = value
if key in ['-P', '--previous']:
previousCluster = True
#
# do exeCmd command
#
if not execCmd == "":
if restful:
tAdapter.init(deployPath)
@ -191,6 +232,9 @@ if __name__ == "__main__":
exec(execCmd)
quit()
#
# do stop option
#
if (stop != 0):
if (valgrind == 0):
toBeKilled = "taosd"
@ -248,6 +292,9 @@ if __name__ == "__main__":
tdLog.info('stop All dnodes')
#
# get hostname
#
if masterIp == "":
host = socket.gethostname()
else:
@ -256,8 +303,20 @@ if __name__ == "__main__":
host = config["host"]
except Exception as r:
host = masterIp
tdLog.info("Procedures for tdengine deployed in %s" % (host))
#
# do previousCluster option
#
if previousCluster:
tdDnodes.init(deployPath, masterIp)
runOnPreviousCluster(host, tdDnodes.getSimCfgPath(), fileName)
tdLog.info("run on previous cluster end.")
quit()
#
# windows run
#
if platform.system().lower() == 'windows':
fileName = fileName.replace("/", os.sep)
if (masterIp == "" and not fileName == "0-others\\udf_create.py"):
@ -387,6 +446,10 @@ if __name__ == "__main__":
tdCases.runOneWindows(conn, fileName)
else:
tdCases.runAllWindows(conn)
#
# linux run
#
else:
tdDnodes.setKillValgrind(killValgrind)
tdDnodes.init(deployPath, masterIp)
@ -418,6 +481,7 @@ if __name__ == "__main__":
tAdapter.stop(force_kill=True)
if dnodeNums == 1 :
# dnode is one
tdDnodes.deploy(1,updateCfgDict)
tdDnodes.start(1)
tdCases.logSql(logSql)
@ -458,6 +522,7 @@ if __name__ == "__main__":
tdLog.exit(f"alter queryPolicy to {queryPolicy} failed")
else :
# dnode > 1 cluster
tdLog.debug("create an cluster with %s nodes and make %s dnode as independent mnode"%(dnodeNums,mnodeNums))
dnodeslist = cluster.configure_cluster(dnodeNums=dnodeNums, mnodeNums=mnodeNums, independentMnode=independentMnode)
tdDnodes = ClusterDnodes(dnodeslist)
@ -476,6 +541,7 @@ if __name__ == "__main__":
tAdapter.deploy(adapter_cfg_dict)
tAdapter.start()
# create taos connect
if not restful:
conn = taos.connect(host,config=tdDnodes.getSimCfgPath())
else:
@ -494,6 +560,7 @@ if __name__ == "__main__":
except Exception as r:
print(r)
# do queryPolicy option
if queryPolicy != 1:
queryPolicy=int(queryPolicy)
if restful:
@ -515,6 +582,7 @@ if __name__ == "__main__":
tdLog.exit(f"alter queryPolicy to {queryPolicy} failed")
# run case
if testCluster:
tdLog.info("Procedures for testing cluster")
if fileName == "all":
@ -533,6 +601,7 @@ if __name__ == "__main__":
else:
tdCases.runOneLinux(conn, fileName, replicaVar)
# do restart option
if restart:
if fileName == "all":
tdLog.info("not need to query ")
@ -552,6 +621,7 @@ if __name__ == "__main__":
else:
tdLog.info("not need to query")
# close for end
if conn is not None:
conn.close()
if asan:

12
tools/shell/src/shellArguments.c
View File

@ -20,18 +20,6 @@
#include "shellInt.h"
#include "version.h"
#ifndef CUS_NAME
char cusName[] = "TDengine";
#endif
#ifndef CUS_PROMPT
char cusPrompt[] = "taos";
#endif
#ifndef CUS_EMAIL
char cusEmail[] = "<support@taosdata.com>";
#endif
#if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL)
#include "cus_name.h"
#endif

22
tools/shell/src/shellEngine.c
View File

@ -315,7 +315,6 @@ void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, i
quotationStr[0] = '\"';
quotationStr[1] = 0;
int n;
char buf[TSDB_MAX_BYTES_PER_ROW];
switch (field->type) {
case TSDB_DATA_TYPE_BOOL:
@ -346,15 +345,11 @@ void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, i
taosFprintfFile(pFile, "%" PRIu64, *((uint64_t *)val));
break;
case TSDB_DATA_TYPE_FLOAT:
taosFprintfFile(pFile, "%.5f", GET_FLOAT_VAL(val));
taosFprintfFile(pFile, "%e", GET_FLOAT_VAL(val));
break;
case TSDB_DATA_TYPE_DOUBLE:
n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.9f", length, GET_DOUBLE_VAL(val));
if (n > TMAX(25, length)) {
taosFprintfFile(pFile, "%*.15e", length, GET_DOUBLE_VAL(val));
} else {
taosFprintfFile(pFile, "%s", buf);
}
snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.15e", 23, GET_DOUBLE_VAL(val));
taosFprintfFile(pFile, "%s", buf);
break;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
@ -510,7 +505,6 @@ void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t
return;
}
int n;
char buf[TSDB_MAX_BYTES_PER_ROW];
switch (field->type) {
case TSDB_DATA_TYPE_BOOL:
@ -541,15 +535,11 @@ void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t
printf("%*" PRIu64, width, *((uint64_t *)val));
break;
case TSDB_DATA_TYPE_FLOAT:
printf("%*ef", width, GET_FLOAT_VAL(val));
printf("%*e", width, GET_FLOAT_VAL(val));
break;
case TSDB_DATA_TYPE_DOUBLE:
n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.9f", width, GET_DOUBLE_VAL(val));
if (n > TMAX(25, width)) {
printf("%*.15e", width, GET_DOUBLE_VAL(val));
} else {
printf("%s", buf);
}
snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%.9e", GET_DOUBLE_VAL(val));
printf("%*s", width, buf);
break;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:

8
utils/test/c/CMakeLists.txt
View File

@ -6,6 +6,14 @@ add_executable(tmq_taosx_ci tmq_taosx_ci.c)
add_executable(write_raw_block_test write_raw_block_test.c)
add_executable(sml_test sml_test.c)
add_executable(get_db_name_test get_db_name_test.c)
add_executable(tmq_offset tmqOffset.c)
target_link_libraries(
tmq_offset
PUBLIC taos_static
PUBLIC util
PUBLIC common
PUBLIC os
)
target_link_libraries(
create_table
PUBLIC taos_static

64
utils/test/c/tmqOffset.c Normal file
View File

@ -0,0 +1,64 @@
//
// Created by mingming wanng on 2023/4/7.
//
#include <stdio.h>
#include <stdlib.h>
#include "taoserror.h"
#include "tlog.h"
#include "tmsg.h"
typedef struct {
int32_t size;
} STqOffsetHead;
int32_t tqOffsetRestoreFromFile(const char* fname) {
TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ);
if (pFile != NULL) {
STqOffsetHead head = {0};
int32_t code;
while (1) {
if ((code = taosReadFile(pFile, &head, sizeof(STqOffsetHead))) != sizeof(STqOffsetHead)) {
if (code == 0) {
break;
} else {
printf("code:%d != 0\n", code);
return -1;
}
}
int32_t size = htonl(head.size);
void* memBuf = taosMemoryCalloc(1, size);
if (memBuf == NULL) {
printf("memBuf == NULL\n");
return -1;
}
if ((code = taosReadFile(pFile, memBuf, size)) != size) {
taosMemoryFree(memBuf);
printf("code:%d != size:%d\n", code, size);
return -1;
}
STqOffset offset;
SDecoder decoder;
tDecoderInit(&decoder, memBuf, size);
if (tDecodeSTqOffset(&decoder, &offset) < 0) {
taosMemoryFree(memBuf);
tDecoderClear(&decoder);
printf("tDecodeSTqOffset error\n");
return -1;
}
tDecoderClear(&decoder);
printf("subkey:%s, type:%d, uid/version:%"PRId64", ts:%"PRId64"\n",
offset.subKey, offset.val.type, offset.val.uid, offset.val.ts);
taosMemoryFree(memBuf);
}
taosCloseFile(&pFile);
}
return 0;
}
int main(int argc, char *argv[]) {
tqOffsetRestoreFromFile("offset-ver0");
return 0;
}