diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 94af11868a..b65576deaf 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -43,10 +43,8 @@ def pre_test(){ sh ''' cd ${WK} git reset --hard - git fetch || git fetch cd ${WKC} git reset --hard - git fetch || git fetch ''' script { if (env.CHANGE_TARGET == 'master') { @@ -82,6 +80,7 @@ def pre_test(){ if (env.CHANGE_URL =~ /\/TDengine\//) { sh ''' cd ${WKC} + git remote prune origin git pull >/dev/null git log -5 echo "`date "+%Y%m%d-%H%M%S"` ${JOB_NAME}:${BRANCH_NAME}:${BUILD_ID}:${CHANGE_TARGET}" >>${WKDIR}/jenkins.log @@ -107,6 +106,7 @@ def pre_test(){ git log -5 echo "tdinternal log merged: `git log -5`" >>${WKDIR}/jenkins.log cd ${WKC} + git remote prune origin git pull >/dev/null git log -5 echo "community log: `git log -5`" >>${WKDIR}/jenkins.log @@ -137,53 +137,51 @@ def pre_test_win(){ set date /t time /t - rd /s /Q C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\debug || exit 0 + rd /s /Q %WIN_INTERNAL_ROOT%\\debug || exit 0 ''' bat ''' - cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal + cd %WIN_INTERNAL_ROOT% git reset --hard - git fetch || git fetch ''' bat ''' - cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community + cd %WIN_COMMUNITY_ROOT% git reset --hard - git fetch || git fetch ''' script { if (env.CHANGE_TARGET == 'master') { bat ''' - cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal + cd %WIN_INTERNAL_ROOT% git checkout master ''' bat ''' - cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community + cd %WIN_COMMUNITY_ROOT% git checkout master ''' } else if(env.CHANGE_TARGET == '2.0') { bat ''' - cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal + cd %WIN_INTERNAL_ROOT% git checkout 2.0 ''' bat ''' - cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community + cd %WIN_COMMUNITY_ROOT% git checkout 2.0 ''' } else if(env.CHANGE_TARGET == '3.0') { bat ''' - cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal + cd %WIN_INTERNAL_ROOT% git checkout 3.0 ''' bat ''' - cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community + cd %WIN_COMMUNITY_ROOT% git checkout 3.0 ''' } else { bat ''' - cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal + cd %WIN_INTERNAL_ROOT% git checkout develop ''' bat ''' - cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community + cd %WIN_COMMUNITY_ROOT% git checkout develop ''' } @@ -191,36 +189,38 @@ def pre_test_win(){ script { if (env.CHANGE_URL =~ /\/TDengine\//) { bat ''' - cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal + cd %WIN_INTERNAL_ROOT% git pull ''' bat ''' - cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community + cd %WIN_COMMUNITY_ROOT% + git remote prune origin git pull ''' bat ''' - cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community + cd %WIN_COMMUNITY_ROOT% git fetch origin +refs/pull/%CHANGE_ID%/merge ''' bat ''' - cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community + cd %WIN_COMMUNITY_ROOT% git checkout -qf FETCH_HEAD ''' } else if (env.CHANGE_URL =~ /\/TDinternal\//) { bat ''' - cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal + cd %WIN_INTERNAL_ROOT% git pull ''' bat ''' - cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal + cd %WIN_INTERNAL_ROOT% git fetch origin +refs/pull/%CHANGE_ID%/merge ''' bat ''' - cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal + cd %WIN_INTERNAL_ROOT% git checkout -qf FETCH_HEAD ''' bat ''' - cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community + cd %WIN_COMMUNITY_ROOT% + git remote prune origin git pull ''' } else { @@ -230,27 +230,27 @@ def pre_test_win(){ } } bat ''' - cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal + cd %WIN_INTERNAL_ROOT% git branch git log -5 ''' bat ''' - cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community + cd %WIN_COMMUNITY_ROOT% git branch git log -5 ''' bat ''' - cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community + cd %WIN_COMMUNITY_ROOT% git submodule update --init --recursive ''' bat ''' - cd C:\\workspace\\%EXECUTOR_NUMBER%\\taos-connector-python + cd %WIN_CONNECTOR_ROOT% git branch git reset --hard git pull ''' bat ''' - cd C:\\workspace\\%EXECUTOR_NUMBER%\\taos-connector-python + cd %WIN_CONNECTOR_ROOT% git log -5 ''' } @@ -258,7 +258,7 @@ def pre_test_build_win() { bat ''' echo "building ..." time /t - cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal + cd %WIN_INTERNAL_ROOT% mkdir debug cd debug time /t @@ -273,9 +273,9 @@ def pre_test_build_win() { time /t ''' bat ''' - cd C:\\workspace\\%EXECUTOR_NUMBER%\\taos-connector-python + cd %WIN_CONNECTOR_ROOT% python -m pip install . - xcopy /e/y/i/f C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\debug\\build\\lib\\taos.dll C:\\Windows\\System32 + xcopy /e/y/i/f %WIN_INTERNAL_ROOT%\\debug\\build\\lib\\taos.dll C:\\Windows\\System32 ''' return 1 } @@ -283,7 +283,7 @@ def run_win_ctest() { bat ''' echo "windows ctest ..." time /t - cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\debug + cd %WIN_INTERNAL_ROOT%\\debug ctest -j 1 || exit 7 time /t ''' @@ -292,12 +292,12 @@ def run_win_test() { echo "LINUX NODE: ${linux_node_ip} - ${linux_node_pass}" bat ''' echo "windows test ..." - cd C:\\workspace\\%EXECUTOR_NUMBER%\\taos-connector-python + cd %WIN_CONNECTOR_ROOT% python -m pip install . - xcopy /e/y/i/f C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\debug\\build\\lib\\taos.dll C:\\Windows\\System32 + xcopy /e/y/i/f %WIN_INTERNAL_ROOT%\\debug\\build\\lib\\taos.dll C:\\Windows\\System32 ls -l C:\\Windows\\System32\\taos.dll time /t - cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community\\tests\\system-test + cd %WIN_SYSTEM_TEST_ROOT% echo "node: ''' + linux_node_ip + ''':''' + linux_node_pass + '''" echo "testing ..." test-all.bat "{\\\"host\\\":\\\"''' + linux_node_ip + '''\\\",\\\"port\\\":22,\\\"user\\\":\\\"root\\\",\\\"password\\\":\\\"''' + linux_node_pass + '''\\\",\\\"path\\\":\\\"/var/lib/jenkins/workspace/TDinternal\\\"}" @@ -319,6 +319,12 @@ pipeline { parallel { stage('windows test') { agent{label " windows10_01 || windows10_02 || windows10_03 || windows10_04 "} + environment{ + WIN_INTERNAL_ROOT="C:\\workspace\\${env.EXECUTOR_NUMBER}\\TDinternal" + WIN_COMMUNITY_ROOT="C:\\workspace\\${env.EXECUTOR_NUMBER}\\TDinternal\\community" + WIN_SYSTEM_TEST_ROOT="C:\\workspace\\${env.EXECUTOR_NUMBER}\\TDinternal\\community\\tests\\system-test" + WIN_CONNECTOR_ROOT="C:\\workspace\\${env.EXECUTOR_NUMBER}\\taos-connector-python" + } steps { catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { timeout(time: 55, unit: 'MINUTES'){ @@ -357,7 +363,7 @@ pipeline { echo "${linux_node_ip}:${linux_node_pass}" } catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { - timeout(time: 40, unit: 'MINUTES'){ + timeout(time: 120, unit: 'MINUTES'){ pre_test() script { sh ''' @@ -368,11 +374,36 @@ pipeline { rm -f /tmp/cases.task ./collect_cases.sh -e ''' + def extra_param = "" + def log_server_file = "/home/log_server.json" + def timeout_cmd = "" + if (fileExists(log_server_file)) { + def log_server_enabled = sh ( + script: 'jq .enabled ' + log_server_file, + returnStdout: true + ).trim() + def timeout_param = sh ( + script: 'jq .timeout ' + log_server_file, + returnStdout: true + ).trim() + if (timeout_param != "null" && timeout_param != "0") { + timeout_cmd = "timeout " + timeout_param + } + if (log_server_enabled == "1") { + def log_server = sh ( + script: 'jq .server ' + log_server_file + ' | sed "s/\\\"//g"', + returnStdout: true + ).trim() + if (log_server != "null" && log_server != "") { + extra_param = "-w " + log_server + } + } + } sh ''' cd ${WKC}/tests/parallel_test export DEFAULT_RETRY_TIME=2 date - timeout 2100 time ./run.sh -e -m /home/m.json -t /tmp/cases.task -b ${BRANCH_NAME}_${BUILD_ID} -l ${WKDIR}/log -o 480 + ''' + timeout_cmd + ''' time ./run.sh -e -m /home/m.json -t /tmp/cases.task -b ${BRANCH_NAME}_${BUILD_ID} -l ${WKDIR}/log -o 480 ''' + extra_param + ''' ''' } } diff --git a/examples/c/stream_demo.c b/examples/c/stream_demo.c index 961eb6c93a..a4fc30ff65 100644 --- a/examples/c/stream_demo.c +++ b/examples/c/stream_demo.c @@ -25,7 +25,7 @@ int32_t init_env() { return -1; } - TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 1"); + TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 2"); if (taos_errno(pRes) != 0) { printf("error in create db, reason:%s\n", taos_errstr(pRes)); return -1; @@ -68,6 +68,14 @@ int32_t init_env() { return -1; } taos_free_result(pRes); + + pRes = taos_query(pConn, "create table if not exists tu3 using st1 tags(3)"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table tu3, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + return 0; } @@ -90,10 +98,9 @@ int32_t create_stream() { /*const char* sql = "select min(k), max(k), sum(k) as sum_of_k from st1";*/ /*const char* sql = "select sum(k) from tu1 interval(10m)";*/ /*pRes = tmq_create_stream(pConn, "stream1", "out1", sql);*/ - pRes = taos_query( - pConn, - "create stream stream1 trigger window_close watermark 10s into outstb as select _wstartts, sum(k) from st1 " - "interval(10s) "); + pRes = taos_query(pConn, + "create stream stream1 trigger at_once into outstb as select _wstartts, sum(k) from st1 " + "partition by tbname interval(10s) "); if (taos_errno(pRes) != 0) { printf("failed to create stream stream1, reason:%s\n", taos_errstr(pRes)); return -1; diff --git a/include/client/taos.h b/include/client/taos.h index d31d5c582c..5e7f12de0a 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -128,7 +128,6 @@ typedef struct setConfRet { DLL_EXPORT void taos_cleanup(void); DLL_EXPORT int taos_options(TSDB_OPTION option, const void *arg, ...); -DLL_EXPORT setConfRet taos_set_config(const char *config); DLL_EXPORT int taos_init(void); DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port); DLL_EXPORT TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port); diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 09f24cc555..9c84d0dd99 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -50,8 +50,10 @@ typedef enum EStreamType { } EStreamType; typedef struct { + SArray* pGroupList; SArray* pTableList; SHashObj* map; // speedup acquire the tableQueryInfo by table uid + bool needSortTableByGroupId; void* pTagCond; void* pTagIndexCond; uint64_t suid; diff --git a/include/common/tglobal.h b/include/common/tglobal.h index e083ebcf78..41674b7a70 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -151,6 +151,7 @@ void taosCfgDynamicOptions(const char *option, const char *value); void taosAddDataDir(int32_t index, char *v1, int32_t level, int32_t primary); struct SConfig *taosGetCfg(); +int32_t taosSetCfg(SConfig *pCfg, char* name); #ifdef __cplusplus } diff --git a/include/common/tmsg.h b/include/common/tmsg.h index e9ee8b5955..8c4d43d5d3 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -510,7 +510,8 @@ typedef struct { int8_t superUser; int8_t connType; SEpSet epSet; - char sVersion[128]; + char sVer[TSDB_VERSION_LEN]; + char sDetailVer[128]; } SConnectRsp; int32_t tSerializeSConnectRsp(void* buf, int32_t bufLen, SConnectRsp* pRsp); @@ -836,6 +837,20 @@ typedef struct { int32_t tSerializeSDnodeListReq(void* buf, int32_t bufLen, SDnodeListReq* pReq); int32_t tDeserializeSDnodeListReq(void* buf, int32_t bufLen, SDnodeListReq* pReq); +typedef struct { + int32_t useless; // useless +} SServerVerReq; + +int32_t tSerializeSServerVerReq(void* buf, int32_t bufLen, SServerVerReq* pReq); +int32_t tDeserializeSServerVerReq(void* buf, int32_t bufLen, SServerVerReq* pReq); + +typedef struct { + char ver[TSDB_VERSION_LEN]; +} SServerVerRsp; + +int32_t tSerializeSServerVerRsp(void* buf, int32_t bufLen, SServerVerRsp* pRsp); +int32_t tDeserializeSServerVerRsp(void* buf, int32_t bufLen, SServerVerRsp* pRsp); + typedef struct SQueryNodeAddr { int32_t nodeId; // vgId or qnodeId @@ -1229,6 +1244,21 @@ typedef struct { int32_t tSerializeSShowVariablesReq(void* buf, int32_t bufLen, SShowVariablesReq* pReq); int32_t tDeserializeSShowVariablesReq(void* buf, int32_t bufLen, SShowVariablesReq* pReq); +typedef struct { + char name[TSDB_CONFIG_OPTION_LEN + 1]; + char value[TSDB_CONFIG_VALUE_LEN + 1]; +} SVariablesInfo; + +typedef struct { + SArray *variables; //SArray +} SShowVariablesRsp; + +int32_t tSerializeSShowVariablesRsp(void* buf, int32_t bufLen, SShowVariablesRsp* pReq); +int32_t tDeserializeSShowVariablesRsp(void* buf, int32_t bufLen, SShowVariablesRsp* pReq); + +void tFreeSShowVariablesRsp(SShowVariablesRsp* pRsp); + + /* * sql: show tables like '%a_%' * payload is the query condition, e.g., '%a_%' diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 1babb45003..acf08bd47e 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -163,6 +163,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_REDISTRIBUTE_VGROUP, "redistribute-vgroup", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_SPLIT_VGROUP, "split-vgroup", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_SHOW_VARIABLES, "show-variables", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_SERVER_VERSION, "server-version", NULL, NULL) TD_NEW_MSG_SEG(TDMT_VND_MSG) TD_DEF_MSG_TYPE(TDMT_VND_SUBMIT, "submit", SSubmitReq, SSubmitRsp) diff --git a/include/common/tname.h b/include/common/tname.h index c691c2f7b2..3bf1cee870 100644 --- a/include/common/tname.h +++ b/include/common/tname.h @@ -57,6 +57,8 @@ void tNameAssign(SName* dst, const SName* src); int32_t tNameSetDbName(SName* dst, int32_t acctId, const char* dbName, size_t nameLen); +int32_t tNameAddTbName(SName* dst, const char* tbName, size_t nameLen); + int32_t tNameFromString(SName* dst, const char* str, uint32_t type); int32_t tNameSetAcctId(SName* dst, int32_t acctId); diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 20f4765190..8482ae3f3d 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -71,6 +71,7 @@ typedef struct SCatalogReq { SArray* pTableCfg; // element is SNAME bool qNodeRequired; // valid qnode bool dNodeRequired; // valid dnode + bool svrVerRequired; bool forceUpdate; } SCatalogReq; @@ -80,18 +81,19 @@ typedef struct SMetaRes { } SMetaRes; typedef struct SMetaData { - SArray* pDbVgroup; // pRes = SArray* - SArray* pDbCfg; // pRes = SDbCfgInfo* - SArray* pDbInfo; // pRes = SDbInfo* - SArray* pTableMeta; // pRes = STableMeta* - SArray* pTableHash; // pRes = SVgroupInfo* - SArray* pTableIndex; // pRes = SArray* - SArray* pUdfList; // pRes = SFuncInfo* - SArray* pIndex; // pRes = SIndexInfo* - SArray* pUser; // pRes = bool* - SArray* pQnodeList; // pRes = SArray* - SArray* pTableCfg; // pRes = STableCfg* - SArray* pDnodeList; // pRes = SArray* + SArray* pDbVgroup; // pRes = SArray* + SArray* pDbCfg; // pRes = SDbCfgInfo* + SArray* pDbInfo; // pRes = SDbInfo* + SArray* pTableMeta; // pRes = STableMeta* + SArray* pTableHash; // pRes = SVgroupInfo* + SArray* pTableIndex; // pRes = SArray* + SArray* pUdfList; // pRes = SFuncInfo* + SArray* pIndex; // pRes = SIndexInfo* + SArray* pUser; // pRes = bool* + SArray* pQnodeList; // pRes = SArray* + SArray* pTableCfg; // pRes = STableCfg* + SArray* pDnodeList; // pRes = SArray* + SMetaRes* pSvrVer; // pRes = char* } SMetaData; typedef struct SCatalogCfg { @@ -268,7 +270,7 @@ int32_t catalogGetTableHashVgroup(SCatalog* pCatalog, SRequestConnInfo* pConn, c */ int32_t catalogGetAllMeta(SCatalog* pCatalog, SRequestConnInfo* pConn, const SCatalogReq* pReq, SMetaData* pRsp); -int32_t catalogAsyncGetAllMeta(SCatalog* pCtg, SRequestConnInfo* pConn, uint64_t reqId, const SCatalogReq* pReq, catalogCallback fp, void* param, int64_t* jobId); +int32_t catalogAsyncGetAllMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SCatalogReq* pReq, catalogCallback fp, void* param, int64_t* jobId); int32_t catalogGetQnodeList(SCatalog* pCatalog, SRequestConnInfo* pConn, SArray* pQnodeList); @@ -298,6 +300,8 @@ int32_t catalogUpdateUserAuthInfo(SCatalog* pCtg, SGetUserAuthRsp* pAuth); int32_t catalogUpdateVgEpSet(SCatalog* pCtg, const char* dbFName, int32_t vgId, SEpSet *epSet); +int32_t catalogGetServerVersion(SCatalog* pCtg, SRequestConnInfo *pConn, char** pVersion); + int32_t ctgdLaunchAsyncCall(SCatalog* pCtg, SRequestConnInfo* pConn, uint64_t reqId, bool forceUpdate); int32_t catalogClearCache(void); diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 6738fc23bc..00acc4741d 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -36,7 +36,7 @@ typedef struct SReadHandle { void* vnode; void* mnd; SMsgCb* pMsgCb; - int8_t initTsdbReader; +// int8_t initTsdbReader; } SReadHandle; enum { diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 1ca51b3043..1ed78750d1 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -124,6 +124,7 @@ typedef enum EFunctionType { FUNCTION_TYPE_BLOCK_DIST, // block distribution aggregate function FUNCTION_TYPE_BLOCK_DIST_INFO, // block distribution pseudo column function FUNCTION_TYPE_TO_COLUMN, + FUNCTION_TYPE_GROUP_KEY, // distributed splitting functions FUNCTION_TYPE_APERCENTILE_PARTIAL = 4000, diff --git a/include/libs/nodes/cmdnodes.h b/include/libs/nodes/cmdnodes.h index b66dff31c1..e473daad8d 100644 --- a/include/libs/nodes/cmdnodes.h +++ b/include/libs/nodes/cmdnodes.h @@ -36,7 +36,7 @@ extern "C" { #define SHOW_CREATE_TB_RESULT_FIELD1_LEN (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE) #define SHOW_CREATE_TB_RESULT_FIELD2_LEN (TSDB_MAX_BINARY_LEN + VARSTR_HEADER_SIZE) -#define SHOW_LOCAL_VARIABLES_RESULT_COLS 2 +#define SHOW_LOCAL_VARIABLES_RESULT_COLS 2 #define SHOW_LOCAL_VARIABLES_RESULT_FIELD1_LEN (TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE) #define SHOW_LOCAL_VARIABLES_RESULT_FIELD2_LEN (TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE) diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index f8cf58d8cb..b7583b41b9 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -458,6 +458,7 @@ typedef struct SSubplan { int32_t msgType; // message type for subplan, used to denote the send message type to vnode. int32_t level; // the execution level of current subplan, starting from 0 in a top-down manner. char dbFName[TSDB_DB_FNAME_LEN]; + char user[TSDB_USER_LEN]; SQueryNodeAddr execNode; // for the scan/modify subplan, the optional execution node SQueryNodeStat execNodeStat; // only for scan subplan SNodeList* pChildren; // the datasource subplan,from which to fetch the result diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index e45e5bd160..6c2a9bb374 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -51,6 +51,8 @@ typedef struct SParseContext { bool isSuperUser; bool async; int8_t schemalessType; + const char* svrVer; + bool nodeOffline; } SParseContext; int32_t qParseSql(SParseContext* pCxt, SQuery** pQuery); diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index c4f71e57a8..b350837551 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -24,18 +24,19 @@ extern "C" { #include "taos.h" typedef struct SPlanContext { - uint64_t queryId; - int32_t acctId; - SEpSet mgmtEpSet; - SNode* pAstRoot; - bool topicQuery; - bool streamQuery; - bool rSmaQuery; - bool showRewrite; - int8_t triggerType; - int64_t watermark; - char* pMsg; - int32_t msgLen; + uint64_t queryId; + int32_t acctId; + SEpSet mgmtEpSet; + SNode* pAstRoot; + bool topicQuery; + bool streamQuery; + bool rSmaQuery; + bool showRewrite; + int8_t triggerType; + int64_t watermark; + char* pMsg; + int32_t msgLen; + const char* pUser; } SPlanContext; // Create the physical plan for the query, according to the AST. diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 26f2758033..d86a8dae6e 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -266,19 +266,19 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t #define qDebug(...) \ do { \ if (qDebugFlag & DEBUG_DEBUG) { \ - taosPrintLog("QRY ", DEBUG_DEBUG, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \ + taosPrintLog("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); \ } \ } while (0) #define qTrace(...) \ do { \ if (qDebugFlag & DEBUG_TRACE) { \ - taosPrintLog("QRY ", DEBUG_TRACE, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \ + taosPrintLog("QRY ", DEBUG_TRACE, qDebugFlag, __VA_ARGS__); \ } \ } while (0) #define qDebugL(...) \ do { \ if (qDebugFlag & DEBUG_DEBUG) { \ - taosPrintLongString("QRY ", DEBUG_DEBUG, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \ + taosPrintLongString("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); \ } \ } while (0) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 2bf49fa006..52d9b6f810 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -200,6 +200,7 @@ const char* syncGetMyRoleStr(int64_t rid); SyncTerm syncGetMyTerm(int64_t rid); SyncGroupId syncGetVgId(int64_t rid); void syncGetEpSet(int64_t rid, SEpSet* pEpSet); +void syncGetRetryEpSet(int64_t rid, SEpSet* pEpSet); int32_t syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak); bool syncEnvIsStart(); const char* syncStr(ESyncState state); diff --git a/include/util/tconfig.h b/include/util/tconfig.h index fd6360aaf6..102656da2b 100644 --- a/include/util/tconfig.h +++ b/include/util/tconfig.h @@ -33,7 +33,8 @@ typedef enum { CFG_STYPE_ENV_CMD, CFG_STYPE_APOLLO_URL, CFG_STYPE_ARG_LIST, - CFG_STYPE_TAOS_OPTIONS + CFG_STYPE_TAOS_OPTIONS, + CFG_STYPE_ALTER_CMD, } ECfgSrcType; typedef enum { diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index cfe507e505..cca79186d0 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -54,6 +54,11 @@ enum { RES_TYPE__TMQ_META, }; +#define SHOW_VARIABLES_RESULT_COLS 2 +#define SHOW_VARIABLES_RESULT_FIELD1_LEN (TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE) +#define SHOW_VARIABLES_RESULT_FIELD2_LEN (TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE) + + #define TD_RES_QUERY(res) (*(int8_t*)res == RES_TYPE__QUERY) #define TD_RES_TMQ(res) (*(int8_t*)res == RES_TYPE__TMQ) #define TD_RES_TMQ_META(res) (*(int8_t*)res == RES_TYPE__TMQ_META) @@ -104,6 +109,8 @@ typedef struct SHeartBeatInfo { struct SAppInstInfo { int64_t numOfConns; SCorEpSet mgmtEp; + int32_t totalDnodes; + int32_t onlineDnodes; TdThreadMutex qnodeMutex; SArray* pQnodeList; SAppClusterSummary summary; @@ -127,7 +134,8 @@ typedef struct STscObj { char user[TSDB_USER_LEN]; char pass[TSDB_PASSWORD_LEN]; char db[TSDB_DB_FNAME_LEN]; - char ver[128]; + char sVer[TSDB_VERSION_LEN]; + char sDetailVer[128]; int8_t connType; int32_t acctId; uint32_t connId; @@ -215,6 +223,7 @@ typedef struct SRequestObj { SQueryExecMetric metric; SRequestSendRecvBody body; bool stableQuery; + bool validateOnly; bool killed; uint32_t prevCode; // previous error code: todo refactor, add update flag for catalog @@ -235,7 +244,12 @@ int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveT bool freeAfterUse); void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols); void doFreeReqResultInfo(SReqResultInfo* pResInfo); -SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen); +int32_t transferTableNameList(const char* tbList, int32_t acctId, char* dbName, SArray** pReq); +void syncCatalogFn(SMetaData* pResult, void* param, int32_t code); + +SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen, bool validateOnly); +TAOS_RES *taosQueryImpl(TAOS *taos, const char *sql, bool validateOnly); +void taosAsyncQueryImpl(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param, bool validateOnly); static FORCE_INLINE SReqResultInfo* tmqGetCurResInfo(TAOS_RES* res) { SMqRspObj* msg = (SMqRspObj*)res; @@ -301,7 +315,7 @@ void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet); STscObj* taos_connect_internal(const char* ip, const char* user, const char* pass, const char* auth, const char* db, uint16_t port, int connType); -SRequestObj* launchQuery(STscObj* pTscObj, const char* sql, int sqlLen); +SRequestObj* launchQuery(STscObj* pTscObj, const char* sql, int sqlLen, bool validateOnly); int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtCallback* pStmtCb); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index c6d0d6a860..86562fea97 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -161,6 +161,9 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { updateEpSet_s(&pTscObj->pAppInfo->mgmtEp, &pRsp->query->epSet); } + + pTscObj->pAppInfo->totalDnodes = pRsp->query->totalDnodes; + pTscObj->pAppInfo->onlineDnodes = pRsp->query->onlineDnodes; pTscObj->connId = pRsp->query->connId; if (pRsp->query->killRid) { diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 191bfe088a..489966b636 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -59,7 +59,7 @@ static STscObj* taosConnectImpl(const char* user, const char* auth, const char* SAppInstInfo* pAppInfo, int connType); STscObj* taos_connect_internal(const char* ip, const char* user, const char* pass, const char* auth, const char* db, - uint16_t port, int connType) { + uint16_t port, int connType) { if (taos_init() != TSDB_CODE_SUCCESS) { return NULL; } @@ -178,7 +178,9 @@ int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtC .pStmtCb = pStmtCb, .pUser = pTscObj->user, .schemalessType = pTscObj->schemalessType, - .isSuperUser = (0 == strcmp(pTscObj->user, TSDB_DEFAULT_USER))}; + .isSuperUser = (0 == strcmp(pTscObj->user, TSDB_DEFAULT_USER)), + .svrVer = pTscObj->sVer, + .nodeOffline = (pTscObj->pAppInfo->onlineDnodes < pTscObj->pAppInfo->totalDnodes)}; cxt.mgmtEpSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); int32_t code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &cxt.pCatalog); @@ -237,6 +239,10 @@ static SAppInstInfo* getAppInfo(SRequestObj* pRequest) { return pRequest->pTscOb void asyncExecLocalCmd(SRequestObj* pRequest, SQuery* pQuery) { SRetrieveTableRsp* pRsp = NULL; + if (pRequest->validateOnly) { + pRequest->body.queryFp(pRequest->body.param, pRequest, 0); + return; + } int32_t code = qExecCommand(pQuery->pRoot, &pRsp); if (TSDB_CODE_SUCCESS == code && NULL != pRsp) { @@ -261,6 +267,11 @@ void asyncExecLocalCmd(SRequestObj* pRequest, SQuery* pQuery) { } int32_t asyncExecDdlQuery(SRequestObj* pRequest, SQuery* pQuery) { + if (pRequest->validateOnly) { + pRequest->body.queryFp(pRequest->body.param, pRequest, 0); + return TSDB_CODE_SUCCESS; + } + // drop table if exists not_exists_table if (NULL == pQuery->pCmdMsg) { pRequest->body.queryFp(pRequest->body.param, pRequest, 0); @@ -276,8 +287,11 @@ int32_t asyncExecDdlQuery(SRequestObj* pRequest, SQuery* pQuery) { SMsgSendInfo* pSendMsg = buildMsgInfoImpl(pRequest); int64_t transporterId = 0; - asyncSendMsgToServer(pAppInfo->pTransporter, &pMsgInfo->epSet, &transporterId, pSendMsg); - return TSDB_CODE_SUCCESS; + int32_t code = asyncSendMsgToServer(pAppInfo->pTransporter, &pMsgInfo->epSet, &transporterId, pSendMsg); + if (code) { + pRequest->body.queryFp(pRequest->body.param, pRequest, code); + } + return code; } int compareQueryNodeLoad(const void* elem1, const void* elem2) { @@ -313,8 +327,8 @@ bool qnodeRequired(SRequestObj* pRequest) { } SAppInstInfo* pInfo = pRequest->pTscObj->pAppInfo; - bool required = false; - + bool required = false; + taosThreadMutexLock(&pInfo->qnodeMutex); required = (NULL == pInfo->pQnodeList); taosThreadMutexUnlock(&pInfo->qnodeMutex); @@ -362,7 +376,8 @@ int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArra .pAstRoot = pQuery->pRoot, .showRewrite = pQuery->showRewrite, .pMsg = pRequest->msgBuf, - .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE}; + .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE, + .pUser = pRequest->pTscObj->user}; return qCreateQueryPlan(&cxt, pPlan, pNodeList); } @@ -419,11 +434,11 @@ int32_t buildVnodePolicyNodeList(SRequestObj* pRequest, SArray** pNodeList, SArr } for (int32_t j = 0; j < vgNum; ++j) { - SVgroupInfo* pInfo = taosArrayGet(pVg, j); + SVgroupInfo* pInfo = taosArrayGet(pVg, j); SQueryNodeLoad load = {0}; load.addr.nodeId = pInfo->vgId; load.addr.epSet = pInfo->epSet; - + taosArrayPush(nodeList, &load); } } @@ -481,17 +496,16 @@ _return: return TSDB_CODE_SUCCESS; } - -int32_t buildAsyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray* pMnodeList, SMetaData *pResultMeta) { +int32_t buildAsyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray* pMnodeList, SMetaData* pResultMeta) { SArray* pDbVgList = NULL; SArray* pQnodeList = NULL; int32_t code = 0; - + switch (tsQueryPolicy) { case QUERY_POLICY_VNODE: { if (pResultMeta) { pDbVgList = taosArrayInit(4, POINTER_BYTES); - + int32_t dbNum = taosArrayGetSize(pResultMeta->pDbVgroup); for (int32_t i = 0; i < dbNum; ++i) { SMetaRes* pRes = taosArrayGet(pResultMeta->pDbVgroup, i); @@ -500,9 +514,9 @@ int32_t buildAsyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray } taosArrayPush(pDbVgList, &pRes->pRes); - } + } } - + code = buildVnodePolicyNodeList(pRequest, pNodeList, pMnodeList, pDbVgList); break; } @@ -523,7 +537,7 @@ int32_t buildAsyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray } taosThreadMutexUnlock(&pInst->qnodeMutex); } - + code = buildQnodePolicyNodeList(pRequest, pNodeList, pMnodeList, pQnodeList); break; } @@ -534,7 +548,7 @@ int32_t buildAsyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray taosArrayDestroy(pDbVgList); taosArrayDestroy(pQnodeList); - + return code; } @@ -542,43 +556,43 @@ int32_t buildSyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray* SArray* pDbVgList = NULL; SArray* pQnodeList = NULL; int32_t code = 0; - + switch (tsQueryPolicy) { case QUERY_POLICY_VNODE: { int32_t dbNum = taosArrayGetSize(pRequest->dbList); if (dbNum > 0) { - SCatalog* pCtg = NULL; + SCatalog* pCtg = NULL; SAppInstInfo* pInst = pRequest->pTscObj->pAppInfo; code = catalogGetHandle(pInst->clusterId, &pCtg); if (code != TSDB_CODE_SUCCESS) { goto _return; } - pDbVgList = taosArrayInit(dbNum, POINTER_BYTES); + pDbVgList = taosArrayInit(dbNum, POINTER_BYTES); SArray* pVgList = NULL; for (int32_t i = 0; i < dbNum; ++i) { - char* dbFName = taosArrayGet(pRequest->dbList, i); + char* dbFName = taosArrayGet(pRequest->dbList, i); SRequestConnInfo conn = {.pTrans = pInst->pTransporter, .requestId = pRequest->requestId, .requestObjRefId = pRequest->self, - .mgmtEps = getEpSet_s(&pInst->mgmtEp)}; - + .mgmtEps = getEpSet_s(&pInst->mgmtEp)}; + code = catalogGetDBVgInfo(pCtg, &conn, dbFName, &pVgList); if (code) { goto _return; } - + taosArrayPush(pDbVgList, &pVgList); - } + } } - + code = buildVnodePolicyNodeList(pRequest, pNodeList, pMnodeList, pDbVgList); break; } case QUERY_POLICY_HYBRID: case QUERY_POLICY_QNODE: { getQnodeList(pRequest, &pQnodeList); - + code = buildQnodePolicyNodeList(pRequest, pNodeList, pMnodeList, pQnodeList); break; } @@ -591,11 +605,10 @@ _return: taosArrayDestroy(pDbVgList); taosArrayDestroy(pQnodeList); - + return code; } - int32_t scheduleAsyncQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList) { tsem_init(&schdRspSem, 0, 0); @@ -819,8 +832,8 @@ void schedulerExecCb(SQueryResult* pResult, void* param, int32_t code) { } } - tscDebug("0x%" PRIx64 " enter scheduler exec cb, code:%d - %s, reqId:0x%" PRIx64, - pRequest->self, code, tstrerror(code), pRequest->requestId); + tscDebug("0x%" PRIx64 " enter scheduler exec cb, code:%d - %s, reqId:0x%" PRIx64, pRequest->self, code, + tstrerror(code), pRequest->requestId); STscObj* pTscObj = pRequest->pTscObj; if (code != TSDB_CODE_SUCCESS && NEED_CLIENT_HANDLE_ERROR(code)) { @@ -851,18 +864,22 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQue switch (pQuery->execMode) { case QUERY_EXEC_MODE_LOCAL: - code = execLocalCmd(pRequest, pQuery); + if (!pRequest->validateOnly) { + code = execLocalCmd(pRequest, pQuery); + } break; case QUERY_EXEC_MODE_RPC: - code = execDdlQuery(pRequest, pQuery); + if (!pRequest->validateOnly) { + code = execDdlQuery(pRequest, pQuery); + } break; case QUERY_EXEC_MODE_SCHEDULE: { SArray* pMnodeList = taosArrayInit(4, sizeof(SQueryNodeLoad)); code = getPlan(pRequest, pQuery, &pRequest->body.pDag, pMnodeList); - if (TSDB_CODE_SUCCESS == code) { + if (TSDB_CODE_SUCCESS == code && !pRequest->validateOnly) { SArray* pNodeList = NULL; buildSyncExecNodeList(pRequest, &pNodeList, pMnodeList); - + code = scheduleQuery(pRequest, pRequest->body.pDag, pNodeList); taosArrayDestroy(pNodeList); } @@ -894,7 +911,7 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQue return pRequest; } -SRequestObj* launchQuery(STscObj* pTscObj, const char* sql, int sqlLen) { +SRequestObj* launchQuery(STscObj* pTscObj, const char* sql, int sqlLen, bool validateOnly) { SRequestObj* pRequest = NULL; SQuery* pQuery = NULL; @@ -904,6 +921,8 @@ SRequestObj* launchQuery(STscObj* pTscObj, const char* sql, int sqlLen) { return NULL; } + pRequest->validateOnly = validateOnly; + code = parseSql(pRequest, false, &pQuery, NULL); if (code != TSDB_CODE_SUCCESS) { pRequest->code = code; @@ -915,7 +934,7 @@ SRequestObj* launchQuery(STscObj* pTscObj, const char* sql, int sqlLen) { return launchQueryImpl(pRequest, pQuery, false, NULL); } -void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData *pResultMeta) { +void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultMeta) { int32_t code = 0; switch (pQuery->execMode) { @@ -936,7 +955,8 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData *pResultM .pAstRoot = pQuery->pRoot, .showRewrite = pQuery->showRewrite, .pMsg = pRequest->msgBuf, - .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE}; + .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE, + .pUser = pRequest->pTscObj->user}; SAppInstInfo* pAppInfo = getAppInfo(pRequest); code = qCreateQueryPlan(&cxt, &pRequest->body.pDag, pMnodeList); @@ -945,10 +965,10 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData *pResultM pRequest->requestId); } - if (TSDB_CODE_SUCCESS == code) { + if (TSDB_CODE_SUCCESS == code && !pRequest->validateOnly) { SArray* pNodeList = NULL; buildAsyncExecNodeList(pRequest, &pNodeList, pMnodeList, pResultMeta); - + SRequestConnInfo conn = { .pTrans = pAppInfo->pTransporter, .requestId = pRequest->requestId, .requestObjRefId = pRequest->self}; SSchedulerReq req = {.pConn = &conn, @@ -962,7 +982,7 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData *pResultM code = schedulerAsyncExecJob(&req, &pRequest->body.queryJob); taosArrayDestroy(pNodeList); } else { - tscError("0x%" PRIx64 " failed to create query plan, code:%s 0x%" PRIx64, pRequest->self, tstrerror(code), + tscDebug("0x%" PRIx64 " plan not executed, code:%s 0x%" PRIx64, pRequest->self, tstrerror(code), pRequest->requestId); pRequest->body.queryFp(pRequest->body.param, pRequest, code); } @@ -1045,14 +1065,14 @@ int32_t removeMeta(STscObj* pTscObj, SArray* tbList) { return TSDB_CODE_SUCCESS; } -SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen) { +SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen, bool validateOnly) { SRequestObj* pRequest = NULL; int32_t retryNum = 0; int32_t code = 0; do { destroyRequest(pRequest); - pRequest = launchQuery(pTscObj, sql, sqlLen); + pRequest = launchQuery(pTscObj, sql, sqlLen, validateOnly); if (pRequest == NULL || TSDB_CODE_SUCCESS == pRequest->code || !NEED_CLIENT_HANDLE_ERROR(pRequest->code)) { break; } @@ -1308,7 +1328,7 @@ TAOS* taos_connect_auth(const char* ip, const char* user, const char* auth, cons if (pObj) { return pObj->id; } - + return NULL; } @@ -1480,10 +1500,10 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int return TSDB_CODE_SUCCESS; } -static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, int32_t numOfRows){ +static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, int32_t numOfRows) { char* p = (char*)pResultInfo->pData; - int32_t len = sizeof(int32_t) + sizeof(uint64_t) + numOfCols * (sizeof(int16_t) + sizeof(int32_t)); + int32_t len = sizeof(int32_t) + sizeof(uint64_t) + numOfCols * (sizeof(int16_t) + sizeof(int32_t)); int32_t* colLength = (int32_t*)(p + len); len += sizeof(int32_t) * numOfCols; @@ -1493,7 +1513,7 @@ static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, i if (pResultInfo->fields[i].type == TSDB_DATA_TYPE_JSON) { int32_t* offset = (int32_t*)pStart; - int32_t lenTmp = numOfRows * sizeof(int32_t); + int32_t lenTmp = numOfRows * sizeof(int32_t); len += lenTmp; pStart += lenTmp; @@ -1518,7 +1538,6 @@ static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, i } else { ASSERT(0); } - } } else if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) { int32_t lenTmp = numOfRows * sizeof(int32_t); @@ -1542,13 +1561,13 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int break; } } - if(!needConvert) return TSDB_CODE_SUCCESS; + if (!needConvert) return TSDB_CODE_SUCCESS; - char* p = (char*)pResultInfo->pData; + char* p = (char*)pResultInfo->pData; int32_t dataLen = estimateJsonLen(pResultInfo, numOfCols, numOfRows); pResultInfo->convertJson = taosMemoryCalloc(1, dataLen); - if(pResultInfo->convertJson == NULL) return TSDB_CODE_OUT_OF_MEMORY; + if (pResultInfo->convertJson == NULL) return TSDB_CODE_OUT_OF_MEMORY; char* p1 = pResultInfo->convertJson; int32_t len = sizeof(int32_t) + sizeof(uint64_t) + numOfCols * (sizeof(int16_t) + sizeof(int32_t)); @@ -1617,7 +1636,7 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int ASSERT(0); } - offset1[j]= len; + offset1[j] = len; memcpy(pStart1 + len, dst, varDataTLen(dst)); len += varDataTLen(dst); } @@ -1635,7 +1654,6 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int pStart += len; pStart1 += len; memcpy(pStart1, pStart, colLen); - } pStart += colLen; pStart1 += colLen1; @@ -1703,7 +1721,7 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32 pStart += colLength[i]; } - if(convertUcs4){ + if (convertUcs4) { code = doConvertUCS4(pResultInfo, numOfRows, numOfCols, colLength); } @@ -1819,3 +1837,246 @@ _OVER: } return code; } + +int32_t appendTbToReq(SArray* pList, int32_t pos1, int32_t len1, int32_t pos2, int32_t len2, const char* str, + int32_t acctId, char* db) { + SName name; + + if (len1 <= 0) { + return -1; + } + + const char* dbName = db; + const char* tbName = NULL; + int32_t dbLen = 0; + int32_t tbLen = 0; + if (len2 > 0) { + dbName = str + pos1; + dbLen = len1; + tbName = str + pos2; + tbLen = len2; + } else { + dbLen = strlen(db); + tbName = str + pos1; + tbLen = len1; + } + + if (tNameSetDbName(&name, acctId, dbName, dbLen)) { + return -1; + } + + if (tNameAddTbName(&name, tbName, tbLen)) { + return -1; + } + + taosArrayPush(pList, &name); + + return TSDB_CODE_SUCCESS; +} + +int32_t transferTableNameList(const char* tbList, int32_t acctId, char* dbName, SArray** pReq) { + *pReq = taosArrayInit(10, sizeof(SName)); + if (NULL == *pReq) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return terrno; + } + + bool inEscape = false; + int32_t code = 0; + + int32_t vIdx = 0; + int32_t vPos[2]; + int32_t vLen[2]; + + memset(vPos, -1, sizeof(vPos)); + memset(vLen, 0, sizeof(vLen)); + + for (int32_t i = 0;; ++i) { + if (0 == *(tbList + i)) { + if (vPos[vIdx] >= 0 && vLen[vIdx] <= 0) { + vLen[vIdx] = i - vPos[vIdx]; + } + + code = appendTbToReq(*pReq, vPos[0], vLen[0], vPos[1], vLen[1], tbList, acctId, dbName); + if (code) { + goto _return; + } + + break; + } + + if ('`' == *(tbList + i)) { + inEscape = !inEscape; + if (!inEscape) { + if (vPos[vIdx] >= 0) { + vLen[vIdx] = i - vPos[vIdx]; + } else { + goto _return; + } + } + + continue; + } + + if (inEscape) { + if (vPos[vIdx] < 0) { + vPos[vIdx] = i; + } + continue; + } + + if ('.' == *(tbList + i)) { + if (vPos[vIdx] < 0) { + goto _return; + } + if (vLen[vIdx] <= 0) { + vLen[vIdx] = i - vPos[vIdx]; + } + vIdx++; + if (vIdx >= 2) { + goto _return; + } + continue; + } + + if (',' == *(tbList + i)) { + if (vPos[vIdx] < 0) { + goto _return; + } + if (vLen[vIdx] <= 0) { + vLen[vIdx] = i - vPos[vIdx]; + } + + code = appendTbToReq(*pReq, vPos[0], vLen[0], vPos[1], vLen[1], tbList, acctId, dbName); + if (code) { + goto _return; + } + + memset(vPos, -1, sizeof(vPos)); + memset(vLen, 0, sizeof(vLen)); + vIdx = 0; + continue; + } + + if (' ' == *(tbList + i) || '\r' == *(tbList + i) || '\t' == *(tbList + i) || '\n' == *(tbList + i)) { + if (vPos[vIdx] >= 0 && vLen[vIdx] <= 0) { + vLen[vIdx] = i - vPos[vIdx]; + } + continue; + } + + if (('a' <= *(tbList + i) && 'z' >= *(tbList + i)) || ('A' <= *(tbList + i) && 'Z' >= *(tbList + i)) || + ('0' <= *(tbList + i) && '9' >= *(tbList + i))) { + if (vLen[vIdx] > 0) { + goto _return; + } + if (vPos[vIdx] < 0) { + vPos[vIdx] = i; + } + continue; + } + + goto _return; + } + + return TSDB_CODE_SUCCESS; + +_return: + + terrno = TSDB_CODE_TSC_INVALID_OPERATION; + + taosArrayDestroy(*pReq); + *pReq = NULL; + + return terrno; +} + +void syncCatalogFn(SMetaData* pResult, void* param, int32_t code) { + SSyncQueryParam* pParam = param; + pParam->pRequest->code = code; + + tsem_post(&pParam->sem); +} + +void syncQueryFn(void* param, void* res, int32_t code) { + SSyncQueryParam* pParam = param; + pParam->pRequest = res; + pParam->pRequest->code = code; + + tsem_post(&pParam->sem); +} + +void taosAsyncQueryImpl(TAOS* taos, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly) { + STscObj* pTscObj = acquireTscObj(*(int64_t*)taos); + if (pTscObj == NULL || sql == NULL || NULL == fp) { + terrno = TSDB_CODE_INVALID_PARA; + if (pTscObj) { + releaseTscObj(*(int64_t*)taos); + } else { + terrno = TSDB_CODE_TSC_DISCONNECTED; + } + fp(param, NULL, terrno); + return; + } + + size_t sqlLen = strlen(sql); + if (sqlLen > (size_t)TSDB_MAX_ALLOWED_SQL_LEN) { + tscError("sql string exceeds max length:%d", TSDB_MAX_ALLOWED_SQL_LEN); + terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT; + + fp(param, NULL, terrno); + return; + } + + SRequestObj* pRequest = NULL; + int32_t code = buildRequest(pTscObj, sql, sqlLen, &pRequest); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + fp(param, NULL, terrno); + return; + } + + pRequest->validateOnly = validateOnly; + pRequest->body.queryFp = fp; + pRequest->body.param = param; + doAsyncQuery(pRequest, false); +} + +TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) { + if (NULL == taos) { + terrno = TSDB_CODE_TSC_DISCONNECTED; + return NULL; + } + + STscObj* pTscObj = acquireTscObj(*(int64_t*)taos); + if (pTscObj == NULL || sql == NULL) { + terrno = TSDB_CODE_TSC_DISCONNECTED; + return NULL; + } + +#if SYNC_ON_TOP_OF_ASYNC + SSyncQueryParam* param = taosMemoryCalloc(1, sizeof(SSyncQueryParam)); + tsem_init(¶m->sem, 0, 0); + + taosAsyncQueryImpl(taos, sql, syncQueryFn, param, validateOnly); + tsem_wait(¶m->sem); + + releaseTscObj(*(int64_t*)taos); + + return param->pRequest; +#else + size_t sqlLen = strlen(sql); + if (sqlLen > (size_t)TSDB_MAX_ALLOWED_SQL_LEN) { + releaseTscObj(*(int64_t*)taos); + tscError("sql string exceeds max length:%d", TSDB_MAX_ALLOWED_SQL_LEN); + terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT; + return NULL; + } + + TAOS_RES* pRes = execQuery(pTscObj, sql, sqlLen, validateOnly); + + releaseTscObj(*(int64_t*)taos); + + return pRes; +#endif +} diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 22747425f0..f5dfe2de36 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -81,12 +81,6 @@ void taos_cleanup(void) { taosCloseLog(); } -setConfRet taos_set_config(const char *config) { - // TODO - setConfRet ret = {SET_CONF_RET_SUCC, {0}}; - return ret; -} - TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port) { tscDebug("try to connect to %s:%u, user:%s db:%s", ip, port, user, db); if (user == NULL) { @@ -205,51 +199,9 @@ TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) { return pResInfo->userFields; } -static void syncQueryFn(void *param, void *res, int32_t code) { - SSyncQueryParam *pParam = param; - pParam->pRequest = res; - pParam->pRequest->code = code; - - tsem_post(&pParam->sem); -} TAOS_RES *taos_query(TAOS *taos, const char *sql) { - if (NULL == taos) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - return NULL; - } - - STscObj *pTscObj = acquireTscObj(*(int64_t *)taos); - if (pTscObj == NULL || sql == NULL) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - return NULL; - } - -#if SYNC_ON_TOP_OF_ASYNC - SSyncQueryParam *param = taosMemoryCalloc(1, sizeof(SSyncQueryParam)); - tsem_init(¶m->sem, 0, 0); - - taos_query_a(taos, sql, syncQueryFn, param); - tsem_wait(¶m->sem); - - releaseTscObj(*(int64_t *)taos); - - return param->pRequest; -#else - size_t sqlLen = strlen(sql); - if (sqlLen > (size_t)TSDB_MAX_ALLOWED_SQL_LEN) { - releaseTscObj(*(int64_t *)taos); - tscError("sql string exceeds max length:%d", TSDB_MAX_ALLOWED_SQL_LEN); - terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT; - return NULL; - } - - TAOS_RES *pRes = execQuery(pTscObj, sql, sqlLen); - - releaseTscObj(*(int64_t *)taos); - - return pRes; -#endif + return taosQueryImpl(taos, sql, false); } TAOS_ROW taos_fetch_row(TAOS_RES *res) { @@ -641,7 +593,14 @@ int *taos_get_column_data_offset(TAOS_RES *res, int columnIndex) { return pResInfo->pCol[columnIndex].offset; } -int taos_validate_sql(TAOS *taos, const char *sql) { return true; } +int taos_validate_sql(TAOS *taos, const char *sql) { + TAOS_RES* pObj = taosQueryImpl(taos, sql, true); + + int code = taos_errno(pObj); + + taos_free_result(pObj); + return code; +} void taos_reset_current_db(TAOS *taos) { STscObj *pTscObj = acquireTscObj(*(int64_t *)taos); @@ -664,7 +623,7 @@ const char *taos_get_server_info(TAOS *taos) { releaseTscObj(*(int64_t *)taos); - return pTscObj->ver; + return pTscObj->sDetailVer; } typedef struct SqlParseWrapper { @@ -731,38 +690,7 @@ void retrieveMetaCallback(SMetaData *pResultMeta, void *param, int32_t code) { } void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param) { - STscObj *pTscObj = acquireTscObj(*(int64_t *)taos); - if (pTscObj == NULL || sql == NULL || NULL == fp) { - terrno = TSDB_CODE_INVALID_PARA; - if (pTscObj) { - releaseTscObj(*(int64_t *)taos); - } else { - terrno = TSDB_CODE_TSC_DISCONNECTED; - } - fp(param, NULL, terrno); - return; - } - - size_t sqlLen = strlen(sql); - if (sqlLen > (size_t)TSDB_MAX_ALLOWED_SQL_LEN) { - tscError("sql string exceeds max length:%d", TSDB_MAX_ALLOWED_SQL_LEN); - terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT; - - fp(param, NULL, terrno); - return; - } - - SRequestObj *pRequest = NULL; - int32_t code = buildRequest(pTscObj, sql, sqlLen, &pRequest); - if (code != TSDB_CODE_SUCCESS) { - terrno = code; - fp(param, NULL, terrno); - return; - } - - pRequest->body.queryFp = fp; - pRequest->body.param = param; - doAsyncQuery(pRequest, false); + taosAsyncQueryImpl(taos, sql, fp, param, false); } int32_t createParseContext(const SRequestObj *pRequest, SParseContext **pCxt) { @@ -838,7 +766,7 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) { .requestObjRefId = pCxt->requestRid, .mgmtEps = pCxt->mgmtEpSet}; - code = catalogAsyncGetAllMeta(pCxt->pCatalog, &conn, pRequest->requestId, &catalogReq, retrieveMetaCallback, pWrapper, + code = catalogAsyncGetAllMeta(pCxt->pCatalog, &conn, &catalogReq, retrieveMetaCallback, pWrapper, &pRequest->body.queryJob); if (code == TSDB_CODE_SUCCESS) { return; @@ -955,10 +883,75 @@ void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress) { } int taos_load_table_info(TAOS *taos, const char *tableNameList) { - // TODO - return -1; + const int32_t MAX_TABLE_NAME_LENGTH = 12 * 1024 * 1024; // 12MB list + int32_t code = 0; + SRequestObj *pRequest = NULL; + SCatalogReq catalogReq = {0}; + + if (NULL == tableNameList) { + return TSDB_CODE_SUCCESS; + } + + int32_t length = (int32_t)strlen(tableNameList); + if (0 == length) { + return TSDB_CODE_SUCCESS; + } else if (length > MAX_TABLE_NAME_LENGTH) { + tscError("tableNameList too long, length:%d, maximum allowed:%d", length, MAX_TABLE_NAME_LENGTH); + return TSDB_CODE_TSC_INVALID_OPERATION; + } + + STscObj *pTscObj = acquireTscObj(*(int64_t *)taos); + if (pTscObj == NULL) { + terrno = TSDB_CODE_TSC_DISCONNECTED; + return terrno; + } + + code = transferTableNameList(tableNameList, pTscObj->acctId, pTscObj->db, &catalogReq.pTableMeta); + if (code) { + goto _return; + } + + SCatalog* pCtg = NULL; + code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCtg); + if (code != TSDB_CODE_SUCCESS) { + goto _return; + } + + char* sql = "taos_load_table_info"; + code = buildRequest(pTscObj, sql, strlen(sql), &pRequest); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + goto _return; + } + + SSyncQueryParam param = {0}; + tsem_init(¶m.sem, 0, 0); + param.pRequest = pRequest; + + SRequestConnInfo conn = {.pTrans = pTscObj->pAppInfo->pTransporter, + .requestId = pRequest->requestId, + .requestObjRefId = pRequest->self}; + + conn.mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); + + code = catalogAsyncGetAllMeta(pCtg, &conn, &catalogReq, syncCatalogFn, ¶m, NULL); + if (code) { + goto _return; + } + + tsem_wait(¶m.sem); + +_return: + + taosArrayDestroy(catalogReq.pTableMeta); + destroyRequest(pRequest); + + releaseTscObj(*(int64_t *)taos); + + return code; } + TAOS_STMT *taos_stmt_init(TAOS *taos) { STscObj *pObj = acquireTscObj(*(int64_t *)taos); if (NULL == pObj) { diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index db8aebb322..45a525d124 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -82,7 +82,8 @@ int32_t processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) { pTscObj->connId = connectRsp.connId; pTscObj->acctId = connectRsp.acctId; - tstrncpy(pTscObj->ver, connectRsp.sVersion, tListLen(pTscObj->ver)); + tstrncpy(pTscObj->sVer, connectRsp.sVer, tListLen(pTscObj->sVer)); + tstrncpy(pTscObj->sDetailVer, connectRsp.sDetailVer, tListLen(pTscObj->sDetailVer)); // update the appInstInfo pTscObj->pAppInfo->clusterId = connectRsp.clusterId; @@ -287,6 +288,103 @@ int32_t processAlterStbRsp(void* param, const SDataBuf* pMsg, int32_t code) { return code; } +static int32_t buildShowVariablesBlock(SArray* pVars, SSDataBlock** block) { + SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); + pBlock->info.hasVarCol = true; + + pBlock->pDataBlock = taosArrayInit(SHOW_VARIABLES_RESULT_COLS, sizeof(SColumnInfoData)); + + SColumnInfoData infoData = {0}; + infoData.info.type = TSDB_DATA_TYPE_VARCHAR; + infoData.info.bytes = SHOW_VARIABLES_RESULT_FIELD1_LEN; + + taosArrayPush(pBlock->pDataBlock, &infoData); + + infoData.info.type = TSDB_DATA_TYPE_VARCHAR; + infoData.info.bytes = SHOW_VARIABLES_RESULT_FIELD2_LEN; + taosArrayPush(pBlock->pDataBlock, &infoData); + + int32_t numOfCfg = taosArrayGetSize(pVars); + blockDataEnsureCapacity(pBlock, numOfCfg); + + for (int32_t i = 0, c = 0; i < numOfCfg; ++i, c = 0) { + SVariablesInfo *pInfo = taosArrayGet(pVars, i); + + char name[TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(name, pInfo->name, TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE); + SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, c++); + colDataAppend(pColInfo, i, name, false); + + char value[TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(value, pInfo->value, TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE); + pColInfo = taosArrayGet(pBlock->pDataBlock, c++); + colDataAppend(pColInfo, i, value, false); + } + + pBlock->info.rows = numOfCfg; + + *block = pBlock; + + return TSDB_CODE_SUCCESS; +} + + +static int32_t buildShowVariablesRsp(SArray* pVars, SRetrieveTableRsp** pRsp) { + SSDataBlock* pBlock = NULL; + int32_t code = buildShowVariablesBlock(pVars, &pBlock); + if (code) { + return code; + } + + size_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock); + *pRsp = taosMemoryCalloc(1, rspSize); + if (NULL == *pRsp) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + (*pRsp)->useconds = 0; + (*pRsp)->completed = 1; + (*pRsp)->precision = 0; + (*pRsp)->compressed = 0; + (*pRsp)->compLen = 0; + (*pRsp)->numOfRows = htonl(pBlock->info.rows); + (*pRsp)->numOfCols = htonl(SHOW_VARIABLES_RESULT_COLS); + + int32_t len = 0; + blockCompressEncode(pBlock, (*pRsp)->data, &len, SHOW_VARIABLES_RESULT_COLS, false); + ASSERT(len == rspSize - sizeof(SRetrieveTableRsp)); + + blockDataDestroy(pBlock); + return TSDB_CODE_SUCCESS; +} + +int32_t processShowVariablesRsp(void* param, const SDataBuf* pMsg, int32_t code) { + SRequestObj* pRequest = param; + if (code != TSDB_CODE_SUCCESS) { + setErrno(pRequest, code); + } else { + SShowVariablesRsp rsp = {0}; + SRetrieveTableRsp* pRes = NULL; + code = tDeserializeSShowVariablesRsp(pMsg->pData, pMsg->len, &rsp); + if (TSDB_CODE_SUCCESS == code) { + code = buildShowVariablesRsp(rsp.variables, &pRes); + } + if (TSDB_CODE_SUCCESS == code) { + code = setQueryResultFromRsp(&pRequest->body.resInfo, pRes, false, false); + } + + tFreeSShowVariablesRsp(&rsp); + } + + if (pRequest->body.queryFp != NULL) { + pRequest->body.queryFp(pRequest->body.param, pRequest, code); + } else { + tsem_post(&pRequest->body.rspSem); + } + return code; +} + + __async_send_cb_fn_t getMsgRspHandle(int32_t msgType) { switch (msgType) { case TDMT_MND_CONNECT: @@ -301,6 +399,8 @@ __async_send_cb_fn_t getMsgRspHandle(int32_t msgType) { return processDropDbRsp; case TDMT_MND_ALTER_STB: return processAlterStbRsp; + case TDMT_MND_SHOW_VARIABLES: + return processShowVariablesRsp; default: return genericRspCallback; } diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 2172a6fe50..b104e1c2be 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -636,6 +636,377 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { return 0; } +int32_t taosSetCfg(SConfig *pCfg, char* name) { + int32_t len = strlen(name); + char lowcaseName[CFG_NAME_MAX_LEN + 1] = {0}; + strntolower(lowcaseName, name, TMIN(CFG_NAME_MAX_LEN, len)); + + switch (lowcaseName[0]) { + case 'a': { + if (strcasecmp("asyncLog", name) == 0) { + tsAsyncLog = cfgGetItem(pCfg, "asyncLog")->bval; + } + break; + } + case 'b': { + if (strcasecmp("bnodeShmSize", name) == 0) { + tsBnodeShmSize = cfgGetItem(pCfg, "bnodeShmSize")->i32; + } + break; + } + case 'c': { + if (strcasecmp("charset", name) == 0) { + const char *locale = cfgGetItem(pCfg, "locale")->str; + const char *charset = cfgGetItem(pCfg, "charset")->str; + taosSetSystemLocale(locale, charset); + osSetSystemLocale(locale, charset); + } else if (strcasecmp("compressMsgSize", name) == 0) { + tsCompressMsgSize = cfgGetItem(pCfg, "compressMsgSize")->i32; + } else if (strcasecmp("compressColData", name) == 0) { + tsCompressColData = cfgGetItem(pCfg, "compressColData")->i32; + } else if (strcasecmp("countAlwaysReturnValue", name) == 0) { + tsCountAlwaysReturnValue = cfgGetItem(pCfg, "countAlwaysReturnValue")->i32; + } else if (strcasecmp("cDebugFlag", name) == 0) { + cDebugFlag = cfgGetItem(pCfg, "cDebugFlag")->i32; + } + break; + } + case 'd': { + if (strcasecmp("deadLockKillQuery", name) == 0) { + tsDeadLockKillQuery = cfgGetItem(pCfg, "deadLockKillQuery")->i32; + } else if (strcasecmp("dDebugFlag", name) == 0) { + dDebugFlag = cfgGetItem(pCfg, "dDebugFlag")->i32; + } + break; + } + case 'e': { + if (strcasecmp("enableCoreFile", name) == 0) { + bool enableCore = cfgGetItem(pCfg, "enableCoreFile")->bval; + taosSetConsoleEcho(enableCore); + } + break; + } + case 'f': { + if (strcasecmp("fqdn", name) == 0) { + tstrncpy(tsLocalFqdn, cfgGetItem(pCfg, "fqdn")->str, TSDB_FQDN_LEN); + tsServerPort = (uint16_t)cfgGetItem(pCfg, "serverPort")->i32; + snprintf(tsLocalEp, sizeof(tsLocalEp), "%s:%u", tsLocalFqdn, tsServerPort); + + char defaultFirstEp[TSDB_EP_LEN] = {0}; + snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%u", tsLocalFqdn, tsServerPort); + + SConfigItem *pFirstEpItem = cfgGetItem(pCfg, "firstEp"); + SEp firstEp = {0}; + taosGetFqdnPortFromEp(strlen(pFirstEpItem->str) == 0 ? defaultFirstEp : pFirstEpItem->str, &firstEp); + snprintf(tsFirst, sizeof(tsFirst), "%s:%u", firstEp.fqdn, firstEp.port); + cfgSetItem(pCfg, "firstEp", tsFirst, pFirstEpItem->stype); + } else if (strcasecmp("firstEp", name) == 0) { + tstrncpy(tsLocalFqdn, cfgGetItem(pCfg, "fqdn")->str, TSDB_FQDN_LEN); + tsServerPort = (uint16_t)cfgGetItem(pCfg, "serverPort")->i32; + snprintf(tsLocalEp, sizeof(tsLocalEp), "%s:%u", tsLocalFqdn, tsServerPort); + + char defaultFirstEp[TSDB_EP_LEN] = {0}; + snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%u", tsLocalFqdn, tsServerPort); + + SConfigItem *pFirstEpItem = cfgGetItem(pCfg, "firstEp"); + SEp firstEp = {0}; + taosGetFqdnPortFromEp(strlen(pFirstEpItem->str) == 0 ? defaultFirstEp : pFirstEpItem->str, &firstEp); + snprintf(tsFirst, sizeof(tsFirst), "%s:%u", firstEp.fqdn, firstEp.port); + cfgSetItem(pCfg, "firstEp", tsFirst, pFirstEpItem->stype); + } else if (strcasecmp("fsDebugFlag", name) == 0) { + fsDebugFlag = cfgGetItem(pCfg, "fsDebugFlag")->i32; + } else if (strcasecmp("fnDebugFlag", name) == 0) { + fnDebugFlag = cfgGetItem(pCfg, "fnDebugFlag")->i32; + } + break; + } + case 'i': { + if (strcasecmp("idxDebugFlag", name) == 0) { + idxDebugFlag = cfgGetItem(pCfg, "idxDebugFlag")->i32; + } + break; + } + case 'j': { + if (strcasecmp("jniDebugFlag", name) == 0) { + jniDebugFlag = cfgGetItem(pCfg, "jniDebugFlag")->i32; + } + break; + } + case 'k': { + if (strcasecmp("keepColumnName", name) == 0) { + tsKeepOriginalColumnName = cfgGetItem(pCfg, "keepColumnName")->bval; + } + break; + } + case 'l': { + if (strcasecmp("locale", name) == 0) { + const char *locale = cfgGetItem(pCfg, "locale")->str; + const char *charset = cfgGetItem(pCfg, "charset")->str; + taosSetSystemLocale(locale, charset); + osSetSystemLocale(locale, charset); + } else if (strcasecmp("logDir", name) == 0) { + tstrncpy(tsLogDir, cfgGetItem(pCfg, "logDir")->str, PATH_MAX); + taosExpandDir(tsLogDir, tsLogDir, PATH_MAX); + } else if (strcasecmp("logKeepDays", name) == 0) { + tsLogKeepDays = cfgGetItem(pCfg, "logKeepDays")->i32; + } + break; + } + case 'm': { + switch (lowcaseName[1]) { + case 'a': { + if (strcasecmp("maxShellConns", name) == 0) { + tsMaxShellConns = cfgGetItem(pCfg, "maxShellConns")->i32; + } else if (strcasecmp("maxNumOfDistinctRes", name) == 0) { + tsMaxNumOfDistinctResults = cfgGetItem(pCfg, "maxNumOfDistinctRes")->i32; + } else if (strcasecmp("maxStreamCompDelay", name) == 0) { + tsMaxStreamComputDelay = cfgGetItem(pCfg, "maxStreamCompDelay")->i32; + } else if (strcasecmp("maxFirstStreamCompDelay", name) == 0) { + tsStreamCompStartDelay = cfgGetItem(pCfg, "maxFirstStreamCompDelay")->i32; + } + break; + } + case 'd': { + if (strcasecmp("mDebugFlag", name) == 0) { + mDebugFlag = cfgGetItem(pCfg, "mDebugFlag")->i32; + } + break; + } + case 'i': { + if (strcasecmp("minimalTempDirGB", name) == 0) { + tsTempSpace.reserved = cfgGetItem(pCfg, "minimalTempDirGB")->fval; + } else if (strcasecmp("minimalDataDirGB", name) == 0) { + tsDataSpace.reserved = cfgGetItem(pCfg, "minimalDataDirGB")->fval; + } else if (strcasecmp("minSlidingTime", name) == 0) { + tsMinSlidingTime = cfgGetItem(pCfg, "minSlidingTime")->i32; + } else if (strcasecmp("minIntervalTime", name) == 0) { + tsMinIntervalTime = cfgGetItem(pCfg, "minIntervalTime")->i32; + } else if (strcasecmp("minimalLogDirGB", name) == 0) { + tsLogSpace.reserved = cfgGetItem(pCfg, "minimalLogDirGB")->fval; + } + break; + } + case 'n': { + if (strcasecmp("mnodeShmSize", name) == 0) { + tsMnodeShmSize = cfgGetItem(pCfg, "mnodeShmSize")->i32; + } + break; + } + case 'o': { + if (strcasecmp("monitor", name) == 0) { + tsEnableMonitor = cfgGetItem(pCfg, "monitor")->bval; + } else if (strcasecmp("monitorInterval", name) == 0) { + tsMonitorInterval = cfgGetItem(pCfg, "monitorInterval")->i32; + } else if (strcasecmp("monitorFqdn", name) == 0) { + tstrncpy(tsMonitorFqdn, cfgGetItem(pCfg, "monitorFqdn")->str, TSDB_FQDN_LEN); + } else if (strcasecmp("monitorPort", name) == 0) { + tsMonitorPort = (uint16_t)cfgGetItem(pCfg, "monitorPort")->i32; + } else if (strcasecmp("monitorMaxLogs", name) == 0) { + tsMonitorMaxLogs = cfgGetItem(pCfg, "monitorMaxLogs")->i32; + } else if (strcasecmp("monitorComp", name) == 0) { + tsMonitorComp = cfgGetItem(pCfg, "monitorComp")->bval; + } + break; + } + case 'q': { + if (strcasecmp("mqRebalanceInterval", name) == 0) { + tsMqRebalanceInterval = cfgGetItem(pCfg, "mqRebalanceInterval")->i32; + } + break; + } + case 'u': { + if (strcasecmp("multiProcess", name) == 0) { + tsMultiProcess = cfgGetItem(pCfg, "multiProcess")->bval; + } + break; + } + default: + terrno = TSDB_CODE_CFG_NOT_FOUND; + return -1; + } + break; + } + case 'n': { + if (strcasecmp("numOfTaskQueueThreads", name) == 0) { + tsNumOfTaskQueueThreads = cfgGetItem(pCfg, "numOfTaskQueueThreads")->i32; + } else if (strcasecmp("numOfRpcThreads", name) == 0) { + tsNumOfRpcThreads = cfgGetItem(pCfg, "numOfRpcThreads")->i32; + } else if (strcasecmp("numOfCommitThreads", name) == 0) { + tsNumOfCommitThreads = cfgGetItem(pCfg, "numOfCommitThreads")->i32; + } else if (strcasecmp("numOfMnodeQueryThreads", name) == 0) { + tsNumOfMnodeQueryThreads = cfgGetItem(pCfg, "numOfMnodeQueryThreads")->i32; + } else if (strcasecmp("numOfMnodeReadThreads", name) == 0) { + tsNumOfMnodeReadThreads = cfgGetItem(pCfg, "numOfMnodeReadThreads")->i32; + } else if (strcasecmp("numOfVnodeQueryThreads", name) == 0) { + tsNumOfVnodeQueryThreads = cfgGetItem(pCfg, "numOfVnodeQueryThreads")->i32; + } else if (strcasecmp("numOfVnodeFetchThreads", name) == 0) { + tsNumOfVnodeFetchThreads = cfgGetItem(pCfg, "numOfVnodeFetchThreads")->i32; + } else if (strcasecmp("numOfVnodeWriteThreads", name) == 0) { + tsNumOfVnodeWriteThreads = cfgGetItem(pCfg, "numOfVnodeWriteThreads")->i32; + } else if (strcasecmp("numOfVnodeSyncThreads", name) == 0) { + tsNumOfVnodeSyncThreads = cfgGetItem(pCfg, "numOfVnodeSyncThreads")->i32; + } else if (strcasecmp("numOfVnodeMergeThreads", name) == 0) { + tsNumOfVnodeMergeThreads = cfgGetItem(pCfg, "numOfVnodeMergeThreads")->i32; + } else if (strcasecmp("numOfQnodeQueryThreads", name) == 0) { + tsNumOfQnodeQueryThreads = cfgGetItem(pCfg, "numOfQnodeQueryThreads")->i32; + } else if (strcasecmp("numOfQnodeFetchThreads", name) == 0) { + tsNumOfQnodeFetchThreads = cfgGetItem(pCfg, "numOfQnodeFetchThreads")->i32; + } else if (strcasecmp("numOfSnodeSharedThreads", name) == 0) { + tsNumOfSnodeSharedThreads = cfgGetItem(pCfg, "numOfSnodeSharedThreads")->i32; + } else if (strcasecmp("numOfSnodeUniqueThreads", name) == 0) { + tsNumOfSnodeUniqueThreads = cfgGetItem(pCfg, "numOfSnodeUniqueThreads")->i32; + } else if (strcasecmp("numOfLogLines", name) == 0) { + tsNumOfLogLines = cfgGetItem(pCfg, "numOfLogLines")->i32; + } + break; + } + case 'p': { + if (strcasecmp("printAuth", name) == 0) { + tsPrintAuth = cfgGetItem(pCfg, "printAuth")->bval; + } + break; + } + case 'q': { + if (strcasecmp("queryPolicy", name) == 0) { + tsQueryPolicy = cfgGetItem(pCfg, "queryPolicy")->i32; + } else if (strcasecmp("querySmaOptimize", name) == 0) { + tsQuerySmaOptimize = cfgGetItem(pCfg, "querySmaOptimize")->i32; + } else if (strcasecmp("queryBufferSize", name) == 0) { + tsQueryBufferSize = cfgGetItem(pCfg, "queryBufferSize")->i32; + if (tsQueryBufferSize >= 0) { + tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL; + } + } else if (strcasecmp("qnodeShmSize", name) == 0) { + tsQnodeShmSize = cfgGetItem(pCfg, "qnodeShmSize")->i32; + } else if (strcasecmp("qDebugFlag", name) == 0) { + qDebugFlag = cfgGetItem(pCfg, "qDebugFlag")->i32; + } + break; + } + case 'r': { + if (strcasecmp("retryStreamCompDelay", name) == 0) { + tsRetryStreamCompDelay = cfgGetItem(pCfg, "retryStreamCompDelay")->i32; + } else if (strcasecmp("retrieveBlockingModel", name) == 0) { + tsRetrieveBlockingModel = cfgGetItem(pCfg, "retrieveBlockingModel")->bval; + } else if (strcasecmp("rpcQueueMemoryAllowed", name) == 0) { + tsRpcQueueMemoryAllowed = cfgGetItem(pCfg, "rpcQueueMemoryAllowed")->i64; + } else if (strcasecmp("rpcDebugFlag", name) == 0) { + rpcDebugFlag = cfgGetItem(pCfg, "rpcDebugFlag")->i32; + } + break; + } + case 's': { + if (strcasecmp("secondEp", name) == 0) { + SConfigItem *pSecondpItem = cfgGetItem(pCfg, "secondEp"); + SEp secondEp = {0}; + taosGetFqdnPortFromEp(strlen(pSecondpItem->str) == 0 ? tsFirst : pSecondpItem->str, &secondEp); + snprintf(tsSecond, sizeof(tsSecond), "%s:%u", secondEp.fqdn, secondEp.port); + cfgSetItem(pCfg, "secondEp", tsSecond, pSecondpItem->stype); + } else if (strcasecmp("smlChildTableName", name) == 0) { + tstrncpy(tsSmlChildTableName, cfgGetItem(pCfg, "smlChildTableName")->str, TSDB_TABLE_NAME_LEN); + } else if (strcasecmp("smlTagName", name) == 0) { + tstrncpy(tsSmlTagName, cfgGetItem(pCfg, "smlTagName")->str, TSDB_COL_NAME_LEN); + } else if (strcasecmp("smlDataFormat", name) == 0) { + tsSmlDataFormat = cfgGetItem(pCfg, "smlDataFormat")->bval; + } else if (strcasecmp("shellActivityTimer", name) == 0) { + tsShellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->i32; + } else if (strcasecmp("supportVnodes", name) == 0) { + tsNumOfSupportVnodes = cfgGetItem(pCfg, "supportVnodes")->i32; + } else if (strcasecmp("statusInterval", name) == 0) { + tsStatusInterval = cfgGetItem(pCfg, "statusInterval")->i32; + } else if (strcasecmp("streamCompDelayRatio", name) == 0) { + tsStreamComputDelayRatio = cfgGetItem(pCfg, "streamCompDelayRatio")->fval; + } else if (strcasecmp("slaveQuery", name) == 0) { + tsEnableSlaveQuery = cfgGetItem(pCfg, "slaveQuery")->bval; + } else if (strcasecmp("snodeShmSize", name) == 0) { + tsSnodeShmSize = cfgGetItem(pCfg, "snodeShmSize")->i32; + } else if (strcasecmp("serverPort", name) == 0) { + tstrncpy(tsLocalFqdn, cfgGetItem(pCfg, "fqdn")->str, TSDB_FQDN_LEN); + tsServerPort = (uint16_t)cfgGetItem(pCfg, "serverPort")->i32; + snprintf(tsLocalEp, sizeof(tsLocalEp), "%s:%u", tsLocalFqdn, tsServerPort); + + char defaultFirstEp[TSDB_EP_LEN] = {0}; + snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%u", tsLocalFqdn, tsServerPort); + + SConfigItem *pFirstEpItem = cfgGetItem(pCfg, "firstEp"); + SEp firstEp = {0}; + taosGetFqdnPortFromEp(strlen(pFirstEpItem->str) == 0 ? defaultFirstEp : pFirstEpItem->str, &firstEp); + snprintf(tsFirst, sizeof(tsFirst), "%s:%u", firstEp.fqdn, firstEp.port); + cfgSetItem(pCfg, "firstEp", tsFirst, pFirstEpItem->stype); + } else if (strcasecmp("sDebugFlag", name) == 0) { + sDebugFlag = cfgGetItem(pCfg, "sDebugFlag")->i32; + } else if (strcasecmp("smaDebugFlag", name) == 0) { + smaDebugFlag = cfgGetItem(pCfg, "smaDebugFlag")->i32; + } + break; + } + case 't': { + if (strcasecmp("timezone", name) == 0) { + SConfigItem *pItem = cfgGetItem(pCfg, "timezone"); + osSetTimezone(pItem->str); + uDebug("timezone format changed from %s to %s", pItem->str, tsTimezoneStr); + cfgSetItem(pCfg, "timezone", tsTimezoneStr, pItem->stype); + } else if (strcasecmp("tempDir", name) == 0) { + tstrncpy(tsTempDir, cfgGetItem(pCfg, "tempDir")->str, PATH_MAX); + taosExpandDir(tsTempDir, tsTempDir, PATH_MAX); + if (taosMulMkDir(tsTempDir) != 0) { + uError("failed to create tempDir:%s since %s", tsTempDir, terrstr()); + return -1; + } + } else if (strcasecmp("telemetryReporting", name) == 0) { + tsEnableTelem = cfgGetItem(pCfg, "telemetryReporting")->bval; + } else if (strcasecmp("telemetryInterval", name) == 0) { + tsTelemInterval = cfgGetItem(pCfg, "telemetryInterval")->i32; + } else if (strcasecmp("telemetryServer", name) == 0) { + tstrncpy(tsTelemServer, cfgGetItem(pCfg, "telemetryServer")->str, TSDB_FQDN_LEN); + } else if (strcasecmp("telemetryPort", name) == 0) { + tsTelemPort = (uint16_t)cfgGetItem(pCfg, "telemetryPort")->i32; + } else if (strcasecmp("transPullupInterval", name) == 0) { + tsTransPullupInterval = cfgGetItem(pCfg, "transPullupInterval")->i32; + } else if (strcasecmp("ttlUnit", name) == 0) { + tsTtlUnit = cfgGetItem(pCfg, "ttlUnit")->i32; + } else if (strcasecmp("ttlPushInterval", name) == 0) { + tsTtlPushInterval = cfgGetItem(pCfg, "ttlPushInterval")->i32; + } else if (strcasecmp("tmrDebugFlag", name) == 0) { + tmrDebugFlag = cfgGetItem(pCfg, "tmrDebugFlag")->i32; + } else if (strcasecmp("tsdbDebugFlag", name) == 0) { + tsdbDebugFlag = cfgGetItem(pCfg, "tsdbDebugFlag")->i32; + } else if (strcasecmp("tqDebugFlag", name) == 0) { + tqDebugFlag = cfgGetItem(pCfg, "tqDebugFlag")->i32; + } + break; + } + case 'u': { + if (strcasecmp("udf", name) == 0) { + tsStartUdfd = cfgGetItem(pCfg, "udf")->bval; + } else if (strcasecmp("uDebugFlag", name) == 0) { + uDebugFlag = cfgGetItem(pCfg, "uDebugFlag")->i32; + } + break; + } + case 'v': { + if (strcasecmp("vnodeShmSize", name) == 0) { + tsVnodeShmSize = cfgGetItem(pCfg, "vnodeShmSize")->i32; + } else if (strcasecmp("vDebugFlag", name) == 0) { + vDebugFlag = cfgGetItem(pCfg, "vDebugFlag")->i32; + } + break; + } + case 'w': { + if (strcasecmp("wDebugFlag", name) == 0) { + wDebugFlag = cfgGetItem(pCfg, "wDebugFlag")->i32; + } + break; + } + default: + terrno = TSDB_CODE_CFG_NOT_FOUND; + return -1; + } + + return 0; +} + + int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc) { if (tsCfg == NULL) osDefaultInit(); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index dde282c4e3..d96a5ec574 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -2250,6 +2250,56 @@ int32_t tDeserializeSDnodeListReq(void *buf, int32_t bufLen, SDnodeListReq *pReq return 0; } +int32_t tSerializeSServerVerReq(void *buf, int32_t bufLen, SServerVerReq *pReq) { + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI32(&encoder, pReq->useless) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSServerVerReq(void *buf, int32_t bufLen, SServerVerReq *pReq) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->useless) < 0) return -1; + + tEndDecode(&decoder); + tDecoderClear(&decoder); + return 0; +} + +int32_t tSerializeSServerVerRsp(void *buf, int32_t bufLen, SServerVerRsp *pRsp) { + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pRsp->ver) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSServerVerRsp(void *buf, int32_t bufLen, SServerVerRsp *pRsp) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pRsp->ver) < 0) return -1; + + tEndDecode(&decoder); + tDecoderClear(&decoder); + return 0; +} + int32_t tSerializeSQnodeListRsp(void *buf, int32_t bufLen, SQnodeListRsp *pRsp) { SEncoder encoder = {0}; @@ -2859,6 +2909,67 @@ int32_t tDeserializeSShowVariablesReq(void *buf, int32_t bufLen, SShowVariablesR return 0; } +int32_t tEncodeSVariablesInfo(SEncoder* pEncoder, SVariablesInfo* pInfo) { + if (tEncodeCStr(pEncoder, pInfo->name) < 0) return -1; + if (tEncodeCStr(pEncoder, pInfo->value) < 0) return -1; + return 0; +} + +int32_t tDecodeSVariablesInfo(SDecoder* pDecoder, SVariablesInfo* pInfo) { + if (tDecodeCStrTo(pDecoder, pInfo->name) < 0) return -1; + if (tDecodeCStrTo(pDecoder, pInfo->value) < 0) return -1; + return 0; +} + + +int32_t tSerializeSShowVariablesRsp(void* buf, int32_t bufLen, SShowVariablesRsp* pRsp) { + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + + if (tStartEncode(&encoder) < 0) return -1; + int32_t varNum = taosArrayGetSize(pRsp->variables); + if (tEncodeI32(&encoder, varNum) < 0) return -1; + for (int32_t i = 0; i < varNum; ++i) { + SVariablesInfo* pInfo = taosArrayGet(pRsp->variables, i); + if (tEncodeSVariablesInfo(&encoder, pInfo) < 0) return -1; + } + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSShowVariablesRsp(void* buf, int32_t bufLen, SShowVariablesRsp* pRsp) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + + if (tStartDecode(&decoder) < 0) return -1; + int32_t varNum = 0; + if (tDecodeI32(&decoder, &varNum) < 0) return -1; + if (varNum > 0) { + pRsp->variables = taosArrayInit(varNum, sizeof(SVariablesInfo)); + if (NULL == pRsp->variables) return -1; + for (int32_t i = 0; i < varNum; ++i) { + SVariablesInfo info = {0}; + if (tDecodeSVariablesInfo(&decoder, &info) < 0) return -1; + if (NULL == taosArrayPush(pRsp->variables, &info)) return -1; + } + } + + tEndDecode(&decoder); + tDecoderClear(&decoder); + return 0; +} + +void tFreeSShowVariablesRsp(SShowVariablesRsp* pRsp) { + if (NULL == pRsp) { + return; + } + + taosArrayDestroy(pRsp->variables); +} + int32_t tSerializeSShowReq(void *buf, int32_t bufLen, SShowReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -3400,7 +3511,8 @@ int32_t tSerializeSConnectRsp(void *buf, int32_t bufLen, SConnectRsp *pRsp) { if (tEncodeI8(&encoder, pRsp->superUser) < 0) return -1; if (tEncodeI8(&encoder, pRsp->connType) < 0) return -1; if (tEncodeSEpSet(&encoder, &pRsp->epSet) < 0) return -1; - if (tEncodeCStr(&encoder, pRsp->sVersion) < 0) return -1; + if (tEncodeCStr(&encoder, pRsp->sVer) < 0) return -1; + if (tEncodeCStr(&encoder, pRsp->sDetailVer) < 0) return -1; tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -3420,7 +3532,8 @@ int32_t tDeserializeSConnectRsp(void *buf, int32_t bufLen, SConnectRsp *pRsp) { if (tDecodeI8(&decoder, &pRsp->superUser) < 0) return -1; if (tDecodeI8(&decoder, &pRsp->connType) < 0) return -1; if (tDecodeSEpSet(&decoder, &pRsp->epSet) < 0) return -1; - if (tDecodeCStrTo(&decoder, pRsp->sVersion) < 0) return -1; + if (tDecodeCStrTo(&decoder, pRsp->sVer) < 0) return -1; + if (tDecodeCStrTo(&decoder, pRsp->sDetailVer) < 0) return -1; tEndDecode(&decoder); tDecoderClear(&decoder); diff --git a/source/common/src/tname.c b/source/common/src/tname.c index ce454d439d..fb3a2d7464 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -214,6 +214,20 @@ int32_t tNameSetDbName(SName* dst, int32_t acct, const char* dbName, size_t name return 0; } +int32_t tNameAddTbName(SName* dst, const char* tbName, size_t nameLen) { + assert(dst != NULL && tbName != NULL && nameLen > 0); + + // too long account id or too long db name + if (nameLen >= tListLen(dst->tname) || nameLen <= 0) { + return -1; + } + + dst->type = TSDB_TABLE_NAME_T; + tstrncpy(dst->tname, tbName, nameLen + 1); + return 0; +} + + int32_t tNameSetAcctId(SName* dst, int32_t acctId) { assert(dst != NULL); dst->acctId = acctId; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index f9c54674f3..3bf6d9b04f 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -206,6 +206,8 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MND_SYSTABLE_RETRIEVE, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_GRANT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_AUTH, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_SHOW_VARIABLES, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_SERVER_VERSION, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_CONTINUE, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/node_util/src/dmFile.c b/source/dnode/mgmt/node_util/src/dmFile.c index 2185adc18b..9ec17a18b5 100644 --- a/source/dnode/mgmt/node_util/src/dmFile.c +++ b/source/dnode/mgmt/node_util/src/dmFile.c @@ -133,10 +133,10 @@ TdFilePtr dmCheckRunning(const char *dataDir) { ret = taosLockFile(pFile); if (ret == 0) break; terrno = TAOS_SYSTEM_ERROR(errno); - taosMsleep(100); + taosMsleep(1000); retryTimes++; dError("failed to lock file:%s since %s, retryTimes:%d", filepath, terrstr(), retryTimes); - } while (retryTimes < 120); + } while (retryTimes < 12); if (ret < 0) { terrno = TAOS_SYSTEM_ERROR(errno); diff --git a/source/dnode/mnode/impl/inc/mndAuth.h b/source/dnode/mnode/impl/inc/mndAuth.h index 45841ca367..81a776b652 100644 --- a/source/dnode/mnode/impl/inc/mndAuth.h +++ b/source/dnode/mnode/impl/inc/mndAuth.h @@ -23,13 +23,15 @@ extern "C" { #endif typedef enum { - MND_OPER_CREATE_USER = 1, + MND_OPER_CONNECT = 1, + MND_OPER_CREATE_USER, MND_OPER_DROP_USER, MND_OPER_ALTER_USER, MND_OPER_CREATE_BNODE, MND_OPER_DROP_BNODE, MND_OPER_CREATE_DNODE, MND_OPER_DROP_DNODE, + MND_OPER_CONFIG_DNODE, MND_OPER_CREATE_MNODE, MND_OPER_DROP_MNODE, MND_OPER_CREATE_QNODE, @@ -37,6 +39,7 @@ typedef enum { MND_OPER_CREATE_SNODE, MND_OPER_DROP_SNODE, MND_OPER_REDISTRIBUTE_VGROUP, + MND_OPER_MERGE_VGROUP, MND_OPER_SPLIT_VGROUP, MND_OPER_BALANCE_VGROUP, MND_OPER_CREATE_FUNC, diff --git a/source/dnode/mnode/impl/inc/mndScheduler.h b/source/dnode/mnode/impl/inc/mndScheduler.h index 15d2c6cd5e..db81d8843e 100644 --- a/source/dnode/mnode/impl/inc/mndScheduler.h +++ b/source/dnode/mnode/impl/inc/mndScheduler.h @@ -27,12 +27,10 @@ void mndCleanupScheduler(SMnode* pMnode); int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscribeObj* pSub); -int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream); - int32_t mndConvertRsmaTask(char** pDst, int32_t* pDstLen, const char* ast, int64_t uid, int8_t triggerType, int64_t watermark); -int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream); +int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndAuth.c b/source/dnode/mnode/impl/src/mndAuth.c index d47fb9dfb4..4445e3b9f7 100644 --- a/source/dnode/mnode/impl/src/mndAuth.c +++ b/source/dnode/mnode/impl/src/mndAuth.c @@ -93,8 +93,13 @@ int32_t mndCheckOperAuth(SMnode *pMnode, const char *user, EOperType operType) { goto _OVER; } - terrno = TSDB_CODE_MND_NO_RIGHTS; - code = -1; + switch (operType) { + case MND_OPER_CONNECT: + break; + default: + terrno = TSDB_CODE_MND_NO_RIGHTS; + code = -1; + } _OVER: mndReleaseUser(pMnode, pUser); @@ -102,7 +107,13 @@ _OVER: } int32_t mndCheckAlterUserAuth(SUserObj *pOperUser, SUserObj *pUser, SAlterUserReq *pAlter) { + if (pUser->superUser && pAlter->alterType != TSDB_ALTER_USER_PASSWD) { + terrno = TSDB_CODE_MND_NO_RIGHTS; + return -1; + } + if (pOperUser->superUser) return 0; + if (!pOperUser->enable) { terrno = TSDB_CODE_MND_USER_DISABLED; return -1; diff --git a/source/dnode/mnode/impl/src/mndBnode.c b/source/dnode/mnode/impl/src/mndBnode.c index 0de40ca671..e2b1aad008 100644 --- a/source/dnode/mnode/impl/src/mndBnode.c +++ b/source/dnode/mnode/impl/src/mndBnode.c @@ -277,6 +277,9 @@ static int32_t mndProcessCreateBnodeReq(SRpcMsg *pReq) { } mDebug("bnode:%d, start to create", createReq.dnodeId); + if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_BNODE) != 0) { + goto _OVER; + } pObj = mndAcquireBnode(pMnode, createReq.dnodeId); if (pObj != NULL) { @@ -292,10 +295,6 @@ static int32_t mndProcessCreateBnodeReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_BNODE) != 0) { - goto _OVER; - } - code = mndCreateBnode(pMnode, pReq, pDnode, &createReq); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; @@ -383,6 +382,9 @@ static int32_t mndProcessDropBnodeReq(SRpcMsg *pReq) { } mDebug("bnode:%d, start to drop", dropReq.dnodeId); + if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_BNODE) != 0) { + goto _OVER; + } if (dropReq.dnodeId <= 0) { terrno = TSDB_CODE_INVALID_MSG; @@ -394,10 +396,6 @@ static int32_t mndProcessDropBnodeReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_BNODE) != 0) { - goto _OVER; - } - code = mndDropBnode(pMnode, pReq, pObj); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index d3723326b9..113777bc1f 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -48,6 +48,7 @@ static int32_t mndDnodeActionInsert(SSdb *pSdb, SDnodeObj *pDnode); static int32_t mndDnodeActionDelete(SSdb *pSdb, SDnodeObj *pDnode); static int32_t mndDnodeActionUpdate(SSdb *pSdb, SDnodeObj *pOld, SDnodeObj *pNew); static int32_t mndProcessDnodeListReq(SRpcMsg *pReq); +static int32_t mndProcessShowVariablesReq(SRpcMsg *pReq); static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq); static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq); @@ -78,6 +79,7 @@ int32_t mndInitDnode(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_DND_CONFIG_DNODE_RSP, mndProcessConfigDnodeRsp); mndSetMsgHandle(pMnode, TDMT_MND_STATUS, mndProcessStatusReq); mndSetMsgHandle(pMnode, TDMT_MND_DNODE_LIST, mndProcessDnodeListReq); + mndSetMsgHandle(pMnode, TDMT_MND_SHOW_VARIABLES, mndProcessShowVariablesReq); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_CONFIGS, mndRetrieveConfigs); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_CONFIGS, mndCancelGetNextConfig); @@ -554,6 +556,59 @@ _OVER: return code; } +static int32_t mndProcessShowVariablesReq(SRpcMsg *pReq) { + SShowVariablesRsp rsp = {0}; + int32_t code = -1; + + rsp.variables = taosArrayInit(4, sizeof(SVariablesInfo)); + if (NULL == rsp.variables) { + mError("failed to alloc SVariablesInfo array while process show variables req"); + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _OVER; + } + + SVariablesInfo info = {0}; + + strcpy(info.name, "statusInterval"); + snprintf(info.value, TSDB_CONFIG_VALUE_LEN, "%d", tsStatusInterval); + taosArrayPush(rsp.variables, &info); + + strcpy(info.name, "timezone"); + snprintf(info.value, TSDB_CONFIG_VALUE_LEN, "%s", tsTimezoneStr); + taosArrayPush(rsp.variables, &info); + + strcpy(info.name, "locale"); + snprintf(info.value, TSDB_CONFIG_VALUE_LEN, "%s", tsLocale); + taosArrayPush(rsp.variables, &info); + + strcpy(info.name, "charset"); + snprintf(info.value, TSDB_CONFIG_VALUE_LEN, "%s", tsCharset); + taosArrayPush(rsp.variables, &info); + + int32_t rspLen = tSerializeSShowVariablesRsp(NULL, 0, &rsp); + void *pRsp = rpcMallocCont(rspLen); + if (pRsp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _OVER; + } + + tSerializeSShowVariablesRsp(pRsp, rspLen, &rsp); + + pReq->info.rspLen = rspLen; + pReq->info.rsp = pRsp; + code = 0; + +_OVER: + + if (code != 0) { + mError("failed to get show variables info since %s", terrstr()); + } + + tFreeSShowVariablesRsp(&rsp); + + return code; +} + static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; int32_t code = -1; @@ -566,6 +621,9 @@ static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) { } mInfo("dnode:%s:%d, start to create", createReq.fqdn, createReq.port); + if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_DNODE) != 0) { + goto _OVER; + } if (createReq.fqdn[0] == 0 || createReq.port <= 0 || createReq.port > UINT16_MAX) { terrno = TSDB_CODE_MND_INVALID_DNODE_EP; @@ -579,10 +637,6 @@ static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_DNODE) != 0) { - goto _OVER; - } - code = mndCreateDnode(pMnode, pReq, &createReq); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; @@ -661,6 +715,9 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) { } mInfo("dnode:%d, start to drop, ep:%s:%d", dropReq.dnodeId, dropReq.fqdn, dropReq.port); + if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_MNODE) != 0) { + goto _OVER; + } pDnode = mndAcquireDnode(pMnode, dropReq.dnodeId); if (pDnode == NULL) { @@ -697,10 +754,6 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) { } } - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_MNODE) != 0) { - goto _OVER; - } - code = mndDropDnode(pMnode, pReq, pDnode, pMObj, pQObj, pSObj, numOfVnodes); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; @@ -726,6 +779,10 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { } mInfo("dnode:%d, start to config, option:%s, value:%s", cfgReq.dnodeId, cfgReq.config, cfgReq.value); + if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CONFIG_DNODE) != 0) { + return -1; + } + SDnodeObj *pDnode = mndAcquireDnode(pMnode, cfgReq.dnodeId); if (pDnode == NULL) { mError("dnode:%d, failed to config since %s ", cfgReq.dnodeId, terrstr()); diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index 832f1b8e68..37e0a719dd 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -283,6 +283,9 @@ static int32_t mndProcessCreateFuncReq(SRpcMsg *pReq) { } mDebug("func:%s, start to create", createReq.name); + if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_FUNC) != 0) { + goto _OVER; + } pFunc = mndAcquireFunc(pMnode, createReq.name); if (pFunc != NULL) { @@ -318,10 +321,6 @@ static int32_t mndProcessCreateFuncReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_FUNC) != 0) { - goto _OVER; - } - code = mndCreateFunc(pMnode, pReq, &createReq); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; @@ -347,6 +346,9 @@ static int32_t mndProcessDropFuncReq(SRpcMsg *pReq) { } mDebug("func:%s, start to drop", dropReq.name); + if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_FUNC) != 0) { + goto _OVER; + } if (dropReq.name[0] == 0) { terrno = TSDB_CODE_MND_INVALID_FUNC_NAME; @@ -365,10 +367,6 @@ static int32_t mndProcessDropFuncReq(SRpcMsg *pReq) { } } - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_FUNC) != 0) { - goto _OVER; - } - code = mndDropFunc(pMnode, pReq, pFunc); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 000e1041d0..9f78e21dd1 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -100,7 +100,7 @@ static void *mndThreadFp(void *param) { taosMsleep(100); if (mndGetStop(pMnode)) break; - if (lastTime % (tsTransPullupInterval * 10) == 1) { + if (lastTime % (tsTtlPushInterval * 10) == 1) { mndTtlTimer(pMnode); } @@ -529,24 +529,33 @@ static int32_t mndCheckMnodeState(SRpcMsg *pMsg) { if (!IsReq(pMsg)) return 0; if (mndAcquireRpcRef(pMsg->info.node) == 0) return 0; if (pMsg->msgType == TDMT_MND_MQ_TIMER || pMsg->msgType == TDMT_MND_TELEM_TIMER || - pMsg->msgType == TDMT_MND_TRANS_TIMER || TDMT_MND_TTL_TIMER) { + pMsg->msgType == TDMT_MND_TRANS_TIMER || pMsg->msgType == TDMT_MND_TTL_TIMER) { return -1; } - const STraceId *trace = &pMsg->info.traceId; - mError("msg:%p, failed to check mnode state since %s, type:%s", pMsg, terrstr(), TMSG_INFO(pMsg->msgType)); - SEpSet epSet = {0}; mndGetMnodeEpSet(pMsg->info.node, &epSet); - int32_t contLen = tSerializeSEpSet(NULL, 0, &epSet); - pMsg->info.rsp = rpcMallocCont(contLen); - if (pMsg->info.rsp != NULL) { - tSerializeSEpSet(pMsg->info.rsp, contLen, &epSet); - pMsg->info.rspLen = contLen; - terrno = TSDB_CODE_RPC_REDIRECT; + const STraceId *trace = &pMsg->info.traceId; + mError("msg:%p, failed to check mnode state since %s, type:%s, numOfMnodes:%d inUse:%d", pMsg, terrstr(), + TMSG_INFO(pMsg->msgType), epSet.numOfEps, epSet.inUse); + + if (epSet.numOfEps > 0) { + for (int32_t i = 0; i < epSet.numOfEps; ++i) { + mInfo("mnode index:%d, ep:%s:%u", i, epSet.eps[i].fqdn, epSet.eps[i].port); + } + + int32_t contLen = tSerializeSEpSet(NULL, 0, &epSet); + pMsg->info.rsp = rpcMallocCont(contLen); + if (pMsg->info.rsp != NULL) { + tSerializeSEpSet(pMsg->info.rsp, contLen, &epSet); + pMsg->info.rspLen = contLen; + terrno = TSDB_CODE_RPC_REDIRECT; + } else { + terrno = TSDB_CODE_OUT_OF_MEMORY; + } } else { - terrno = TSDB_CODE_OUT_OF_MEMORY; + terrno = TSDB_CODE_APP_NOT_READY; } return -1; @@ -555,10 +564,10 @@ static int32_t mndCheckMnodeState(SRpcMsg *pMsg) { static int32_t mndCheckMsgContent(SRpcMsg *pMsg) { if (!IsReq(pMsg)) return 0; if (pMsg->contLen != 0 && pMsg->pCont != NULL) return 0; - + const STraceId *trace = &pMsg->info.traceId; mGError("msg:%p, failed to check msg, cont:%p contLen:%d, app:%p type:%s", pMsg, pMsg->pCont, pMsg->contLen, - pMsg->info.ahandle, TMSG_INFO(pMsg->msgType)); + pMsg->info.ahandle, TMSG_INFO(pMsg->msgType)); terrno = TSDB_CODE_INVALID_MSG_LEN; return -1; } @@ -723,7 +732,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr pIter = sdbFetch(pSdb, SDB_STB, pIter, (void **)&pStb); if (pIter == NULL) break; - SMonStbDesc desc = {0}; + SMonStbDesc desc = {0}; SName name1 = {0}; tNameFromString(&name1, pStb->db, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index b40cd713e5..bc3d23282c 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -389,6 +389,9 @@ static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq) { } mDebug("mnode:%d, start to create", createReq.dnodeId); + if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_MNODE) != 0) { + goto _OVER; + } pObj = mndAcquireMnode(pMnode, createReq.dnodeId); if (pObj != NULL) { @@ -414,10 +417,6 @@ static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_MNODE) != 0) { - goto _OVER; - } - code = mndCreateMnode(pMnode, pReq, pDnode, &createReq); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; @@ -495,7 +494,7 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode { int32_t contLen = tSerializeSSetStandbyReq(NULL, 0, &standbyReq) + sizeof(SMsgHead); void *pReq = taosMemoryMalloc(contLen); - tSerializeSSetStandbyReq((char*)pReq + sizeof(SMsgHead), contLen, &standbyReq); + tSerializeSSetStandbyReq((char *)pReq + sizeof(SMsgHead), contLen, &standbyReq); SMsgHead *pHead = pReq; pHead->contLen = htonl(contLen); pHead->vgId = htonl(MNODE_HANDLE); @@ -595,6 +594,9 @@ static int32_t mndProcessDropMnodeReq(SRpcMsg *pReq) { } mDebug("mnode:%d, start to drop", dropReq.dnodeId); + if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_MNODE) != 0) { + goto _OVER; + } if (dropReq.dnodeId <= 0) { terrno = TSDB_CODE_INVALID_MSG; @@ -621,10 +623,6 @@ static int32_t mndProcessDropMnodeReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_MNODE) != 0) { - goto _OVER; - } - code = mndDropMnode(pMnode, pReq, pObj); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 832e328d96..e9df4ae1d0 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "mndProfile.h" +#include "mndAuth.h" #include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" @@ -70,6 +71,7 @@ static void mndCancelGetNextQuery(SMnode *pMnode, void *pIter); static void mndFreeApp(SAppObj *pApp); static int32_t mndRetrieveApps(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextApp(SMnode *pMnode, void *pIter); +static int32_t mndProcessSvrVerReq(SRpcMsg *pReq); int32_t mndInitProfile(SMnode *pMnode) { SProfileMgmt *pMgmt = &pMnode->profileMgmt; @@ -94,6 +96,7 @@ int32_t mndInitProfile(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_CONNECT, mndProcessConnectReq); mndSetMsgHandle(pMnode, TDMT_MND_KILL_QUERY, mndProcessKillQueryReq); mndSetMsgHandle(pMnode, TDMT_MND_KILL_CONN, mndProcessKillConnReq); + mndSetMsgHandle(pMnode, TDMT_MND_SERVER_VERSION, mndProcessSvrVerReq); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_CONNS, mndRetrieveConns); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_CONNS, mndCancelGetNextConn); @@ -215,36 +218,42 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { SConnObj *pConn = NULL; int32_t code = -1; SConnectReq connReq = {0}; - char ip[30] = {0}; + char ip[24] = {0}; const STraceId *trace = &pReq->info.traceId; if (tDeserializeSConnectReq(pReq->pCont, pReq->contLen, &connReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; - goto CONN_OVER; + goto _OVER; } taosIp2String(pReq->info.conn.clientIp, ip); pUser = mndAcquireUser(pMnode, pReq->info.conn.user); if (pUser == NULL) { - mGError("user:%s, failed to login while acquire user since %s", pReq->info.conn.user, terrstr()); - goto CONN_OVER; + mGError("user:%s, failed to login from %s while acquire user since %s", pReq->info.conn.user, ip, terrstr()); + goto _OVER; } - if (0 != strncmp(connReq.passwd, pUser->pass, TSDB_PASSWORD_LEN - 1)) { - mGError("user:%s, failed to auth while acquire user, input:%s", pReq->info.conn.user, connReq.passwd); + + if (strncmp(connReq.passwd, pUser->pass, TSDB_PASSWORD_LEN - 1) != 0) { + mGError("user:%s, failed to login from %s since invalid pass, input:%s", pReq->info.conn.user, ip, connReq.passwd); code = TSDB_CODE_RPC_AUTH_FAILURE; - goto CONN_OVER; + goto _OVER; + } + + if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CONNECT) != 0) { + mGError("user:%s, failed to login from %s since %s", pReq->info.conn.user, ip, terrstr()); + goto _OVER; } if (connReq.db[0]) { - char db[TSDB_DB_FNAME_LEN]; + char db[TSDB_DB_FNAME_LEN] = {0}; snprintf(db, TSDB_DB_FNAME_LEN, "%d%s%s", pUser->acctId, TS_PATH_DELIMITER, connReq.db); pDb = mndAcquireDb(pMnode, db); if (pDb == NULL) { terrno = TSDB_CODE_MND_INVALID_DB; mGError("user:%s, failed to login from %s while use db:%s since %s", pReq->info.conn.user, ip, connReq.db, terrstr()); - goto CONN_OVER; + goto _OVER; } } @@ -252,7 +261,7 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { pReq->info.conn.clientPort, connReq.pid, connReq.app, connReq.startTime); if (pConn == NULL) { mGError("user:%s, failed to login from %s while create connection since %s", pReq->info.conn.user, ip, terrstr()); - goto CONN_OVER; + goto _OVER; } SConnectRsp connectRsp = {0}; @@ -262,15 +271,16 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { connectRsp.connId = pConn->id; connectRsp.connType = connReq.connType; connectRsp.dnodeNum = mndGetDnodeSize(pMnode); - - snprintf(connectRsp.sVersion, sizeof(connectRsp.sVersion), "ver:%s\nbuild:%s\ngitinfo:%s", version, buildinfo, + + strcpy(connectRsp.sVer, version); + snprintf(connectRsp.sDetailVer, sizeof(connectRsp.sDetailVer), "ver:%s\nbuild:%s\ngitinfo:%s", version, buildinfo, gitinfo); mndGetMnodeEpSet(pMnode, &connectRsp.epSet); int32_t contLen = tSerializeSConnectRsp(NULL, 0, &connectRsp); - if (contLen < 0) goto CONN_OVER; + if (contLen < 0) goto _OVER; void *pRsp = rpcMallocCont(contLen); - if (pRsp == NULL) goto CONN_OVER; + if (pRsp == NULL) goto _OVER; tSerializeSConnectRsp(pRsp, contLen, &connectRsp); pReq->info.rspLen = contLen; @@ -280,7 +290,7 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { code = 0; -CONN_OVER: +_OVER: mndReleaseUser(pMnode, pUser); mndReleaseDb(pMnode, pDb); @@ -460,6 +470,27 @@ static int32_t mndUpdateAppInfo(SMnode *pMnode, SClientHbReq *pHbReq, SRpcConnIn return TSDB_CODE_SUCCESS; } +static int32_t mndGetOnlineDnodeNum(SMnode *pMnode, int32_t *num) { + SSdb *pSdb = pMnode->pSdb; + SDnodeObj *pDnode = NULL; + int64_t curMs = taosGetTimestampMs(); + void *pIter = NULL; + + while (true) { + pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pDnode); + if (pIter == NULL) break; + + bool online = mndIsDnodeOnline(pDnode, curMs); + if (online) { + (*num)++; + } + + sdbRelease(pSdb, pDnode); + } + + return TSDB_CODE_SUCCESS; +} + static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHbReq *pHbReq, SClientHbBatchRsp *pBatchRsp) { SProfileMgmt *pMgmt = &pMnode->profileMgmt; @@ -503,7 +534,7 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb rspBasic->connId = pConn->id; rspBasic->totalDnodes = mndGetDnodeSize(pMnode); - rspBasic->onlineDnodes = 1; // TODO + mndGetOnlineDnodeNum(pMnode, &rspBasic->onlineDnodes); mndGetMnodeEpSet(pMnode, &rspBasic->epSet); mndCreateQnodeList(pMnode, &rspBasic->pQnodeList, -1); @@ -694,6 +725,28 @@ static int32_t mndProcessKillConnReq(SRpcMsg *pReq) { } } +static int32_t mndProcessSvrVerReq(SRpcMsg *pReq) { + int32_t code = -1; + SServerVerRsp rsp = {0}; + strcpy(rsp.ver, version); + + int32_t contLen = tSerializeSServerVerRsp(NULL, 0, &rsp); + if (contLen < 0) goto _over; + void *pRsp = rpcMallocCont(contLen); + if (pRsp == NULL) goto _over; + tSerializeSServerVerRsp(pRsp, contLen, &rsp); + + pReq->info.rspLen = contLen; + pReq->info.rsp = pRsp; + + code = 0; + +_over: + + return code; +} + + static int32_t mndRetrieveConns(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { SMnode *pMnode = pReq->info.node; SSdb *pSdb = pMnode->pSdb; diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index 0a6c97e63c..9f1eb4ee24 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -279,6 +279,9 @@ static int32_t mndProcessCreateQnodeReq(SRpcMsg *pReq) { } mDebug("qnode:%d, start to create", createReq.dnodeId); + if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_QNODE) != 0) { + goto _OVER; + } pObj = mndAcquireQnode(pMnode, createReq.dnodeId); if (pObj != NULL) { @@ -294,10 +297,6 @@ static int32_t mndProcessCreateQnodeReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_QNODE) != 0) { - goto _OVER; - } - code = mndCreateQnode(pMnode, pReq, pDnode, &createReq); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; @@ -391,6 +390,9 @@ static int32_t mndProcessDropQnodeReq(SRpcMsg *pReq) { } mDebug("qnode:%d, start to drop", dropReq.dnodeId); + if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_QNODE) != 0) { + goto _OVER; + } if (dropReq.dnodeId <= 0) { terrno = TSDB_CODE_INVALID_MSG; @@ -402,10 +404,6 @@ static int32_t mndProcessDropQnodeReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_QNODE) != 0) { - goto _OVER; - } - code = mndDropQnode(pMnode, pReq, pObj); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; @@ -418,19 +416,19 @@ _OVER: return code; } -int32_t mndCreateQnodeList(SMnode *pMnode, SArray** pList, int32_t limit) { - SSdb *pSdb = pMnode->pSdb; - void *pIter = NULL; - SQnodeObj *pObj = NULL; - int32_t numOfRows = 0; +int32_t mndCreateQnodeList(SMnode *pMnode, SArray **pList, int32_t limit) { + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; + SQnodeObj *pObj = NULL; + int32_t numOfRows = 0; - SArray* qnodeList = taosArrayInit(5, sizeof(SQueryNodeLoad)); + SArray *qnodeList = taosArrayInit(5, sizeof(SQueryNodeLoad)); if (NULL == qnodeList) { mError("failed to alloc epSet while process qnode list req"); terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno; } - + while (1) { pIter = sdbFetch(pSdb, SDB_QNODE, pIter, (void **)&pObj); if (pIter == NULL) break; @@ -457,7 +455,6 @@ int32_t mndCreateQnodeList(SMnode *pMnode, SArray** pList, int32_t limit) return TSDB_CODE_SUCCESS; } - static int32_t mndProcessQnodeListReq(SRpcMsg *pReq) { int32_t code = -1; SMnode *pMnode = pReq->info.node; diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index 9d8ed3701e..eb0ccbb5f7 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -97,37 +97,7 @@ END: return terrno; } -int32_t mndPersistTaskDeployReq(STrans* pTrans, SStreamTask* pTask, const SEpSet* pEpSet, tmsg_t type, int32_t nodeId) { - SEncoder encoder; - tEncoderInit(&encoder, NULL, 0); - tEncodeSStreamTask(&encoder, pTask); - int32_t size = encoder.pos; - int32_t tlen = sizeof(SMsgHead) + size; - tEncoderClear(&encoder); - void* buf = taosMemoryCalloc(1, tlen); - if (buf == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - ((SMsgHead*)buf)->vgId = htonl(nodeId); - void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); - tEncoderInit(&encoder, abuf, size); - tEncodeSStreamTask(&encoder, pTask); - tEncoderClear(&encoder); - - STransAction action = {0}; - memcpy(&action.epSet, pEpSet, sizeof(SEpSet)); - action.pCont = buf; - action.contLen = tlen; - action.msgType = type; - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(buf); - return -1; - } - return 0; -} - -int32_t mndAddSinkToTask(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream, SStreamTask* pTask) { +int32_t mndAddSinkToTask(SMnode* pMnode, SStreamObj* pStream, SStreamTask* pTask) { pTask->dispatchType = TASK_DISPATCH__NONE; // sink if (pStream->smaId != 0) { @@ -142,7 +112,7 @@ int32_t mndAddSinkToTask(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream, SS return 0; } -int32_t mndAddDispatcherToInnerTask(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream, SStreamTask* pTask) { +int32_t mndAddDispatcherToInnerTask(SMnode* pMnode, SStreamObj* pStream, SStreamTask* pTask) { pTask->sinkType = TASK_SINK__NONE; if (pStream->fixedSinkVgId == 0) { pTask->dispatchType = TASK_DISPATCH__SHUFFLE; @@ -187,7 +157,7 @@ int32_t mndAddDispatcherToInnerTask(SMnode* pMnode, STrans* pTrans, SStreamObj* return 0; } -int32_t mndAssignTaskToVg(SMnode* pMnode, STrans* pTrans, SStreamTask* pTask, SSubplan* plan, const SVgObj* pVgroup) { +int32_t mndAssignTaskToVg(SMnode* pMnode, SStreamTask* pTask, SSubplan* plan, const SVgObj* pVgroup) { int32_t msgLen; pTask->nodeId = pVgroup->vgId; pTask->epSet = mndGetVgroupEpset(pMnode, pVgroup); @@ -196,11 +166,11 @@ int32_t mndAssignTaskToVg(SMnode* pMnode, STrans* pTrans, SStreamTask* pTask, SS plan->execNode.epSet = pTask->epSet; if (qSubPlanToString(plan, &pTask->exec.qmsg, &msgLen) < 0) { + ASSERT(0); terrno = TSDB_CODE_QRY_INVALID_INPUT; return -1; } ASSERT(pTask->dispatchType != TASK_DISPATCH__NONE || pTask->sinkType != TASK_SINK__NONE); - mndPersistTaskDeployReq(pTrans, pTask, &plan->execNode.epSet, TDMT_STREAM_TASK_DEPLOY, pVgroup->vgId); return 0; } @@ -212,8 +182,7 @@ SSnodeObj* mndSchedFetchOneSnode(SMnode* pMnode) { return pObj; } -int32_t mndAssignTaskToSnode(SMnode* pMnode, STrans* pTrans, SStreamTask* pTask, SSubplan* plan, - const SSnodeObj* pSnode) { +int32_t mndAssignTaskToSnode(SMnode* pMnode, SStreamTask* pTask, SSubplan* plan, const SSnodeObj* pSnode) { int32_t msgLen; pTask->nodeId = SNODE_HANDLE; @@ -223,10 +192,10 @@ int32_t mndAssignTaskToSnode(SMnode* pMnode, STrans* pTrans, SStreamTask* pTask, plan->execNode.epSet = pTask->epSet; if (qSubPlanToString(plan, &pTask->exec.qmsg, &msgLen) < 0) { + ASSERT(0); terrno = TSDB_CODE_QRY_INVALID_INPUT; return -1; } - mndPersistTaskDeployReq(pTrans, pTask, &plan->execNode.epSet, TDMT_STREAM_TASK_DEPLOY, SNODE_HANDLE); return 0; } @@ -245,7 +214,7 @@ SVgObj* mndSchedFetchOneVg(SMnode* pMnode, int64_t dbUid) { return pVgroup; } -int32_t mndAddShuffleSinkTasksToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { +int32_t mndAddShuffleSinkTasksToStream(SMnode* pMnode, SStreamObj* pStream) { SSdb* pSdb = pMnode->pSdb; void* pIter = NULL; SArray* tasks = taosArrayGetP(pStream->tasks, 0); @@ -262,6 +231,7 @@ int32_t mndAddShuffleSinkTasksToStream(SMnode* pMnode, STrans* pTrans, SStreamOb } SStreamTask* pTask = tNewSStreamTask(pStream->uid); if (pTask == NULL) { + sdbRelease(pSdb, pVgroup); terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -290,13 +260,11 @@ int32_t mndAddShuffleSinkTasksToStream(SMnode* pMnode, STrans* pTrans, SStreamOb // dispatch pTask->dispatchType = TASK_DISPATCH__NONE; - - mndPersistTaskDeployReq(pTrans, pTask, &pTask->epSet, TDMT_STREAM_TASK_DEPLOY, pVgroup->vgId); } return 0; } -int32_t mndAddFixedSinkTaskToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { +int32_t mndAddFixedSinkTaskToStream(SMnode* pMnode, SStreamObj* pStream) { ASSERT(pStream->fixedSinkVgId != 0); SArray* tasks = taosArrayGetP(pStream->tasks, 0); SStreamTask* pTask = tNewSStreamTask(pStream->uid); @@ -337,13 +305,10 @@ int32_t mndAddFixedSinkTaskToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* // dispatch pTask->dispatchType = TASK_DISPATCH__NONE; - - mndPersistTaskDeployReq(pTrans, pTask, &pTask->epSet, TDMT_STREAM_TASK_DEPLOY, pStream->fixedSinkVg.vgId); - return 0; } -int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { +int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { SSdb* pSdb = pMnode->pSdb; SQueryPlan* pPlan = qStringToQueryPlan(pStream->physicalPlan); if (pPlan == NULL) { @@ -368,9 +333,15 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { // add extra sink hasExtraSink = true; if (pStream->fixedSinkVgId == 0) { - mndAddShuffleSinkTasksToStream(pMnode, pTrans, pStream); + if (mndAddShuffleSinkTasksToStream(pMnode, pStream) < 0) { + // TODO free + return -1; + } } else { - mndAddFixedSinkTaskToStream(pMnode, pTrans, pStream); + if (mndAddFixedSinkTaskToStream(pMnode, pStream) < 0) { + // TODO free + return -1; + } } } @@ -386,6 +357,11 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { ASSERT(plan->subplanType == SUBPLAN_TYPE_MERGE); pInnerTask = tNewSStreamTask(pStream->uid); + if (pInnerTask == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + qDestroyQueryPlan(pPlan); + return -1; + } mndAddTaskToTaskSet(taskInnerLevel, pInnerTask); pInnerTask->childEpInfo = taosArrayInit(0, sizeof(void*)); @@ -397,7 +373,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { pInnerTask->triggerParam = pStream->triggerParam; // dispatch - if (mndAddDispatcherToInnerTask(pMnode, pTrans, pStream, pInnerTask) < 0) { + if (mndAddDispatcherToInnerTask(pMnode, pStream, pInnerTask) < 0) { qDestroyQueryPlan(pPlan); return -1; } @@ -409,14 +385,13 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { SSnodeObj* pSnode = mndSchedFetchOneSnode(pMnode); if (pSnode == NULL) { SVgObj* pVgroup = mndSchedFetchOneVg(pMnode, pStream->sourceDbUid); - if (mndAssignTaskToVg(pMnode, pTrans, pInnerTask, plan, pVgroup) < 0) { + if (mndAssignTaskToVg(pMnode, pInnerTask, plan, pVgroup) < 0) { sdbRelease(pSdb, pVgroup); qDestroyQueryPlan(pPlan); return -1; } } else { - if (mndAssignTaskToSnode(pMnode, pTrans, pInnerTask, plan, pSnode) < 0) { - ASSERT(0); + if (mndAssignTaskToSnode(pMnode, pInnerTask, plan, pSnode) < 0) { sdbRelease(pSdb, pSnode); qDestroyQueryPlan(pPlan); return -1; @@ -424,7 +399,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { } } else { SVgObj* pVgroup = mndSchedFetchOneVg(pMnode, pStream->sourceDbUid); - if (mndAssignTaskToVg(pMnode, pTrans, pInnerTask, plan, pVgroup) < 0) { + if (mndAssignTaskToVg(pMnode, pInnerTask, plan, pVgroup) < 0) { sdbRelease(pSdb, pVgroup); qDestroyQueryPlan(pPlan); return -1; @@ -450,6 +425,12 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { continue; } SStreamTask* pTask = tNewSStreamTask(pStream->uid); + if (pInnerTask == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + sdbRelease(pSdb, pVgroup); + qDestroyQueryPlan(pPlan); + return -1; + } mndAddTaskToTaskSet(taskSourceLevel, pTask); // source @@ -466,7 +447,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { // exec pTask->execType = TASK_EXEC__PIPE; - if (mndAssignTaskToVg(pMnode, pTrans, pTask, plan, pVgroup) < 0) { + if (mndAssignTaskToVg(pMnode, pTask, plan, pVgroup) < 0) { sdbRelease(pSdb, pVgroup); qDestroyQueryPlan(pPlan); return -1; @@ -507,6 +488,11 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { continue; } SStreamTask* pTask = tNewSStreamTask(pStream->uid); + if (pTask == NULL) { + sdbRelease(pSdb, pVgroup); + qDestroyQueryPlan(pPlan); + return -1; + } mndAddTaskToTaskSet(taskOneLevel, pTask); // source @@ -517,14 +503,14 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { // sink or dispatch if (hasExtraSink) { - mndAddDispatcherToInnerTask(pMnode, pTrans, pStream, pTask); + mndAddDispatcherToInnerTask(pMnode, pStream, pTask); } else { - mndAddSinkToTask(pMnode, pTrans, pStream, pTask); + mndAddSinkToTask(pMnode, pStream, pTask); } // exec pTask->execType = TASK_EXEC__PIPE; - if (mndAssignTaskToVg(pMnode, pTrans, pTask, plan, pVgroup) < 0) { + if (mndAssignTaskToVg(pMnode, pTask, plan, pVgroup) < 0) { sdbRelease(pSdb, pVgroup); qDestroyQueryPlan(pPlan); return -1; diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index c44fb03be2..05603f8554 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -533,7 +533,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea #if 0 smaObj.timezone = pCreate->timezone; #endif - smaObj.timezone = tsTimezone; // use timezone of server + smaObj.timezone = tsTimezone; // use timezone of server smaObj.interval = pCreate->interval; smaObj.offset = pCreate->offset; smaObj.sliding = pCreate->sliding; @@ -623,7 +623,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea if (mndSetUpdateSmaStbCommitLogs(pMnode, pTrans, pStb) != 0) goto _OVER; // if (mndSetCreateSmaRedoActions(pMnode, pTrans, pDb, &smaObj) != 0) goto _OVER; if (mndSetCreateSmaVgroupRedoActions(pMnode, pTrans, pDb, &streamObj.fixedSinkVg, &smaObj) != 0) goto _OVER; - if (mndScheduleStream(pMnode, pTrans, &streamObj) != 0) goto _OVER; + if (mndScheduleStream(pMnode, &streamObj) != 0) goto _OVER; if (mndPersistStream(pMnode, pTrans, &streamObj) != 0) goto _OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c index df1330197a..a638bdf61f 100644 --- a/source/dnode/mnode/impl/src/mndSnode.c +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -285,6 +285,9 @@ static int32_t mndProcessCreateSnodeReq(SRpcMsg *pReq) { } mDebug("snode:%d, start to create", createReq.dnodeId); + if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_SNODE) != 0) { + goto _OVER; + } pObj = mndAcquireSnode(pMnode, createReq.dnodeId); if (pObj != NULL) { @@ -300,10 +303,6 @@ static int32_t mndProcessCreateSnodeReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_SNODE) != 0) { - goto _OVER; - } - code = mndCreateSnode(pMnode, pReq, pDnode, &createReq); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; @@ -398,6 +397,9 @@ static int32_t mndProcessDropSnodeReq(SRpcMsg *pReq) { } mDebug("snode:%d, start to drop", dropReq.dnodeId); + if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_SNODE) != 0) { + goto _OVER; + } if (dropReq.dnodeId <= 0) { terrno = TSDB_CODE_INVALID_MSG; @@ -409,10 +411,6 @@ static int32_t mndProcessDropSnodeReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_SNODE) != 0) { - goto _OVER; - } - // check deletable code = mndDropSnode(pMnode, pReq, pObj); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 7e9069d5fc..5e2f5bc2dd 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -320,11 +320,62 @@ FAIL: return 0; } +int32_t mndPersistTaskDeployReq(STrans *pTrans, const SStreamTask *pTask) { + ASSERT(pTask->isDataScan == 0 || pTask->isDataScan == 1); + if (pTask->isDataScan == 0 && pTask->sinkType == TASK_SINK__NONE) { + ASSERT(taosArrayGetSize(pTask->childEpInfo) != 0); + } + SEncoder encoder; + tEncoderInit(&encoder, NULL, 0); + tEncodeSStreamTask(&encoder, pTask); + int32_t size = encoder.pos; + int32_t tlen = sizeof(SMsgHead) + size; + tEncoderClear(&encoder); + void *buf = taosMemoryCalloc(1, tlen); + if (buf == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + ((SMsgHead *)buf)->vgId = htonl(pTask->nodeId); + void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + tEncoderInit(&encoder, abuf, size); + tEncodeSStreamTask(&encoder, pTask); + tEncoderClear(&encoder); + + STransAction action = {0}; + memcpy(&action.epSet, &pTask->epSet, sizeof(SEpSet)); + action.pCont = buf; + action.contLen = tlen; + action.msgType = TDMT_STREAM_TASK_DEPLOY; + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(buf); + return -1; + } + return 0; +} + +int32_t mndPersistStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) { + int32_t level = taosArrayGetSize(pStream->tasks); + for (int32_t i = 0; i < level; i++) { + SArray *pLevel = taosArrayGetP(pStream->tasks, i); + int32_t sz = taosArrayGetSize(pLevel); + for (int32_t j = 0; j < sz; j++) { + SStreamTask *pTask = taosArrayGetP(pLevel, j); + if (mndPersistTaskDeployReq(pTrans, pTask) < 0) { + return -1; + } + } + } + return 0; +} + int32_t mndPersistStream(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) { + if (mndPersistStreamTasks(pMnode, pTrans, pStream) < 0) { + return -1; + } SSdbRaw *pCommitRaw = mndStreamActionEncode(pStream); if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); return -1; } sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); @@ -342,51 +393,6 @@ int32_t mndPersistDropStreamLog(SMnode *pMnode, STrans *pTrans, SStreamObj *pStr return 0; } -int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast, STrans *pTrans) { - SNode *pAst = NULL; - - if (nodesStringToNode(ast, &pAst) < 0) { - return -1; - } - - if (qExtractResultSchema(pAst, (int32_t *)&pStream->outputSchema.nCols, &pStream->outputSchema.pSchema) != 0) { - nodesDestroyNode(pAst); - return -1; - } - // free - nodesDestroyNode(pAst); - -#if 0 - printf("|"); - for (int i = 0; i < pStream->outputSchema.nCols; i++) { - printf(" %15s |", (char *)pStream->outputSchema.pSchema[i].name); - } - printf("\n=======================================================\n"); - -#endif - - if (TSDB_CODE_SUCCESS != mndStreamGetPlanString(ast, pStream->trigger, pStream->watermark, &pStream->physicalPlan)) { - mError("topic:%s, failed to get plan since %s", pStream->name, terrstr()); - return -1; - } - - if (mndScheduleStream(pMnode, pTrans, pStream) < 0) { - mError("stream:%ld, schedule stream since %s", pStream->uid, terrstr()); - return -1; - } - mDebug("trans:%d, used to create stream:%s", pTrans->id, pStream->name); - - SSdbRaw *pCommitRaw = mndStreamActionEncode(pStream); - if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { - mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; - } - sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); - - return 0; -} - static int32_t mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStreamObj *pStream, const char *user) { SStbObj *pStb = NULL; SDbObj *pDb = NULL; @@ -503,64 +509,6 @@ static int32_t mndDropStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pS return 0; } -static int32_t mndCreateStream(SMnode *pMnode, SRpcMsg *pReq, SCMCreateStreamReq *pCreate, SDbObj *pDb) { - mDebug("stream:%s to create", pCreate->name); - SStreamObj streamObj = {0}; - tstrncpy(streamObj.name, pCreate->name, TSDB_STREAM_FNAME_LEN); - tstrncpy(streamObj.sourceDb, pDb->name, TSDB_DB_FNAME_LEN); - tstrncpy(streamObj.targetSTbName, pCreate->targetStbFullName, TSDB_TABLE_FNAME_LEN); - streamObj.createTime = taosGetTimestampMs(); - streamObj.updateTime = streamObj.createTime; - streamObj.uid = mndGenerateUid(pCreate->name, strlen(pCreate->name)); - streamObj.targetStbUid = mndGenerateUid(pCreate->targetStbFullName, TSDB_TABLE_FNAME_LEN); - streamObj.sourceDbUid = pDb->uid; - streamObj.version = 1; - streamObj.sql = pCreate->sql; - // TODO - streamObj.fixedSinkVgId = 0; - streamObj.smaId = 0; - streamObj.trigger = pCreate->triggerType; - streamObj.watermark = pCreate->watermark; - streamObj.triggerParam = pCreate->maxDelay; - - if (streamObj.targetSTbName[0]) { - pDb = mndAcquireDbByStb(pMnode, streamObj.targetSTbName); - if (pDb == NULL) { - terrno = TSDB_CODE_MND_DB_NOT_SELECTED; - return -1; - } - tstrncpy(streamObj.targetDb, pDb->name, TSDB_DB_FNAME_LEN); - } - - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq); - if (pTrans == NULL) { - mError("stream:%s, failed to create since %s", pCreate->name, terrstr()); - return -1; - } - mDebug("trans:%d, used to create stream:%s", pTrans->id, pCreate->name); - - if (mndAddStreamToTrans(pMnode, &streamObj, pCreate->ast, pTrans) != 0) { - mError("trans:%d, failed to add stream since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; - } - - if (streamObj.targetSTbName[0] && mndCreateStbForStream(pMnode, pTrans, &streamObj, pReq->info.conn.user) < 0) { - mError("trans:%d, failed to create stb for stream since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; - } - - if (mndTransPrepare(pMnode, pTrans) != 0) { - mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; - } - - mndTransDrop(pTrans); - return 0; -} - static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; int32_t code = -1; @@ -631,7 +579,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { } // schedule stream task for stream obj - if (mndScheduleStream(pMnode, pTrans, &streamObj) < 0) { + if (mndScheduleStream(pMnode, &streamObj) < 0) { mError("stream:%s, failed to schedule since %s", createStreamReq.name, terrstr()); mndTransDrop(pTrans); goto _OVER; @@ -653,8 +601,6 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { mndTransDrop(pTrans); - /*code = mndCreateStream(pMnode, pReq, &createStreamReq, pDb);*/ - /*if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;*/ code = TSDB_CODE_ACTION_IN_PROGRESS; _OVER: diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index e3358b41df..c58ce97588 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -198,6 +198,10 @@ int32_t mndInitSync(SMnode *pMnode) { return -1; } + // decrease election timer + setElectTimerMS(pMgmt->sync, 600); + setHeartbeatTimerMS(pMgmt->sync, 300); + mDebug("mnode-sync is opened, id:%" PRId64, pMgmt->sync); return 0; } @@ -261,7 +265,8 @@ bool mndIsMaster(SMnode *pMnode) { SSyncMgmt *pMgmt = &pMnode->syncMgmt; if (!syncIsReady(pMgmt->sync)) { - terrno = TSDB_CODE_SYN_NOT_LEADER; + // get terrno from syncIsReady + // terrno = TSDB_CODE_SYN_NOT_LEADER; return false; } diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index eb0a818a60..03c9647bfe 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -295,7 +295,7 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate tstrncpy(userObj.acct, acct, TSDB_USER_LEN); userObj.createdTime = taosGetTimestampMs(); userObj.updateTime = userObj.createdTime; - userObj.superUser = pCreate->superUser; + userObj.superUser = 0;//pCreate->superUser; userObj.sysInfo = pCreate->sysInfo; userObj.enable = pCreate->enable; diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index ae13987d25..1c5d73031f 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -1212,8 +1212,9 @@ static int32_t mndProcessRedistributeVgroupMsg(SRpcMsg *pReq) { } mInfo("vgId:%d, start to redistribute vgroup to dnode %d:%d:%d", req.vgId, req.dnodeId1, req.dnodeId2, req.dnodeId3); - - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_REDISTRIBUTE_VGROUP) != 0) goto _OVER; + if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_REDISTRIBUTE_VGROUP) != 0) { + goto _OVER; + } pVgroup = mndAcquireVgroup(pMnode, req.vgId); if (pVgroup == NULL) goto _OVER; @@ -1506,6 +1507,9 @@ static int32_t mndProcessSplitVgroupMsg(SRpcMsg *pReq) { SDbObj *pDb = NULL; mDebug("vgId:%d, start to split", vgId); + if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_SPLIT_VGROUP) != 0) { + goto _OVER; + } pVgroup = mndAcquireVgroup(pMnode, vgId); if (pVgroup == NULL) goto _OVER; @@ -1513,8 +1517,6 @@ static int32_t mndProcessSplitVgroupMsg(SRpcMsg *pReq) { pDb = mndAcquireDb(pMnode, pVgroup->dbName); if (pDb == NULL) goto _OVER; - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_SPLIT_VGROUP) != 0) goto _OVER; - code = mndSplitVgroup(pMnode, pReq, pDb, pVgroup); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; @@ -1655,8 +1657,9 @@ static int32_t mndProcessBalanceVgroupMsg(SRpcMsg *pReq) { } mInfo("start to balance vgroup"); - - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_BALANCE_VGROUP) != 0) goto _OVER; + if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_BALANCE_VGROUP) != 0) { + goto _OVER; + } while (1) { SDnodeObj *pDnode = NULL; diff --git a/source/dnode/mnode/impl/test/db/CMakeLists.txt b/source/dnode/mnode/impl/test/db/CMakeLists.txt index e28cdd4f61..3f6a80835f 100644 --- a/source/dnode/mnode/impl/test/db/CMakeLists.txt +++ b/source/dnode/mnode/impl/test/db/CMakeLists.txt @@ -5,9 +5,7 @@ target_link_libraries( PUBLIC sut ) -if(NOT TD_WINDOWS) - add_test( - NAME dbTest - COMMAND dbTest - ) -endif(NOT TD_WINDOWS) +add_test( + NAME dbTest + COMMAND dbTest +) diff --git a/source/dnode/mnode/impl/test/func/CMakeLists.txt b/source/dnode/mnode/impl/test/func/CMakeLists.txt index 17adfde801..3b6f2b5782 100644 --- a/source/dnode/mnode/impl/test/func/CMakeLists.txt +++ b/source/dnode/mnode/impl/test/func/CMakeLists.txt @@ -5,9 +5,7 @@ target_link_libraries( PUBLIC sut ) -# if(NOT TD_WINDOWS) - add_test( - NAME funcTest - COMMAND funcTest - ) -# endif(NOT TD_WINDOWS) +add_test( + NAME funcTest + COMMAND funcTest +) \ No newline at end of file diff --git a/source/dnode/mnode/impl/test/profile/CMakeLists.txt b/source/dnode/mnode/impl/test/profile/CMakeLists.txt index 049a4f3a9f..8b811ebfed 100644 --- a/source/dnode/mnode/impl/test/profile/CMakeLists.txt +++ b/source/dnode/mnode/impl/test/profile/CMakeLists.txt @@ -5,9 +5,7 @@ target_link_libraries( PUBLIC sut ) -# if(NOT TD_WINDOWS) - add_test( - NAME profileTest - COMMAND profileTest - ) -# endif(NOT TD_WINDOWS) +add_test( + NAME profileTest + COMMAND profileTest +) diff --git a/source/dnode/mnode/impl/test/show/CMakeLists.txt b/source/dnode/mnode/impl/test/show/CMakeLists.txt index 33ba0c5697..69e93e7086 100644 --- a/source/dnode/mnode/impl/test/show/CMakeLists.txt +++ b/source/dnode/mnode/impl/test/show/CMakeLists.txt @@ -5,9 +5,7 @@ target_link_libraries( PUBLIC sut ) -# if(NOT TD_WINDOWS) - add_test( - NAME showTest - COMMAND showTest - ) -# endif(NOT TD_WINDOWS) +add_test( + NAME showTest + COMMAND showTest +) diff --git a/source/dnode/mnode/impl/test/sma/CMakeLists.txt b/source/dnode/mnode/impl/test/sma/CMakeLists.txt index fd596c5021..3f9ec123a8 100644 --- a/source/dnode/mnode/impl/test/sma/CMakeLists.txt +++ b/source/dnode/mnode/impl/test/sma/CMakeLists.txt @@ -5,9 +5,7 @@ target_link_libraries( PUBLIC sut ) -if(NOT TD_WINDOWS) - add_test( - NAME smaTest - COMMAND smaTest - ) -endif(NOT TD_WINDOWS) +add_test( + NAME smaTest + COMMAND smaTest +) diff --git a/source/dnode/mnode/impl/test/stb/CMakeLists.txt b/source/dnode/mnode/impl/test/stb/CMakeLists.txt index 857c404c1c..dcfbe658fc 100644 --- a/source/dnode/mnode/impl/test/stb/CMakeLists.txt +++ b/source/dnode/mnode/impl/test/stb/CMakeLists.txt @@ -5,9 +5,7 @@ target_link_libraries( PUBLIC sut ) -if(NOT TD_WINDOWS) - add_test( - NAME stbTest - COMMAND stbTest - ) -endif(NOT TD_WINDOWS) \ No newline at end of file +add_test( + NAME stbTest + COMMAND stbTest +) \ No newline at end of file diff --git a/source/dnode/mnode/impl/test/user/CMakeLists.txt b/source/dnode/mnode/impl/test/user/CMakeLists.txt index ed4d964617..b39ea0e73f 100644 --- a/source/dnode/mnode/impl/test/user/CMakeLists.txt +++ b/source/dnode/mnode/impl/test/user/CMakeLists.txt @@ -5,9 +5,7 @@ target_link_libraries( PUBLIC sut ) -if(NOT TD_WINDOWS) - add_test( - NAME userTest - COMMAND userTest - ) -endif(NOT TD_WINDOWS) +add_test( + NAME userTest + COMMAND userTest +) diff --git a/source/dnode/mnode/impl/test/user/user.cpp b/source/dnode/mnode/impl/test/user/user.cpp index 6aa28a9007..3b1a5fa3c5 100644 --- a/source/dnode/mnode/impl/test/user/user.cpp +++ b/source/dnode/mnode/impl/test/user/user.cpp @@ -33,6 +33,8 @@ TEST_F(MndTestUser, 01_Show_User) { TEST_F(MndTestUser, 02_Create_User) { { SCreateUserReq createReq = {0}; + createReq.enable = 1; + createReq.sysInfo = 1; strcpy(createReq.user, ""); strcpy(createReq.pass, "p1"); @@ -47,6 +49,8 @@ TEST_F(MndTestUser, 02_Create_User) { { SCreateUserReq createReq = {0}; + createReq.enable = 1; + createReq.sysInfo = 1; strcpy(createReq.user, "u1"); strcpy(createReq.pass, ""); @@ -61,6 +65,8 @@ TEST_F(MndTestUser, 02_Create_User) { { SCreateUserReq createReq = {0}; + createReq.enable = 1; + createReq.sysInfo = 1; strcpy(createReq.user, "root"); strcpy(createReq.pass, "1"); @@ -75,6 +81,8 @@ TEST_F(MndTestUser, 02_Create_User) { { SCreateUserReq createReq = {0}; + createReq.enable = 1; + createReq.sysInfo = 1; strcpy(createReq.user, "u1"); strcpy(createReq.pass, "p1"); @@ -108,9 +116,11 @@ TEST_F(MndTestUser, 02_Create_User) { { SCreateUserReq createReq = {0}; + createReq.enable = 1; + createReq.sysInfo = 1; strcpy(createReq.user, "u2"); strcpy(createReq.pass, "p1"); - createReq.superUser = 1; + createReq.superUser = 0; int32_t contLen = tSerializeSCreateUserReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); @@ -144,9 +154,11 @@ TEST_F(MndTestUser, 02_Create_User) { TEST_F(MndTestUser, 03_Alter_User) { { SCreateUserReq createReq = {0}; + createReq.enable = 1; + createReq.sysInfo = 1; strcpy(createReq.user, "u3"); strcpy(createReq.pass, "p1"); - createReq.superUser = 1; + createReq.superUser = 0; int32_t contLen = tSerializeSCreateUserReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); @@ -225,7 +237,7 @@ TEST_F(MndTestUser, 03_Alter_User) { alterReq.alterType = TSDB_ALTER_USER_SUPERUSER; strcpy(alterReq.user, "u3"); strcpy(alterReq.pass, "1"); - alterReq.superUser = 1; + alterReq.superUser = 0; int32_t contLen = tSerializeSAlterUserReq(NULL, 0, &alterReq); void* pReq = rpcMallocCont(contLen); @@ -361,7 +373,7 @@ TEST_F(MndTestUser, 03_Alter_User) { SGetUserAuthRsp authRsp = {0}; tDeserializeSGetUserAuthRsp(pRsp->pCont, pRsp->contLen, &authRsp); EXPECT_STREQ(authRsp.user, "u3"); - EXPECT_EQ(authRsp.superAuth, 1); + EXPECT_EQ(authRsp.superAuth, 0); int32_t numOfReadDbs = taosHashGetSize(authRsp.readDbs); int32_t numOfWriteDbs = taosHashGetSize(authRsp.writeDbs); EXPECT_EQ(numOfReadDbs, 1); @@ -436,6 +448,8 @@ TEST_F(MndTestUser, 05_Drop_User) { { SCreateUserReq createReq = {0}; + createReq.enable = 1; + createReq.sysInfo = 1; strcpy(createReq.user, "u1"); strcpy(createReq.pass, "p1"); @@ -468,6 +482,8 @@ TEST_F(MndTestUser, 05_Drop_User) { TEST_F(MndTestUser, 06_Create_Drop_Alter_User) { { SCreateUserReq createReq = {0}; + createReq.enable = 1; + createReq.sysInfo = 1; strcpy(createReq.user, "u1"); strcpy(createReq.pass, "p1"); @@ -482,6 +498,8 @@ TEST_F(MndTestUser, 06_Create_Drop_Alter_User) { { SCreateUserReq createReq = {0}; + createReq.enable = 1; + createReq.sysInfo = 1; strcpy(createReq.user, "u2"); strcpy(createReq.pass, "p2"); diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index c7e8e8dc86..a32bf0ecdb 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -116,7 +116,8 @@ typedef void *tsdbReaderT; #define BLOCK_LOAD_TABLE_SEQ_ORDER 2 #define BLOCK_LOAD_TABLE_RR_ORDER 3 -tsdbReaderT tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, STableListInfo *tableInfoGroup, uint64_t qId, +int32_t tsdbSetTableList(tsdbReaderT reader, SArray* tableList); +tsdbReaderT tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, SArray *tableList, uint64_t qId, uint64_t taskId); tsdbReaderT tsdbQueryCacheLast(SVnode *pVnode, SQueryTableDataCond *pCond, STableListInfo *groupList, uint64_t qId, void *pMemRef); @@ -195,7 +196,6 @@ struct SVnodeCfg { typedef struct { TSKEY lastKey; uint64_t uid; - uint64_t groupId; } STableKeyInfo; struct SMetaEntry { diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index f7965f0902..3b30224e17 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -121,7 +121,7 @@ int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSub int32_t tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, SSubmitBlkRsp* pRsp); int32_t tsdbDeleteTableData(STsdb* pTsdb, int64_t version, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey); -tsdbReaderT tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, STableListInfo* tableList, uint64_t qId, +tsdbReaderT tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* tableList, uint64_t qId, uint64_t taskId); tsdbReaderT tsdbQueryCacheLastT(STsdb* tsdb, SQueryTableDataCond* pCond, STableListInfo* tableList, uint64_t qId, void* pMemRef); diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index ab512f7774..4a3feca8d0 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -381,6 +381,7 @@ int metaTtlDropTable(SMeta *pMeta, int64_t ttl, SArray *tbUids) { for (int i = 0; i < taosArrayGetSize(tbUids); ++i) { tb_uid_t *uid = (tb_uid_t *)taosArrayGet(tbUids, i); metaDropTableByUid(pMeta, *uid, NULL); + metaDebug("ttl drop table:%"PRId64, *uid); } metaULock(pMeta); return 0; @@ -443,7 +444,6 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) { // drop schema.db (todo) } - metaError("ttl drop table:%s", e.name); tDecoderClear(&dc); tdbFree(pData); @@ -976,7 +976,9 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { SDecoder dc = {0}; // get super table - tdbTbGet(pMeta->pUidIdx, &pCtbEntry->ctbEntry.suid, sizeof(tb_uid_t), &pData, &nData); + if(tdbTbGet(pMeta->pUidIdx, &pCtbEntry->ctbEntry.suid, sizeof(tb_uid_t), &pData, &nData) != 0){ + return -1; + } tbDbKey.uid = pCtbEntry->ctbEntry.suid; tbDbKey.version = *(int64_t *)pData; tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pData, &nData); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index cca3a58588..766cf7af35 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -403,7 +403,7 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) { .reader = pHandle->execHandle.pExecReader[i], .meta = pTq->pVnode->pMeta, .vnode = pTq->pVnode, - .initTsdbReader = 1, +// .initTsdbReader = 1, }; pHandle->execHandle.execCol.task[i] = qCreateStreamExecTaskInfo(pHandle->execHandle.execCol.qmsg, &handle); ASSERT(pHandle->execHandle.execCol.task[i]); @@ -455,6 +455,9 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, char* msg, int32_t msgLen) { } tDecoderClear(&decoder); ASSERT(pTask->isDataScan == 0 || pTask->isDataScan == 1); + if (pTask->isDataScan == 0 && pTask->sinkType == TASK_SINK__NONE) { + ASSERT(taosArrayGetSize(pTask->childEpInfo) != 0); + } pTask->execStatus = TASK_EXEC_STATUS__IDLE; @@ -476,7 +479,7 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, char* msg, int32_t msgLen) { .reader = pStreamReader, .meta = pTq->pVnode->pMeta, .vnode = pTq->pVnode, - .initTsdbReader = 1, +// .initTsdbReader = 1, }; /*pTask->exec.inputHandle = pStreamReader;*/ pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle); diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 5d2fb81cf0..6184d8f810 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -146,8 +146,7 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle* pHandle, uint64_ int32_t* pNumOfRows) { *pUid = 0; - // TODO set to real sversion - /*int32_t sversion = 1;*/ + // TODO: cache multiple schema int32_t sversion = htonl(pHandle->pBlock->sversion); if (pHandle->cachedSchemaSuid == 0 || pHandle->cachedSchemaVer != sversion || pHandle->cachedSchemaSuid != pHandle->msgIter.suid) { @@ -161,7 +160,6 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle* pHandle, uint64_ return -1; } - // this interface use suid instead of uid if (pHandle->pSchemaWrapper) tDeleteSSchemaWrapper(pHandle->pSchemaWrapper); pHandle->pSchemaWrapper = metaGetTableSchema(pHandle->pVnodeMeta, pHandle->msgIter.uid, sversion, true); if (pHandle->pSchemaWrapper == NULL) { @@ -245,7 +243,7 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle* pHandle, uint64_ if (!tdSTSRowIterNext(&iter, pColData->info.colId, pColData->info.type, &sVal)) { break; } - if (colDataAppend(pColData, curRow, sVal.val, sVal.valType == TD_VTYPE_NULL) < 0) { + if (colDataAppend(pColData, curRow, sVal.val, sVal.valType != TD_VTYPE_NORM) < 0) { goto FAIL; } } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index e16360a58b..540810f876 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -223,9 +223,8 @@ int64_t tsdbGetNumOfRowsInMemTable(tsdbReaderT* pHandle) { return rows; } -static SArray* createCheckInfoFromTableGroup(STsdbReadHandle* pTsdbReadHandle, STableListInfo* pTableList) { - size_t tableSize = taosArrayGetSize(pTableList->pTableList); - assert(tableSize >= 1); +static SArray* createCheckInfoFromTableGroup(STsdbReadHandle* pTsdbReadHandle, SArray* pTableList) { + size_t tableSize = taosArrayGetSize(pTableList); // allocate buffer in order to load data blocks from file SArray* pTableCheckInfo = taosArrayInit(tableSize, sizeof(STableCheckInfo)); @@ -235,7 +234,7 @@ static SArray* createCheckInfoFromTableGroup(STsdbReadHandle* pTsdbReadHandle, S // todo apply the lastkey of table check to avoid to load header file for (int32_t j = 0; j < tableSize; ++j) { - STableKeyInfo* pKeyInfo = (STableKeyInfo*)taosArrayGet(pTableList->pTableList, j); + STableKeyInfo* pKeyInfo = (STableKeyInfo*)taosArrayGet(pTableList, j); STableCheckInfo info = {.lastKey = pKeyInfo->lastKey, .tableId = pKeyInfo->uid}; info.suid = pTsdbReadHandle->suid; @@ -254,8 +253,6 @@ static SArray* createCheckInfoFromTableGroup(STsdbReadHandle* pTsdbReadHandle, S pTsdbReadHandle->idStr); } - // TODO group table according to the tag value. - taosArraySort(pTableCheckInfo, tsdbCheckInfoCompar); return pTableCheckInfo; } @@ -497,8 +494,21 @@ static int32_t setCurrentSchema(SVnode* pVnode, STsdbReadHandle* pTsdbReadHandle return TSDB_CODE_SUCCESS; } -tsdbReaderT tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, STableListInfo* tableList, uint64_t qId, +int32_t tsdbSetTableList(tsdbReaderT reader, SArray* tableList){ + STsdbReadHandle* pTsdbReadHandle = reader; + if(pTsdbReadHandle->pTableCheckInfo) taosArrayDestroy(pTsdbReadHandle->pTableCheckInfo); + pTsdbReadHandle->pTableCheckInfo = createCheckInfoFromTableGroup(pTsdbReadHandle, tableList); + if (pTsdbReadHandle->pTableCheckInfo == NULL) { + return TSDB_CODE_TDB_OUT_OF_MEMORY; + } + return TDB_CODE_SUCCESS; +} + +tsdbReaderT tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* tableList, uint64_t qId, uint64_t taskId) { + if(taosArrayGetSize(tableList) == 0){ + return NULL; + } STsdbReadHandle* pTsdbReadHandle = tsdbQueryTablesImpl(pVnode, pCond, qId, taskId); if (pTsdbReadHandle == NULL) { return NULL; @@ -543,7 +553,7 @@ tsdbReaderT tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, STableLis } tsdbDebug("%p total numOfTable:%" PRIzu " in this query, table %" PRIzu " %s", pTsdbReadHandle, - taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo), taosArrayGetSize(tableList->pTableList), + taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo), taosArrayGetSize(tableList), pTsdbReadHandle->idStr); return (tsdbReaderT)pTsdbReadHandle; @@ -639,7 +649,7 @@ tsdbReaderT tsdbQueryLastRow(SVnode* pVnode, SQueryTableDataCond* pCond, STableL return NULL; } - STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*)tsdbReaderOpen(pVnode, pCond, pList, qId, taskId); + STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*)tsdbReaderOpen(pVnode, pCond, pList->pTableList, qId, taskId); if (pTsdbReadHandle == NULL) { return NULL; } @@ -2842,7 +2852,7 @@ int32_t tsdbGetAllTableList(SMeta* pMeta, uint64_t uid, SArray* list) { break; } - STableKeyInfo info = {.lastKey = TSKEY_INITIAL_VAL, uid = id, .groupId = 0}; + STableKeyInfo info = {.lastKey = TSKEY_INITIAL_VAL, uid = id}; taosArrayPush(list, &info); } @@ -3644,17 +3654,6 @@ SArray* tsdbRetrieveDataBlock(tsdbReaderT* pTsdbReadHandle, SArray* pIdList) { } } -static int tsdbCheckInfoCompar(const void* key1, const void* key2) { - if (((STableCheckInfo*)key1)->tableId < ((STableCheckInfo*)key2)->tableId) { - return -1; - } else if (((STableCheckInfo*)key1)->tableId > ((STableCheckInfo*)key2)->tableId) { - return 1; - } else { - ASSERT(false); - return 0; - } -} - static void* doFreeColumnInfoData(SArray* pColumnInfoData) { if (pColumnInfoData == NULL) { return NULL; diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 32090b774e..4c56180979 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -144,11 +144,15 @@ void vnodeProposeMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { vnodeAccumBlockMsg(pVnode, pMsg->msgType); } else if (code == -1 && terrno == TSDB_CODE_SYN_NOT_LEADER) { SEpSet newEpSet = {0}; + syncGetRetryEpSet(pVnode->sync, &newEpSet); + + /* syncGetEpSet(pVnode->sync, &newEpSet); SEp *pEp = &newEpSet.eps[newEpSet.inUse]; if (pEp->port == tsServerPort && strcmp(pEp->fqdn, tsLocalFqdn) == 0) { newEpSet.inUse = (newEpSet.inUse + 1) % newEpSet.numOfEps; } + */ vGTrace("vgId:%d, msg:%p is redirect since not leader, numOfEps:%d inUse:%d", vgId, pMsg, newEpSet.numOfEps, newEpSet.inUse); diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 384c3f19e5..74caa717b8 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -76,6 +76,7 @@ typedef enum { CTG_TASK_GET_INDEX, CTG_TASK_GET_UDF, CTG_TASK_GET_USER, + CTG_TASK_GET_SVR_VER, } CTG_TASK_TYPE; typedef enum { @@ -224,6 +225,7 @@ typedef struct SCtgJob { int32_t dbInfoNum; int32_t tbIndexNum; int32_t tbCfgNum; + int32_t svrVerNum; } SCtgJob; typedef struct SCtgMsgCtx { @@ -578,8 +580,9 @@ int32_t ctgGetTbMetaFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const SNa int32_t ctgGetTbMetaFromVnode(SCatalog* pCtg, SRequestConnInfo *pConn, const SName* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* out, SCtgTask* pTask); int32_t ctgGetTableCfgFromVnode(SCatalog* pCtg, SRequestConnInfo *pConn, const SName* pTableName, SVgroupInfo *vgroupInfo, STableCfg **out, SCtgTask* pTask); int32_t ctgGetTableCfgFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const SName* pTableName, STableCfg **out, SCtgTask* pTask); +int32_t ctgGetSvrVerFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, char **out, SCtgTask* pTask); -int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgJob** job, uint64_t reqId, const SCatalogReq* pReq, catalogCallback fp, void* param, int32_t* taskNum); +int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgJob** job, const SCatalogReq* pReq, catalogCallback fp, void* param, int32_t* taskNum); int32_t ctgLaunchJob(SCtgJob *pJob); int32_t ctgMakeAsyncRes(SCtgJob *pJob); int32_t ctgLaunchSubTask(SCtgTask *pTask, CTG_TASK_TYPE type, ctgSubTaskCbFp fp, void* param); diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 931a944adf..abead99cb0 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -1051,7 +1051,7 @@ _return: CTG_API_LEAVE(code); } -int32_t catalogAsyncGetAllMeta(SCatalog* pCtg, SRequestConnInfo *pConn, uint64_t reqId, const SCatalogReq* pReq, catalogCallback fp, void* param, int64_t* jobId) { +int32_t catalogAsyncGetAllMeta(SCatalog* pCtg, SRequestConnInfo *pConn, const SCatalogReq* pReq, catalogCallback fp, void* param, int64_t* jobId) { CTG_API_ENTER(); if (NULL == pCtg || NULL == pConn || NULL == pReq || NULL == fp || NULL == param) { @@ -1060,7 +1060,7 @@ int32_t catalogAsyncGetAllMeta(SCatalog* pCtg, SRequestConnInfo *pConn, uint64_t int32_t code = 0, taskNum = 0; SCtgJob *pJob = NULL; - CTG_ERR_JRET(ctgInitJob(pCtg, pConn, &pJob, reqId, pReq, fp, param, &taskNum)); + CTG_ERR_JRET(ctgInitJob(pCtg, pConn, &pJob, pReq, fp, param, &taskNum)); if (taskNum <= 0) { SMetaData* pMetaData = taosMemoryCalloc(1, sizeof(SMetaData)); fp(pMetaData, param, TSDB_CODE_SUCCESS); @@ -1247,6 +1247,22 @@ _return: CTG_API_LEAVE(code); } +int32_t catalogGetServerVersion(SCatalog* pCtg, SRequestConnInfo *pConn, char** pVersion) { + CTG_API_ENTER(); + + if (NULL == pCtg || NULL == pConn || NULL == pVersion) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + int32_t code = 0; + CTG_ERR_JRET(ctgGetSvrVerFromMnode(pCtg, pConn, pVersion, NULL)); + +_return: + + CTG_API_LEAVE(code); +} + + int32_t catalogUpdateUserAuthInfo(SCatalog* pCtg, SGetUserAuthRsp* pAuth) { CTG_API_ENTER(); diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 8928a7e028..18c95397dd 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -255,6 +255,20 @@ int32_t ctgInitGetUserTask(SCtgJob *pJob, int32_t taskIdx, void* param) { return TSDB_CODE_SUCCESS; } +int32_t ctgInitGetSvrVerTask(SCtgJob *pJob, int32_t taskIdx, void* param) { + SCtgTask task = {0}; + + task.type = CTG_TASK_GET_SVR_VER; + task.taskId = taskIdx; + task.pJob = pJob; + + taosArrayPush(pJob->pTasks, &task); + + qDebug("QID:0x%" PRIx64 " [%dth] task type %s initialized", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type)); + + return TSDB_CODE_SUCCESS; +} + int32_t ctgInitGetTbIndexTask(SCtgJob *pJob, int32_t taskIdx, void* param) { SName *name = (SName*)param; SCtgTask task = {0}; @@ -413,7 +427,7 @@ int32_t ctgInitTask(SCtgJob *pJob, CTG_TASK_TYPE type, void* param, int32_t *tas return TSDB_CODE_SUCCESS; } -int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgJob** job, uint64_t reqId, const SCatalogReq* pReq, catalogCallback fp, void* param, int32_t* taskNum) { +int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgJob** job, const SCatalogReq* pReq, catalogCallback fp, void* param, int32_t* taskNum) { int32_t code = 0; int32_t tbMetaNum = (int32_t)taosArrayGetSize(pReq->pTableMeta); int32_t dbVgNum = (int32_t)taosArrayGetSize(pReq->pDbVgroup); @@ -421,6 +435,7 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgJob** job, uint6 int32_t udfNum = (int32_t)taosArrayGetSize(pReq->pUdf); int32_t qnodeNum = pReq->qNodeRequired ? 1 : 0; int32_t dnodeNum = pReq->dNodeRequired ? 1 : 0; + int32_t svrVerNum = pReq->svrVerRequired ? 1 : 0; int32_t dbCfgNum = (int32_t)taosArrayGetSize(pReq->pDbCfg); int32_t indexNum = (int32_t)taosArrayGetSize(pReq->pIndex); int32_t userNum = (int32_t)taosArrayGetSize(pReq->pUser); @@ -428,21 +443,21 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgJob** job, uint6 int32_t tbIndexNum = (int32_t)taosArrayGetSize(pReq->pTableIndex); int32_t tbCfgNum = (int32_t)taosArrayGetSize(pReq->pTableCfg); - *taskNum = tbMetaNum + dbVgNum + udfNum + tbHashNum + qnodeNum + dnodeNum + dbCfgNum + indexNum + userNum + dbInfoNum + tbIndexNum + tbCfgNum; + *taskNum = tbMetaNum + dbVgNum + udfNum + tbHashNum + qnodeNum + dnodeNum + svrVerNum + dbCfgNum + indexNum + userNum + dbInfoNum + tbIndexNum + tbCfgNum; if (*taskNum <= 0) { - ctgDebug("Empty input for job, no need to retrieve meta, reqId:0x%" PRIx64, reqId); + ctgDebug("Empty input for job, no need to retrieve meta, reqId:0x%" PRIx64, pConn->requestId); return TSDB_CODE_SUCCESS; } *job = taosMemoryCalloc(1, sizeof(SCtgJob)); if (NULL == *job) { - ctgError("failed to calloc, size:%d, reqId:0x%" PRIx64, (int32_t)sizeof(SCtgJob), reqId); + ctgError("failed to calloc, size:%d, reqId:0x%" PRIx64, (int32_t)sizeof(SCtgJob), pConn->requestId); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } SCtgJob *pJob = *job; - pJob->queryId = reqId; + pJob->queryId = pConn->requestId; pJob->userFp = fp; pJob->pCtg = pCtg; pJob->conn = *pConn; @@ -460,6 +475,7 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgJob** job, uint6 pJob->dbInfoNum = dbInfoNum; pJob->tbIndexNum = tbIndexNum; pJob->tbCfgNum = tbCfgNum; + pJob->svrVerNum = svrVerNum; pJob->pTasks = taosArrayInit(*taskNum, sizeof(SCtgTask)); @@ -530,6 +546,10 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgJob** job, uint6 CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_DNODE, NULL, NULL)); } + if (svrVerNum) { + CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_SVR_VER, NULL, NULL)); + } + pJob->refId = taosAddRef(gCtgMgmt.jobPool, pJob); if (pJob->refId < 0) { ctgError("add job to ref failed, error: %s", tstrerror(terrno)); @@ -728,6 +748,21 @@ int32_t ctgDumpUserRes(SCtgTask* pTask) { return TSDB_CODE_SUCCESS; } +int32_t ctgDumpSvrVer(SCtgTask* pTask) { + SCtgJob* pJob = pTask->pJob; + if (NULL == pJob->jobRes.pSvrVer) { + pJob->jobRes.pSvrVer = taosMemoryCalloc(1, sizeof(SMetaRes)); + if (NULL == pJob->jobRes.pSvrVer) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + } + + pJob->jobRes.pSvrVer->code = pTask->code; + pJob->jobRes.pSvrVer->pRes = pTask->res; + + return TSDB_CODE_SUCCESS; +} + int32_t ctgInvokeSubCb(SCtgTask *pTask) { int32_t code = 0; @@ -1156,6 +1191,20 @@ _return: CTG_RET(code); } +int32_t ctgHandleGetSvrVerRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { + int32_t code = 0; + + CTG_ERR_JRET(ctgProcessRspMsg(&pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); + + TSWAP(pTask->res, pTask->msgCtx.out); + +_return: + + ctgHandleTaskEnd(pTask, code); + + CTG_RET(code); +} + int32_t ctgAsyncRefreshTbMeta(SCtgTask *pTask) { SCatalog* pCtg = pTask->pJob->pCtg; SRequestConnInfo* pConn = &pTask->pJob->conn; @@ -1459,6 +1508,15 @@ int32_t ctgLaunchGetUserTask(SCtgTask *pTask) { return TSDB_CODE_SUCCESS; } +int32_t ctgLaunchGetSvrVerTask(SCtgTask *pTask) { + SCatalog* pCtg = pTask->pJob->pCtg; + SRequestConnInfo* pConn = &pTask->pJob->conn; + + CTG_ERR_RET(ctgGetSvrVerFromMnode(pCtg, pConn, NULL, pTask)); + + return TSDB_CODE_SUCCESS; +} + int32_t ctgRelaunchGetTbMetaTask(SCtgTask *pTask) { ctgResetTbMetaTask(pTask); @@ -1532,6 +1590,7 @@ SCtgAsyncFps gCtgAsyncFps[] = { {ctgInitGetIndexTask, ctgLaunchGetIndexTask, ctgHandleGetIndexRsp, ctgDumpIndexRes, NULL, NULL}, {ctgInitGetUdfTask, ctgLaunchGetUdfTask, ctgHandleGetUdfRsp, ctgDumpUdfRes, NULL, NULL}, {ctgInitGetUserTask, ctgLaunchGetUserTask, ctgHandleGetUserRsp, ctgDumpUserRes, NULL, NULL}, + {ctgInitGetSvrVerTask, ctgLaunchGetSvrVerTask, ctgHandleGetSvrVerRsp, ctgDumpSvrVer, NULL, NULL}, }; int32_t ctgMakeAsyncRes(SCtgJob *pJob) { @@ -1633,7 +1692,7 @@ int32_t ctgLaunchJob(SCtgJob *pJob) { for (int32_t i = 0; i < taskNum; ++i) { SCtgTask *pTask = taosArrayGet(pJob->pTasks, i); - qDebug("QID:0x%" PRIx64 " ctg start to launch task %d", pJob->queryId, pTask->taskId); + qDebug("QID:0x%" PRIx64 " ctg launch [%dth] task", pJob->queryId, pTask->taskId); CTG_ERR_RET((*gCtgAsyncFps[pTask->type].launchFp)(pTask)); pTask->status = CTG_TASK_LAUNCHED; } diff --git a/source/libs/catalog/src/ctgDbg.c b/source/libs/catalog/src/ctgDbg.c index 7f2b919f17..5f54362d8e 100644 --- a/source/libs/catalog/src/ctgDbg.c +++ b/source/libs/catalog/src/ctgDbg.c @@ -210,7 +210,7 @@ int32_t ctgdLaunchAsyncCall(SCatalog* pCtg, SRequestConnInfo* pConn, uint64_t re int64_t jobId = 0; - CTG_ERR_JRET(catalogAsyncGetAllMeta(pCtg, pConn, reqId, &req, ctgdUserCallback, param, &jobId)); + CTG_ERR_JRET(catalogAsyncGetAllMeta(pCtg, pConn, &req, ctgdUserCallback, param, &jobId)); _return: diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index 97edb1b837..304da88888 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -217,6 +217,21 @@ int32_t ctgProcessRspMsg(void* out, int32_t reqType, char* msg, int32_t msgSize, qDebug("Got stb cfg from mnode, tbFName:%s", target); break; } + case TDMT_MND_SERVER_VERSION: { + if (TSDB_CODE_SUCCESS != rspCode) { + qError("error rsp for svr ver from mnode, error:%s", tstrerror(rspCode)); + CTG_ERR_RET(rspCode); + } + + code = queryProcessMsgRsp[TMSG_INDEX(reqType)](out, msg, msgSize); + if (code) { + qError("Process svr ver rsp failed, error:%s", tstrerror(code)); + CTG_ERR_RET(code); + } + + qDebug("Got svr ver from mnode"); + break; + } default: qError("invalid req type %s", TMSG_INFO(reqType)); return TSDB_CODE_APP_ERROR; @@ -811,4 +826,38 @@ int32_t ctgGetTableCfgFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const S return TSDB_CODE_SUCCESS; } +int32_t ctgGetSvrVerFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, char **out, SCtgTask* pTask) { + char *msg = NULL; + int32_t msgLen = 0; + int32_t reqType = TDMT_MND_SERVER_VERSION; + void*(*mallocFp)(int32_t) = pTask ? taosMemoryMalloc : rpcMallocCont; + + qDebug("try to get svr ver from mnode"); + + int32_t code = queryBuildMsg[TMSG_INDEX(reqType)](NULL, &msg, 0, &msgLen, mallocFp); + if (code) { + ctgError("Build get svr ver msg failed, code:%s", tstrerror(code)); + CTG_ERR_RET(code); + } + + if (pTask) { + CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, NULL, NULL)); + + CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask, reqType, msg, msgLen)); + } + + SRpcMsg rpcMsg = { + .msgType = reqType, + .pCont = msg, + .contLen = msgLen, + }; + + SRpcMsg rpcRsp = {0}; + rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp); + + CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, NULL)); + + return TSDB_CODE_SUCCESS; +} + diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index cc823adad0..0c10b69a2d 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -45,6 +45,8 @@ char *ctgTaskTypeStr(CTG_TASK_TYPE type) { return "[get udf]"; case CTG_TASK_GET_USER: return "[get user]"; + case CTG_TASK_GET_SVR_VER: + return "[get svr ver]"; default: return "unknown"; } @@ -103,8 +105,13 @@ void ctgFreeSMetaData(SMetaData* pData) { taosArrayDestroy(pData->pQnodeList); pData->pQnodeList = NULL; + taosArrayDestroy(pData->pDnodeList); + pData->pDnodeList = NULL; + taosArrayDestroy(pData->pTableCfg); pData->pTableCfg = NULL; + + taosMemoryFreeClear(pData->pSvrVer); } void ctgFreeSCtgUserAuth(SCtgUserAuth *userCache) { @@ -346,20 +353,8 @@ void ctgResetTbMetaTask(SCtgTask* pTask) { void ctgFreeTaskRes(CTG_TASK_TYPE type, void **pRes) { switch (type) { - case CTG_TASK_GET_QNODE: { - taosArrayDestroy((SArray*)*pRes); - *pRes = NULL; - break; - } - case CTG_TASK_GET_DNODE: { - taosArrayDestroy((SArray*)*pRes); - *pRes = NULL; - break; - } - case CTG_TASK_GET_TB_META: { - taosMemoryFreeClear(*pRes); - break; - } + case CTG_TASK_GET_QNODE: + case CTG_TASK_GET_DNODE: case CTG_TASK_GET_DB_VGROUP: { taosArrayDestroy((SArray*)*pRes); *pRes = NULL; @@ -373,14 +368,6 @@ void ctgFreeTaskRes(CTG_TASK_TYPE type, void **pRes) { } break; } - case CTG_TASK_GET_DB_INFO: { - taosMemoryFreeClear(*pRes); - break; - } - case CTG_TASK_GET_TB_HASH: { - taosMemoryFreeClear(*pRes); - break; - } case CTG_TASK_GET_TB_INDEX: { taosArrayDestroyEx(*pRes, tFreeSTableIndexInfo); *pRes = NULL; @@ -394,15 +381,13 @@ void ctgFreeTaskRes(CTG_TASK_TYPE type, void **pRes) { } break; } - case CTG_TASK_GET_INDEX: { - taosMemoryFreeClear(*pRes); - break; - } - case CTG_TASK_GET_UDF: { - taosMemoryFreeClear(*pRes); - break; - } - case CTG_TASK_GET_USER: { + case CTG_TASK_GET_TB_HASH: + case CTG_TASK_GET_DB_INFO: + case CTG_TASK_GET_INDEX: + case CTG_TASK_GET_UDF: + case CTG_TASK_GET_USER: + case CTG_TASK_GET_SVR_VER: + case CTG_TASK_GET_TB_META: { taosMemoryFreeClear(*pRes); break; } @@ -415,20 +400,12 @@ void ctgFreeTaskRes(CTG_TASK_TYPE type, void **pRes) { void ctgFreeSubTaskRes(CTG_TASK_TYPE type, void **pRes) { switch (type) { - case CTG_TASK_GET_QNODE: { - taosArrayDestroy((SArray*)*pRes); - *pRes = NULL; - break; - } + case CTG_TASK_GET_QNODE: case CTG_TASK_GET_DNODE: { taosArrayDestroy((SArray*)*pRes); *pRes = NULL; break; } - case CTG_TASK_GET_TB_META: { - taosMemoryFreeClear(*pRes); - break; - } case CTG_TASK_GET_DB_VGROUP: { if (*pRes) { SDBVgInfo* pInfo = (SDBVgInfo*)*pRes; @@ -445,14 +422,6 @@ void ctgFreeSubTaskRes(CTG_TASK_TYPE type, void **pRes) { } break; } - case CTG_TASK_GET_DB_INFO: { - taosMemoryFreeClear(*pRes); - break; - } - case CTG_TASK_GET_TB_HASH: { - taosMemoryFreeClear(*pRes); - break; - } case CTG_TASK_GET_TB_INDEX: { taosArrayDestroyEx(*pRes, tFreeSTableIndexInfo); *pRes = NULL; @@ -466,14 +435,12 @@ void ctgFreeSubTaskRes(CTG_TASK_TYPE type, void **pRes) { } break; } - case CTG_TASK_GET_INDEX: { - taosMemoryFreeClear(*pRes); - break; - } - case CTG_TASK_GET_UDF: { - taosMemoryFreeClear(*pRes); - break; - } + case CTG_TASK_GET_TB_META: + case CTG_TASK_GET_DB_INFO: + case CTG_TASK_GET_TB_HASH: + case CTG_TASK_GET_INDEX: + case CTG_TASK_GET_UDF: + case CTG_TASK_GET_SVR_VER: case CTG_TASK_GET_USER: { taosMemoryFreeClear(*pRes); break; @@ -497,10 +464,6 @@ void ctgClearSubTaskRes(SCtgSubRes *pRes) { void ctgFreeTaskCtx(SCtgTask* pTask) { switch (pTask->type) { - case CTG_TASK_GET_QNODE: { - taosMemoryFreeClear(pTask->taskCtx); - break; - } case CTG_TASK_GET_TB_META: { SCtgTbMetaCtx* taskCtx = (SCtgTbMetaCtx*)pTask->taskCtx; taosMemoryFreeClear(taskCtx->pName); @@ -511,18 +474,6 @@ void ctgFreeTaskCtx(SCtgTask* pTask) { taosMemoryFreeClear(pTask->taskCtx); break; } - case CTG_TASK_GET_DB_VGROUP: { - taosMemoryFreeClear(pTask->taskCtx); - break; - } - case CTG_TASK_GET_DB_CFG: { - taosMemoryFreeClear(pTask->taskCtx); - break; - } - case CTG_TASK_GET_DB_INFO: { - taosMemoryFreeClear(pTask->taskCtx); - break; - } case CTG_TASK_GET_TB_HASH: { SCtgTbHashCtx* taskCtx = (SCtgTbHashCtx*)pTask->taskCtx; taosMemoryFreeClear(taskCtx->pName); @@ -542,14 +493,12 @@ void ctgFreeTaskCtx(SCtgTask* pTask) { taosMemoryFreeClear(pTask->taskCtx); break; } - case CTG_TASK_GET_INDEX: { - taosMemoryFreeClear(pTask->taskCtx); - break; - } - case CTG_TASK_GET_UDF: { - taosMemoryFreeClear(pTask->taskCtx); - break; - } + case CTG_TASK_GET_DB_VGROUP: + case CTG_TASK_GET_DB_CFG: + case CTG_TASK_GET_DB_INFO: + case CTG_TASK_GET_INDEX: + case CTG_TASK_GET_UDF: + case CTG_TASK_GET_QNODE: case CTG_TASK_GET_USER: { taosMemoryFreeClear(pTask->taskCtx); break; diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 8d01ed7315..cb14b916af 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -520,7 +520,17 @@ static int32_t execShowCreateSTable(SShowCreateTableStmt* pStmt, SRetrieveTableR return execShowCreateTable(pStmt, pRsp); } -static int32_t execAlterLocal(SAlterLocalStmt* pStmt) { return TSDB_CODE_FAILED; } +static int32_t execAlterLocal(SAlterLocalStmt* pStmt) { + if (cfgSetItem(tsCfg, pStmt->config, pStmt->value, CFG_STYPE_ALTER_CMD)) { + return terrno; + } + + if (taosSetCfg(tsCfg, pStmt->config)) { + return terrno; + } + + return TSDB_CODE_SUCCESS; +} static SSDataBlock* buildLocalVariablesResultDataBlock() { SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 57a2c57b16..eb7beb19db 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -273,6 +273,10 @@ typedef struct STableScanInfo { SSampleExecInfo sample; // sample execution info int32_t curTWinIdx; + + int32_t currentGroupId; + uint64_t queryId; + uint64_t taskId; } STableScanInfo; typedef struct STagScanInfo { @@ -706,7 +710,7 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode* pExNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, tsdbReaderT pDataReader, SReadHandle* pHandle, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, SExecTaskInfo* pTaskInfo, uint64_t queryId, uint64_t taskId); SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode *pScanPhyNode, SExecTaskInfo* pTaskInfo); @@ -749,8 +753,8 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SReadHandle* readHandle, uint64_t uid, SBlockDistScanPhysiNode* pBlockScanNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createStreamScanOperatorInfo(void* pDataReader, SReadHandle* pHandle, - STableScanPhysiNode* pTableScanNode, SExecTaskInfo* pTaskInfo, STimeWindowAggSupp* pTwSup); +SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, + STableScanPhysiNode* pTableScanNode, SExecTaskInfo* pTaskInfo, STimeWindowAggSupp* pTwSup, uint64_t queryId, uint64_t taskId); SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* pPhyFillNode, bool multigroupResult, SExecTaskInfo* pTaskInfo); @@ -845,7 +849,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex); -int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SArray* groupKey); +int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* groupKey); #ifdef __cplusplus } diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 8669c2c28c..5ac5957f2b 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -287,6 +287,7 @@ static bool isTableOk(STableKeyInfo* info, SNode *pTagCond, SMeta *metaHandle){ int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo* pListInfo) { int32_t code = TSDB_CODE_SUCCESS; pListInfo->pTableList = taosArrayInit(8, sizeof(STableKeyInfo)); + if(pListInfo->pTableList == NULL) return TSDB_CODE_OUT_OF_MEMORY; uint64_t tableUid = pScanNode->uid; @@ -314,7 +315,7 @@ int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo } for (int i = 0; i < taosArrayGetSize(res); i++) { - STableKeyInfo info = {.lastKey = TSKEY_INITIAL_VAL, .uid = *(uint64_t*)taosArrayGet(res, i), .groupId = 0}; + STableKeyInfo info = {.lastKey = TSKEY_INITIAL_VAL, .uid = *(uint64_t*)taosArrayGet(res, i)}; taosArrayPush(pListInfo->pTableList, &info); } taosArrayDestroy(res); @@ -335,9 +336,14 @@ int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo } } }else { // Create one table group. - STableKeyInfo info = {.lastKey = 0, .uid = tableUid, .groupId = 0}; + STableKeyInfo info = {.lastKey = 0, .uid = tableUid}; taosArrayPush(pListInfo->pTableList, &info); } + pListInfo->pGroupList = taosArrayInit(4, POINTER_BYTES); + if(pListInfo->pGroupList == NULL) return TSDB_CODE_OUT_OF_MEMORY; + + //put into list as default group, remove it if grouping sorting is required later + taosArrayPush(pListInfo->pGroupList, &pListInfo->pTableList); return code; } diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 1bd1ce14b1..6de364e63a 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -40,15 +40,8 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu SStreamBlockScanInfo* pInfo = pOperator->info; pInfo->assignBlockUid = assignUid; - // no need to check -#if 0 - if (pInfo->blockType == 0) { - pInfo->blockType = type; - } else if (pInfo->blockType != type) { - ASSERT(0); - return TSDB_CODE_QRY_APP_ERROR; - } -#endif + // TODO: if a block was set but not consumed, + // prevent setting a different type of block pInfo->blockType = type; if (type == STREAM_DATA_TYPE_SUBMIT_BLOCK) { @@ -152,10 +145,10 @@ static SArray* filterQualifiedChildTables(const SStreamBlockScanInfo* pScanInfo, continue; } - ASSERT(mr.me.type == TSDB_CHILD_TABLE); - if (mr.me.ctbEntry.suid != pScanInfo->tableUid) { + if (mr.me.type != TSDB_CHILD_TABLE || mr.me.ctbEntry.suid != pScanInfo->tableUid) { continue; } + // TODO handle ntb case taosArrayPush(qa, id); } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 1105cb3156..c23d9a5040 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1321,7 +1321,7 @@ void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock) { // todo move to the initialization function int32_t code = filterInitFromNode((SNode*)pFilterNode, &filter, 0); - size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); SFilterColumnParam param1 = {.numOfCols = numOfCols, .pDataBlock = pBlock->pDataBlock}; code = filterSetDataFromSlotId(filter, ¶m1); @@ -2048,7 +2048,7 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLo SArray* pColList) { if (pColList == NULL) { // data from other sources blockDataCleanup(pRes); -// blockDataEnsureCapacity(pRes, numOfRows); + // blockDataEnsureCapacity(pRes, numOfRows); blockCompressDecode(pRes, numOfOutput, numOfRows, pData); } else { // extract data according to pColList ASSERT(numOfOutput == taosArrayGetSize(pColList)); @@ -2402,14 +2402,14 @@ SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode tsem_init(&pInfo->ready, 0, 0); - pInfo->seqLoadData = false; + pInfo->seqLoadData = false; pInfo->pTransporter = pTransporter; - pInfo->pResult = createResDataBlock(pExNode->node.pOutputDataBlockDesc); - pOperator->name = "ExchangeOperator"; + pInfo->pResult = createResDataBlock(pExNode->node.pOutputDataBlockDesc); + pOperator->name = "ExchangeOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_EXCHANGE; pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pResult->pDataBlock); pOperator->pTaskInfo = pTaskInfo; @@ -3862,9 +3862,6 @@ static SExecTaskInfo* createExecTaskInfo(uint64_t queryId, uint64_t taskId, EOPT return pTaskInfo; } -static tsdbReaderT doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, - STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId); - static SArray* extractColumnInfo(SNodeList* pNodeList); int32_t extractTableSchemaVersion(SReadHandle* pHandle, uint64_t uid, SExecTaskInfo* pTaskInfo) { @@ -3895,8 +3892,67 @@ int32_t extractTableSchemaVersion(SReadHandle* pHandle, uint64_t uid, SExecTaskI return TSDB_CODE_SUCCESS; } -int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SArray* groupKey) { - if (groupKey == NULL) { +static int32_t sortTableGroup(STableListInfo* pTableListInfo, int32_t groupNum){ + taosArrayClear(pTableListInfo->pGroupList); + SArray *sortSupport = taosArrayInit(groupNum, sizeof(uint64_t)); + if(sortSupport == NULL) return TSDB_CODE_OUT_OF_MEMORY; + for (int32_t i = 0; i < taosArrayGetSize(pTableListInfo->pTableList); i++) { + STableKeyInfo* info = taosArrayGet(pTableListInfo->pTableList, i); + uint64_t* groupId = taosHashGet(pTableListInfo->map, &info->uid, sizeof(uint64_t)); + + int32_t index = taosArraySearchIdx(sortSupport, groupId, compareUint64Val, TD_EQ); + if (index == -1){ + void *p = taosArraySearch(sortSupport, groupId, compareUint64Val, TD_GT); + SArray *tGroup = taosArrayInit(8, sizeof(STableKeyInfo)); + if(tGroup == NULL) { + taosArrayDestroy(sortSupport); + return TSDB_CODE_OUT_OF_MEMORY; + } + if(taosArrayPush(tGroup, info) == NULL){ + qError("taos push info array error"); + taosArrayDestroy(sortSupport); + return TSDB_CODE_QRY_APP_ERROR; + } + if(p == NULL){ + if(taosArrayPush(sortSupport, groupId) != NULL){ + qError("taos push support array error"); + taosArrayDestroy(sortSupport); + return TSDB_CODE_QRY_APP_ERROR; + } + if(taosArrayPush(pTableListInfo->pGroupList, &tGroup) != NULL){ + qError("taos push group array error"); + taosArrayDestroy(sortSupport); + return TSDB_CODE_QRY_APP_ERROR; + } + }else{ + int32_t pos = TARRAY_ELEM_IDX(sortSupport, p); + if(taosArrayInsert(sortSupport, pos, groupId) == NULL){ + qError("taos insert support array error"); + taosArrayDestroy(sortSupport); + return TSDB_CODE_QRY_APP_ERROR; + } + if(taosArrayInsert(pTableListInfo->pGroupList, pos, &tGroup) == NULL){ + qError("taos insert group array error"); + taosArrayDestroy(sortSupport); + return TSDB_CODE_QRY_APP_ERROR; + } + } + }else{ + SArray* tGroup = (SArray*)taosArrayGetP(pTableListInfo->pGroupList, index); + if(taosArrayPush(tGroup, info) == NULL){ + qError("taos push uid array error"); + taosArrayDestroy(sortSupport); + return TSDB_CODE_QRY_APP_ERROR; + } + } + + } + taosArrayDestroy(sortSupport); + return TDB_CODE_SUCCESS; +} + +int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* group) { + if (group == NULL) { return TDB_CODE_SUCCESS; } @@ -3906,13 +3962,14 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, } int32_t keyLen = 0; void* keyBuf = NULL; - int32_t numOfGroupCols = taosArrayGetSize(groupKey); - for (int32_t j = 0; j < numOfGroupCols; ++j) { - SColumn* pCol = taosArrayGet(groupKey, j); - keyLen += pCol->bytes; // actual data + null_flag + + SNode* node; + FOREACH(node, group) { + SExprNode *pExpr = (SExprNode *)node; + keyLen += pExpr->resType.bytes; } - int32_t nullFlagSize = sizeof(int8_t) * numOfGroupCols; + int32_t nullFlagSize = sizeof(int8_t) * LIST_LENGTH(group); keyLen += nullFlagSize; keyBuf = taosMemoryCalloc(1, keyLen); @@ -3920,59 +3977,68 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, return TSDB_CODE_OUT_OF_MEMORY; } + int32_t groupNum = 0; for (int32_t i = 0; i < taosArrayGetSize(pTableListInfo->pTableList); i++) { STableKeyInfo* info = taosArrayGet(pTableListInfo->pTableList, i); SMetaReader mr = {0}; metaReaderInit(&mr, pHandle->meta, 0); metaGetTableEntryByUid(&mr, info->uid); - char* isNull = (char*)keyBuf; - char* pStart = (char*)keyBuf + sizeof(int8_t) * numOfGroupCols; - for (int32_t j = 0; j < numOfGroupCols; ++j) { - SColumn* pCol = taosArrayGet(groupKey, j); + SNodeList *groupNew = nodesCloneList(group); - if (strcmp(pCol->name, "tbname") == 0) { - isNull[i] = 0; - memcpy(pStart, mr.me.name, strlen(mr.me.name)); - pStart += strlen(mr.me.name); + nodesRewriteExprsPostOrder(groupNew, doTranslateTagExpr, &mr); + char* isNull = (char*)keyBuf; + char* pStart = (char*)keyBuf + nullFlagSize; + + SNode* pNode; + int32_t index = 0; + FOREACH(pNode, groupNew){ + SNode* pNew = NULL; + int32_t code = scalarCalculateConstants(pNode, &pNew); + if (TSDB_CODE_SUCCESS == code) { + REPLACE_NODE(pNew); } else { - STagVal tagVal = {0}; - tagVal.cid = pCol->colId; - const char* p = metaGetTableTagVal(&mr.me, pCol->type, &tagVal); - if (p == NULL) { - isNull[j] = 1; - continue; - } - isNull[i] = 0; - if (pCol->type == TSDB_DATA_TYPE_JSON) { - // int32_t dataLen = getJsonValueLen(pkey->pData); - // memcpy(pStart, (pkey->pData), dataLen); - // pStart += dataLen; - } else if (IS_VAR_DATA_TYPE(pCol->type)) { - memcpy(pStart, tagVal.pData, tagVal.nData); - pStart += tagVal.nData; - ASSERT(tagVal.nData <= pCol->bytes); + taosMemoryFree(keyBuf); + nodesClearList(groupNew); + return code; + } + + ASSERT(nodeType(pNew) == QUERY_NODE_VALUE); + SValueNode *pValue = (SValueNode *)pNew; + + if (pValue->node.resType.type == TSDB_DATA_TYPE_NULL) { + isNull[index++] = 1; + continue; + } else { + isNull[index++] = 0; + char* data = nodesGetValueFromNode(pValue); + if (pValue->node.resType.type == TSDB_DATA_TYPE_JSON){ + int32_t len = getJsonValueLen(data); + memcpy(pStart, data, len); + pStart += len; + } else if (IS_VAR_DATA_TYPE(pValue->node.resType.type)) { + memcpy(pStart, data, varDataTLen(data)); + pStart += varDataTLen(data); } else { - memcpy(pStart, &(tagVal.i64), pCol->bytes); - pStart += pCol->bytes; + memcpy(pStart, data, pValue->node.resType.bytes); + pStart += pValue->node.resType.bytes; } } } - int32_t len = (int32_t)(pStart - (char*)keyBuf); - - uint64_t* pGroupId = taosHashGet(pTableListInfo->map, keyBuf, len); - - if (!pGroupId) { - uint64_t tmpId = calcGroupId(keyBuf, len); - info->groupId = tmpId; - taosHashPut(pTableListInfo->map, &(info->uid), sizeof(uint64_t), &tmpId, sizeof(uint64_t)); - } else { - info->groupId = *pGroupId; - } + int32_t len = (int32_t)(pStart - (char*)keyBuf); + uint64_t groupId = calcGroupId(keyBuf, len); + taosHashPut(pTableListInfo->map, &(info->uid), sizeof(uint64_t), &groupId, sizeof(uint64_t)); + groupNum++; + nodesClearList(groupNew); metaReaderClear(&mr); } taosMemoryFree(keyBuf); + + if(pTableListInfo->needSortTableByGroupId){ + return sortTableGroup(pTableListInfo, groupNum); + } + return TDB_CODE_SUCCESS; } @@ -3984,39 +4050,36 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == type) { STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode; - tsdbReaderT pDataReader = doCreateDataReader(pTableScanNode, pHandle, pTableListInfo, (uint64_t)queryId, taskId); - if (pDataReader == NULL && terrno != 0) { - pTaskInfo->code = terrno; + int32_t code = createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId); + if(code){ + pTaskInfo->code = code; return NULL; } - - int32_t code = extractTableSchemaVersion(pHandle, pTableScanNode->scan.uid, pTaskInfo); + code = extractTableSchemaVersion(pHandle, pTableScanNode->scan.uid, pTaskInfo); if (code) { - tsdbCleanupReadHandle(pDataReader); pTaskInfo->code = terrno; return NULL; } - SArray* groupKeys = extractPartitionColInfo(pTableScanNode->pPartitionTags); - code = generateGroupIdMap(pTableListInfo, pHandle, groupKeys); // todo for json - taosArrayDestroy(groupKeys); - if (code) { - tsdbCleanupReadHandle(pDataReader); - pTaskInfo->code = terrno; - return NULL; - } - - SOperatorInfo* pOperator = createTableScanOperatorInfo(pTableScanNode, pDataReader, pHandle, pTaskInfo); + SOperatorInfo* pOperator = createTableScanOperatorInfo(pTableScanNode, pHandle, pTaskInfo, queryId, taskId); STableScanInfo* pScanInfo = pOperator->info; pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder; return pOperator; } else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN == type) { STableMergeScanPhysiNode* pTableScanNode = (STableMergeScanPhysiNode*)pPhyNode; - createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId); - extractTableSchemaVersion(pHandle, pTableScanNode->scan.uid, pTaskInfo); - SOperatorInfo* pOperator = - createTableMergeScanOperatorInfo(pTableScanNode, pTableListInfo, pHandle, pTaskInfo, queryId, taskId); + int32_t code = createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId); + if(code){ + return NULL; + } + code = extractTableSchemaVersion(pHandle, pTableScanNode->scan.uid, pTaskInfo); + if (code) { + pTaskInfo->code = terrno; + return NULL; + } + + SOperatorInfo* pOperator = createTableMergeScanOperatorInfo(pTableScanNode, pTableListInfo, pHandle, pTaskInfo, queryId, taskId); + STableScanInfo* pScanInfo = pOperator->info; pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder; return pOperator; @@ -4025,46 +4088,17 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo return createExchangeOperatorInfo(pHandle->pMsgCb->clientRpc, (SExchangePhysiNode*)pPhyNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN == type) { - SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; // simple child table. STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode; STimeWindowAggSupp twSup = { .waterMark = pTableScanNode->watermark, .calTrigger = pTableScanNode->triggerType, .maxTs = INT64_MIN, }; - tsdbReaderT pDataReader = NULL; - if (pHandle) { - if (pHandle->initTsdbReader) { - // for stream - ASSERT(pHandle->vnode); - pDataReader = doCreateDataReader(pTableScanNode, pHandle, pTableListInfo, (uint64_t)queryId, taskId); - } else { - // for tq - ASSERT(pHandle->meta); - getTableList(pHandle->meta, pScanPhyNode, pTableListInfo); - } + createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId); } -#if 0 - if (pDataReader == NULL && terrno != 0) { - qDebug("%s pDataReader is NULL", GET_TASKID(pTaskInfo)); - // return NULL; - } else { - qDebug("%s pDataReader is not NULL", GET_TASKID(pTaskInfo)); - } -#endif - - SArray* groupKeys = extractPartitionColInfo(pTableScanNode->pPartitionTags); - int32_t code = generateGroupIdMap(pTableListInfo, pHandle, groupKeys); // todo for json - taosArrayDestroy(groupKeys); - if (code) { - tsdbCleanupReadHandle(pDataReader); - return NULL; - } - - SOperatorInfo* pOperator = createStreamScanOperatorInfo(pDataReader, pHandle, pTableScanNode, pTaskInfo, &twSup); - + SOperatorInfo* pOperator = createStreamScanOperatorInfo(pHandle, pTableScanNode, pTaskInfo, &twSup, queryId, taskId); return pOperator; } else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) { @@ -4093,7 +4127,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo return NULL; } } else { // Create one table group. - STableKeyInfo info = {.lastKey = 0, .uid = pBlockNode->uid, .groupId = 0}; + STableKeyInfo info = {.lastKey = 0, .uid = pBlockNode->uid}; taosArrayPush(pTableListInfo->pTableList, &info); } @@ -4118,7 +4152,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo cond.suid = pBlockNode->suid; cond.type = BLOCK_LOAD_OFFSET_SEQ_ORDER; } - tsdbReaderT* pReader = tsdbReaderOpen(pHandle->vnode, &cond, pTableListInfo, queryId, taskId); + tsdbReaderT* pReader = tsdbReaderOpen(pHandle->vnode, &cond, pTableListInfo->pTableList, queryId, taskId); cleanupQueryTableDataCond(&cond); return createDataBlockInfoScanOperator(pReader, pHandle, cond.suid, pBlockNode, pTaskInfo); @@ -4355,7 +4389,7 @@ tsdbReaderT doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle* goto _error; } - tsdbReaderT pReader = tsdbReaderOpen(pHandle->vnode, &cond, pTableListInfo, queryId, taskId); + tsdbReaderT pReader = tsdbReaderOpen(pHandle->vnode, &cond, pTableListInfo->pTableList, queryId, taskId); cleanupQueryTableDataCond(&cond); return pReader; @@ -4365,6 +4399,72 @@ _error: return NULL; } +static int32_t extractTbscanInStreamOpTree(SOperatorInfo* pOperator, STableScanInfo** ppInfo) { + if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { + if (pOperator->numOfDownstream == 0) { + qError("failed to find stream scan operator"); + return TSDB_CODE_QRY_APP_ERROR; + } + + if (pOperator->numOfDownstream > 1) { + qError("join not supported for stream block scan"); + return TSDB_CODE_QRY_APP_ERROR; + } + return extractTbscanInStreamOpTree(pOperator->pDownstream[0], ppInfo); + } else { + SStreamBlockScanInfo* pInfo = pOperator->info; + ASSERT(pInfo->pSnapshotReadOp->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN); + *ppInfo = pInfo->pSnapshotReadOp->info; + return 0; + } +} + +int32_t extractTableScanNode(SPhysiNode* pNode, STableScanPhysiNode** ppNode) { + if (pNode->pChildren == NULL || LIST_LENGTH(pNode->pChildren) == 0) { + if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == pNode->type) { + *ppNode = (STableScanPhysiNode*)pNode; + return 0; + } else { + ASSERT(0); + terrno = TSDB_CODE_QRY_APP_ERROR; + return -1; + } + } else { + if (LIST_LENGTH(pNode->pChildren) != 1) { + ASSERT(0); + terrno = TSDB_CODE_QRY_APP_ERROR; + return -1; + } + SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pNode->pChildren, 0); + return extractTableScanNode(pChildNode, ppNode); + } + return -1; +} + +int32_t rebuildReader(SOperatorInfo* pOperator, SSubplan* plan, SReadHandle* pHandle, int64_t uid, int64_t ts) { + STableScanInfo* pTableScanInfo = NULL; + if (extractTbscanInStreamOpTree(pOperator, &pTableScanInfo) < 0) { + return -1; + } + + STableScanPhysiNode* pNode = NULL; + if (extractTableScanNode(plan->pNode, &pNode) < 0) { + ASSERT(0); + } + + tsdbCleanupReadHandle(pTableScanInfo->dataReader); + + STableListInfo info = {0}; + pTableScanInfo->dataReader = doCreateDataReader(pNode, pHandle, &info, 0, 0); + if (pTableScanInfo->dataReader == NULL) { + ASSERT(0); + qError("failed to create data reader"); + return TSDB_CODE_QRY_APP_ERROR; + } + // TODO: set uid and ts to data reader + return 0; +} + int32_t encodeOperator(SOperatorInfo* ops, char** result, int32_t* length) { int32_t code = TDB_CODE_SUCCESS; char* pCurrent = NULL; @@ -4518,6 +4618,13 @@ _complete: static void doDestroyTableList(STableListInfo* pTableqinfoList) { taosArrayDestroy(pTableqinfoList->pTableList); taosHashCleanup(pTableqinfoList->map); + if(pTableqinfoList->needSortTableByGroupId){ + for(int32_t i = 0; i < taosArrayGetSize(pTableqinfoList->pGroupList); i++){ + SArray* tmp = taosArrayGetP(pTableqinfoList->pGroupList, i); + taosArrayDestroy(tmp); + } + } + taosArrayDestroy(pTableqinfoList->pGroupList); pTableqinfoList->pTableList = NULL; pTableqinfoList->map = NULL; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 07212bc018..9c0ed40c30 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -418,7 +418,7 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { return NULL; } -static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { +static SSDataBlock* doTableScanGroup(SOperatorInfo* pOperator) { STableScanInfo* pTableScanInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -500,6 +500,48 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { } } + return NULL; +} + +static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { + STableScanInfo* pInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + if(pInfo->currentGroupId == -1){ + pInfo->currentGroupId++; + if (pInfo->currentGroupId >= taosArrayGetSize(pTaskInfo->tableqinfoList.pGroupList)) { + setTaskStatus(pTaskInfo, TASK_COMPLETED); + return NULL; + } + SArray *tableList = taosArrayGetP(pTaskInfo->tableqinfoList.pGroupList, pInfo->currentGroupId); + tsdbCleanupReadHandle(pInfo->dataReader); + tsdbReaderT* pReader = tsdbReaderOpen(pInfo->readHandle.vnode, &pInfo->cond, tableList, pInfo->queryId, pInfo->taskId); + pInfo->dataReader = pReader; + } + + SSDataBlock* result = doTableScanGroup(pOperator); + if(result){ + return result; + } + + pInfo->currentGroupId++; + if (pInfo->currentGroupId >= taosArrayGetSize(pTaskInfo->tableqinfoList.pGroupList)) { + setTaskStatus(pTaskInfo, TASK_COMPLETED); + return NULL; + } + + SArray *tableList = taosArrayGetP(pTaskInfo->tableqinfoList.pGroupList, pInfo->currentGroupId); + tsdbSetTableList(pInfo->dataReader, tableList); + + tsdbResetReadHandle(pInfo->dataReader, &pInfo->cond, 0); + pInfo->curTWinIdx = 0; + pInfo->scanTimes = 0; + + result = doTableScanGroup(pOperator); + if(result){ + return result; + } + setTaskStatus(pTaskInfo, TASK_COMPLETED); return NULL; } @@ -525,8 +567,8 @@ static void destroyTableScanOperatorInfo(void* param, int32_t numOfOutput) { } } -SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, tsdbReaderT pDataReader, - SReadHandle* readHandle, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* readHandle, + SExecTaskInfo* pTaskInfo, uint64_t queryId, uint64_t taskId) { STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -561,10 +603,12 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, pInfo->dataBlockLoadFlag = pTableScanNode->dataRequired; pInfo->pResBlock = createResDataBlock(pDescNode); pInfo->pFilterNode = pTableScanNode->scan.node.pConditions; - pInfo->dataReader = pDataReader; pInfo->scanFlag = MAIN_SCAN; pInfo->pColMatchInfo = pColList; pInfo->curTWinIdx = 0; + pInfo->queryId = queryId; + pInfo->taskId = taskId; + pInfo->currentGroupId = -1; pOperator->name = "TableScanOperator"; // for debug purpose pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN; @@ -778,8 +822,9 @@ static bool prepareDataScan(SStreamBlockScanInfo* pInfo) { STableScanInfo* pTableScanInfo = pInfo->pSnapshotReadOp->info; pTableScanInfo->cond.twindows[0] = win; pTableScanInfo->curTWinIdx = 0; - tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0); +// tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0); pTableScanInfo->scanTimes = 0; + pTableScanInfo->currentGroupId = -1; return true; } @@ -1087,9 +1132,9 @@ static SArray* extractTableIdList(const STableListInfo* pTableGroupInfo) { return tableIdList; } -SOperatorInfo* createStreamScanOperatorInfo(void* pDataReader, SReadHandle* pHandle, +SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, SExecTaskInfo* pTaskInfo, - STimeWindowAggSupp* pTwSup) { + STimeWindowAggSupp* pTwSup, uint64_t queryId, uint64_t taskId) { SStreamBlockScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamBlockScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); @@ -1129,7 +1174,7 @@ SOperatorInfo* createStreamScanOperatorInfo(void* pDataReader, SReadHandle* pHan } if (pHandle) { - SOperatorInfo* pTableScanDummy = createTableScanOperatorInfo(pTableScanNode, pDataReader, pHandle, pTaskInfo); + SOperatorInfo* pTableScanDummy = createTableScanOperatorInfo(pTableScanNode, pHandle, pTaskInfo, queryId, taskId); STableScanInfo* pSTInfo = (STableScanInfo*)pTableScanDummy->info; if (pSTInfo->interval.interval > 0) { pInfo->pUpdateInfo = updateInfoInitP(&pSTInfo->interval, pTwSup->waterMark); @@ -1889,11 +1934,12 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi goto _error; } - pInfo->pTableList = pTableListInfo; - pInfo->pColMatchInfo = colList; - pInfo->pRes = createResDataBlock(pDescNode); - pInfo->readHandle = *pReadHandle; - pInfo->curPos = 0; + pInfo->pTableList = pTableListInfo; + pInfo->pColMatchInfo = colList; + pInfo->pRes = createResDataBlock(pDescNode); + pInfo->readHandle = *pReadHandle; + pInfo->curPos = 0; + pOperator->name = "TagScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN; @@ -1919,10 +1965,7 @@ _error: typedef struct STableMergeScanInfo { STableListInfo* tableListInfo; - int32_t tableStartIndex; - int32_t tableEndIndex; - bool hasGroupId; - uint64_t groupId; + int32_t currentGroupId; SArray* dataReaders; // array of tsdbReaderT* SReadHandle readHandle; @@ -1968,12 +2011,6 @@ typedef struct STableMergeScanInfo { SSampleExecInfo sample; // sample execution info } STableMergeScanInfo; -int32_t compareTableKeyInfoByGid(const void* p1, const void* p2) { - const STableKeyInfo* info1 = p1; - const STableKeyInfo* info2 = p2; - return info1->groupId - info2->groupId; -} - int32_t createScanTableListInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId) { int32_t code = getTableList(pHandle->meta, &pTableScanNode->scan, pTableListInfo); @@ -1985,55 +2022,9 @@ int32_t createScanTableListInfo(STableScanPhysiNode* pTableScanNode, SReadHandle qDebug("no table qualified for query, TID:0x%" PRIx64 ", QID:0x%" PRIx64, taskId, queryId); return TSDB_CODE_SUCCESS; } - SArray* groupKeys = extractPartitionColInfo(pTableScanNode->pPartitionTags); - generateGroupIdMap(pTableListInfo, pHandle, groupKeys); // todo for json - if (groupKeys) { - taosArraySort(pTableListInfo->pTableList, compareTableKeyInfoByGid); - } - taosArrayDestroy(groupKeys); - return TSDB_CODE_SUCCESS; -} - -int32_t doCreateMultipleDataReaders(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, - STableListInfo* pTableListInfo, SArray* arrayReader, uint64_t queryId, - uint64_t taskId) { - SQueryTableDataCond cond = {0}; - int32_t code = initQueryTableDataCond(&cond, pTableScanNode); + code = generateGroupIdMap(pTableListInfo, pHandle, pTableScanNode->pPartitionTags); if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - for (int32_t i = 0; i < taosArrayGetSize(pTableListInfo->pTableList); ++i) { - STableListInfo* subListInfo = taosMemoryCalloc(1, sizeof(subListInfo)); - subListInfo->pTableList = taosArrayInit(1, sizeof(STableKeyInfo)); - taosArrayPush(subListInfo->pTableList, taosArrayGet(pTableListInfo->pTableList, i)); - - tsdbReaderT* pReader = tsdbReaderOpen(pHandle->vnode, &cond, subListInfo, queryId, taskId); - taosArrayPush(arrayReader, &pReader); - - taosArrayDestroy(subListInfo->pTableList); - taosMemoryFree(subListInfo); - } - cleanupQueryTableDataCond(&cond); - - return TSDB_CODE_SUCCESS; - -_error: - return code; -} - -int32_t createMultipleDataReaders(SQueryTableDataCond* pQueryCond, SReadHandle* pHandle, STableListInfo* pTableListInfo, - int32_t tableStartIdx, int32_t tableEndIdx, SArray* arrayReader, uint64_t queryId, - uint64_t taskId) { - for (int32_t i = tableStartIdx; i <= tableEndIdx; ++i) { - STableListInfo* subListInfo = taosMemoryCalloc(1, sizeof(subListInfo)); - subListInfo->pTableList = taosArrayInit(1, sizeof(STableKeyInfo)); - taosArrayPush(subListInfo->pTableList, taosArrayGet(pTableListInfo->pTableList, i)); - - tsdbReaderT* pReader = tsdbReaderOpen(pHandle->vnode, pQueryCond, subListInfo, queryId, taskId); - taosArrayPush(arrayReader, &pReader); - - taosArrayDestroy(subListInfo->pTableList); - taosMemoryFree(subListInfo); + return code; } return TSDB_CODE_SUCCESS; @@ -2225,32 +2216,34 @@ SArray* generateSortByTsInfo(int32_t order) { return pList; } +static int32_t createMultipleDataReaders(SQueryTableDataCond* pQueryCond, SReadHandle* pHandle, SArray* tableList, SArray* arrayReader, uint64_t queryId, + uint64_t taskId) { + for (int32_t i = 0; i < taosArrayGetSize(tableList); ++i) { + SArray* tmp = taosArrayInit(1, sizeof(STableKeyInfo)); + taosArrayPush(tmp, taosArrayGet(tableList, i)); + + tsdbReaderT* pReader = tsdbReaderOpen(pHandle->vnode, pQueryCond, tmp, queryId, taskId); + taosArrayPush(arrayReader, &pReader); + + taosArrayDestroy(tmp); + } + + return TSDB_CODE_SUCCESS; +} + int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) { STableMergeScanInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - { - size_t tableListSize = taosArrayGetSize(pInfo->tableListInfo->pTableList); - int32_t i = pInfo->tableStartIndex + 1; - for (; i < tableListSize; ++i) { - STableKeyInfo* tableKeyInfo = taosArrayGet(pInfo->tableListInfo->pTableList, i); - if (tableKeyInfo->groupId != pInfo->groupId) { - break; - } - } - pInfo->tableEndIndex = i - 1; - } + SArray* tableList = taosArrayGetP(pInfo->tableListInfo->pGroupList, pInfo->currentGroupId); - int32_t tableStartIdx = pInfo->tableStartIndex; - int32_t tableEndIdx = pInfo->tableEndIndex; - - STableListInfo* tableListInfo = pInfo->tableListInfo; - createMultipleDataReaders(&pInfo->cond, &pInfo->readHandle, tableListInfo, tableStartIdx, tableEndIdx, + createMultipleDataReaders(&pInfo->cond, &pInfo->readHandle, tableList, pInfo->dataReaders, pInfo->queryId, pInfo->taskId); // todo the total available buffer should be determined by total capacity of buffer of this task. // the additional one is reserved for merge result - pInfo->sortBufSize = pInfo->bufPageSize * (tableEndIdx - tableStartIdx + 1 + 1); + int32_t tableLen = taosArrayGetSize(tableList); + pInfo->sortBufSize = pInfo->bufPageSize * ((tableLen==0?1:tableLen) + 1); int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_MULTISOURCE_MERGE, pInfo->bufPageSize, numOfBufPage, pInfo->pSortInputBlock, pTaskInfo->id.str); @@ -2337,38 +2330,43 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) { if (code != TSDB_CODE_SUCCESS) { longjmp(pTaskInfo->env, code); } - size_t tableListSize = taosArrayGetSize(pInfo->tableListInfo->pTableList); - if (!pInfo->hasGroupId) { - pInfo->hasGroupId = true; - if (tableListSize == 0) { + if (pInfo->currentGroupId == -1) { + pInfo->currentGroupId++; + if (pInfo->currentGroupId >= taosArrayGetSize(pInfo->tableListInfo->pGroupList)) { doSetOperatorCompleted(pOperator); return NULL; } - pInfo->tableStartIndex = 0; - pInfo->groupId = ((STableKeyInfo*)taosArrayGet(pInfo->tableListInfo->pTableList, pInfo->tableStartIndex))->groupId; startGroupTableMergeScan(pOperator); } - SSDataBlock* pBlock = NULL; - while (pInfo->tableStartIndex < tableListSize) { - pBlock = getSortedTableMergeScanBlockData(pInfo->pSortHandle, pOperator->resultInfo.capacity, pOperator); - if (pBlock != NULL) { - pBlock->info.groupId = pInfo->groupId; - pOperator->resultInfo.totalRows += pBlock->info.rows; - return pBlock; - } else { - stopGroupTableMergeScan(pOperator); - if (pInfo->tableEndIndex >= tableListSize - 1) { - doSetOperatorCompleted(pOperator); - break; - } - pInfo->tableStartIndex = pInfo->tableEndIndex + 1; - pInfo->groupId = - ((STableKeyInfo*)taosArrayGet(pInfo->tableListInfo->pTableList, pInfo->tableStartIndex))->groupId; - startGroupTableMergeScan(pOperator); - } + SSDataBlock* pBlock = getSortedTableMergeScanBlockData(pInfo->pSortHandle, pOperator->resultInfo.capacity, pOperator); + if (pBlock != NULL) { + uint64_t* groupId = taosHashGet(pInfo->tableListInfo->map, &(pBlock->info.uid), sizeof(uint64_t)); + if(groupId) pBlock->info.groupId = *groupId; + + pOperator->resultInfo.totalRows += pBlock->info.rows; + return pBlock; } + stopGroupTableMergeScan(pOperator); + pInfo->currentGroupId++; + if (pInfo->currentGroupId >= taosArrayGetSize(pInfo->tableListInfo->pGroupList)) { + doSetOperatorCompleted(pOperator); + return NULL; + } + startGroupTableMergeScan(pOperator); + + pBlock = getSortedTableMergeScanBlockData(pInfo->pSortHandle, pOperator->resultInfo.capacity, pOperator); + if (pBlock != NULL) { + uint64_t* groupId = taosHashGet(pInfo->tableListInfo->map, &(pBlock->info.uid), sizeof(uint64_t)); + if(groupId) pBlock->info.groupId = *groupId; + + pOperator->resultInfo.totalRows += pBlock->info.rows; + return pBlock; + } + + doSetOperatorCompleted(pOperator); + return pBlock; } @@ -2445,6 +2443,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN pInfo->dataReaders = taosArrayInit(64, POINTER_BYTES); pInfo->queryId = queryId; pInfo->taskId = taskId; + pInfo->currentGroupId = -1; pInfo->sortSourceParams = taosArrayInit(64, sizeof(STableMergeScanSortSourceParam)); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 1a56eacda8..48b0b1c071 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -3902,6 +3902,7 @@ typedef struct SMergeIntervalAggOperatorInfo { SIntervalAggOperatorInfo intervalAggOperatorInfo; SHashObj* groupIntervalHash; + void* groupIntervalIter; bool hasGroupId; uint64_t groupId; SSDataBlock* prefetchedBlock; @@ -3914,6 +3915,23 @@ void destroyMergeIntervalOperatorInfo(void* param, int32_t numOfOutput) { destroyIntervalOperatorInfo(&miaInfo->intervalAggOperatorInfo, numOfOutput); } +static int32_t finalizeWindowResult(SOperatorInfo* pOperatorInfo, uint64_t tableGroupId, STimeWindow* win, SSDataBlock* pResultBlock) { + SMergeIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info; + SIntervalAggOperatorInfo* iaInfo = &miaInfo->intervalAggOperatorInfo; + SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; + bool ascScan = (iaInfo->order == TSDB_ORDER_ASC); + SExprSupp* pExprSup = &pOperatorInfo->exprSupp; + + SET_RES_WINDOW_KEY(iaInfo->aggSup.keyBuf, &win->skey, TSDB_KEYSIZE, tableGroupId); + SResultRowPosition* p1 = (SResultRowPosition*)taosHashGet(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, + GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE)); + ASSERT(p1 != NULL); + finalizeResultRowIntoResultDataBlock(iaInfo->aggSup.pResultBuf, p1, pExprSup->pCtx, pExprSup->pExprInfo, + pExprSup->numOfExprs, pExprSup->rowEntryInfoOffset, pResultBlock, pTaskInfo); + taosHashRemove(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE)); + return TSDB_CODE_SUCCESS; +} + static int32_t outputPrevIntervalResult(SOperatorInfo* pOperatorInfo, uint64_t tableGroupId, SSDataBlock* pResultBlock, STimeWindow* newWin) { SMergeIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info; @@ -3928,20 +3946,9 @@ static int32_t outputPrevIntervalResult(SOperatorInfo* pOperatorInfo, uint64_t t return 0; } - if (newWin == NULL || (ascScan && newWin->skey > prevWin->ekey || (!ascScan) && newWin->skey < prevWin->ekey)) { - SET_RES_WINDOW_KEY(iaInfo->aggSup.keyBuf, &prevWin->skey, TSDB_KEYSIZE, tableGroupId); - SResultRowPosition* p1 = (SResultRowPosition*)taosHashGet(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, - GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE)); - ASSERT(p1 != NULL); - - finalizeResultRowIntoResultDataBlock(iaInfo->aggSup.pResultBuf, p1, pExprSup->pCtx, pExprSup->pExprInfo, - pExprSup->numOfExprs, pExprSup->rowEntryInfoOffset, pResultBlock, pTaskInfo); - taosHashRemove(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE)); - if (newWin == NULL) { - taosHashRemove(miaInfo->groupIntervalHash, &tableGroupId, sizeof(tableGroupId)); - } else { - taosHashPut(miaInfo->groupIntervalHash, &tableGroupId, sizeof(tableGroupId), newWin, sizeof(STimeWindow)); - } + if ((ascScan && newWin->skey > prevWin->skey || (!ascScan) && newWin->skey < prevWin->skey)) { + finalizeWindowResult(pOperatorInfo, tableGroupId, prevWin, pResultBlock); + taosHashPut(miaInfo->groupIntervalHash, &tableGroupId, sizeof(tableGroupId), newWin, sizeof(STimeWindow)); } return 0; @@ -4090,12 +4097,17 @@ static SSDataBlock* doMergeIntervalAgg(SOperatorInfo* pOperator) { } pRes->info.groupId = miaInfo->groupId; - } else { - void* p = taosHashIterate(miaInfo->groupIntervalHash, NULL); - if (p != NULL) { + } + + if (miaInfo->inputBlocksFinished) { + void* win = taosHashIterate(miaInfo->groupIntervalHash, miaInfo->groupIntervalIter); + if (win != NULL) { + miaInfo->groupIntervalIter = win; + size_t len = 0; - uint64_t* pKey = taosHashGetKey(p, &len); - outputPrevIntervalResult(pOperator, *pKey, pRes, NULL); + uint64_t* pTableGroupId = taosHashGetKey(win, &len); + finalizeWindowResult(pOperator, *pTableGroupId, win, pRes); + pRes->info.groupId = *pTableGroupId; } } @@ -4118,6 +4130,7 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SExprI } miaInfo->groupIntervalHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_NO_LOCK); + miaInfo->groupIntervalIter = NULL; SIntervalAggOperatorInfo* iaInfo = &miaInfo->intervalAggOperatorInfo; diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index 4d2f926c12..0820e884ce 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -202,6 +202,10 @@ bool blockDistSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); int32_t blockDistFunction(SqlFunctionCtx *pCtx); int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); +bool getGroupKeyFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv); +int32_t groupKeyFunction(SqlFunctionCtx* pCtx); +int32_t groupKeyFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); + #ifdef __cplusplus } #endif diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 6516a10795..6744ebe4b9 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1515,6 +1515,16 @@ static bool getBlockDistFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv return true; } +static int32_t translateGroupKey(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + if (1 != LIST_LENGTH(pFunc->pParameterList)) { + return TSDB_CODE_SUCCESS; + } + + SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0); + pFunc->node.resType = ((SExprNode*)pPara)->resType; + return TSDB_CODE_SUCCESS; +} + // clang-format off const SBuiltinFuncDefinition funcMgtBuiltins[] = { { @@ -2499,7 +2509,17 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .type = FUNCTION_TYPE_BLOCK_DIST_INFO, .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_SCAN_PC_FUNC, .translateFunc = translateBlockDistInfoFunc, - } + }, + { + .name = "_group_key", + .type = FUNCTION_TYPE_GROUP_KEY, + .classification = FUNC_MGT_AGG_FUNC, + .translateFunc = translateGroupKey, + .getEnvFunc = getGroupKeyFuncEnv, + .initFunc = functionSetup, + .processFunc = groupKeyFunction, + .finalizeFunc = groupKeyFinalize, + }, }; // clang-format on diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 4ec6432fc8..61f3154811 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -262,6 +262,12 @@ typedef struct SRateInfo { int8_t hasResult; // flag to denote has value } SRateInfo; +typedef struct SGroupKeyInfo{ + bool hasResult; + char data[]; +} SGroupKeyInfo; + + #define SET_VAL(_info, numOfElem, res) \ do { \ if ((numOfElem) <= 0) { \ @@ -2402,6 +2408,12 @@ bool getSelectivityFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { return true; } +bool getGroupKeyFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { + SColumnNode* pNode = (SColumnNode*)nodesListGetNode(pFunc->pParameterList, 0); + pEnv->calcMemSize = sizeof(SGroupKeyInfo) + pNode->node.resType.bytes; + return true; +} + static FORCE_INLINE TSKEY getRowPTs(SColumnInfoData* pTsColInfo, int32_t rowIndex) { if (pTsColInfo == NULL) { return 0; @@ -5349,6 +5361,43 @@ int32_t irateFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { return pResInfo->numOfRes; } +int32_t groupKeyFunction(SqlFunctionCtx* pCtx) { + SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); + SGroupKeyInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo); + + SInputColumnInfoData* pInput = &pCtx->input; + SColumnInfoData* pInputCol = pInput->pData[0]; + + int32_t bytes = pInputCol->info.bytes; + + int32_t startIndex = pInput->startRowIndex; + if (colDataIsNull_s(pInputCol, startIndex)) { + pInfo->hasResult = false; + goto _group_key_over; + } + + pInfo->hasResult = true; + char* data = colDataGetData(pInputCol, startIndex); + memcpy(pInfo->data, data, bytes); + +_group_key_over: + + SET_VAL(pResInfo, 1, 1); + return TSDB_CODE_SUCCESS; +} + +int32_t groupKeyFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { + int32_t slotId = pCtx->pExpr->base.resSchema.slotId; + SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); + + SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); + + SGroupKeyInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo); + colDataAppend(pCol, pBlock->info.rows, pInfo->data, pInfo->hasResult ? false : true); + + return pResInfo->numOfRes; +} + int32_t interpFunction(SqlFunctionCtx* pCtx) { #if 0 int32_t fillType = (int32_t) pCtx->param[2].i64; diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 1858918e4a..514d65d52e 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -2294,6 +2294,7 @@ static const char* jkSubplanType = "SubplanType"; static const char* jkSubplanMsgType = "MsgType"; static const char* jkSubplanLevel = "Level"; static const char* jkSubplanDbFName = "DbFName"; +static const char* jkSubplanUser = "User"; static const char* jkSubplanNodeAddr = "NodeAddr"; static const char* jkSubplanRootNode = "RootNode"; static const char* jkSubplanDataSink = "DataSink"; @@ -2316,6 +2317,9 @@ static int32_t subplanToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddStringToObject(pJson, jkSubplanDbFName, pNode->dbFName); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkSubplanUser, pNode->user); + } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkSubplanNodeAddr, queryNodeAddrToJson, &pNode->execNode); } @@ -2352,6 +2356,9 @@ static int32_t jsonToSubplan(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetStringValue(pJson, jkSubplanDbFName, pNode->dbFName); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkSubplanUser, pNode->user); + } if (TSDB_CODE_SUCCESS == code) { code = tjsonToObject(pJson, jkSubplanNodeAddr, jsonToQueryNodeAddr, &pNode->execNode); } diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 1d8baf5373..227914449a 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -1220,6 +1220,7 @@ int32_t nodesSetValueNodeValue(SValueNode* pNode, void* value) { case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: + case TSDB_DATA_TYPE_JSON: pNode->datum.p = (char*)value; break; default: diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index 2b505d4bf7..56301c072c 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -26,6 +26,7 @@ typedef struct SBlockKeyTuple { TSKEY skey; void* payloadAddr; + int16_t index; } SBlockKeyTuple; typedef struct SBlockKeyInfo { @@ -36,7 +37,6 @@ typedef struct SBlockKeyInfo { static int32_t rowDataCompar(const void* lhs, const void* rhs) { TSKEY left = *(TSKEY*)lhs; TSKEY right = *(TSKEY*)rhs; - if (left == right) { return 0; } else { @@ -44,6 +44,16 @@ static int32_t rowDataCompar(const void* lhs, const void* rhs) { } } +static int32_t rowDataComparStable(const void* lhs, const void* rhs) { + TSKEY left = *(TSKEY*)lhs; + TSKEY right = *(TSKEY*)rhs; + if (left == right) { + return ((SBlockKeyTuple*)lhs)->index - ((SBlockKeyTuple*)rhs)->index; + } else { + return left > right ? 1 : -1; + } +} + void setBoundColumnInfo(SParsedDataColInfo* pColList, SSchema* pSchema, col_id_t numOfCols) { pColList->numOfCols = numOfCols; pColList->numOfBound = numOfCols; @@ -343,6 +353,7 @@ int sortRemoveDataBlockDupRows(STableDataBlocks* dataBuf, SBlockKeyInfo* pBlkKey while (n < nRows) { pBlkKeyTuple->skey = TD_ROW_KEY((STSRow*)pBlockData); pBlkKeyTuple->payloadAddr = pBlockData; + pBlkKeyTuple->index = n; // next loop pBlockData += extendedRowSize; @@ -354,7 +365,7 @@ int sortRemoveDataBlockDupRows(STableDataBlocks* dataBuf, SBlockKeyInfo* pBlkKey pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; // todo. qsort is unstable, if timestamp is same, should get the last one - qsort(pBlkKeyTuple, nRows, sizeof(SBlockKeyTuple), rowDataCompar); + qsort(pBlkKeyTuple, nRows, sizeof(SBlockKeyTuple), rowDataComparStable); pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; int32_t i = 0; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 424968ac80..d4bde5d624 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -4641,7 +4641,7 @@ static int32_t extractShowCreateTableResultSchema(int32_t* numOfCols, SSchema** return TSDB_CODE_SUCCESS; } -static int32_t extractShowLocalVariablesResultSchema(int32_t* numOfCols, SSchema** pSchema) { +static int32_t extractShowVariablesResultSchema(int32_t* numOfCols, SSchema** pSchema) { *numOfCols = 2; *pSchema = taosMemoryCalloc((*numOfCols), sizeof(SSchema)); if (NULL == (*pSchema)) { @@ -4678,7 +4678,8 @@ int32_t extractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pS case QUERY_NODE_SHOW_CREATE_STABLE_STMT: return extractShowCreateTableResultSchema(numOfCols, pSchema); case QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT: - return extractShowLocalVariablesResultSchema(numOfCols, pSchema); + case QUERY_NODE_SHOW_VARIABLES_STMT: + return extractShowVariablesResultSchema(numOfCols, pSchema); default: break; } @@ -5909,7 +5910,6 @@ static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) { case QUERY_NODE_SHOW_CLUSTER_STMT: case QUERY_NODE_SHOW_TOPICS_STMT: case QUERY_NODE_SHOW_TRANSACTIONS_STMT: - case QUERY_NODE_SHOW_VARIABLES_STMT: case QUERY_NODE_SHOW_APPS_STMT: case QUERY_NODE_SHOW_CONSUMERS_STMT: case QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT: @@ -6011,6 +6011,14 @@ static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) { case QUERY_NODE_ALTER_LOCAL_STMT: pQuery->execMode = QUERY_EXEC_MODE_LOCAL; break; + case QUERY_NODE_SHOW_VARIABLES_STMT: + pQuery->haveResultSet = true; + pQuery->execMode = QUERY_EXEC_MODE_RPC; + if (NULL != pCxt->pCmdMsg) { + TSWAP(pQuery->pCmdMsg, pCxt->pCmdMsg); + pQuery->msgType = pQuery->pCmdMsg->msgType; + } + break; default: pQuery->execMode = QUERY_EXEC_MODE_RPC; if (NULL != pCxt->pCmdMsg) { diff --git a/source/libs/parser/test/parTestUtil.h b/source/libs/parser/test/parTestUtil.h index 16c3d05b38..3e5fab5927 100644 --- a/source/libs/parser/test/parTestUtil.h +++ b/source/libs/parser/test/parTestUtil.h @@ -54,12 +54,6 @@ class ParserDdlTest : public ParserTestBase { virtual void checkDdl(const SQuery* pQuery, ParserStage stage) { ASSERT_NE(pQuery, nullptr); ASSERT_NE(pQuery->pRoot, nullptr); - if (QUERY_EXEC_MODE_RPC == pQuery->execMode) { - ASSERT_EQ(pQuery->haveResultSet, false); - ASSERT_EQ(pQuery->numOfResCols, 0); - ASSERT_EQ(pQuery->pResSchema, nullptr); - ASSERT_EQ(pQuery->precision, 0); - } if (nullptr != checkDdl_) { checkDdl_(pQuery, stage); } diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 7862ad5445..10275ffd9d 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1061,6 +1061,7 @@ static EDealRes partTagsOptRebuildTbanmeImpl(SNode** pNode, void* pContext) { } strcpy(pFunc->functionName, "tbname"); pFunc->funcType = FUNCTION_TYPE_TBNAME; + pFunc->node.resType = ((SColumnNode*)*pNode)->node.resType; nodesDestroyNode(*pNode); *pNode = (SNode*)pFunc; return DEAL_RES_IGNORE_CHILD; @@ -1188,7 +1189,7 @@ static const SOptimizeRule optimizeRuleSet[] = { {.pName = "ConditionPushDown", .optimizeFunc = cpdOptimize}, {.pName = "OrderByPrimaryKey", .optimizeFunc = opkOptimize}, {.pName = "SmaIndex", .optimizeFunc = smaOptimize}, - // {.pName = "PartitionTags", .optimizeFunc = partTagsOptimize}, + {.pName = "PartitionTags", .optimizeFunc = partTagsOptimize}, {.pName = "EliminateProject", .optimizeFunc = eliminateProjOptimize} }; // clang-format on diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index efb0f790e1..37b69ffdfe 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -1453,6 +1453,9 @@ static SSubplan* makeSubplan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogicSubpl pSubplan->id = pLogicSubplan->id; pSubplan->subplanType = pLogicSubplan->subplanType; pSubplan->level = pLogicSubplan->level; + if (NULL != pCxt->pPlanCxt->pUser) { + strcpy(pSubplan->user, pCxt->pPlanCxt->pUser); + } return pSubplan; } diff --git a/source/libs/planner/test/CMakeLists.txt b/source/libs/planner/test/CMakeLists.txt index b34ffa7bb1..a21b36fef6 100644 --- a/source/libs/planner/test/CMakeLists.txt +++ b/source/libs/planner/test/CMakeLists.txt @@ -32,9 +32,7 @@ if(${BUILD_WINGETOPT}) target_link_libraries(plannerTest PUBLIC wingetopt) endif() -# if(NOT TD_WINDOWS) - add_test( - NAME plannerTest - COMMAND plannerTest - ) -# endif(NOT TD_WINDOWS) +add_test( + NAME plannerTest + COMMAND plannerTest +) diff --git a/source/libs/planner/test/planTestUtil.cpp b/source/libs/planner/test/planTestUtil.cpp index f1ab9c8196..c197a02dd1 100644 --- a/source/libs/planner/test/planTestUtil.cpp +++ b/source/libs/planner/test/planTestUtil.cpp @@ -85,8 +85,9 @@ class PlannerTestBaseImpl { public: PlannerTestBaseImpl() : sqlNo_(0) {} - void useDb(const string& acctId, const string& db) { - caseEnv_.acctId_ = acctId; + void useDb(const string& user, const string& db) { + caseEnv_.acctId_ = 0; + caseEnv_.user_ = user; caseEnv_.db_ = db; caseEnv_.nsql_ = g_skipSql; } @@ -193,7 +194,8 @@ class PlannerTestBaseImpl { private: struct caseEnv { - string acctId_; + int32_t acctId_; + string user_; string db_; int32_t nsql_; @@ -295,7 +297,7 @@ class PlannerTestBaseImpl { transform(stmtEnv_.sql_.begin(), stmtEnv_.sql_.end(), stmtEnv_.sql_.begin(), ::tolower); SParseContext cxt = {0}; - cxt.acctId = atoi(caseEnv_.acctId_.c_str()); + cxt.acctId = caseEnv_.acctId_; cxt.db = caseEnv_.db_.c_str(); cxt.pSql = stmtEnv_.sql_.c_str(); cxt.sqlLen = stmtEnv_.sql_.length(); @@ -319,12 +321,13 @@ class PlannerTestBaseImpl { void doParseBoundSql(SQuery* pQuery) { SParseContext cxt = {0}; - cxt.acctId = atoi(caseEnv_.acctId_.c_str()); + cxt.acctId = caseEnv_.acctId_; cxt.db = caseEnv_.db_.c_str(); cxt.pSql = stmtEnv_.sql_.c_str(); cxt.sqlLen = stmtEnv_.sql_.length(); cxt.pMsg = stmtEnv_.msgBuf_.data(); cxt.msgLen = stmtEnv_.msgBuf_.max_size(); + cxt.pUser = caseEnv_.user_.c_str(); DO_WITH_THROW(qStmtParseQuerySql, &cxt, pQuery); res_.ast_ = toString(pQuery->pRoot); @@ -364,6 +367,7 @@ class PlannerTestBaseImpl { void setPlanContext(SQuery* pQuery, SPlanContext* pCxt) { pCxt->queryId = 1; + pCxt->pUser = caseEnv_.user_.c_str(); if (QUERY_NODE_CREATE_TOPIC_STMT == nodeType(pQuery->pRoot)) { pCxt->pAstRoot = ((SCreateTopicStmt*)pQuery->pRoot)->pQuery; pCxt->topicQuery = true; @@ -403,7 +407,7 @@ PlannerTestBase::PlannerTestBase() : impl_(new PlannerTestBaseImpl()) {} PlannerTestBase::~PlannerTestBase() {} -void PlannerTestBase::useDb(const std::string& acctId, const std::string& db) { impl_->useDb(acctId, db); } +void PlannerTestBase::useDb(const std::string& user, const std::string& db) { impl_->useDb(user, db); } void PlannerTestBase::run(const std::string& sql) { return impl_->run(sql); } diff --git a/source/libs/planner/test/planTestUtil.h b/source/libs/planner/test/planTestUtil.h index b188a7a054..7d2a9e533f 100644 --- a/source/libs/planner/test/planTestUtil.h +++ b/source/libs/planner/test/planTestUtil.h @@ -30,7 +30,7 @@ class PlannerTestBase : public testing::Test { PlannerTestBase(); virtual ~PlannerTestBase(); - void useDb(const std::string& acctId, const std::string& db); + void useDb(const std::string& user, const std::string& db); void run(const std::string& sql); // stmt mode APIs void prepare(const std::string& sql); diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index df608412b0..b433a88001 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -144,6 +144,23 @@ int32_t queryBuildDnodeListMsg(void *input, char **msg, int32_t msgSize, int32_t return TSDB_CODE_SUCCESS; } +int32_t queryBuildGetSerVerMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void*(*mallcFp)(int32_t)) { + if (NULL == msg || NULL == msgLen) { + return TSDB_CODE_TSC_INVALID_INPUT; + } + + SServerVerReq req = {0}; + + int32_t bufLen = tSerializeSServerVerReq(NULL, 0, &req); + void *pBuf = (*mallcFp)(bufLen); + tSerializeSServerVerReq(pBuf, bufLen, &req); + + *msg = pBuf; + *msgLen = bufLen; + + return TSDB_CODE_SUCCESS; +} + int32_t queryBuildGetDBCfgMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void*(*mallcFp)(int32_t)) { if (NULL == msg || NULL == msgLen) { @@ -467,6 +484,26 @@ int32_t queryProcessDnodeListRsp(void *output, char *msg, int32_t msgSize) { return code; } +int32_t queryProcessGetSerVerRsp(void *output, char *msg, int32_t msgSize) { + SServerVerRsp out = {0}; + int32_t code = 0; + + if (NULL == output || NULL == msg || msgSize <= 0) { + code = TSDB_CODE_TSC_INVALID_INPUT; + return code; + } + + if (tDeserializeSServerVerRsp(msg, msgSize, &out) != 0) { + qError("invalid svr ver rsp msg, msgSize:%d", msgSize); + code = TSDB_CODE_INVALID_MSG; + return code; + } + + *(char**)output = strdup(out.ver); + + return code; +} + int32_t queryProcessGetDbCfgRsp(void *output, char *msg, int32_t msgSize) { SDbCfgRsp out = {0}; @@ -583,6 +620,7 @@ void initQueryModuleMsgHandle() { queryBuildMsg[TMSG_INDEX(TDMT_MND_GET_TABLE_INDEX)] = queryBuildGetTbIndexMsg; queryBuildMsg[TMSG_INDEX(TDMT_VND_TABLE_CFG)] = queryBuildGetTbCfgMsg; queryBuildMsg[TMSG_INDEX(TDMT_MND_TABLE_CFG)] = queryBuildGetTbCfgMsg; + queryBuildMsg[TMSG_INDEX(TDMT_MND_SERVER_VERSION)] = queryBuildGetSerVerMsg; queryProcessMsgRsp[TMSG_INDEX(TDMT_VND_TABLE_META)] = queryProcessTableMetaRsp; queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_TABLE_META)] = queryProcessTableMetaRsp; @@ -596,6 +634,7 @@ void initQueryModuleMsgHandle() { queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_GET_TABLE_INDEX)] = queryProcessGetTbIndexRsp; queryProcessMsgRsp[TMSG_INDEX(TDMT_VND_TABLE_CFG)] = queryProcessGetTbCfgRsp; queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_TABLE_CFG)] = queryProcessGetTbCfgRsp; + queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_SERVER_VERSION)] = queryProcessGetSerVerRsp; } #pragma GCC diagnostic pop diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 11cd089606..a8b18210dd 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -63,27 +63,27 @@ int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq) { } int32_t tEncodeStreamRetrieveReq(SEncoder* pEncoder, const SStreamRetrieveReq* pReq) { - // if (tStartEncode(pEncoder) < 0) return -1; if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1; if (tEncodeI32(pEncoder, pReq->dstNodeId) < 0) return -1; if (tEncodeI32(pEncoder, pReq->dstTaskId) < 0) return -1; if (tEncodeI32(pEncoder, pReq->srcNodeId) < 0) return -1; if (tEncodeI32(pEncoder, pReq->srcTaskId) < 0) return -1; - if (tEncodeBinary(pEncoder, (const uint8_t*)&pReq->pRetrieve, pReq->retrieveLen) < 0) return -1; + if (tEncodeBinary(pEncoder, (const uint8_t*)pReq->pRetrieve, pReq->retrieveLen) < 0) return -1; tEndEncode(pEncoder); return pEncoder->pos; } int32_t tDecodeStreamRetrieveReq(SDecoder* pDecoder, SStreamRetrieveReq* pReq) { - int32_t tlen = 0; if (tStartDecode(pDecoder) < 0) return -1; if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1; if (tDecodeI32(pDecoder, &pReq->dstNodeId) < 0) return -1; if (tDecodeI32(pDecoder, &pReq->dstTaskId) < 0) return -1; if (tDecodeI32(pDecoder, &pReq->srcNodeId) < 0) return -1; if (tDecodeI32(pDecoder, &pReq->srcTaskId) < 0) return -1; - if (tDecodeBinary(pDecoder, (uint8_t**)&pReq->pRetrieve, &pReq->retrieveLen) < 0) return -1; + uint64_t len = 0; + if (tDecodeBinaryAlloc(pDecoder, (void**)&pReq->pRetrieve, &len) < 0) return -1; + pReq->retrieveLen = (int32_t)len; tEndDecode(pDecoder); return 0; } diff --git a/source/libs/sync/src/syncElection.c b/source/libs/sync/src/syncElection.c index e0f2b0fed2..738b17b9cb 100644 --- a/source/libs/sync/src/syncElection.c +++ b/source/libs/sync/src/syncElection.c @@ -105,9 +105,16 @@ int32_t syncNodeElect(SSyncNode* pSyncNode) { } int32_t syncNodeRequestVote(SSyncNode* pSyncNode, const SRaftId* destRaftId, const SyncRequestVote* pMsg) { - sTrace("syncNodeRequestVote pSyncNode:%p ", pSyncNode); int32_t ret = 0; + do { + char host[128]; + uint16_t port; + syncUtilU642Addr(destRaftId->addr, host, sizeof(host), &port); + sDebug("vgId:%d, send sync-request-vote to %s:%d, {term:%lu, last-index:%ld, last-term:%lu}", pSyncNode->vgId, host, + port, pMsg->term, pMsg->lastLogTerm, pMsg->lastLogIndex); + } while (0); + SRpcMsg rpcMsg; syncRequestVote2RpcMsg(pMsg, &rpcMsg); syncNodeSendMsgById(destRaftId, pSyncNode, &rpcMsg); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 42ba2b85b1..c5af72c971 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -370,6 +370,15 @@ bool syncIsReady(int64_t rid) { ASSERT(rid == pSyncNode->rid); bool b = (pSyncNode->state == TAOS_SYNC_STATE_LEADER) && pSyncNode->restoreFinish; taosReleaseRef(tsNodeRefId, pSyncNode->rid); + + // if false, set error code + if (false == b) { + if (pSyncNode->state != TAOS_SYNC_STATE_LEADER) { + terrno = TSDB_CODE_SYN_NOT_LEADER; + } else { + terrno = TSDB_CODE_APP_NOT_READY; + } + } return b; } @@ -480,12 +489,30 @@ void syncGetEpSet(int64_t rid, SEpSet* pEpSet) { snprintf(pEpSet->eps[i].fqdn, sizeof(pEpSet->eps[i].fqdn), "%s", (pSyncNode->pRaftCfg->cfg.nodeInfo)[i].nodeFqdn); pEpSet->eps[i].port = (pSyncNode->pRaftCfg->cfg.nodeInfo)[i].nodePort; (pEpSet->numOfEps)++; - - sInfo("syncGetEpSet index:%d %s:%d", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); + sInfo("vgId:%d sync get epset: index:%d %s:%d", pSyncNode->vgId, i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); } pEpSet->inUse = pSyncNode->pRaftCfg->cfg.myIndex; + sInfo("vgId:%d sync get epset in-use:%d", pSyncNode->vgId, pEpSet->inUse); - sInfo("syncGetEpSet pEpSet->inUse:%d ", pEpSet->inUse); + taosReleaseRef(tsNodeRefId, pSyncNode->rid); +} + +void syncGetRetryEpSet(int64_t rid, SEpSet* pEpSet) { + SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + if (pSyncNode == NULL) { + memset(pEpSet, 0, sizeof(*pEpSet)); + return; + } + ASSERT(rid == pSyncNode->rid); + pEpSet->numOfEps = 0; + for (int i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) { + snprintf(pEpSet->eps[i].fqdn, sizeof(pEpSet->eps[i].fqdn), "%s", (pSyncNode->pRaftCfg->cfg.nodeInfo)[i].nodeFqdn); + pEpSet->eps[i].port = (pSyncNode->pRaftCfg->cfg.nodeInfo)[i].nodePort; + (pEpSet->numOfEps)++; + sInfo("vgId:%d sync get retry epset: index:%d %s:%d", pSyncNode->vgId, i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); + } + pEpSet->inUse = (pSyncNode->pRaftCfg->cfg.myIndex + 1) % pEpSet->numOfEps; + sInfo("vgId:%d sync get retry epset in-use:%d", pSyncNode->vgId, pEpSet->inUse); taosReleaseRef(tsNodeRefId, pSyncNode->rid); } diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index 02cd977e1e..46850758f1 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -224,8 +224,8 @@ int32_t syncNodeAppendEntries(SSyncNode* pSyncNode, const SRaftId* destRaftId, c uint16_t port; syncUtilU642Addr(destRaftId->addr, host, sizeof(host), &port); sDebug( - "vgId:%d, send sync-append-entries to %s:%d, term:%lu, pre-index:%ld, pre-term:%lu, pterm:%lu, commit:%ld, " - "datalen:%d", + "vgId:%d, send sync-append-entries to %s:%d, {term:%lu, pre-index:%ld, pre-term:%lu, pterm:%lu, commit:%ld, " + "datalen:%d}", pSyncNode->vgId, host, port, pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->privateTerm, pMsg->commitIndex, pMsg->dataLen); } while (0); diff --git a/source/libs/sync/src/syncRequestVote.c b/source/libs/sync/src/syncRequestVote.c index def5c321ad..3270b5c0e4 100644 --- a/source/libs/sync/src/syncRequestVote.c +++ b/source/libs/sync/src/syncRequestVote.c @@ -99,15 +99,20 @@ static bool syncNodeOnRequestVoteLogOK(SSyncNode* pSyncNode, SyncRequestVote* pM int32_t syncNodeOnRequestVoteSnapshotCb(SSyncNode* ths, SyncRequestVote* pMsg) { int32_t ret = 0; - // print log - char logBuf[128] = {0}; - snprintf(logBuf, sizeof(logBuf), "recv SyncRequestVote, currentTerm:%lu", ths->pRaftStore->currentTerm); - syncRequestVoteLog2(logBuf, pMsg); - // if already drop replica, do not process if (!syncNodeInRaftGroup(ths, &(pMsg->srcId)) && !ths->pRaftCfg->isStandBy) { - sInfo("recv SyncRequestVote maybe replica already dropped"); - return ret; + do { + char logBuf[256]; + char host[64]; + uint16_t port; + syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); + snprintf(logBuf, sizeof(logBuf), + "recv sync-request-vote from %s:%d, term:%lu, lindex:%ld, lterm:%lu, maybe replica already dropped", + host, port, pMsg->term, pMsg->lastLogIndex, pMsg->lastLogTerm); + syncNodeEventLog(ths, logBuf); + } while (0); + + return -1; } // maybe update term @@ -135,10 +140,22 @@ int32_t syncNodeOnRequestVoteSnapshotCb(SSyncNode* ths, SyncRequestVote* pMsg) { pReply->term = ths->pRaftStore->currentTerm; pReply->voteGranted = grant; + // trace log + do { + char logBuf[256]; + char host[64]; + uint16_t port; + syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); + snprintf(logBuf, sizeof(logBuf), + "recv sync-request-vote from %s:%d, term:%lu, lindex:%ld, lterm:%lu, reply-grant:%d", host, port, + pMsg->term, pMsg->lastLogIndex, pMsg->lastLogTerm, pReply->voteGranted); + syncNodeEventLog(ths, logBuf); + } while (0); + SRpcMsg rpcMsg; syncRequestVoteReply2RpcMsg(pReply, &rpcMsg); syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); syncRequestVoteReplyDestroy(pReply); - return ret; + return 0; } \ No newline at end of file diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 327fe50814..bbd9ce3a78 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -125,6 +125,7 @@ typedef struct { STransMsg* pRsp; // for synchronous API tsem_t* pSem; // for synchronous API SCvtAddr cvtAddr; + bool setMaxRetry; int hThrdIdx; } STransConnCtx; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 0dd00e0dc2..e563bf0ddd 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -980,7 +980,7 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { tTrace("try to send req to next node"); pMsg->st = taosGetTimestampUs(); pCtx->retryCount += 1; - if (pResp->code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { + if (pResp->code == TSDB_CODE_RPC_NETWORK_UNAVAIL && pCtx->setMaxRetry == false) { if (pCtx->retryCount < pEpSet->numOfEps * 3) { pEpSet->inUse = (++pEpSet->inUse) % pEpSet->numOfEps; if (pThrd->quit == false) { @@ -997,6 +997,7 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { } } } else if (pCtx->retryCount < TRANS_RETRY_COUNT_LIMIT) { + pCtx->setMaxRetry = true; if (pResp->contLen == 0) { pEpSet->inUse = (++pEpSet->inUse) % pEpSet->numOfEps; transPrintEpSet(&pCtx->epSet); @@ -1012,8 +1013,10 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { pCtx->retryCount + 1, TRANS_RETRY_COUNT_LIMIT); } if (pThrd->quit == false) { - if (pConn->status != ConnInPool) { - addConnToPool(pThrd->pool, pConn); + if (pResp->code != TSDB_CODE_RPC_NETWORK_UNAVAIL) { + if (pConn->status != ConnInPool) addConnToPool(pThrd->pool, pConn); + } else { + transUnrefCliHandle(pConn); } STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg)); arg->param1 = pMsg; diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index f82f27e3af..1d169a0891 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -23,7 +23,7 @@ int32_t walRestoreFromSnapshot(SWal *pWal, int64_t ver) { void *pIter = NULL; while (1) { - taosHashIterate(pWal->pRefHash, pIter); + pIter = taosHashIterate(pWal->pRefHash, pIter); if (pIter == NULL) break; SWalRef *pRef = (SWalRef *)pIter; if (pRef->ver != -1) { @@ -309,7 +309,7 @@ static int walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) { SWalIdxEntry entry = {.ver = ver, .offset = offset}; /*int64_t idxOffset = taosLSeekFile(pWal->pWriteIdxTFile, 0, SEEK_CUR);*/ /*wDebug("write index: ver: %ld, offset: %ld, at %ld", ver, offset, idxOffset);*/ - int size = taosWriteFile(pWal->pWriteIdxTFile, &entry, sizeof(SWalIdxEntry)); + int64_t size = taosWriteFile(pWal->pWriteIdxTFile, &entry, sizeof(SWalIdxEntry)); if (size != sizeof(SWalIdxEntry)) { terrno = TAOS_SYSTEM_ERROR(errno); // TODO truncate diff --git a/tests/parallel_test/run.sh b/tests/parallel_test/run.sh index e9871637bd..26f481e571 100755 --- a/tests/parallel_test/run.sh +++ b/tests/parallel_test/run.sh @@ -8,11 +8,12 @@ function usage() { echo -e "\t -l log dir" echo -e "\t -e enterprise edition" echo -e "\t -o default timeout value" + echo -e "\t -w log web server" echo -e "\t -h help" } ent=0 -while getopts "m:t:b:l:o:eh" opt; do +while getopts "m:t:b:l:o:w:eh" opt; do case $opt in m) config_file=$OPTARG @@ -32,6 +33,9 @@ while getopts "m:t:b:l:o:eh" opt; do o) timeout_param="-o $OPTARG" ;; + w) + web_server=$OPTARG + ;; h) usage exit 0 @@ -64,10 +68,11 @@ if [ ! -f $t_file ]; then exit 1 fi date_tag=`date +%Y%m%d-%H%M%S` +test_log_dir=${branch}_${date_tag} if [ -z $log_dir ]; then - log_dir="log/${branch}_${date_tag}" + log_dir="log/${test_log_dir}" else - log_dir="$log_dir/${branch}_${date_tag}" + log_dir="$log_dir/${test_log_dir}" fi hosts=() @@ -190,44 +195,54 @@ function run_thread() { # echo "$thread_no $count $cmd" local ret=0 local redo_count=1 + local case_log_file=$log_dir/${case_file}.txt start_time=`date +%s` + local case_index=`flock -x $lock_file -c "sh -c \"echo \\\$(( \\\$( cat $index_file ) + 1 )) | tee $index_file\""` + case_index=`printf "%5d" $case_index` + local case_info=`echo "$line"|cut -d, -f 3,4` while [ ${redo_count} -lt 6 ]; do - if [ -f $log_dir/$case_file.log ]; then - cp $log_dir/$case_file.log $log_dir/$case_file.${redo_count}.redolog + if [ -f $case_log_file ]; then + cp $case_log_file $log_dir/$case_file.${redo_count}.redotxt fi - echo "${hosts[index]}-${thread_no} order:${count}, redo:${redo_count} task:${line}" >$log_dir/$case_file.log - echo -e "\e[33m >>>>> \e[0m ${case_cmd}" - date >>$log_dir/$case_file.log - # $cmd 2>&1 | tee -a $log_dir/$case_file.log + echo "${hosts[index]}-${thread_no} order:${count}, redo:${redo_count} task:${line}" >$case_log_file + local current_time=`date "+%Y-%m-%d %H:%M:%S"` + echo -e "$case_index \e[33m START >>>>> \e[0m ${case_info} \e[33m[$current_time]\e[0m" + echo "$current_time" >>$case_log_file + local real_start_time=`date +%s` + # $cmd 2>&1 | tee -a $case_log_file # ret=${PIPESTATUS[0]} - $cmd >>$log_dir/$case_file.log 2>&1 + $cmd >>$case_log_file 2>&1 ret=$? - echo "${hosts[index]} `date` ret:${ret}" >>$log_dir/$case_file.log + local real_end_time=`date +%s` + local time_elapsed=$(( real_end_time - real_start_time )) + echo "execute time: ${time_elapsed}s" >>$case_log_file + current_time=`date "+%Y-%m-%d %H:%M:%S"` + echo "${hosts[index]} $current_time exit code:${ret}" >>$case_log_file if [ $ret -eq 0 ]; then break fi redo=0 - grep -q "wait too long for taosd start" $log_dir/$case_file.log + grep -q "wait too long for taosd start" $case_log_file if [ $? -eq 0 ]; then redo=1 fi - grep -q "kex_exchange_identification: Connection closed by remote host" $log_dir/$case_file.log + grep -q "kex_exchange_identification: Connection closed by remote host" $case_log_file if [ $? -eq 0 ]; then redo=1 fi - grep -q "ssh_exchange_identification: Connection closed by remote host" $log_dir/$case_file.log + grep -q "ssh_exchange_identification: Connection closed by remote host" $case_log_file if [ $? -eq 0 ]; then redo=1 fi - grep -q "kex_exchange_identification: read: Connection reset by peer" $log_dir/$case_file.log + grep -q "kex_exchange_identification: read: Connection reset by peer" $case_log_file if [ $? -eq 0 ]; then redo=1 fi - grep -q "Database not ready" $log_dir/$case_file.log + grep -q "Database not ready" $case_log_file if [ $? -eq 0 ]; then redo=1 fi - grep -q "Unable to establish connection" $log_dir/$case_file.log + grep -q "Unable to establish connection" $case_log_file if [ $? -eq 0 ]; then redo=1 fi @@ -240,11 +255,18 @@ function run_thread() { redo_count=$(( redo_count + 1 )) done end_time=`date +%s` - echo >>$log_dir/$case_file.log - echo "${hosts[index]} execute time: $(( end_time - start_time ))s" >>$log_dir/$case_file.log + echo >>$case_log_file + total_time=$(( end_time - start_time )) + echo "${hosts[index]} total time: ${total_time}s" >>$case_log_file # echo "$thread_no ${line} DONE" - if [ $ret -ne 0 ]; then - flock -x $lock_file -c "echo \"${hosts[index]} ret:${ret} ${line}\" >>$log_dir/failed.log" + if [ $ret -eq 0 ]; then + echo -e "$case_index \e[34m DONE <<<<< \e[0m ${case_info} \e[34m[${total_time}s]\e[0m \e[32m success\e[0m" + else + if [ ! -z ${web_server} ]; then + flock -x $lock_file -c "echo -e \"${hosts[index]} ret:${ret} ${line}\n ${web_server}/$test_log_dir/${case_file}.txt\" >>${failed_case_file}" + else + flock -x $lock_file -c "echo -e \"${hosts[index]} ret:${ret} ${line}\n log file: ${case_log_file}\" >>${failed_case_file}" + fi mkdir -p $log_dir/${case_file}.coredump local remote_coredump_dir="${workdirs[index]}/tmp/thread_volume/$thread_no/coredump" local scpcmd="sshpass -p ${passwords[index]} scp -o StrictHostKeyChecking=no -r ${usernames[index]}@${hosts[index]}" @@ -253,13 +275,15 @@ function run_thread() { fi cmd="$scpcmd:${remote_coredump_dir}/* $log_dir/${case_file}.coredump/" $cmd # 2>/dev/null - local case_info=`echo "$line"|cut -d, -f 3,4` local corefile=`ls $log_dir/${case_file}.coredump/` - echo -e "$case_info \e[31m failed\e[0m" + echo -e "$case_index \e[34m DONE <<<<< \e[0m ${case_info} \e[34m[${total_time}s]\e[0m \e[31m failed\e[0m" echo "=========================log============================" - cat $log_dir/$case_file.log + cat $case_log_file echo "=====================================================" - echo -e "\e[34m log file: $log_dir/$case_file.log \e[0m" + echo -e "\e[34m log file: $case_log_file \e[0m" + if [ ! -z "${web_server}" ]; then + echo "${web_server}/$test_log_dir/${case_file}.txt" + fi if [ ! -z "$corefile" ]; then echo -e "\e[34m corefiles: $corefile \e[0m" local build_dir=$log_dir/build_${hosts[index]} @@ -325,6 +349,10 @@ mkdir -p $log_dir rm -rf $log_dir/* task_file=$log_dir/$$.task lock_file=$log_dir/$$.lock +index_file=$log_dir/case_index.txt +stat_file=$log_dir/stat.txt +failed_case_file=$log_dir/failed.txt +echo "0" >$index_file i=0 j=0 @@ -350,15 +378,45 @@ rm -f $lock_file rm -f $task_file # docker ps -a|grep -v CONTAINER|awk '{print $1}'|xargs docker rm -f +echo "=====================================================================" +echo "log dir: $log_dir" +total_cases=`cat $index_file` +failed_cases=0 +if [ -f $failed_case_file ]; then + if [ ! -z "$web_server" ]; then + failed_cases=`grep -v "$web_server" $failed_case_file|wc -l` + else + failed_cases=`grep -v "log file:" $failed_case_file|wc -l` + fi +fi +success_cases=$(( total_cases - failed_cases )) +echo "Total Cases: $total_cases" >$stat_file +echo "Successful: $success_cases" >>$stat_file +echo "Failed: $failed_cases" >>$stat_file +cat $stat_file + RET=0 i=1 -if [ -f "$log_dir/failed.log" ]; then +if [ -f "${failed_case_file}" ]; then echo "=====================================================" while read line; do + if [ ! -z "${web_server}" ]; then + echo "$line"|grep -q "${web_server}" + if [ $? -eq 0 ]; then + echo " $line" + continue + fi + else + echo "$line"|grep -q "log file:" + if [ $? -eq 0 ]; then + echo " $line" + continue + fi + fi line=`echo "$line"|cut -d, -f 3,4` echo -e "$i. $line \e[31m failed\e[0m" >&2 i=$(( i + 1 )) - done <$log_dir/failed.log + done <${failed_case_file} RET=1 fi diff --git a/tests/pytest/util/sqlset.py b/tests/pytest/util/sqlset.py index 939f53154a..1897b3cf23 100644 --- a/tests/pytest/util/sqlset.py +++ b/tests/pytest/util/sqlset.py @@ -15,7 +15,7 @@ from util.sql import tdSql class TDSetSql: def init(self, conn, logSql): - tdSql.init(conn.cursor(), logSql) + self.stbname = 'stb' def set_create_normaltable_sql(self, ntbname='ntb', diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index cc28b19de9..a21da2c212 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -2,12 +2,10 @@ #======================b1-start=============== # ---- user -./test.sh -f tsim/user/basic1.sim -./test.sh -f tsim/user/pass_alter.sim -./test.sh -f tsim/user/pass_len.sim -./test.sh -f tsim/user/user_len.sim -./test.sh -f tsim/user/privilege1.sim -./test.sh -f tsim/user/privilege2.sim +./test.sh -f tsim/user/basic.sim +./test.sh -f tsim/user/password.sim +./test.sh -f tsim/user/privilege_db.sim +./test.sh -f tsim/user/privilege_sysinfo.sim ## ---- db ./test.sh -f tsim/db/create_all_options.sim @@ -90,6 +88,7 @@ ./test.sh -f tsim/stream/triggerInterval0.sim # ./test.sh -f tsim/stream/triggerSession0.sim ./test.sh -f tsim/stream/partitionby.sim +./test.sh -f tsim/stream/partitionby1.sim ./test.sh -f tsim/stream/schedSnode.sim ./test.sh -f tsim/stream/windowClose.sim @@ -134,7 +133,7 @@ ./test.sh -f tsim/stable/tag_filter.sim # --- for multi process mode -./test.sh -f tsim/user/basic1.sim -m +./test.sh -f tsim/user/basic.sim -m ./test.sh -f tsim/db/basic3.sim -m ./test.sh -f tsim/db/error1.sim -m ./test.sh -f tsim/insert/backquote.sim -m diff --git a/tests/script/sh/cfg.bat b/tests/script/sh/cfg.bat index 49ab34383a..c25bebd159 100644 --- a/tests/script/sh/cfg.bat +++ b/tests/script/sh/cfg.bat @@ -16,12 +16,12 @@ rem echo NODE: %NODE% set SCRIPT_DIR=%~dp0..\ rem echo SCRIPT_DIR: %SCRIPT_DIR% -set BUILD_DIR=%SCRIPT_DIR%..\..\..\debug\build\bin\ +echo %cd% | grep community > nul && set "BUILD_DIR=%SCRIPT_DIR%..\..\..\debug\build\bin\" || set "BUILD_DIR=%SCRIPT_DIR%..\..\debug\build\bin\" set TSIM=%BUILD_DIR%tsim rem echo BUILD_DIR: %BUILD_DIR% rem echo TSIM: %TSIM% -set SIM_DIR=%SCRIPT_DIR%..\..\..\sim\ +echo %cd% | grep community > nul && set "SIM_DIR=%SCRIPT_DIR%..\..\..\sim\" || set "SIM_DIR=%SCRIPT_DIR%..\..\sim\" rem echo SIM_DIR: %SIM_DIR% set NODE_DIR=%SIM_DIR%%NODE_NAME%\ diff --git a/tests/script/sh/copy_udf.bat b/tests/script/sh/copy_udf.bat index 5144cf8d25..597b77792e 100644 --- a/tests/script/sh/copy_udf.bat +++ b/tests/script/sh/copy_udf.bat @@ -4,7 +4,7 @@ echo Executing copy_udf.bat set SCRIPT_DIR=%cd% echo SCRIPT_DIR: %SCRIPT_DIR% -cd ..\..\.. +echo %cd% | grep community > nul && cd ..\..\.. || cd ..\.. set TAOS_DIR=%cd% echo find udf library in %TAOS_DIR% set UDF1_DIR=%TAOS_DIR%\debug\build\lib\udf1.dll diff --git a/tests/script/sh/deploy.bat b/tests/script/sh/deploy.bat index f0d8c10b7e..4b221dd479 100644 --- a/tests/script/sh/deploy.bat +++ b/tests/script/sh/deploy.bat @@ -13,12 +13,12 @@ rem echo NODE: %NODE% set SCRIPT_DIR=%~dp0..\ rem echo SCRIPT_DIR: %SCRIPT_DIR% -set BUILD_DIR=%SCRIPT_DIR%..\..\..\debug\build\bin\ +echo %cd% | grep community > nul && set "BUILD_DIR=%SCRIPT_DIR%..\..\..\debug\build\bin\" || set "BUILD_DIR=%SCRIPT_DIR%..\..\debug\build\bin\" set TSIM=%BUILD_DIR%tsim rem echo BUILD_DIR: %BUILD_DIR% rem echo TSIM: %TSIM% -set SIM_DIR=%SCRIPT_DIR%..\..\..\sim\ +echo %cd% | grep community > nul && set "SIM_DIR=%SCRIPT_DIR%..\..\..\sim\" || set "SIM_DIR=%SCRIPT_DIR%..\..\sim\" rem echo SIM_DIR: %SIM_DIR% set NODE_DIR=%SIM_DIR%%NODE_NAME%\ diff --git a/tests/script/sh/exec.bat b/tests/script/sh/exec.bat index 7a2c58fc66..0f3d0dcf49 100644 --- a/tests/script/sh/exec.bat +++ b/tests/script/sh/exec.bat @@ -13,12 +13,12 @@ rem echo NODE: %EXEC_OPTON% set SCRIPT_DIR=%~dp0..\ rem echo SCRIPT_DIR: %SCRIPT_DIR% -set BUILD_DIR=%SCRIPT_DIR%..\..\..\debug\build\bin\ +echo %cd% | grep community > nul && set "BUILD_DIR=%SCRIPT_DIR%..\..\..\debug\build\bin\" || set "BUILD_DIR=%SCRIPT_DIR%..\..\debug\build\bin\" set TAOSD=%BUILD_DIR%taosd rem echo BUILD_DIR: %BUILD_DIR% rem echo TAOSD: %TAOSD% -set SIM_DIR=%SCRIPT_DIR%..\..\..\sim\ +echo %cd% | grep community > nul && set "SIM_DIR=%SCRIPT_DIR%..\..\..\sim\" || set "SIM_DIR=%SCRIPT_DIR%..\..\sim\" rem echo SIM_DIR: %SIM_DIR% set NODE_DIR=%SIM_DIR%%NODE_NAME%\ diff --git a/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim index 788a23991a..9b11e16616 100644 --- a/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim +++ b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim @@ -112,7 +112,7 @@ if $data(5)[4] != ready then goto step2 endi -print =============== step31: move follower +print =============== step3: create database $leaderExist = 0 $leaderVnode = 0 $follower1 = 0 @@ -165,7 +165,7 @@ if $rows != 1 then return -1 endi -print =============== step32: move follower2 +print =============== step31: print redistribute vgroup 2 dnode $leaderVnode dnode $follower2 dnode 5 sql redistribute vgroup 2 dnode $leaderVnode dnode $follower2 dnode 5 sql show d1.vgroups @@ -178,7 +178,7 @@ endi return -print =============== step33: move follower1 +print =============== step32: print redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 5 sql redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 5 sql show d1.vgroups @@ -189,7 +189,7 @@ if $rows != 1 then return -1 endi -print =============== step34: move follower2 +print =============== step33: print redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode $follower2 sql redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode $follower2 sql show d1.vgroups @@ -200,7 +200,7 @@ if $rows != 1 then return -1 endi -print =============== step35: move follower1 +print =============== step34: print redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode $follower1 sql redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode $follower1 sql show d1.vgroups @@ -211,9 +211,9 @@ if $rows != 1 then return -1 endi -print =============== step36: move follower2 -print redistribute vgroup 2 dnode $leaderVnode dnode $follower2 dnode 5 -sql redistribute vgroup 2 dnode $leaderVnode dnode $follower2 dnode 5 +print =============== step34: +print redistribute vgroup 2 dnode $leaderVnode dnode $follower2 dnode $follower1 +sql redistribute vgroup 2 dnode $leaderVnode dnode $follower2 dnode $follower1 sql show d1.vgroups print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 @@ -222,7 +222,7 @@ if $rows != 1 then return -1 endi -print =============== step37: move follower1 +print =============== step36: print redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 5 sql redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 5 sql show d1.vgroups @@ -233,7 +233,7 @@ if $rows != 1 then return -1 endi -print =============== step38: move follower2 +print =============== step37: print redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode $follower2 sql redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode $follower2 sql show d1.vgroups @@ -244,7 +244,7 @@ if $rows != 1 then return -1 endi -print =============== step39: move follower1 +print =============== step38: print redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode $follower1 sql redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode $follower1 sql show d1.vgroups @@ -255,6 +255,17 @@ if $rows != 1 then return -1 endi +print =============== step39: +print redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode $follower2 +sql redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode $follower2 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT diff --git a/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim index e9dd82cad9..947482b3d4 100644 --- a/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim +++ b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim @@ -91,7 +91,7 @@ if $data(5)[4] != ready then goto step2 endi -print =============== step31: move follower +print =============== step3: create database $leaderExist = 0 $leaderVnode = 0 $follower1 = 0 @@ -144,7 +144,7 @@ if $rows != 1 then return -1 endi -print =============== step32: move leader +print =============== step40: move leader print redistribute vgroup 2 dnode $follower1 dnode $follower2 dnode 5 sql redistribute vgroup 2 dnode $follower1 dnode $follower2 dnode 5 sql show d1.vgroups @@ -155,9 +155,7 @@ if $rows != 1 then return -1 endi -return - -print =============== step33: move follower1 +print =============== step41: print redistribute vgroup 2 dnode $leaderVnode dnode $follower2 dnode 5 sql redistribute vgroup 2 dnode $leaderVnode dnode $follower2 dnode 5 sql show d1.vgroups @@ -168,7 +166,7 @@ if $rows != 1 then return -1 endi -print =============== step34: move follower2 +print =============== step42: print redistribute vgroup 2 dnode $follower1 dnode 5 dnode $leaderVnode sql redistribute vgroup 2 dnode $follower1 dnode 5 dnode $leaderVnode sql show d1.vgroups @@ -179,7 +177,18 @@ if $rows != 1 then return -1 endi -print =============== step35: move leader +print =============== step43: +print redistribute vgroup 2 dnode $follower1 dnode $follower2 dnode $leaderVnode +sql redistribute vgroup 2 dnode $follower1 dnode $follower2 dnode $leaderVnode +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step44: print redistribute vgroup 2 dnode $follower2 dnode 5 dnode $follower1 sql redistribute vgroup 2 dnode $follower2 dnode 5 dnode $follower1 sql show d1.vgroups @@ -190,7 +199,7 @@ if $rows != 1 then return -1 endi -print =============== step36: move follower1 +print =============== step45: print redistribute vgroup 2 dnode $leaderVnode dnode $follower2 dnode 5 sql redistribute vgroup 2 dnode $leaderVnode dnode $follower2 dnode 5 sql show d1.vgroups @@ -201,7 +210,7 @@ if $rows != 1 then return -1 endi -print =============== step37: move follower2 +print =============== step46: print redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 5 sql redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 5 sql show d1.vgroups @@ -212,7 +221,7 @@ if $rows != 1 then return -1 endi -print =============== step38: move leader +print =============== step47: print redistribute vgroup 2 dnode $follower1 dnode 5 dnode $follower2 sql redistribute vgroup 2 dnode $follower1 dnode 5 dnode $follower2 sql show d1.vgroups @@ -223,7 +232,18 @@ if $rows != 1 then return -1 endi -print =============== step39: move follower1 +print =============== step48: +print redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode $follower2 +sql redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode $follower2 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step49: print redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode $follower2 sql redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode $follower2 sql show d1.vgroups diff --git a/tests/script/tsim/mnode/basic5.sim b/tests/script/tsim/mnode/basic5.sim index 23f5f6d782..c017d7f23f 100644 --- a/tests/script/tsim/mnode/basic5.sim +++ b/tests/script/tsim/mnode/basic5.sim @@ -157,7 +157,7 @@ step61: if $x == 10 then return -1 endi -sql show mnodes +sql show mnodes -x step61 print ===> $data00 $data01 $data02 $data03 $data04 $data05 print ===> $data10 $data11 $data12 $data13 $data14 $data15 print ===> $data20 $data21 $data22 $data23 $data24 $data25 diff --git a/tests/script/tsim/stream/partitionby1.sim b/tests/script/tsim/stream/partitionby1.sim new file mode 100644 index 0000000000..3e823b05d6 --- /dev/null +++ b/tests/script/tsim/stream/partitionby1.sim @@ -0,0 +1,124 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sleep 50 +sql connect + +sql create database test vgroups 4; +sql use test; +sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int); +sql create table ts1 using st tags(1,1,1); +sql create table ts2 using st tags(2,2,2); +sql create table ts3 using st tags(3,2,2); +sql create table ts4 using st tags(4,2,2); +sql create stream stream_t1 trigger at_once into streamtST1 as select _wstartts, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from st partition by tbname interval(10s); + +sql insert into ts1 values(1648791213001,1,12,3,1.0); +sql insert into ts2 values(1648791213001,1,12,3,1.0); + +sql insert into ts3 values(1648791213001,1,12,3,1.0); +sql insert into ts4 values(1648791213001,1,12,3,1.0); +$loop_count = 0 + +loop0: +sleep 300 +sql select * from streamtST1; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 4 then +print =====rows=$rows +goto loop0 +endi + +print =====loop0 + +sql create database test1 vgroups 1; +sql use test1; +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table ts1 using st tags(1,2,3); +sql create table ts2 using st tags(1,3,4); +sql create table ts3 using st tags(1,4,5); + +sql create stream streams1 trigger at_once into streamt as select _wstartts, count(*) c1, count(a) c2 from st partition by tbname interval(10s); + + +sql insert into ts1 values(1648791211000,1,2,3); + +sql insert into ts2 values(1648791211000,1,2,3); + +$loop_count = 0 + +loop1: +sleep 300 +sql select * from streamt; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 2 then +print =====rows=$rows +goto loop1 +endi + +print =====loop1 + +sql create database test2 vgroups 1; +sql use test2; +sql create stable st(ts timestamp,a int,b int,c int,id int) tags(ta int,tb int,tc int); +sql create table ts1 using st tags(1,1,1); +sql create table ts2 using st tags(2,2,2); + +sql create stream stream_t2 trigger at_once into streamtST as select _wstartts, count(*) c1, count(a) c2 , sum(a) c3 , max(b) c5, min(c) c6, max(id) c7 from st partition by tbname interval(10s) ; +sql insert into ts1 values(1648791211000,1,2,3,1); +sql insert into ts1 values(1648791222001,2,2,3,2); +sql insert into ts2 values(1648791211000,1,2,3,3); +sql insert into ts2 values(1648791222001,2,2,3,4); + +sql insert into ts2 values(1648791222002,2,2,3,5); +sql insert into ts2 values(1648791222002,2,2,3,6); + +sql insert into ts1 values(1648791211000,1,2,3,1); +sql insert into ts1 values(1648791222001,2,2,3,2); +sql insert into ts2 values(1648791211000,1,2,3,3); +sql insert into ts2 values(1648791222001,2,2,3,4); + +$loop_count = 0 + +loop2: +sleep 300 +sql select * from streamtST; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 1 then +print =====data01=$data01 +goto loop2 +endi + +if $data02 != 1 then +print =====data02=$data02 +goto loop2 +endi + +if $data03 != 1 then +print =====data03=$data03 +goto loop2 +endi + +if $data04 != 2 then +print =====data04=$data04 +goto loop2 +endi + +print =====loop2 + +system sh/stop_dnodes.sh diff --git a/tests/script/tsim/sync/3Replica1VgElect.sim b/tests/script/tsim/sync/3Replica1VgElect.sim index e531fa82ad..6649b9c335 100644 --- a/tests/script/tsim/sync/3Replica1VgElect.sim +++ b/tests/script/tsim/sync/3Replica1VgElect.sim @@ -197,22 +197,22 @@ print $switch_loop_cnt if $switch_loop_cnt == 1 then sql show vgroups - $dnodeId = $data[1][3] + $dnodeId = $data[0][3] $dnodeId = dnode . $dnodeId goto switch_leader_to_offine_loop elif $switch_loop_cnt == 2 then sql show vgroups - $dnodeId = $data[2][3] + $dnodeId = $data[0][3] $dnodeId = dnode . $dnodeId goto switch_leader_to_offine_loop elif $switch_loop_cnt == 3 then sql show vgroups - $dnodeId = $data[3][3] + $dnodeId = $data[0][3] $dnodeId = dnode . $dnodeId goto switch_leader_to_offine_loop elif $switch_loop_cnt == 4 then sql show vgroups - $dnodeId = $data[4][3] + $dnodeId = $data[0][3] $dnodeId = dnode . $dnodeId goto switch_leader_to_offine_loop else diff --git a/tests/script/tsim/sync/3Replica5VgElect.sim b/tests/script/tsim/sync/3Replica5VgElect.sim index 1243cf8124..d96ad1655d 100644 --- a/tests/script/tsim/sync/3Replica5VgElect.sim +++ b/tests/script/tsim/sync/3Replica5VgElect.sim @@ -290,22 +290,22 @@ print $switch_loop_cnt if $switch_loop_cnt == 1 then sql show vgroups - $dnodeId = $data[1][3] + $dnodeId = $data[0][3] $dnodeId = dnode . $dnodeId goto switch_leader_to_offine_loop elif $switch_loop_cnt == 2 then sql show vgroups - $dnodeId = $data[2][3] + $dnodeId = $data[0][3] $dnodeId = dnode . $dnodeId goto switch_leader_to_offine_loop elif $switch_loop_cnt == 3 then sql show vgroups - $dnodeId = $data[3][3] + $dnodeId = $data[0][3] $dnodeId = dnode . $dnodeId goto switch_leader_to_offine_loop elif $switch_loop_cnt == 4 then sql show vgroups - $dnodeId = $data[4][3] + $dnodeId = $data[0][3] $dnodeId = dnode . $dnodeId goto switch_leader_to_offine_loop else diff --git a/tests/script/tsim/sync/oneReplica5VgElect.sim b/tests/script/tsim/sync/oneReplica5VgElect.sim index 5af48c7491..71c26e17c7 100644 --- a/tests/script/tsim/sync/oneReplica5VgElect.sim +++ b/tests/script/tsim/sync/oneReplica5VgElect.sim @@ -227,22 +227,22 @@ print $switch_loop_cnt if $switch_loop_cnt == 1 then sql show vgroups - $dnodeId = $data[1][3] + $dnodeId = $data[0][3] $dnodeId = dnode . $dnodeId goto switch_leader_to_offine_loop elif $switch_loop_cnt == 2 then sql show vgroups - $dnodeId = $data[2][3] + $dnodeId = $data[0][3] $dnodeId = dnode . $dnodeId goto switch_leader_to_offine_loop elif $switch_loop_cnt == 3 then sql show vgroups - $dnodeId = $data[3][3] + $dnodeId = $data[0][3] $dnodeId = dnode . $dnodeId goto switch_leader_to_offine_loop elif $switch_loop_cnt == 4 then sql show vgroups - $dnodeId = $data[4][3] + $dnodeId = $data[0][3] $dnodeId = dnode . $dnodeId goto switch_leader_to_offine_loop else diff --git a/tests/script/tsim/tmq/consume.bat b/tests/script/tsim/tmq/consume.bat new file mode 100644 index 0000000000..7698ebc865 --- /dev/null +++ b/tests/script/tsim/tmq/consume.bat @@ -0,0 +1,67 @@ + +@echo off + +set EXEC_OPTON=start +set DB_NAME=db +set CDB_NAME=db +set /a POLL_DELAY=5 +set /a VALGRIND=0 +set SIGNAL=SIGINT +set /a SHOW_MSG=0 +set /a SHOW_ROW=0 +set /a EXP_USE_SNAPSHOT=0 + +:param +if "%1"=="" ( + goto :end +) +if %1 == -d ( set "DB_NAME=%2" && shift && shift && goto :param ) +if %1 == -g ( set "SHOW_MSG=%2" && shift && shift && goto :param ) +if %1 == -r ( set "SHOW_ROW=%2" && shift && shift && goto :param ) +if %1 == -s ( set "EXEC_OPTON=%2" && shift && shift && goto :param ) +if %1 == -v ( set "VALGRIND=1" && shift && goto :param ) +if %1 == -y ( set "POLL_DELAY=%2" && shift && shift && goto :param ) +if %1 == -x ( set "SIGNAL=%2" && shift && shift && goto :param ) +if %1 == -w ( set "CDB_NAME=%2" && shift && shift && goto :param ) +if %1 == -e ( set "EXP_USE_SNAPSHOT=%2" && shift && shift && goto :param ) +echo unkown argument %1 +goto :eof +:end + +echo EXEC_OPTON %EXEC_OPTON% +echo DB_NAME %DB_NAME% +echo CDB_NAME %CDB_NAME% +echo POLL_DELAY %POLL_DELAY% +echo VALGRIND %VALGRIND% +echo SIGNAL %SIGNAL% +echo SHOW_MSG %SHOW_MSG% +echo SHOW_ROW %SHOW_ROW% +echo EXP_USE_SNAPSHOT %EXP_USE_SNAPSHOT% + +echo %cd% | grep community > nul && cd ..\..\.. || cd ..\.. +set BUILD_DIR=%cd%\debug\build\bin +set SIM_DIR=%cd%\sim +set PRG_DIR=%SIM_DIR%\tsim +set CFG_DIR=%PRG_DIR%\cfg +set LOG_DIR=%PRG_DIR%\log +set PROGRAM=%BUILD_DIR%\tmq_sim.exe + +echo ------------------------------------------------------------------------ +echo BUILD_DIR : %BUILD_DIR% +echo SIM_DIR : %SIM_DIR% +echo CFG_DIR : %CFG_DIR% +echo PRG_DIR : %PRG_DIR% +echo CFG_DIR : %CFG_DIR% +echo LOG_DIR : %LOG_DIR% +echo PROGRAM : %PROGRAM% +echo POLL_DELAY : %POLL_DELAY% +echo DB_NAME : %DB_NAME% +echo ------------------------------------------------------------------------ + +if "%EXEC_OPTON%" == "start" ( + echo mintty -h never %PROGRAM% -c %CFG_DIR% -y %POLL_DELAY% -d %DB_NAME% -g %SHOW_MSG% -r %SHOW_ROW% -w %CDB_NAME% -e %EXP_USE_SNAPSHOT% + mintty -h never %PROGRAM% -c %CFG_DIR% -y %POLL_DELAY% -d %DB_NAME% -g %SHOW_MSG% -r %SHOW_ROW% -w %CDB_NAME% -e %EXP_USE_SNAPSHOT% +) else ( + if "%SIGNAL%" == "SIGKILL" ( ps | grep tmq_sim | awk '{print $2}' | xargs kill -9 ) else ( ps | grep tmq_sim | awk '{print $2}' | xargs kill -SIGINT ) +) +goto :eof diff --git a/tests/script/tsim/user/basic.sim b/tests/script/tsim/user/basic.sim new file mode 100644 index 0000000000..85d5f8375e --- /dev/null +++ b/tests/script/tsim/user/basic.sim @@ -0,0 +1,157 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +print =============== step0 +sql show users +if $data(root)[1] != 1 then + return -1 +endi +if $data(root)[2] != 1 then + return -1 +endi +if $data(root)[3] != 1 then + return -1 +endi + +sql alter user root pass 'taosdata' + +sql_error ALTER USER root SYSINFO 0 +sql_error ALTER USER root SYSINFO 1 +sql_error ALTER USER root enable 0 +sql_error ALTER USER root enable 1 + +sql_error create database db vgroups 1; +sql_error GRANT read ON db.* to root; +sql_error GRANT read ON *.* to root; +sql_error REVOKE read ON db.* from root; +sql_error REVOKE read ON *.* from root; +sql_error GRANT write ON db.* to root; +sql_error GRANT write ON *.* to root; +sql_error REVOKE write ON db.* from root; +sql_error REVOKE write ON *.* from root; +sql_error REVOKE write ON *.* from root; + +sql_error GRANT all ON *.* to root; +sql_error REVOKE all ON *.* from root; +sql_error GRANT read,write ON *.* to root; +sql_error REVOKE read,write ON *.* from root; + +print =============== step1: sysinfo create +sql CREATE USER u1 PASS 'taosdata' SYSINFO 0; +sql show users +if $rows != 2 then + return -1 +endi +if $data(u1)[1] != 0 then + return -1 +endi +if $data(u1)[2] != 1 then + return -1 +endi +if $data(u1)[3] != 0 then + return -1 +endi + +sql CREATE USER u2 PASS 'taosdata' SYSINFO 1; +sql show users +if $rows != 3 then + return -1 +endi +if $data(u2)[1] != 0 then + return -1 +endi +if $data(u2)[2] != 1 then + return -1 +endi +if $data(u2)[3] != 1 then + return -1 +endi + +print =============== step2: sysinfo alter +sql ALTER USER u1 SYSINFO 1 +sql show users +if $data(u1)[1] != 0 then + return -1 +endi +if $data(u1)[2] != 1 then + return -1 +endi +if $data(u1)[3] != 1 then + return -1 +endi + +sql ALTER USER u1 SYSINFO 0 +sql show users +if $data(u1)[1] != 0 then + return -1 +endi +if $data(u1)[2] != 1 then + return -1 +endi +if $data(u1)[3] != 0 then + return -1 +endi + +sql ALTER USER u1 SYSINFO 0 +sql ALTER USER u1 SYSINFO 0 + +sql drop user u1 +sql show users +if $rows != 2 then + return -1 +endi + +print =============== step3: enable alter +sql ALTER USER u2 enable 0 +sql show users +if $rows != 2 then + return -1 +endi +if $data(u2)[1] != 0 then + return -1 +endi +if $data(u2)[2] != 0 then + return -1 +endi +if $data(u2)[3] != 1 then + return -1 +endi + +sql ALTER USER u2 enable 1 +sql show users +if $data(u2)[1] != 0 then + return -1 +endi +if $data(u2)[2] != 1 then + return -1 +endi +if $data(u2)[3] != 1 then + return -1 +endi + +sql ALTER USER u2 enable 1 +sql ALTER USER u2 enable 1 + +print =============== restart taosd +system sh/exec.sh -n dnode1 -s stop +system sh/exec.sh -n dnode1 -s start + +print =============== step4: enable privilege +sql show users +if $rows != 2 then + return -1 +endi +if $data(u2)[1] != 0 then + return -1 +endi +if $data(u2)[2] != 1 then + return -1 +endi +if $data(u2)[3] != 1 then + return -1 +endi + + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/user/basic1.sim b/tests/script/tsim/user/basic1.sim deleted file mode 100644 index 06a52c6604..0000000000 --- a/tests/script/tsim/user/basic1.sim +++ /dev/null @@ -1,74 +0,0 @@ -system sh/stop_dnodes.sh -system sh/deploy.sh -n dnode1 -i 1 -system sh/exec.sh -n dnode1 -s start -sql connect - -print =============== show users -sql show users -if $rows != 1 then - return -1 -endi - -print $data[0][0] $data[0][1] $data[0][2] -print $data[1][0] $data[1][1] $data[1][2] -print $data[2][0] $data[1][2] $data[2][2] - -sql_error show accounts; -sql_error create account a pass "a" -sql_error drop account a -sql_error drop account root - -print =============== create user1 -sql create user user1 PASS 'user1' -sql show users -if $rows != 2 then - return -1 -endi - -print $data[0][0] $data[0][1] $data[0][2] -print $data[1][0] $data[1][1] $data[1][2] -print $data[2][0] $data[1][2] $data[2][2] -print $data[3][0] $data[3][1] $data[3][2] - -print =============== create user2 -sql create user user2 PASS 'user2' -sql show users -if $rows != 3 then - return -1 -endi - -print $data[0][0] $data[0][1] $data[0][2] -print $data[1][0] $data[1][1] $data[1][2] -print $data[2][0] $data[1][2] $data[2][2] -print $data[3][0] $data[3][1] $data[3][2] -print $data40 $data41 $data42 - -print =============== drop user1 -sql drop user user1 -sql show users -if $rows != 2 then - return -1 -endi - -print $data[0][0] $data[0][1] $data[0][2] -print $data[1][0] $data[1][1] $data[1][2] -print $data[2][0] $data[1][2] $data[2][2] -print $data[3][0] $data[3][1] $data[3][2] - -print =============== restart taosd -system sh/exec.sh -n dnode1 -s stop -sleep 1000 -system sh/exec.sh -n dnode1 -s start - -print =============== show users -sql show users -if $rows != 2 then - return -1 -endi - -print $data[0][0] $data[0][1] $data[0][2] -print $data[1][0] $data[1][1] $data[1][2] -print $data[2][0] $data[1][2] $data[2][2] -print $data[3][0] $data[3][1] $data[3][2] - -system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/user/pass_alter.sim b/tests/script/tsim/user/pass_alter.sim deleted file mode 100644 index 33fc9e51bd..0000000000 --- a/tests/script/tsim/user/pass_alter.sim +++ /dev/null @@ -1,66 +0,0 @@ -system sh/stop_dnodes.sh -system sh/deploy.sh -n dnode1 -i 1 -system sh/exec.sh -n dnode1 -s start -sql connect - -print ============= step1 -sql create user u_read pass 'taosdata1' -sql create user u_write pass 'taosdata1' - -sql alter user u_read pass 'taosdata' -sql alter user u_write pass 'taosdata' - -sql show users -if $rows != 3 then - return -1 -endi - -print ============= step2 -sql close -sleep 2500 -print user u_read login -sql connect u_read -sql alter user u_read pass 'taosdata' -sql alter user u_write pass 'taosdata1' -x step2 - return -1 -step2: - -sql_error create user read1 pass 'taosdata1' -sql_error create user write1 pass 'taosdata1' - -sql show users -if $rows != 3 then - return -1 -endi - -print ============= step3 -sql close -sleep 2500 -print user u_write login -sql connect u_write - -sql_error create user read2 pass 'taosdata1' -sql_error create user write2 pass 'taosdata1' -sql alter user u_write pass 'taosdata' -sql alter user u_read pass 'taosdata' -x step3 - return -1 -step3: - -sql show users -if $rows != 3 then - return -1 -endi - -print ============= step4 -sql close -sleep 2500 -print user root login -sql connect -sql create user oroot pass 'taosdata' - -sql show users -if $rows != 4 then - return -1 -endi - -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/user/pass_len.sim b/tests/script/tsim/user/pass_len.sim deleted file mode 100644 index 66c378c6cb..0000000000 --- a/tests/script/tsim/user/pass_len.sim +++ /dev/null @@ -1,79 +0,0 @@ -system sh/stop_dnodes.sh -system sh/deploy.sh -n dnode1 -i 1 -system sh/exec.sh -n dnode1 -s start -sql connect - -$i = 0 -$dbPrefix = apdb -$tbPrefix = aptb -$db = $dbPrefix . $i -$tb = $tbPrefix . $i -$userPrefix = apusr - -print =============== step1 -$i = 0 -$user = $userPrefix . $i - -sql drop user $user -x step11 - return -1 -step11: - -sql create user $user PASS -x step12 - return -1 -step12: - -sql create user $user PASS 'taosdata' - -sql show users -if $rows != 2 then - return -1 -endi - -print =============== step2 -$i = 1 -$user = $userPrefix . $i -sql drop user $user -x step2 -step2: -sql create user $user PASS '1' -sql show users -if $rows != 3 then - return -1 -endi - -print =============== step3 -$i = 2 -$user = $userPrefix . $i -sql drop user $user -x step3 -step3: - -sql create user $user PASS 'abc0123456789' -sql show users -if $rows != 4 then - return -1 -endi - -print =============== step4 -$i = 3 -$user = $userPrefix . $i -sql create user $user PASS 'abcd012345678901234567891234567890abcd012345678901234567891234567890abcd012345678901234567891234567890abcd012345678901234567891234567890123' -x step4 - return -1 - -step4: -sql show users -if $rows != 4 then - return -1 -endi - -$i = 0 -while $i < 3 - $user = $userPrefix . $i - sql drop user $user - $i = $i + 1 -endw - -sql show users -if $rows != 1 then - return -1 -endi - -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/user/password.sim b/tests/script/tsim/user/password.sim new file mode 100644 index 0000000000..d26b9dbc2e --- /dev/null +++ b/tests/script/tsim/user/password.sim @@ -0,0 +1,87 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +print ============= step1 +sql create user u_read pass 'taosdata1' +sql create user u_write pass 'taosdata1' + +sql alter user u_read pass 'taosdata' +sql alter user u_write pass 'taosdata' + +sql show users +if $rows != 3 then + return -1 +endi + +print ============= step2 +print user u_read login +sql close +sql connect u_read + +sql alter user u_read pass 'taosdata' +sql_error alter user u_write pass 'taosdata1' + +sql_error create user read1 pass 'taosdata1' +sql_error create user write1 pass 'taosdata1' + +sql show users +if $rows != 3 then + return -1 +endi + +print ============= step3 +print user u_write login +sql close +sql connect u_write + +sql_error create user read2 pass 'taosdata1' +sql_error create user write2 pass 'taosdata1' +sql alter user u_write pass 'taosdata' +sql_error alter user u_read pass 'taosdata' + +sql show users +if $rows != 3 then + return -1 +endi + +print ============= step4 +print user root login +sql close +sql connect +sql create user oroot pass 'taosdata' +sql_error create user $user PASS 'abcd012345678901234567891234567890abcd012345678901234567891234567890abcd012345678901234567891234567890abcd012345678901234567891234567890123' +sql_error create userabcd012345678901234567891234567890abcd01234567890123456789123456789 PASS 'taosdata' +sql_error create user abcd0123456789012345678901234567890111 PASS '123' +sql create user abc01234567890123456789 PASS '123' + +sql show users +if $rows != 5 then + return -1 +endi + +print ============= step5 +sql create database db vgroups 1 +sql_error ALTER USER o_root SYSINFO 0 +sql_error ALTER USER o_root SYSINFO 1 +sql_error ALTER USER o_root enable 0 +sql_error ALTER USER o_root enable 1 + +sql_error create database db vgroups 1; +sql_error GRANT read ON db.* to o_root; +sql_error GRANT read ON *.* to o_root; +sql_error REVOKE read ON db.* from o_root; +sql_error REVOKE read ON *.* from o_root; +sql_error GRANT write ON db.* to o_root; +sql_error GRANT write ON *.* to o_root; +sql_error REVOKE write ON db.* from o_root; +sql_error REVOKE write ON *.* from o_root; +sql_error REVOKE write ON *.* from o_root; + +sql_error GRANT all ON *.* to o_root; +sql_error REVOKE all ON *.* from o_root; +sql_error GRANT read,write ON *.* to o_root; +sql_error REVOKE read,write ON *.* from o_root; + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/user/privilege2.sim b/tests/script/tsim/user/privilege2.sim deleted file mode 100644 index 470f167c50..0000000000 --- a/tests/script/tsim/user/privilege2.sim +++ /dev/null @@ -1,38 +0,0 @@ -system sh/stop_dnodes.sh -system sh/deploy.sh -n dnode1 -i 1 -system sh/exec.sh -n dnode1 -s start -sql connect - -print =============== show users -sql create database d1 vgroups 1; -sql create database d2 vgroups 1; -sql create database d3 vgroups 1; -sql show databases -if $rows != 5 then - return -1 -endi - -print =============== create users -sql create user user1 PASS 'taosdata' -sql create user user2 PASS 'taosdata' -sql show users -if $rows != 3 then - return -1 -endi - -sql GRANT read ON d1.* to user1; -sql GRANT write ON d2.* to user1; - -print =============== re connect -sql close -sleep 2500 -print user user1 login -sql connect user1 - -sql_error drop database d1; -sql_error drop database d2; - -sql_error create stable d1.st (ts timestamp, i int) tags (j int) -sql create stable d2.st (ts timestamp, i int) tags (j int) - -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/user/privilege1.sim b/tests/script/tsim/user/privilege_db.sim similarity index 78% rename from tests/script/tsim/user/privilege1.sim rename to tests/script/tsim/user/privilege_db.sim index a7c5d9d13d..a694d21f2f 100644 --- a/tests/script/tsim/user/privilege1.sim +++ b/tests/script/tsim/user/privilege_db.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start sql connect -print =============== show users +print =============== create db sql create database d1 vgroups 1; sql create database d2 vgroups 1; sql create database d3 vgroups 1; @@ -68,4 +68,26 @@ sql REVOKE read,write ON d1.* from user1; sql REVOKE read,write ON d2.* from user1; sql REVOKE read,write ON *.* from user1; + +print =============== create users +sql create user u1 PASS 'taosdata' +sql show users +if $rows != 4 then + return -1 +endi + +sql GRANT read ON d1.* to u1; +sql GRANT write ON d2.* to u1; + +print =============== re connect +print user u1 login +sql close +sql connect u1 + +sql_error drop database d1; +sql_error drop database d2; + +sql_error create stable d1.st (ts timestamp, i int) tags (j int) +sql create stable d2.st (ts timestamp, i int) tags (j int) + system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/user/privilege_sysinfo.sim b/tests/script/tsim/user/privilege_sysinfo.sim new file mode 100644 index 0000000000..9ddfce8a97 --- /dev/null +++ b/tests/script/tsim/user/privilege_sysinfo.sim @@ -0,0 +1,26 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +print =============== create user and login +sql create user sysinfo0 pass 'taosdata' +sql create user sysinfo1 pass 'taosdata' +sql alter user sysinfo0 sysinfo 0 +sql alter user sysinfo1 sysinfo 1 + +print user sysinfo0 login +sql close +sql connect sysinfo0 + +system sh/exec.sh -n dnode1 -s stop +return + +print =============== check oper +sql_error create user u1 pass 'u1' +sql_error drop user sysinfo1 +sql_error alter user sysinfo1 pass '1' +sql_error alter user sysinfo0 pass '1' + + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/user/user_len.sim b/tests/script/tsim/user/user_len.sim deleted file mode 100644 index 0e44f94294..0000000000 --- a/tests/script/tsim/user/user_len.sim +++ /dev/null @@ -1,85 +0,0 @@ -system sh/stop_dnodes.sh -system sh/deploy.sh -n dnode1 -i 1 -system sh/exec.sh -n dnode1 -s start -sql connect - -$i = 0 -$dbPrefix = lm_us_db -$tbPrefix = lm_us_tb -$db = $dbPrefix . $i -$tb = $tbPrefix . $i - -print =============== step1 -sql drop user ac -x step0 - return -1 -step0: - -sql create user PASS '123' -x step1 - return -1 -step1: - -sql show users -if $rows != 1 then - return -1 -endi - -print =============== step2 -sql drop user a -x step2 -step2: -sql create user a PASS '123' -sql show users -if $rows != 2 then - return -1 -endi - -sql drop user a -sql show users -if $rows != 1 then - return -1 -endi - -print =============== step3 -sql drop user abc01234567890123456789 -x step3 -step3: - -sql create user abc01234567890123456789 PASS '123' -sql show users -if $rows != 2 then - return -1 -endi - -sql drop user abc01234567890123456789 -sql show users -if $rows != 1 then - return -1 -endi - -print =============== step4 -sql create user abcd0123456789012345678901234567890111 PASS '123' -x step4 - return -1 -step4: -sql show users -if $rows != 1 then - return -1 -endi - -print =============== step5 -sql drop user 123 -x step5 -step5: -sql create user 123 PASS '123' -x step61 - return -1 -step61: - -sql create user a123 PASS '123' -sql show users -if $rows != 2 then - return -1 -endi - -sql drop user a123 -sql show users -if $rows != 1 then - return -1 -endi - -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/wtest.bat b/tests/script/wtest.bat index 1f089c9d86..79daf01295 100644 --- a/tests/script/wtest.bat +++ b/tests/script/wtest.bat @@ -6,12 +6,12 @@ rem echo Start TDengine Testing Case ... set "SCRIPT_DIR=%~dp0" rem echo SCRIPT_DIR: %SCRIPT_DIR% -set "BUILD_DIR=%SCRIPT_DIR%..\..\..\debug\build\bin\" +echo %cd% | grep community > nul && set "BUILD_DIR=%SCRIPT_DIR%..\..\..\debug\build\bin\" || set "BUILD_DIR=%SCRIPT_DIR%..\..\debug\build\bin\" set "TSIM=%BUILD_DIR%tsim" rem echo BUILD_DIR: %BUILD_DIR% rem echo TSIM: %TSIM% -set "SIM_DIR=%SCRIPT_DIR%..\..\..\sim\" +echo %cd% | grep community > nul && set "SIM_DIR=%SCRIPT_DIR%..\..\..\sim\" || set "SIM_DIR=%SCRIPT_DIR%..\..\sim\" rem echo SIM_DIR: %SIM_DIR% set "TSIM_DIR=%SIM_DIR%tsim\" diff --git a/tests/system-test/0-others/taosShell.py b/tests/system-test/0-others/taosShell.py index 046db93c49..e03b34adca 100644 --- a/tests/system-test/0-others/taosShell.py +++ b/tests/system-test/0-others/taosShell.py @@ -44,13 +44,13 @@ def taos_command (buildPath, key, value, expectString, cfgDir, sqlString='', key tdLog.info ("taos cmd: %s" % taosCmd) - child = taosExpect.spawn(taosCmd, timeout=3) + child = taosExpect.spawn(taosCmd, timeout=10) #output = child.readline() #print (output.decode()) if len(expectString) != 0: - i = child.expect([expectString, taosExpect.TIMEOUT, taosExpect.EOF], timeout=6) + i = child.expect([expectString, taosExpect.TIMEOUT, taosExpect.EOF], timeout=10) else: - i = child.expect([taosExpect.TIMEOUT, taosExpect.EOF], timeout=6) + i = child.expect([taosExpect.TIMEOUT, taosExpect.EOF], timeout=10) if platform.system().lower() == 'windows': retResult = child.before diff --git a/tests/system-test/2-query/Now.py b/tests/system-test/2-query/Now.py index 9d073eb4f7..6785fddc6f 100644 --- a/tests/system-test/2-query/Now.py +++ b/tests/system-test/2-query/Now.py @@ -3,437 +3,95 @@ from util.dnodes import * from util.log import * from util.sql import * from util.cases import * - +from util.sqlset import * class TDTestCase: def init(self, conn, logSql): tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(),True) + self.setsql = TDSetSql() + # name of normal table + self.ntbname = 'ntb' + # name of stable + self.stbname = 'stb' + # structure of column + self.column_dict = { + 'ts':'timestamp', + 'c1':'int', + 'c2':'float', + 'c3':'double' + } + # structure of tag + self.tag_dict = { + 't0':'int' + } + # number of child tables + self.tbnum = 2 + # values of tag,the number of values should equal to tbnum + self.tag_values = [ + f'10', + f'100' + ] + self.values_list = [ + f'now,10,99.99,11.111111', + f'today(),100,11.111,22.222222' + ] + self.time_unit = ['b','u','a','s','m','h','d','w'] + self.symbol = ['+','-','*','/'] + self.error_values = [1.5,'abc','"abc"','!@','today()'] + def tbtype_check(self,tb_type): + if tb_type == 'normal table' or tb_type == 'child table': + tdSql.checkRows(len(self.values_list)) + elif tb_type == 'stable': + tdSql.checkRows(len(self.values_list) * self.tbnum) + def data_check(self,tbname,tb_type): + tdSql.query(f'select now() from {tbname}') + self.tbtype_check(tb_type) + for unit in self.time_unit: + for symbol in self.symbol: + if symbol in ['+','-']: + tdSql.query(f'select now() {symbol}1{unit} from {tbname}') + self.tbtype_check(tb_type) + for k,v in self.column_dict.items(): + if v.lower() != 'timestamp': + continue + else: + tdSql.query(f'select * from {tbname} where {k}>=now()') + tdSql.checkRows(0) + tdSql.query(f'select * from {tbname} where {k}=now()") - tdSql.checkRows(0) - tdSql.query("select * from db.ntb where ts>=now()") - tdSql.checkRows(0) - tdSql.query("select * from ntb where ts>now()") - tdSql.checkRows(0) - tdSql.query("select * from db.ntb where ts>now()") - tdSql.checkRows(0) - tdSql.query("select now() from ntb where ts=today()") - tdSql.checkRows(1) - tdSql.query("select now() from db.ntb where ts=today()") - tdSql.checkRows(1) - tdSql.query("select now()+1 from ntb") - tdSql.checkRows(3) - tdSql.query("select now()+1 from db.ntb") - tdSql.checkRows(3) - # tdSql.query("select now()+9223372036854775807 from ntb") - # tdSql.checkRows(3) - - tdSql.error("select now()+1.5 from ntb") - tdSql.error("select now()+1.5 from db.ntb") - tdSql.error("select now()-1.5 from ntb") - tdSql.error("select now()-1.5 from db.ntb") - tdSql.error("select now()*1.5 from ntb") - tdSql.error("select now()*1.5 from db.ntb") - tdSql.error("select now()/1.5 from ntb") - tdSql.error("select now()/1.5 from db.ntb") - tdSql.error("select now()+'abc' from ntb") - tdSql.error("select now()+'abc' from db.ntb") - tdSql.error("select now()+abc from ntb") - tdSql.error("select now()+abc from db.ntb") - tdSql.error("select now()+! from ntb") - tdSql.error("select now()+! from db.ntb") - - tdSql.query("select now()+null from ntb") - tdSql.checkData(0,0,None) - tdSql.query("select now()+null from db.ntb") - tdSql.checkData(0,0,None) - tdSql.query("select now()-null from ntb") - tdSql.checkData(0,0,None) - tdSql.query("select now()-null from db.ntb") - tdSql.checkData(0,0,None) - tdSql.query("select now()*null from ntb") - tdSql.checkData(0,0,None) - tdSql.query("select now()*null from db.ntb") - tdSql.checkData(0,0,None) - tdSql.query("select now()/null from ntb") - tdSql.checkData(0,0,None) - tdSql.query("select now()/null from db.ntb") - tdSql.checkData(0,0,None) - - tdSql.error("select now() +today() from ntb") - tdSql.error("select now() +today() from db.ntb") + self.now_check_ntb() + self.now_check_stb() - # stable - tdSql.query("select now() from stb") - tdSql.checkRows(3) - tdSql.query("select now() from db.stb") - tdSql.checkRows(3) - tdSql.query("select now() +1w from stb") - tdSql.checkRows(3) - tdSql.query("select now() +1w from db.stb") - tdSql.checkRows(3) - tdSql.query("select now() +1d from stb") - tdSql.checkRows(3) - tdSql.query("select now() +1d from db.stb") - tdSql.checkRows(3) - tdSql.query("select now() +1h from stb") - tdSql.checkRows(3) - tdSql.query("select now() +1h from db.stb") - tdSql.checkRows(3) - tdSql.query("select now() +1m from stb") - tdSql.checkRows(3) - tdSql.query("select now() +1m from db.stb") - tdSql.checkRows(3) - tdSql.query("select now() +1s from stb") - tdSql.checkRows(3) - tdSql.query("select now() +1s from db.stb") - tdSql.checkRows(3) - tdSql.query("select now() +1a from stb") - tdSql.checkRows(3) - tdSql.query("select now() +1a from db.stb") - tdSql.checkRows(3) - tdSql.query("select now() +1u from stb") - tdSql.checkRows(3) - tdSql.query("select now() +1u from db.stb") - tdSql.checkRows(3) - tdSql.query("select now() +1b from stb") - tdSql.checkRows(3) - tdSql.query("select now() +1b from db.stb") - tdSql.checkRows(3) - tdSql.query("select now() -1w from stb") - tdSql.checkRows(3) - tdSql.query("select now() -1w from db.stb") - tdSql.checkRows(3) - tdSql.query("select now() -1d from stb") - tdSql.checkRows(3) - tdSql.query("select now() -1d from db.stb") - tdSql.checkRows(3) - tdSql.query("select now() -1h from stb") - tdSql.checkRows(3) - tdSql.query("select now() -1h from db.stb") - tdSql.checkRows(3) - tdSql.query("select now() -1m from stb") - tdSql.checkRows(3) - tdSql.query("select now() -1m from db.stb") - tdSql.checkRows(3) - tdSql.query("select now() -1s from stb") - tdSql.checkRows(3) - tdSql.query("select now() -1s from db.stb") - tdSql.checkRows(3) - tdSql.query("select now() -1a from stb") - tdSql.checkRows(3) - tdSql.query("select now() -1a from db.stb") - tdSql.checkRows(3) - tdSql.query("select now() -1u from stb") - tdSql.checkRows(3) - tdSql.query("select now() -1u from db.stb") - tdSql.checkRows(3) - tdSql.query("select now() -1b from stb") - tdSql.checkRows(3) - tdSql.query("select now() -1b from db.stb") - tdSql.checkRows(3) - # tdSql.query("select * from stb where ts=now()") - # tdSql.checkRows(0) - # tdSql.query("select * from stb where ts>now()") - # tdSql.checkRows(0) - tdSql.query("select now() from stb where ts=today()") - tdSql.checkRows(1) - tdSql.query("select now() from db.stb where ts=today()") - tdSql.checkRows(1) - tdSql.query("select now() +1 from stb") - tdSql.checkRows(3) - tdSql.query("select now() +1 from db.stb") - tdSql.checkRows(3) - tdSql.error("select now() +1.5 from stb") - tdSql.error("select now() -1.5 from stb") - tdSql.error("select now() *1.5 from stb") - tdSql.error("select now() /1.5 from stb") - tdSql.error("select now() +'abc' from stb") - tdSql.error("select now() +'abc' from db.stb") - tdSql.error("select now() + ! from stb") - tdSql.error("select now() + ! from db.stb") - tdSql.error("select now() + today() from stb") - tdSql.error("select now() + today() from db.stb") - tdSql.error("select now() -today() from stb") - tdSql.error("select now() - today() from db.stb") - - - tdSql.query("select now()+null from stb") - tdSql.checkData(0,0,None) - tdSql.query("select now()+null from db.stb") - tdSql.checkData(0,0,None) - tdSql.query("select now()-null from stb") - tdSql.checkData(0,0,None) - tdSql.query("select now()-null from db.stb") - tdSql.checkData(0,0,None) - tdSql.query("select now()*null from stb") - tdSql.checkData(0,0,None) - tdSql.query("select now()*null from db.stb") - tdSql.checkData(0,0,None) - tdSql.query("select now()/null from stb") - tdSql.checkData(0,0,None) - tdSql.query("select now()/null from db.stb") - tdSql.checkData(0,0,None) - - # table - tdSql.query("select now() from stb_1") - tdSql.checkRows(3) - tdSql.query("select now() from db.stb_1") - tdSql.checkRows(3) - tdSql.query("select now() +1w from stb_1") - tdSql.checkRows(3) - tdSql.query("select now() +1w from db.stb_1") - tdSql.checkRows(3) - tdSql.query("select now() +1d from stb_1") - tdSql.checkRows(3) - tdSql.query("select now() +1d from db.stb_1") - tdSql.checkRows(3) - tdSql.query("select now() +1h from stb_1") - tdSql.checkRows(3) - tdSql.query("select now() +1h from db.stb_1") - tdSql.checkRows(3) - tdSql.query("select now() +1m from stb_1") - tdSql.checkRows(3) - tdSql.query("select now() +1m from db.stb_1") - tdSql.checkRows(3) - tdSql.query("select now() +1s from stb_1") - tdSql.checkRows(3) - tdSql.query("select now() +1s from db.stb_1") - tdSql.checkRows(3) - tdSql.query("select now() +1a from stb_1") - tdSql.checkRows(3) - tdSql.query("select now() +1a from db.stb_1") - tdSql.checkRows(3) - tdSql.query("select now() +1u from stb_1") - tdSql.checkRows(3) - tdSql.query("select now() +1u from db.stb_1") - tdSql.checkRows(3) - tdSql.query("select now() +1b from stb_1") - tdSql.checkRows(3) - tdSql.query("select now() +1b from db.stb_1") - tdSql.checkRows(3) - tdSql.query("select now() -1w from stb_1") - tdSql.checkRows(3) - tdSql.query("select now() -1w from db.stb_1") - tdSql.checkRows(3) - tdSql.query("select now() -1d from stb_1") - tdSql.checkRows(3) - tdSql.query("select now() -1d from db.stb_1") - tdSql.checkRows(3) - tdSql.query("select now() -1h from stb_1") - tdSql.checkRows(3) - tdSql.query("select now() -1h from db.stb_1") - tdSql.checkRows(3) - tdSql.query("select now() -1m from stb_1") - tdSql.checkRows(3) - tdSql.query("select now() -1m from db.stb_1") - tdSql.checkRows(3) - tdSql.query("select now() -1s from stb_1") - tdSql.checkRows(3) - tdSql.query("select now() -1s from db.stb_1") - tdSql.checkRows(3) - tdSql.query("select now() -1a from stb_1") - tdSql.checkRows(3) - tdSql.query("select now() -1a from db.stb_1") - tdSql.checkRows(3) - tdSql.query("select now() -1u from stb_1") - tdSql.checkRows(3) - tdSql.query("select now() -1u from db.stb_1") - tdSql.checkRows(3) - tdSql.query("select now() -1b from stb_1") - tdSql.checkRows(3) - tdSql.query("select now() -1b from db.stb_1") - tdSql.checkRows(3) - tdSql.query("select * from stb_1 where ts=now()") - tdSql.checkRows(0) - tdSql.query("select * from db.stb_1 where ts>=now()") - tdSql.checkRows(0) - tdSql.query("select * from stb_1 where ts>now()") - tdSql.checkRows(0) - tdSql.query("select * from db.stb_1 where ts>now()") - tdSql.checkRows(0) - - # tdSql.query("select * from stb_1 where ts=now") - # tdSql.checkRows(0) - # tdSql.query("select * from stb_1 where ts>now") - # tdSql.checkRows(0) - - tdSql.query("select now() from stb_1 where ts=today()") - tdSql.checkRows(1) - - tdSql.error("select now() +'abc' from stb_1") - tdSql.error("select now() +'abc' from db.stb_1") - tdSql.error("select now() + ! from stb_1") - tdSql.error("select now() + ! from db.stb_1") - tdSql.error("select now() + today() from stb_1") - tdSql.error("select now() + today() from db.stb_1") - tdSql.error("select now() - today() from stb_1") - tdSql.error("select now()-today() from db.stb_1") - - tdSql.query("select now()+null from stb_1") - tdSql.checkData(0,0,None) - tdSql.query("select now()+null from db.stb_1") - tdSql.checkData(0,0,None) - tdSql.query("select now()-null from stb_1") - tdSql.checkData(0,0,None) - tdSql.query("select now()-null from db.stb_1") - tdSql.checkData(0,0,None) - tdSql.query("select now()*null from stb_1") - tdSql.checkData(0,0,None) - tdSql.query("select now()*null from db.stb_1") - tdSql.checkData(0,0,None) - tdSql.query("select now()/null from stb_1") - tdSql.checkData(0,0,None) - tdSql.query("select now()/null from db.stb_1") - tdSql.checkData(0,0,None) def stop(self): tdSql.close() tdLog.success(f"{__file__} successfully executed") diff --git a/tests/system-test/2-query/To_unixtimestamp.py b/tests/system-test/2-query/To_unixtimestamp.py index 43315ff0d8..8dc995b178 100644 --- a/tests/system-test/2-query/To_unixtimestamp.py +++ b/tests/system-test/2-query/To_unixtimestamp.py @@ -12,7 +12,37 @@ class TDTestCase: def init(self, conn, logSql): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) + # name of normal table + self.ntbname = 'ntb' + # name of stable + self.stbname = 'stb' + # structure of column + self.column_dict = { + 'ts':'timestamp', + 'c1':'int', + 'c2':'float', + 'c3':'binary(20)', + 'c4':'nchar(20)' + } + # structure of tag + self.tag_dict = { + 't0':'int' + } + # number of child tables + self.tbnum = 2 + # values of tag,the number of values should equal to tbnum + self.tag_values = [ + f'10', + f'100' + ] + # values of rows, structure should be same as column + self.values_list = [ + f'now,10,99.99,"2020-1-1 00:00:00"', + f'today(),100,11.111,22.222222' + ] + self.error_param = [1,'now()'] + def run(self): # sourcery skip: extract-duplicate-method tdSql.prepare() tdLog.printNoPrefix("==========step1:create tables==========") diff --git a/tests/system-test/2-query/first.py b/tests/system-test/2-query/first.py index e9a8cc950b..830b19d8ac 100644 --- a/tests/system-test/2-query/first.py +++ b/tests/system-test/2-query/first.py @@ -60,10 +60,10 @@ class TDTestCase: tdSql.checkRows(1) tdSql.checkData(0, 1, None) #!bug TD-16561 - # for i in ['stb','db.stb']: - # tdSql.query(f"select first(*) from {i}") - # tdSql.checkRows(1) - # tdSql.checkData(0, 1, None) + for i in ['stb','db.stb']: + tdSql.query(f"select first(*) from {i}") + tdSql.checkRows(1) + tdSql.checkData(0, 1, None) for i in column_list: for j in ['stb_1','db.stb_1','stb_1','db.stb_1']: tdSql.query(f"select first({i}) from {j}") @@ -125,10 +125,10 @@ class TDTestCase: tdSql.execute(f"create table {stbname}_{i} using {stbname} tags('beijing')") tdSql.execute(f"insert into {stbname}_{i}(ts) values(%d)" % (self.ts - 1-i)) #!bug TD-16561 - # for i in [f'{stbname}', f'{dbname}.{stbname}']: - # tdSql.query(f"select first(*) from {i}") - # tdSql.checkRows(1) - # tdSql.checkData(0, 1, None) + for i in [f'{stbname}', f'{dbname}.{stbname}']: + tdSql.query(f"select first(*) from {i}") + tdSql.checkRows(1) + tdSql.checkData(0, 1, None) tdSql.query('show tables') vgroup_list = [] for i in range(len(tdSql.queryResult)): @@ -170,8 +170,8 @@ class TDTestCase: elif 'nchar' in v: tdSql.checkData(0, 0, f'{self.nchar_str}1') #!bug TD-16569 - # tdSql.query(f"select first(*),last(*) from {stbname} where ts < 23 interval(1s)") - # tdSql.checkRows(0) + tdSql.query(f"select first(*),last(*) from {stbname} where ts < 23 interval(1s)") + tdSql.checkRows(0) tdSql.execute(f'drop database {dbname}') diff --git a/tests/system-test/2-query/function_null.py b/tests/system-test/2-query/function_null.py new file mode 100644 index 0000000000..4de7a7f113 --- /dev/null +++ b/tests/system-test/2-query/function_null.py @@ -0,0 +1,254 @@ +import taos +import sys +import datetime +import inspect + +from util.log import * +from util.sql import * +from util.cases import * +import random + + +class TDTestCase: + updatecfgDict = {'debugFlag': 143, "cDebugFlag": 143, "uDebugFlag": 143, "rpcDebugFlag": 143, "tmrDebugFlag": 143, + "jniDebugFlag": 143, "simDebugFlag": 143, "dDebugFlag": 143, "dDebugFlag": 143, "vDebugFlag": 143, "mDebugFlag": 143, "qDebugFlag": 143, + "wDebugFlag": 143, "sDebugFlag": 143, "tsdbDebugFlag": 143, "tqDebugFlag": 143, "fsDebugFlag": 143, "fnDebugFlag": 143} + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), True) + self.tb_nums = 10 + self.row_nums = 20 + self.ts = 1434938400000 + self.time_step = 1000 + + def prepare_tag_datas(self): + # prepare datas + tdSql.execute( + "create database if not exists testdb keep 3650 duration 1000") + tdSql.execute(" use testdb ") + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t0 timestamp, t1 int, t2 bigint, t3 smallint, t4 tinyint, t5 float, t6 double, t7 bool, t8 binary(16),t9 nchar(32)) + ''' + ) + + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(4): + tdSql.execute( + f'create table ct{i+1} using stb1 tags ( now(), {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, "binary{i}", "nchar{i}" )') + + for i in range(9): + tdSql.execute( + f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + "insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute( + "insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute( + "insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute( + "insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + + tdSql.execute( + "insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute( + "insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute( + "insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + + tdSql.execute( + f'''insert into t1 values + ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) + ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) + ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) + ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) + ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) + ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ''' + ) + + def function_for_null_data(self): + + function_names = ["abs" , "floor" , "ceil" , "round"] + + for function_name in function_names: + + scalar_sql_1 = f"select {function_name}(c1)/0 from t1 group by c1 order by c1" + scalar_sql_2 = f"select {function_name}(c1/0) from t1 group by c1 order by c1" + scalar_sql_3 = f"select {function_name}(NULL) from t1 group by c1 order by c1" + tdSql.query(scalar_sql_1) + tdSql.checkRows(10) + tdSql.checkData(0,0,None) + tdSql.checkData(9,0,None) + tdSql.query(scalar_sql_2) + tdSql.checkRows(10) + tdSql.checkData(0,0,None) + tdSql.checkData(9,0,None) + tdSql.query(scalar_sql_3) + tdSql.checkRows(10) + tdSql.checkData(0,0,None) + tdSql.checkData(9,0,None) + + function_names = ["sin" ,"cos" ,"tan" ,"asin" ,"acos" ,"atan"] + + PI = 3.141592654 + + # sin + tdSql.query(" select sin(c1/0) from t1 group by c1 order by c1") + tdSql.checkData(9,0,None) + + tdSql.query(" select sin(NULL) from t1 group by c1 order by c1") + tdSql.checkData(9,0,None) + + tdSql.query(" select sin(0.00) from t1 group by c1 order by c1") + tdSql.checkData(9,0,0.000000000) + + tdSql.query(f" select sin({PI/2}) from t1 group by c1 order by c1") + tdSql.checkData(9,0,1.0) + + tdSql.query(" select sin(1000) from t1 group by c1 order by c1") + tdSql.checkData(9,0,0.826879541) + + # cos + tdSql.query(" select cos(c1/0) from t1 group by c1 order by c1") + tdSql.checkData(9,0,None) + + tdSql.query(" select cos(NULL) from t1 group by c1 order by c1") + tdSql.checkData(9,0,None) + + tdSql.query(" select cos(0.00) from t1 group by c1 order by c1") + tdSql.checkData(9,0,1.000000000) + + tdSql.query(f" select cos({PI}/2) from t1 group by c1 order by c1") + + tdSql.query(" select cos(1000) from t1 group by c1 order by c1") + tdSql.checkData(9,0,0.562379076) + + + # tan + tdSql.query(" select tan(c1/0) from t1 group by c1 order by c1") + tdSql.checkData(9,0,None) + + tdSql.query(" select tan(NULL) from t1 group by c1 order by c1") + tdSql.checkData(9,0,None) + + tdSql.query(" select tan(0.00) from t1 group by c1 order by c1") + tdSql.checkData(9,0,0.000000000) + + tdSql.query(f" select tan({PI}/2) from t1 group by c1 order by c1") + + tdSql.query(" select tan(1000) from t1 group by c1 order by c1") + tdSql.checkData(9,0,1.470324156) + + # atan + tdSql.query(" select atan(c1/0) from t1 group by c1 order by c1") + tdSql.checkData(9,0,None) + + tdSql.query(" select atan(NULL) from t1 group by c1 order by c1") + tdSql.checkData(9,0,None) + + tdSql.query(" select atan(0.00) from t1 group by c1 order by c1") + tdSql.checkData(9,0,0.000000000) + + tdSql.query(f" select atan({PI}/2) from t1 group by c1 order by c1") + tdSql.checkData(9,0,1.003884822) + + tdSql.query(" select atan(1000) from t1 group by c1 order by c1") + tdSql.checkData(9,0,1.569796327) + + # asin + tdSql.query(" select asin(c1/0) from t1 group by c1 order by c1") + tdSql.checkData(9,0,None) + + tdSql.query(" select asin(NULL) from t1 group by c1 order by c1") + tdSql.checkData(9,0,None) + + tdSql.query(" select asin(0.00) from t1 group by c1 order by c1") + tdSql.checkData(9,0,0.000000000) + + tdSql.query(f" select asin({PI}/2) from t1 group by c1 order by c1") + + tdSql.query(" select asin(1000) from t1 group by c1 order by c1") + tdSql.checkData(9,0,None) + + # acos + + tdSql.query(" select acos(c1/0) from t1 group by c1 order by c1") + tdSql.checkData(9,0,None) + + tdSql.query(" select acos(NULL) from t1 group by c1 order by c1") + tdSql.checkData(9,0,None) + + tdSql.query(" select acos(0.00) from t1 group by c1 order by c1") + tdSql.checkData(9,0,1.570796327) + + tdSql.query(f" select acos({PI}/2) from t1 group by c1 order by c1") + tdSql.checkData(9,0,None) + + tdSql.query(" select acos(1000) from t1 group by c1 order by c1") + tdSql.checkData(9,0,None) + + function_names = ["log" ,"pow"] + + # log + tdSql.query(" select log(-10) from t1 group by c1 order by c1") + tdSql.checkData(9,0,None) + + tdSql.query(" select log(NULL ,2) from t1 group by c1 order by c1") + tdSql.checkData(9,0,None) + + tdSql.query(" select log(c1)/0 from t1 group by c1 order by c1") + tdSql.checkData(9,0,None) + + tdSql.query(f" select log(0.00) from t1 group by c1 order by c1") + tdSql.checkData(9,0,None) + + # pow + tdSql.query(" select pow(c1,10000) from t1 group by c1 order by c1") + tdSql.checkData(9,0,None) + + tdSql.query(" select pow(c1,2)/0 from t1 group by c1 order by c1") + tdSql.checkData(9,0,None) + + tdSql.query(" select pow(NULL,2) from t1 group by c1 order by c1") + tdSql.checkData(9,0,None) + + tdSql.query(f" select pow(c1/0 ,1 ) from t1 group by c1 order by c1") + tdSql.checkData(9,0,None) + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table ==============") + + self.prepare_tag_datas() + + tdLog.printNoPrefix("==========step2:test errors ==============") + + self.function_for_null_data() + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/2-query/irate.py b/tests/system-test/2-query/irate.py new file mode 100644 index 0000000000..3a451d069c --- /dev/null +++ b/tests/system-test/2-query/irate.py @@ -0,0 +1,279 @@ +import taos +import sys +import datetime +import inspect + +from util.log import * +from util.sql import * +from util.cases import * +import random + + +class TDTestCase: + updatecfgDict = {'debugFlag': 143, "cDebugFlag": 143, "uDebugFlag": 143, "rpcDebugFlag": 143, "tmrDebugFlag": 143, + "jniDebugFlag": 143, "simDebugFlag": 143, "dDebugFlag": 143, "dDebugFlag": 143, "vDebugFlag": 143, "mDebugFlag": 143, "qDebugFlag": 143, + "wDebugFlag": 143, "sDebugFlag": 143, "tsdbDebugFlag": 143, "tqDebugFlag": 143, "fsDebugFlag": 143, "fnDebugFlag": 143} + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), True) + self.tb_nums = 10 + self.row_nums = 20 + self.ts = 1434938400000 + self.time_step = 1000 + + def insert_datas_and_check_irate(self ,tbnums , rownums , time_step ): + + tdLog.info(" prepare datas for auto check irate function ") + + tdSql.execute(" create database test ") + tdSql.execute(" use test ") + tdSql.execute(" create stable stb (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint,\ + c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) tags (t1 int)") + for tbnum in range(tbnums): + tbname = "sub_tb_%d"%tbnum + tdSql.execute(" create table %s using stb tags(%d) "%(tbname , tbnum)) + + ts = self.ts + for row in range(rownums): + ts = self.ts + time_step*row + c1 = random.randint(0,1000) + c2 = random.randint(0,100000) + c3 = random.randint(0,125) + c4 = random.randint(0,125) + c5 = random.random()/1.0 + c6 = random.random()/1.0 + c7 = "'true'" + c8 = "'binary_val'" + c9 = "'nchar_val'" + c10 = ts + tdSql.execute(f" insert into {tbname} values ({ts},{c1},{c2},{c3},{c4},{c5},{c6},{c7},{c8},{c9},{c10})") + + tdSql.execute("use test") + tbnames = ["stb", "sub_tb_1"] + support_types = ["BIGINT", "SMALLINT", "TINYINT", "FLOAT", "DOUBLE", "INT"] + for tbname in tbnames: + tdSql.query("desc {}".format(tbname)) + coltypes = tdSql.queryResult + for coltype in coltypes: + colname = coltype[0] + if coltype[1] in support_types and coltype[-1] != "TAG" : + irate_sql = "select irate({}) from (select * from {} order by tbname ) ".format(colname, tbname) + origin_sql = "select ts , {} , cast(ts as bigint) from (select ts , {} from {} order by ts desc limit 2 offset 0 ) order by ts".format(colname,colname, tbname) + + tdSql.query(irate_sql) + irate_result = tdSql.queryResult + tdSql.query(origin_sql) + origin_result = tdSql.queryResult + irate_value = irate_result[0][0] + if origin_result[1][-1] - origin_result[0][-1] == 0: + comput_irate_value = 0 + elif (origin_result[1][1] - origin_result[0][1])<0: + comput_irate_value = origin_result[1][1]*1000/( origin_result[1][-1] - origin_result[0][-1]) + else: + comput_irate_value = (origin_result[1][1] - origin_result[0][1])*1000/( origin_result[1][-1] - origin_result[0][-1]) + if comput_irate_value ==irate_value: + tdLog.info(" irate work as expected , sql is %s "% irate_sql) + else: + tdLog.exit(" irate work not as expected , sql is %s "% irate_sql) + + def prepare_tag_datas(self): + # prepare datas + tdSql.execute( + "create database if not exists testdb keep 3650 duration 1000") + tdSql.execute(" use testdb ") + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t0 timestamp, t1 int, t2 bigint, t3 smallint, t4 tinyint, t5 float, t6 double, t7 bool, t8 binary(16),t9 nchar(32)) + ''' + ) + + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(4): + tdSql.execute( + f'create table ct{i+1} using stb1 tags ( now(), {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, "binary{i}", "nchar{i}" )') + + for i in range(9): + tdSql.execute( + f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + "insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute( + "insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute( + "insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute( + "insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + + tdSql.execute( + "insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute( + "insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute( + "insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + + tdSql.execute( + f'''insert into t1 values + ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) + ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) + ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) + ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) + ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) + ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ''' + ) + + def test_errors(self): + tdSql.execute("use testdb") + error_sql_lists = [ + "select irate from t1", + "select irate(-+--+c1) from t1", + # "select +-irate(c1) from t1", + # "select ++-irate(c1) from t1", + # "select ++--irate(c1) from t1", + # "select - -irate(c1)*0 from t1", + # "select irate(tbname+1) from t1 ", + "select irate(123--123)==1 from t1", + "select irate(c1) as 'd1' from t1", + "select irate(c1 ,c2 ) from t1", + "select irate(c1 ,NULL) from t1", + "select irate(,) from t1;", + "select irate(irate(c1) ab from t1)", + "select irate(c1) as int from t1", + "select irate from stb1", + # "select irate(-+--+c1) from stb1", + # "select +-irate(c1) from stb1", + # "select ++-irate(c1) from stb1", + # "select ++--irate(c1) from stb1", + # "select - -irate(c1)*0 from stb1", + # "select irate(tbname+1) from stb1 ", + "select irate(123--123)==1 from stb1", + "select irate(c1) as 'd1' from stb1", + "select irate(c1 ,c2 ) from stb1", + "select irate(c1 ,NULL) from stb1", + "select irate(,) from stb1;", + "select irate(abs(c1) ab from stb1)", + "select irate(c1) as int from stb1" + ] + for error_sql in error_sql_lists: + tdSql.error(error_sql) + + def support_types(self): + tdSql.execute("use testdb") + tbnames = ["stb1", "t1", "ct1", "ct2"] + support_types = ["BIGINT", "SMALLINT", "TINYINT", "FLOAT", "DOUBLE", "INT"] + for tbname in tbnames: + tdSql.query("desc {}".format(tbname)) + coltypes = tdSql.queryResult + for coltype in coltypes: + colname = coltype[0] + irate_sql = "select irate({}) from {}".format(colname, tbname) + if coltype[1] in support_types: + tdSql.query(irate_sql) + else: + tdSql.error(irate_sql) + + def basic_irate_function(self): + + # used for empty table , ct3 is empty + tdSql.query("select irate(c1) from ct3") + tdSql.checkRows(0) + tdSql.query("select irate(c2) from ct3") + tdSql.checkRows(0) + + # used for regular table + tdSql.query("select irate(c1) from t1") + tdSql.checkData(0, 0, 0.000000386) + + # used for sub table + tdSql.query("select irate(abs(c1+c2)) from ct1") + tdSql.checkData(0, 0, 0.000000000) + + + # mix with common col + tdSql.error("select c1, irate(c1) from ct1") + + # mix with common functions + tdSql.error("select irate(c1), abs(c1) from ct4 ") + + # agg functions mix with agg functions + tdSql.query("select irate(c1), count(c5) from stb1 partition by tbname ") + tdSql.checkData(0, 0, 0.000000000) + tdSql.checkData(1, 0, 0.000000000) + tdSql.checkData(0, 1, 13) + tdSql.checkData(1, 1, 9) + + + def irate_func_filter(self): + tdSql.execute("use testdb") + tdSql.query( + "select irate(c1+2)/2 from ct4 where c1>5 ") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0.000000514) + + tdSql.query( + "select irate(c1+c2)/10 from ct4 where c1=5 ") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0.000000000) + + tdSql.query( + "select irate(c1+c2)/10 from stb1 where c1 = 5 partition by tbname ") + tdSql.checkRows(2) + tdSql.checkData(0, 0, 0.000000000) + + + def irate_Arithmetic(self): + pass + + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table ==============") + + self.prepare_tag_datas() + + tdLog.printNoPrefix("==========step2:test errors ==============") + + self.test_errors() + + tdLog.printNoPrefix("==========step3:support types ============") + + self.support_types() + + tdLog.printNoPrefix("==========step4: irate basic query ============") + + self.basic_irate_function() + + tdLog.printNoPrefix("==========step5: irate filter query ============") + + self.irate_func_filter() + + + tdLog.printNoPrefix("==========step6: check result of query ============") + + self.insert_datas_and_check_irate(self.tb_nums,self.row_nums,self.time_step) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/json_tag.py b/tests/system-test/2-query/json_tag.py index 0c649f2008..2ef1b8dad2 100644 --- a/tests/system-test/2-query/json_tag.py +++ b/tests/system-test/2-query/json_tag.py @@ -412,52 +412,59 @@ class TDTestCase: tdSql.checkColNameList(res, cname_list) # # test group by & order by json tag + tdSql.query("select ts,jtag->'tag1' from jsons1 partition by jtag->'tag1' order by jtag->'tag1' desc") + tdSql.checkRows(11) + tdSql.checkData(0, 1, '"femail"') + tdSql.checkData(2, 1, '"收到货"') + tdSql.checkData(7, 1, "false") + + # tdSql.error("select count(*) from jsons1 group by jtag") # tdSql.error("select count(*) from jsons1 partition by jtag") # tdSql.error("select count(*) from jsons1 group by jtag order by jtag") tdSql.error("select count(*) from jsons1 group by jtag->'tag1' order by jtag->'tag2'") tdSql.error("select count(*) from jsons1 group by jtag->'tag1' order by jtag") - tdSql.query("select count(*),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1' desc") - tdSql.checkRows(8) - tdSql.checkData(0, 0, 2) - tdSql.checkData(0, 1, '"femail"') - tdSql.checkData(1, 0, 2) - tdSql.checkData(1, 1, '"收到货"') - tdSql.checkData(2, 0, 1) - tdSql.checkData(2, 1, "11.000000000") - tdSql.checkData(5, 0, 1) - tdSql.checkData(5, 1, "false") - tdSql.checkData(6, 0, 1) - tdSql.checkData(6, 1, "null") - tdSql.checkData(7, 0, 2) - tdSql.checkData(7, 1, None) + # tdSql.query("select count(*),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1' desc") + # tdSql.checkRows(8) + # tdSql.checkData(0, 0, 2) + # tdSql.checkData(0, 1, '"femail"') + # tdSql.checkData(1, 0, 2) + # tdSql.checkData(1, 1, '"收到货"') + # tdSql.checkData(2, 0, 1) + # tdSql.checkData(2, 1, "11.000000000") + # tdSql.checkData(5, 0, 1) + # tdSql.checkData(5, 1, "false") + # tdSql.checkData(6, 0, 1) + # tdSql.checkData(6, 1, "null") + # tdSql.checkData(7, 0, 2) + # tdSql.checkData(7, 1, None) - tdSql.query("select count(*),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1' asc") - tdSql.checkRows(8) - tdSql.checkData(0, 0, 2) - tdSql.checkData(0, 1, None) - tdSql.checkData(2, 0, 1) - tdSql.checkData(2, 1, "false") - tdSql.checkData(5, 0, 1) - tdSql.checkData(5, 1, "11.000000000") - tdSql.checkData(7, 0, 2) - tdSql.checkData(7, 1, '"femail"') + # tdSql.query("select count(*),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1' asc") + # tdSql.checkRows(8) + # tdSql.checkData(0, 0, 2) + # tdSql.checkData(0, 1, None) + # tdSql.checkData(2, 0, 1) + # tdSql.checkData(2, 1, "false") + # tdSql.checkData(5, 0, 1) + # tdSql.checkData(5, 1, "11.000000000") + # tdSql.checkData(7, 0, 2) + # tdSql.checkData(7, 1, '"femail"') # # test stddev with group by json tag - tdSql.query("select stddev(dataint),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1'") - tdSql.checkRows(8) - tdSql.checkData(0, 0, 10) - tdSql.checkData(0, 1, None) - tdSql.checkData(4, 0, 0) - tdSql.checkData(4, 1, "5.000000000") - tdSql.checkData(7, 0, 11) - tdSql.checkData(7, 1, '"femail"') - - res = tdSql.getColNameList("select stddev(dataint),jsons1.jtag->'tag1' from jsons1 group by jsons1.jtag->'tag1' order by jtag->'tag1'") - cname_list = [] - cname_list.append("stddev(dataint)") - cname_list.append("jsons1.jtag->'tag1'") - tdSql.checkColNameList(res, cname_list) + # tdSql.query("select stddev(dataint),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1'") + # tdSql.checkRows(8) + # tdSql.checkData(0, 0, 10) + # tdSql.checkData(0, 1, None) + # tdSql.checkData(4, 0, 0) + # tdSql.checkData(4, 1, "5.000000000") + # tdSql.checkData(7, 0, 11) + # tdSql.checkData(7, 1, '"femail"') + # + # res = tdSql.getColNameList("select stddev(dataint),jsons1.jtag->'tag1' from jsons1 group by jsons1.jtag->'tag1' order by jtag->'tag1'") + # cname_list = [] + # cname_list.append("stddev(dataint)") + # cname_list.append("jsons1.jtag->'tag1'") + # tdSql.checkColNameList(res, cname_list) # test top/bottom with group by json tag # tdSql.query("select top(dataint,2),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1'") @@ -470,8 +477,8 @@ class TDTestCase: # tdSql.checkData(10, 1, '"femail"') # test having - tdSql.query("select count(*),jtag->'tag1' from jsons1 group by jtag->'tag1' having count(*) > 1") - tdSql.checkRows(3) + # tdSql.query("select count(*),jtag->'tag1' from jsons1 group by jtag->'tag1' having count(*) > 1") + # tdSql.checkRows(3) # subquery with json tag tdSql.query("select * from (select jtag, dataint from jsons1) order by dataint") diff --git a/tests/system-test/2-query/timezone.py b/tests/system-test/2-query/timezone.py index 20ee58feac..ce2cdc062e 100644 --- a/tests/system-test/2-query/timezone.py +++ b/tests/system-test/2-query/timezone.py @@ -2,7 +2,7 @@ from util.log import * from util.sql import * from util.cases import * - +from util.sqlset import * import platform import os if platform.system().lower() == 'windows': @@ -14,10 +14,39 @@ class TDTestCase: def init(self, conn, logSql): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) + self.setsql = TDSetSql() + self.arithmetic_operators = ['+','-','*','/'] + self.arithmetic_values = [0,1,100,15.5] + # name of normal table + self.ntbname = 'ntb' + # name of stable + self.stbname = 'stb' + # structure of column + self.column_dict = { + 'ts':'timestamp', + 'c1':'int', + 'c2':'float', + 'c3':'double' + } + # structure of tag + self.tag_dict = { + 't0':'int' + } + # number of child tables + self.tbnum = 2 + # values of tag,the number of values should equal to tbnum + self.tag_values = [ + f'10', + f'100' + ] + # values of rows, structure should be same as column + self.values_list = [ + f'now,10,99.99,11.111111', + f'today(),100,11.111,22.222222' - def run(self): # sourcery skip: extract-duplicate-method - tdSql.prepare() - # get system timezone + ] + self.error_param = [1,'now()'] + def get_system_timezone(self): if platform.system().lower() == 'windows': time_zone_1 = tzlocal.get_localzone_name() time_zone_2 = time.strftime('(UTC, %z)') @@ -32,122 +61,61 @@ class TDTestCase: time_zone_2 = os.popen('date "+(%Z, %z)"').read().strip() time_zone = time_zone_1 + " " + time_zone_2 print("expected time zone: " + time_zone) - - tdLog.printNoPrefix("==========step1:create tables==========") - tdSql.execute( - '''create table if not exists ntb - (ts timestamp, c1 int, c2 float,c3 double) - ''' - ) - tdSql.execute( - '''create table if not exists stb - (ts timestamp, c1 int, c2 float,c3 double) tags(t0 int) - ''' - ) - tdSql.execute( - '''create table if not exists stb_1 using stb tags(100) - ''' - ) - - tdLog.printNoPrefix("==========step2:insert data==========") - tdSql.execute( - "insert into ntb values(now,10,99.99,11.111111)(today(),100,11.111,22.222222)") - tdSql.execute( - "insert into stb_1 values(now,111,99.99,11.111111)(today(),1,11.111,22.222222)") - - tdLog.printNoPrefix("==========step3:query data==========") + return time_zone + + def tb_type_check(self,tb_type): + if tb_type in ['normal_table','child_table']: + tdSql.checkRows(len(self.values_list)) + elif tb_type == 'stable': + tdSql.checkRows(len(self.values_list*self.tbnum)) + def data_check(self,timezone,tbname,tb_type): + tdSql.query(f"select timezone() from {tbname}") + self.tb_type_check(tb_type) + tdSql.checkData(0,0,timezone) + for symbol in self.arithmetic_operators: + tdSql.query(f"select timezone(){symbol}null from {tbname}") + self.tb_type_check(tb_type) + tdSql.checkData(0,0,None) + for i in self.arithmetic_values: + for symbol in self.arithmetic_operators: + tdSql.query(f"select timezone(){symbol}{i} from {tbname}") + self.tb_type_check(tb_type) + if symbol == '+': + tdSql.checkData(0,0,i) + elif symbol == '-': + tdSql.checkData(0,0,-i) + elif symbol in ['*','/','%']: + if i == 0 and symbol == '/': + tdSql.checkData(0,0,None) + else: + tdSql.checkData(0,0,0) + for param in self.error_param: + tdSql.error(f'select timezone({param}) from {tbname}') + tdSql.query(f"select * from {tbname} where timezone()='{timezone}'") + self.tb_type_check(tb_type) + def timezone_check_ntb(self,timezone): + tdSql.prepare() + tdSql.execute(self.setsql.set_create_normaltable_sql(self.ntbname,self.column_dict)) + for value in self.values_list: + tdSql.execute( + f'insert into {self.ntbname} values({value})') + self.data_check(timezone,self.ntbname,'normal_table') + tdSql.execute('drop database db') + def timezone_check_stb(self,timezone): + tdSql.prepare() + tdSql.execute(self.setsql.set_create_stable_sql(self.stbname,self.column_dict,self.tag_dict)) + for i in range(self.tbnum): + tdSql.execute(f'create table if not exists {self.stbname}_{i} using {self.stbname} tags({self.tag_values[i]})') + for j in self.values_list: + tdSql.execute(f'insert into {self.stbname}_{i} values({j})') + self.data_check(timezone,self.stbname,'stable') + for i in range(self.tbnum): + self.data_check(timezone,f'{self.stbname}_{i}','child_table') + def run(self): # sourcery skip: extract-duplicate-method + timezone = self.get_system_timezone() + self.timezone_check_ntb(timezone) + self.timezone_check_stb(timezone) - tdSql.query("select timezone() from ntb") - tdSql.checkRows(2) - tdSql.checkData(0, 0, time_zone) - tdSql.query("select timezone() from db.ntb") - tdSql.checkRows(2) - tdSql.checkData(0, 0, time_zone) - tdSql.query("select timezone() from stb") - tdSql.checkRows(2) - tdSql.checkData(0, 0, time_zone) - tdSql.query("select timezone() from db.stb") - tdSql.checkRows(2) - tdSql.checkData(0, 0, time_zone) - tdSql.query("select timezone() from stb_1") - tdSql.checkRows(2) - tdSql.checkData(0, 0, time_zone) - tdSql.query("select timezone() from db.stb_1 ") - tdSql.checkRows(2) - tdSql.checkData(0, 0, time_zone) - - tdSql.error("select timezone(1) from stb") - tdSql.error("select timezone(1) from db.stb") - tdSql.error("select timezone(1) from ntb") - tdSql.error("select timezone(1) from db.ntb") - tdSql.error("select timezone(1) from stb_1") - tdSql.error("select timezone(1) from db.stb_1") - tdSql.error("select timezone(now()) from stb") - tdSql.error("select timezone(now()) from db.stb") - - tdSql.query(f"select * from ntb where timezone()='{time_zone}'") - tdSql.checkRows(2) - tdSql.query("select timezone()+1 from ntb") - tdSql.checkRows(2) - tdSql.query("select timezone()+1 from db.ntb") - tdSql.checkRows(2) - tdSql.query("select timezone()+1 from stb") - tdSql.checkRows(2) - tdSql.query("select timezone()+1 from db.stb") - tdSql.checkRows(2) - tdSql.query("select timezone()+1 from stb_1") - tdSql.checkRows(2) - tdSql.query("select timezone()+1 from db.stb_1") - tdSql.checkRows(2) - tdSql.query("select timezone()+1.5 from ntb") - tdSql.checkRows(2) - tdSql.query("select timezone()+1.5 from db.ntb") - tdSql.checkRows(2) - tdSql.query("select timezone()-100 from ntb") - tdSql.checkRows(2) - tdSql.query("select timezone()*100 from ntb") - tdSql.checkRows(2) - tdSql.query("select timezone()/10 from ntb") - # tdSql.query("select timezone()/0 from ntb") - - - tdSql.query("select timezone()+null from ntb") - tdSql.checkRows(2) - tdSql.checkData(0,0,None) - tdSql.query("select timezone()-null from ntb") - tdSql.checkRows(2) - tdSql.checkData(0,0,None) - tdSql.query("select timezone()*null from ntb") - tdSql.checkRows(2) - tdSql.checkData(0,0,None) - tdSql.query("select timezone()/null from ntb") - tdSql.checkRows(2) - tdSql.checkData(0,0,None) - # tdSql.query("select timezone()") - tdSql.query("select timezone()+null from stb") - tdSql.checkRows(2) - tdSql.checkData(0,0,None) - tdSql.query("select timezone()-null from stb") - tdSql.checkRows(2) - tdSql.checkData(0,0,None) - tdSql.query("select timezone()*null from stb") - tdSql.checkRows(2) - tdSql.checkData(0,0,None) - tdSql.query("select timezone()/null from stb") - tdSql.checkRows(2) - tdSql.checkData(0,0,None) - tdSql.query("select timezone()+null from stb_1") - tdSql.checkRows(2) - tdSql.checkData(0,0,None) - tdSql.query("select timezone()-null from stb_1") - tdSql.checkRows(2) - tdSql.checkData(0,0,None) - tdSql.query("select timezone()*null from stb_1") - tdSql.checkRows(2) - tdSql.checkData(0,0,None) - tdSql.query("select timezone()/null from stb_1") - tdSql.checkRows(2) - tdSql.checkData(0,0,None) def stop(self): tdSql.close() tdLog.success(f"{__file__} successfully executed") diff --git a/tests/system-test/6-cluster/5dnode3mnodeStop.py b/tests/system-test/6-cluster/5dnode3mnodeStop.py index 69b9c3d879..3a85207af6 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeStop.py +++ b/tests/system-test/6-cluster/5dnode3mnodeStop.py @@ -227,6 +227,7 @@ class TDTestCase: # fisr add three mnodes; tdSql.execute("create mnode on dnode 2") + time.sleep(10) tdSql.execute("create mnode on dnode 3") # fisrt check statut ready diff --git a/tests/system-test/7-tmq/tmqCommon.py b/tests/system-test/7-tmq/tmqCommon.py index d17e36fc97..b8aa78e3ac 100644 --- a/tests/system-test/7-tmq/tmqCommon.py +++ b/tests/system-test/7-tmq/tmqCommon.py @@ -11,7 +11,9 @@ # -*- coding: utf-8 -*- +from asyncore import loop from collections import defaultdict +import subprocess import random import string import threading @@ -75,7 +77,7 @@ class TMQCom: return resultList - def startTmqSimProcess(self,pollDelay,dbName,showMsg=1,showRow=1,cdbName='cdb',valgrind=0): + def startTmqSimProcess(self,pollDelay,dbName,showMsg=1,showRow=1,cdbName='cdb',valgrind=0,alias=0): buildPath = tdCom.getBuildPath() cfgPath = tdCom.getClientCfgPath() if valgrind == 1: @@ -88,30 +90,53 @@ class TMQCom: shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) shellCmd += "> nul 2>&1 &" else: - shellCmd = 'nohup ' + buildPath + '/build/bin/tmq_sim -c ' + cfgPath + processorName = buildPath + '/build/bin/tmq_sim' + if alias != 0: + processorNameNew = buildPath + '/build/bin/tmq_sim_new' + shellCmd = 'cp %s %s'%(processorName, processorNameNew) + os.system(shellCmd) + processorName = processorNameNew + shellCmd = 'nohup ' + processorName + ' -c ' + cfgPath shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) shellCmd += "> /dev/null 2>&1 &" tdLog.info(shellCmd) - os.system(shellCmd) + os.system(shellCmd) - def getStartConsumeNotifyFromTmqsim(self,cdbName='cdb'): - while 1: + def stopTmqSimProcess(self, processorName): + psCmd = "ps -ef|grep -w %s|grep -v grep | awk '{print $2}'"%(processorName) + processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") + while(processID): + killCmd = "kill -INT %s > /dev/null 2>&1" % processID + os.system(killCmd) + time.sleep(0.2) + processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") + tdLog.debug("%s is stopped by kill -INT" % (processorName)) + + def getStartConsumeNotifyFromTmqsim(self,cdbName='cdb',rows=1): + loopFlag = 1 + while loopFlag: tdSql.query("select * from %s.notifyinfo"%cdbName) #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) - if (tdSql.getRows() == 1) and (tdSql.getData(0, 1) == 0): - break - else: - time.sleep(0.1) + actRows = tdSql.getRows() + if (actRows >= rows): + for i in range(actRows): + if tdSql.getData(i, 1) == 0: + loopFlag = 0 + break + time.sleep(0.1) return - def getStartCommitNotifyFromTmqsim(self,cdbName='cdb'): - while 1: + def getStartCommitNotifyFromTmqsim(self,cdbName='cdb',rows=2): + loopFlag = 1 + while loopFlag: tdSql.query("select * from %s.notifyinfo"%cdbName) #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) - if tdSql.getRows() == 2 : - print(tdSql.getData(0, 1), tdSql.getData(1, 1)) - if tdSql.getData(1, 1) == 1: - break + actRows = tdSql.getRows() + if (actRows >= rows): + for i in range(actRows): + if tdSql.getData(i, 1) == 1: + loopFlag = 0 + break time.sleep(0.1) return diff --git a/tests/system-test/7-tmq/tmqConsumerGroup.py b/tests/system-test/7-tmq/tmqConsumerGroup.py new file mode 100644 index 0000000000..bfd63fd4a2 --- /dev/null +++ b/tests/system-test/7-tmq/tmqConsumerGroup.py @@ -0,0 +1,170 @@ + +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + #tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def checkFileContent(self, consumerId, queryString): + buildPath = tdCom.getBuildPath() + cfgPath = tdCom.getClientCfgPath() + dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId) + cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile) + tdLog.info(cmdStr) + os.system(cmdStr) + + consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId) + tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile)) + + consumeFile = open(consumeRowsFile, mode='r') + queryFile = open(dstFile, mode='r') + + # skip first line for it is schema + queryFile.readline() + + while True: + dst = queryFile.readline() + src = consumeFile.readline() + + if dst: + if dst != src: + tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId) + else: + break + return + + def tmqCase1(self): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'db1', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':2}, {'type': 'binary', 'len':20, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbNum': 10, + 'rowsPerTbl': 1000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 20, + 'showMsg': 1, + 'showRow': 1} + + topicNameList = ['topic1', 'topic2'] + expectRowsList = [] + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) + tdLog.info("create stb") + tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) + tdLog.info("create ctb") + tdCom.create_ctable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix']) + tdLog.info("insert data") + tmqCom.insert_data_2(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"]) + + tdLog.info("create topics from stb with filter") + # queryString = "select ts, log(c1), ceil(pow(c1,3)) from %s.%s where c1 %% 7 == 0" %(paraDict['dbName'], paraDict['stbName']) + queryString = "select ts, log(c1), ceil(pow(c1,3)) from %s.%s" %(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + tdSql.query(queryString) + expectRowsList.append(tdSql.getRows()) + + # create one stb2 + paraDict["stbName"] = 'stb2' + paraDict["ctbPrefix"] = 'ctbx' + paraDict["rowsPerTbl"] = 5000 + tdLog.info("create stb2") + tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) + tdLog.info("create ctb2") + tdCom.create_ctable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix']) + + # queryString = "select ts, sin(c1), abs(pow(c1,3)) from %s.%s where c1 %% 7 == 0" %(paraDict['dbName'], paraDict['stbName']) + queryString = "select ts, sin(c1), abs(pow(c1,3)) from %s.%s" %(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[1], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + # tdSql.query(queryString) + # expectRowsList.append(tdSql.getRows()) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2 + topicList = "%s,%s"%(topicNameList[0],topicNameList[1]) + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:3000, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor 1") + tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow']) + + tdLog.info("start consume processor 2") + tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow'],'cdb',0,1) + + tdLog.info("async insert data") + pThread = tmqCom.asyncInsertData(paraDict) + + tdLog.info("wait consumer commit notify") + tmqCom.getStartCommitNotifyFromTmqsim(rows=4) + + tdLog.info("pkill one consume processor") + tmqCom.stopTmqSimProcess('tmq_sim_new') + + pThread.join() + + tdLog.info("wait the consume result") + expectRows = 2 + resultList = tmqCom.selectConsumeResult(expectRows) + actTotalRows = 0 + for i in range(len(resultList)): + actTotalRows += resultList[i] + + tdSql.query(queryString) + expectRowsList.append(tdSql.getRows()) + expectTotalRows = 0 + for i in range(len(expectRowsList)): + expectTotalRows += expectRowsList[i] + + tdLog.info("act consume rows: %d, expect consume rows: %d"%(actTotalRows, expectTotalRows)) + if expectTotalRows <= resultList[0]: + tdLog.info("act consume rows: %d should >= expect consume rows: %d"%(actTotalRows, expectTotalRows)) + tdLog.exit("0 tmq consume rows error!") + + # time.sleep(10) + # for i in range(len(topicNameList)): + # tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def run(self): + tdSql.prepare() + self.tmqCase1() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 73e6716cad..ef217b828f 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -109,6 +109,9 @@ python3 ./test.py -f 2-query/distribute_agg_apercentile.py python3 ./test.py -f 2-query/distribute_agg_avg.py python3 ./test.py -f 2-query/distribute_agg_stddev.py python3 ./test.py -f 2-query/twa.py +python3 ./test.py -f 2-query/irate.py + +python3 ./test.py -f 2-query/function_null.py python3 ./test.py -f 6-cluster/5dnode1mnode.py python3 ./test.py -f 6-cluster/5dnode2mnode.py @@ -135,3 +138,4 @@ python3 ./test.py -f 7-tmq/stbFilter.py python3 ./test.py -f 7-tmq/tmqCheckData.py python3 ./test.py -f 7-tmq/tmqUdf.py #python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 5 +python3 ./test.py -f 7-tmq/tmqConsumerGroup.py diff --git a/tests/system-test/simpletest.bat b/tests/system-test/simpletest.bat index 0b1f16d216..656828aa1e 100644 --- a/tests/system-test/simpletest.bat +++ b/tests/system-test/simpletest.bat @@ -1,12 +1,12 @@ -@REM python3 .\test.py -f 0-others\taosShell.py -@REM python3 .\test.py -f 0-others\taosShellError.py +python3 .\test.py -f 0-others\taosShell.py +python3 .\test.py -f 0-others\taosShellError.py python3 .\test.py -f 0-others\taosShellNetChk.py python3 .\test.py -f 0-others\telemetry.py python3 .\test.py -f 0-others\taosdMonitor.py python3 .\test.py -f 0-others\udfTest.py -@REM python3 .\test.py -f 0-others\udf_create.py -@REM python3 .\test.py -f 0-others\udf_restart_taosd.py +python3 .\test.py -f 0-others\udf_create.py +python3 .\test.py -f 0-others\udf_restart_taosd.py @REM python3 .\test.py -f 0-others\cachelast.py @REM python3 .\test.py -f 0-others\user_control.py diff --git a/tests/system-test/test.py b/tests/system-test/test.py index 7d78f5ab21..8a8356449c 100644 --- a/tests/system-test/test.py +++ b/tests/system-test/test.py @@ -38,7 +38,9 @@ def checkRunTimeError(): while 1: time.sleep(1) timeCount = timeCount + 1 + print("checkRunTimeError",timeCount) if (timeCount>900): + print("stop the test.") os.system("TASKKILL /F /IM taosd.exe") os.system("TASKKILL /F /IM taos.exe") os.system("TASKKILL /F /IM tmq_sim.exe") diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index e96f5d4793..c2fba52e6e 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -93,8 +93,6 @@ static SConfInfo g_stConfInfo; TdFilePtr g_fp = NULL; static int running = 1; -int8_t useSnapshot = 0; - // char* g_pRowValue = NULL; // TdFilePtr g_fp = NULL; @@ -126,11 +124,23 @@ char* getCurrentTimeString(char* timeString) { return timeString; } +static void tmqStop(int signum, void *info, void *ctx) { + running = 0; + char tmpString[128]; + taosFprintfFile(g_fp, "%s tmqStop() receive stop signal[%d]\n", getCurrentTimeString(tmpString), signum); +} + +static void tmqSetSignalHandle() { + taosSetSignal(SIGINT, tmqStop); +} + void initLogFile() { char filename[256]; char tmpString[128]; - sprintf(filename, "%s/../log/tmqlog_%s.txt", configDir, getCurrentTimeString(tmpString)); + pid_t process_id = getpid(); + + sprintf(filename, "%s/../log/tmqlog-%d-%s.txt", configDir, process_id, getCurrentTimeString(tmpString)); #ifdef WINDOWS for (int i = 2; i < sizeof(filename); i++) { if (filename[i] == ':') filename[i] = '-'; @@ -205,7 +215,7 @@ void parseArgument(int32_t argc, char* argv[]) { } else if (strcmp(argv[i], "-y") == 0) { g_stConfInfo.consumeDelay = atol(argv[++i]); } else if (strcmp(argv[i], "-e") == 0) { - useSnapshot = (int8_t)atol(argv[++i]); + g_stConfInfo.useSnapshot = atol(argv[++i]); } else { pError("%s unknow para: %s %s", GREEN, argv[++i], NC); exit(-1); @@ -519,7 +529,9 @@ static void tmq_commit_cb_print(tmq_t* tmq, int32_t code, void* param) { g_once_commit_flag = 1; notifyMainScript((SThreadInfo*)param, (int32_t)NOTIFY_CMD_START_COMMIT); } - taosFprintfFile(g_fp, "tmq_commit_cb_print() be called\n"); + + char tmpString[128]; + taosFprintfFile(g_fp, "%s tmq_commit_cb_print() be called\n", getCurrentTimeString(tmpString)); } void build_consumer(SThreadInfo* pInfo) { @@ -552,7 +564,7 @@ void build_consumer(SThreadInfo* pInfo) { // tmq_conf_set(conf, "auto.offset.reset", "earliest"); // tmq_conf_set(conf, "auto.offset.reset", "latest"); // - if (useSnapshot) { + if (g_stConfInfo.useSnapshot) { tmq_conf_set(conf, "experiment.use.snapshot", "true"); } @@ -651,6 +663,10 @@ void loop_consume(SThreadInfo* pInfo) { } } + if (0 == running) { + taosFprintfFile(g_fp, "receive stop signal and not continue consume\n"); + } + pInfo->consumeMsgCnt = totalMsgs; pInfo->consumeRowCnt = totalRows; @@ -680,7 +696,7 @@ void* consumeThreadFunc(void* param) { int32_t err = tmq_subscribe(pInfo->tmq, pInfo->topicList); if (err != 0) { pError("tmq_subscribe() fail, reason: %s\n", tmq_err2str(err)); - taosFprintfFile(g_fp, "tmq_subscribe()! reason: %s\n", tmq_err2str(err)); + taosFprintfFile(g_fp, "tmq_subscribe() fail! reason: %s\n", tmq_err2str(err)); assert(0); return NULL; } @@ -829,6 +845,8 @@ int main(int32_t argc, char* argv[]) { getConsumeInfo(); saveConfigToLogFile(); + tmqSetSignalHandle(); + TdThreadAttr thattr; taosThreadAttrInit(&thattr); taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE);