diff --git a/Jenkinsfile2 b/Jenkinsfile2
index b65576deaf..f582461fb2 100644
--- a/Jenkinsfile2
+++ b/Jenkinsfile2
@@ -4,11 +4,6 @@ import jenkins.model.CauseOfInterruption
node {
}
-win_test_stage = 0
-linux_ready = 0
-linux_node_ip = ""
-linux_node_pass = ""
-
def abortPreviousBuilds() {
def currentJobName = env.JOB_NAME
def currentBuildNumber = env.BUILD_NUMBER.toInteger()
@@ -84,6 +79,7 @@ def pre_test(){
git pull >/dev/null
git log -5
echo "`date "+%Y%m%d-%H%M%S"` ${JOB_NAME}:${BRANCH_NAME}:${BUILD_ID}:${CHANGE_TARGET}" >>${WKDIR}/jenkins.log
+ echo "CHANGE_BRANCH:${CHANGE_BRANCH}" >>${WKDIR}/jenkins.log
echo "community log: `git log -5`" >>${WKDIR}/jenkins.log
git fetch origin +refs/pull/${CHANGE_ID}/merge
git checkout -qf FETCH_HEAD
@@ -100,6 +96,7 @@ def pre_test(){
git pull >/dev/null
git log -5
echo "`date "+%Y%m%d-%H%M%S"` ${JOB_NAME}:${BRANCH_NAME}:${BUILD_ID}:${CHANGE_TARGET}" >>${WKDIR}/jenkins.log
+ echo "CHANGE_BRANCH:${CHANGE_BRANCH}" >>${WKDIR}/jenkins.log
echo "tdinternal log: `git log -5`" >>${WKDIR}/jenkins.log
git fetch origin +refs/pull/${CHANGE_ID}/merge
git checkout -qf FETCH_HEAD
@@ -289,7 +286,6 @@ def run_win_ctest() {
'''
}
def run_win_test() {
- echo "LINUX NODE: ${linux_node_ip} - ${linux_node_pass}"
bat '''
echo "windows test ..."
cd %WIN_CONNECTOR_ROOT%
@@ -298,9 +294,8 @@ def run_win_test() {
ls -l C:\\Windows\\System32\\taos.dll
time /t
cd %WIN_SYSTEM_TEST_ROOT%
- echo "node: ''' + linux_node_ip + ''':''' + linux_node_pass + '''"
echo "testing ..."
- test-all.bat "{\\\"host\\\":\\\"''' + linux_node_ip + '''\\\",\\\"port\\\":22,\\\"user\\\":\\\"root\\\",\\\"password\\\":\\\"''' + linux_node_pass + '''\\\",\\\"path\\\":\\\"/var/lib/jenkins/workspace/TDinternal\\\"}"
+ test-all.bat ci
time /t
'''
}
@@ -331,17 +326,9 @@ pipeline {
pre_test_win()
pre_test_build_win()
run_win_ctest()
- script {
- while(linux_ready == 0) {
- sleep(8)
- }
- }
run_win_test()
}
}
- script {
- win_test_stage = 1
- }
}
}
stage('linux test') {
@@ -351,17 +338,6 @@ pipeline {
changeRequest()
}
steps {
- script {
- linux_node_ip = sh (
- script: 'jq .ip /home/node_info.json | sed "s/\\\"//g"',
- returnStdout: true
- ).trim()
- linux_node_pass = sh (
- script: 'jq .password /home/node_info.json | sed "s/\\\"//g" |sed "s/\\!/^^^^^^^^\\!/g"',
- returnStdout: true
- ).trim()
- echo "${linux_node_ip}:${linux_node_pass}"
- }
catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
timeout(time: 120, unit: 'MINUTES'){
pre_test()
@@ -418,38 +394,9 @@ pipeline {
cd ${WKC}/packaging
./release.sh -v cluster -n 3.0.0.100 -s static
'''
- sh '''
- echo "install ..."
- cd ${WKC}/release
- tar xzf TDengine-enterprise-server-3.0.0.100-Linux-x64.tar.gz
- cd TDengine-enterprise-server-3.0.0.100
- service taosd stop || :
- rm -rf /var/lib/taos
- ./install.sh -e no
- '''
- sh '''
- echo "checking ..."
- which taos
- which taosd
- rm -rf ${WK}/debug
- mv ${WKC}/debug ${WK}/
- '''
- sh '''
- echo "install taospy ..."
- cd ${WKPY}
- pip3 install .
- '''
}
}
}
- script {
- linux_ready = 1
- }
- script {
- while(win_test_stage == 0){
- sleep(12)
- }
- }
}
}
}
diff --git a/include/common/tcommon.h b/include/common/tcommon.h
index 9c84d0dd99..928fe0aa0e 100644
--- a/include/common/tcommon.h
+++ b/include/common/tcommon.h
@@ -42,11 +42,13 @@ enum {
typedef enum EStreamType {
STREAM_NORMAL = 1,
STREAM_INVERT,
- STREAM_REPROCESS,
+ STREAM_CLEAR,
STREAM_INVALID,
STREAM_GET_ALL,
STREAM_DELETE,
STREAM_RETRIEVE,
+ STREAM_PUSH_DATA,
+ STREAM_PUSH_EMPTY,
} EStreamType;
typedef struct {
@@ -71,7 +73,7 @@ typedef struct SColumnDataAgg {
typedef struct SDataBlockInfo {
STimeWindow window;
- int32_t rows; // todo hide this attribute
+ int32_t rows; // todo hide this attribute
int32_t rowSize;
uint64_t uid; // the uid of table, from which current data block comes
uint16_t blockId; // block id, generated by physical planner
diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h
index 2a0d4e7ff6..2e2c7d1700 100644
--- a/include/common/tdatablock.h
+++ b/include/common/tdatablock.h
@@ -224,6 +224,7 @@ size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize);
int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n);
int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src);
+int32_t copyDataBlock(SSDataBlock* dst, const SSDataBlock* src);
SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData);
SSDataBlock* createDataBlock();
int32_t blockDataAppendColInfo(SSDataBlock* pBlock, SColumnInfoData* pColInfoData);
@@ -236,6 +237,8 @@ void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t*
const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData);
void blockDebugShowData(const SArray* dataBlocks, const char* flag);
+// for debug
+char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** dumpBuf);
int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks, STSchema* pTSchema, int32_t vgId,
tb_uid_t suid);
diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h
index d86a8dae6e..0b767e96f6 100644
--- a/include/libs/qcom/query.h
+++ b/include/libs/qcom/query.h
@@ -16,6 +16,7 @@
#ifndef _TD_QUERY_H_
#define _TD_QUERY_H_
+// clang-foramt off
#ifdef __cplusplus
extern "C" {
#endif
@@ -71,7 +72,7 @@ typedef struct SIndexMeta {
} SIndexMeta;
typedef struct STbVerInfo {
- char tbFName[TSDB_TABLE_FNAME_LEN];
+ char tbFName[TSDB_TABLE_FNAME_LEN];
int32_t sversion;
int32_t tversion;
} STbVerInfo;
@@ -141,7 +142,7 @@ typedef struct SDataBuf {
typedef struct STargetInfo {
ETargetType type;
- char* dbFName; // used to update db's vgroup epset
+ char* dbFName; // used to update db's vgroup epset
int32_t vgId;
} STargetInfo;
@@ -149,15 +150,15 @@ typedef int32_t (*__async_send_cb_fn_t)(void* param, const SDataBuf* pMsg, int32
typedef int32_t (*__async_exec_fn_t)(void* param);
typedef struct SRequestConnInfo {
- void* pTrans;
- uint64_t requestId;
- int64_t requestObjRefId;
- SEpSet mgmtEps;
+ void* pTrans;
+ uint64_t requestId;
+ int64_t requestObjRefId;
+ SEpSet mgmtEps;
} SRequestConnInfo;
typedef struct SMsgSendInfo {
- __async_send_cb_fn_t fp; // async callback function
- STargetInfo target; // for update epset
+ __async_send_cb_fn_t fp; // async callback function
+ STargetInfo target; // for update epset
void* param;
uint64_t requestId;
uint64_t requestObjRefId;
@@ -206,13 +207,15 @@ int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STabl
char* jobTaskStatusStr(int32_t status);
SSchema createSchema(int8_t type, int32_t bytes, col_id_t colId, const char* name);
-void destroyQueryExecRes(SQueryExecRes* pRes);
-int32_t dataConverToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *len);
-char* parseTagDatatoJson(void* p);
+
+void destroyQueryExecRes(SQueryExecRes* pRes);
+int32_t dataConverToStr(char* str, int type, void* buf, int32_t bufSize, int32_t* len);
+char* parseTagDatatoJson(void* p);
int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst);
int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst);
-extern int32_t (*queryBuildMsg[TDMT_MAX])(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void*(*mallocFp)(int32_t));
+extern int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char** msg, int32_t msgSize, int32_t* msgLen,
+ void* (*mallocFp)(int32_t));
extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t msgSize);
#define SET_META_TYPE_NULL(t) (t) = META_TYPE_NULL_TABLE
@@ -223,7 +226,7 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t
#define NEED_CLIENT_RM_TBLMETA_ERROR(_code) \
((_code) == TSDB_CODE_PAR_TABLE_NOT_EXIST || (_code) == TSDB_CODE_VND_TB_NOT_EXIST || \
(_code) == TSDB_CODE_PAR_INVALID_COLUMNS_NUM || (_code) == TSDB_CODE_PAR_INVALID_COLUMN || \
- (_code) == TSDB_CODE_PAR_TAGS_NOT_MATCHED || (_code) == TSDB_CODE_PAR_VALUE_TOO_LONG || \
+ (_code) == TSDB_CODE_PAR_TAGS_NOT_MATCHED || (_code) == TSDB_CODE_PAR_VALUE_TOO_LONG || \
(_code) == TSDB_CODE_PAR_INVALID_DROP_COL || ((_code) == TSDB_CODE_TDB_INVALID_TABLE_ID))
#define NEED_CLIENT_REFRESH_VG_ERROR(_code) \
((_code) == TSDB_CODE_VND_HASH_MISMATCH || (_code) == TSDB_CODE_VND_INVALID_VGROUP_ID)
@@ -231,11 +234,13 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t
#define NEED_CLIENT_HANDLE_ERROR(_code) \
(NEED_CLIENT_RM_TBLMETA_ERROR(_code) || NEED_CLIENT_REFRESH_VG_ERROR(_code) || \
NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code))
-#define NEED_CLIENT_RM_TBLMETA_REQ(_type) ((_type) == TDMT_VND_CREATE_TABLE || (_type) == TDMT_VND_CREATE_STB \
- || (_type) == TDMT_VND_DROP_TABLE || (_type) == TDMT_VND_DROP_STB)
+#define NEED_CLIENT_RM_TBLMETA_REQ(_type) \
+ ((_type) == TDMT_VND_CREATE_TABLE || (_type) == TDMT_VND_CREATE_STB || (_type) == TDMT_VND_DROP_TABLE || \
+ (_type) == TDMT_VND_DROP_STB)
-#define NEED_SCHEDULER_RETRY_ERROR(_code) \
- ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL || (_code) == TSDB_CODE_SCH_TIMEOUT_ERROR)
+#define NEED_SCHEDULER_RETRY_ERROR(_code) \
+ ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL || \
+ (_code) == TSDB_CODE_SCH_TIMEOUT_ERROR)
#define REQUEST_TOTAL_EXEC_TIMES 2
@@ -312,3 +317,4 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t
#endif
#endif /*_TD_QUERY_H_*/
+ // clang-foramt on
diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h
index a47810e7c1..db928f194c 100644
--- a/include/libs/stream/tstream.h
+++ b/include/libs/stream/tstream.h
@@ -58,6 +58,7 @@ enum {
enum {
STREAM_INPUT__DATA_SUBMIT = 1,
STREAM_INPUT__DATA_BLOCK,
+ STREAM_INPUT__DATA_RETRIEVE,
STREAM_INPUT__TRIGGER,
STREAM_INPUT__CHECKPOINT,
STREAM_INPUT__DROP,
@@ -318,7 +319,7 @@ static FORCE_INLINE int32_t streamTaskInput(SStreamTask* pTask, SStreamQueueItem
return -1;
}
taosWriteQitem(pTask->inputQueue->queue, pSubmitClone);
- } else if (pItem->type == STREAM_INPUT__DATA_BLOCK) {
+ } else if (pItem->type == STREAM_INPUT__DATA_BLOCK || pItem->type == STREAM_INPUT__DATA_RETRIEVE) {
taosWriteQitem(pTask->inputQueue->queue, pItem);
} else if (pItem->type == STREAM_INPUT__CHECKPOINT) {
taosWriteQitem(pTask->inputQueue->queue, pItem);
diff --git a/include/libs/stream/tstreamUpdate.h b/include/libs/stream/tstreamUpdate.h
index 398851a09f..21a1515d8f 100644
--- a/include/libs/stream/tstreamUpdate.h
+++ b/include/libs/stream/tstreamUpdate.h
@@ -32,12 +32,15 @@ typedef struct SUpdateInfo {
int64_t interval;
int64_t watermark;
TSKEY minTS;
+ SScalableBf* pCloseWinSBF;
} SUpdateInfo;
SUpdateInfo *updateInfoInitP(SInterval* pInterval, int64_t watermark);
SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t watermark);
bool updateInfoIsUpdated(SUpdateInfo *pInfo, tb_uid_t tableId, TSKEY ts);
void updateInfoDestroy(SUpdateInfo *pInfo);
+void updateInfoAddCloseWindowSBF(SUpdateInfo *pInfo);
+void updateInfoDestoryColseWinSBF(SUpdateInfo *pInfo);
#ifdef __cplusplus
}
diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h
index 77de5f4756..c2c1a3534d 100644
--- a/include/libs/transport/trpc.h
+++ b/include/libs/transport/trpc.h
@@ -34,10 +34,8 @@ extern int32_t tsRpcHeadSize;
typedef struct {
uint32_t clientIp;
uint16_t clientPort;
- union {
- char user[TSDB_USER_LEN];
- int64_t applyIndex;
- };
+ int64_t applyIndex;
+ char user[TSDB_USER_LEN];
} SRpcConnInfo;
typedef struct SRpcHandleInfo {
diff --git a/include/os/osSocket.h b/include/os/osSocket.h
index 213a6930ee..9dd5b972fa 100644
--- a/include/os/osSocket.h
+++ b/include/os/osSocket.h
@@ -157,7 +157,10 @@ int32_t taosNonblockwrite(TdSocketPtr pSocket, char *ptr, int32_t nbytes);
int64_t taosCopyFds(TdSocketPtr pSrcSocket, TdSocketPtr pDestSocket, int64_t len);
void taosWinSocketInit();
-int taosCreateSocketWithTimeOutOpt(uint32_t conn_timeout_sec);
+/*
+ * set timeout(ms)
+ */
+int32_t taosCreateSocketWithTimeout(uint32_t timeout);
TdSocketPtr taosOpenUdpSocket(uint32_t localIp, uint16_t localPort);
TdSocketPtr taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp);
diff --git a/include/util/ttrace.h b/include/util/ttrace.h
index 206cbbf28d..579768228a 100644
--- a/include/util/ttrace.h
+++ b/include/util/ttrace.h
@@ -45,9 +45,11 @@ typedef struct STraceId {
#define TRACE_GET_MSGID(traceId) (traceId)->msgId
-#define TRACE_TO_STR(traceId, buf) \
- do { \
- sprintf(buf, "0x%" PRIx64 ":0x%" PRIx64 "", traceId->rootId, traceId->msgId); \
+#define TRACE_TO_STR(traceId, buf) \
+ do { \
+ int64_t rootId = (traceId) != NULL ? (traceId)->rootId : 0; \
+ int64_t msgId = (traceId) != NULL ? (traceId)->msgId : 0; \
+ sprintf(buf, "0x%" PRIx64 ":0x%" PRIx64 "", rootId, msgId); \
} while (0)
#ifdef __cplusplus
diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c
index 9f04e89694..d7bf4b60f1 100644
--- a/source/client/src/clientEnv.c
+++ b/source/client/src/clientEnv.c
@@ -13,11 +13,11 @@
* along with this program. If not, see .
*/
-#include "os.h"
#include "catalog.h"
-#include "functionMgt.h"
#include "clientInt.h"
#include "clientLog.h"
+#include "functionMgt.h"
+#include "os.h"
#include "query.h"
#include "scheduler.h"
#include "tcache.h"
@@ -38,7 +38,7 @@ static TdThreadOnce tscinit = PTHREAD_ONCE_INIT;
volatile int32_t tscInitRes = 0;
static void registerRequest(SRequestObj *pRequest) {
- STscObj *pTscObj = acquireTscObj(*(int64_t*)pRequest->pTscObj->id);
+ STscObj *pTscObj = acquireTscObj(*(int64_t *)pRequest->pTscObj->id);
assert(pTscObj != NULL);
@@ -54,14 +54,14 @@ static void registerRequest(SRequestObj *pRequest) {
int32_t currentInst = atomic_add_fetch_64((int64_t *)&pSummary->currentRequests, 1);
tscDebug("0x%" PRIx64 " new Request from connObj:0x%" PRIx64
", current:%d, app current:%d, total:%d, reqId:0x%" PRIx64,
- pRequest->self, *(int64_t*)pRequest->pTscObj->id, num, currentInst, total, pRequest->requestId);
+ pRequest->self, *(int64_t *)pRequest->pTscObj->id, num, currentInst, total, pRequest->requestId);
}
}
static void deregisterRequest(SRequestObj *pRequest) {
assert(pRequest != NULL);
- STscObj *pTscObj = pRequest->pTscObj;
+ STscObj * pTscObj = pRequest->pTscObj;
SAppClusterSummary *pActivity = &pTscObj->pAppInfo->summary;
int32_t currentInst = atomic_sub_fetch_64((int64_t *)&pActivity->currentRequests, 1);
@@ -70,8 +70,8 @@ static void deregisterRequest(SRequestObj *pRequest) {
int64_t duration = taosGetTimestampUs() - pRequest->metric.start;
tscDebug("0x%" PRIx64 " free Request from connObj: 0x%" PRIx64 ", reqId:0x%" PRIx64 " elapsed:%" PRIu64
" ms, current:%d, app current:%d",
- pRequest->self, *(int64_t*)pTscObj->id, pRequest->requestId, duration / 1000, num, currentInst);
- releaseTscObj(*(int64_t*)pTscObj->id);
+ pRequest->self, *(int64_t *)pTscObj->id, pRequest->requestId, duration / 1000, num, currentInst);
+ releaseTscObj(*(int64_t *)pTscObj->id);
}
// todo close the transporter properly
@@ -80,12 +80,13 @@ void closeTransporter(STscObj *pTscObj) {
return;
}
- tscDebug("free transporter:%p in connObj: 0x%" PRIx64, pTscObj->pAppInfo->pTransporter, *(int64_t*)pTscObj->id);
+ tscDebug("free transporter:%p in connObj: 0x%" PRIx64, pTscObj->pAppInfo->pTransporter, *(int64_t *)pTscObj->id);
rpcClose(pTscObj->pAppInfo->pTransporter);
}
static bool clientRpcRfp(int32_t code) {
- if (code == TSDB_CODE_RPC_REDIRECT) {
+ if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED ||
+ code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY) {
return true;
} else {
return false;
@@ -128,16 +129,17 @@ void closeAllRequests(SHashObj *pRequests) {
void destroyTscObj(void *pObj) {
STscObj *pTscObj = pObj;
- SClientHbKey connKey = {.tscRid = *(int64_t*)pTscObj->id, .connType = pTscObj->connType};
+ SClientHbKey connKey = {.tscRid = *(int64_t *)pTscObj->id, .connType = pTscObj->connType};
hbDeregisterConn(pTscObj->pAppInfo->pAppHbMgr, connKey);
int64_t connNum = atomic_sub_fetch_64(&pTscObj->pAppInfo->numOfConns, 1);
closeAllRequests(pTscObj->pRequests);
schedulerStopQueryHb(pTscObj->pAppInfo->pTransporter);
if (0 == connNum) {
- // TODO
- //closeTransporter(pTscObj);
+ // TODO
+ // closeTransporter(pTscObj);
}
- tscDebug("connObj 0x%" PRIx64 " destroyed, totalConn:%" PRId64, *(int64_t*)pTscObj->id, pTscObj->pAppInfo->numOfConns);
+ tscDebug("connObj 0x%" PRIx64 " destroyed, totalConn:%" PRId64, *(int64_t *)pTscObj->id,
+ pTscObj->pAppInfo->numOfConns);
taosThreadMutexDestroy(&pTscObj->mutex);
taosMemoryFreeClear(pTscObj);
}
@@ -167,10 +169,10 @@ void *createTscObj(const char *user, const char *auth, const char *db, int32_t c
taosThreadMutexInit(&pObj->mutex, NULL);
pObj->id = taosMemoryMalloc(sizeof(int64_t));
- *(int64_t*)pObj->id = taosAddRef(clientConnRefPool, pObj);
+ *(int64_t *)pObj->id = taosAddRef(clientConnRefPool, pObj);
pObj->schemalessType = 1;
- tscDebug("connObj created, 0x%" PRIx64, *(int64_t*)pObj->id);
+ tscDebug("connObj created, 0x%" PRIx64, *(int64_t *)pObj->id);
return pObj;
}
@@ -325,7 +327,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
return 0;
}
- SConfig *pCfg = taosGetCfg();
+ SConfig * pCfg = taosGetCfg();
SConfigItem *pItem = NULL;
switch (option) {
diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c
index f5dfe2de36..bbd477fa3b 100644
--- a/source/client/src/clientMain.c
+++ b/source/client/src/clientMain.c
@@ -199,10 +199,7 @@ TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) {
return pResInfo->userFields;
}
-
-TAOS_RES *taos_query(TAOS *taos, const char *sql) {
- return taosQueryImpl(taos, sql, false);
-}
+TAOS_RES *taos_query(TAOS *taos, const char *sql) { return taosQueryImpl(taos, sql, false); }
TAOS_ROW taos_fetch_row(TAOS_RES *res) {
if (res == NULL) {
@@ -593,11 +590,11 @@ int *taos_get_column_data_offset(TAOS_RES *res, int columnIndex) {
return pResInfo->pCol[columnIndex].offset;
}
-int taos_validate_sql(TAOS *taos, const char *sql) {
- TAOS_RES* pObj = taosQueryImpl(taos, sql, true);
+int taos_validate_sql(TAOS *taos, const char *sql) {
+ TAOS_RES *pObj = taosQueryImpl(taos, sql, true);
int code = taos_errno(pObj);
-
+
taos_free_result(pObj);
return code;
}
@@ -884,10 +881,10 @@ void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress) {
int taos_load_table_info(TAOS *taos, const char *tableNameList) {
const int32_t MAX_TABLE_NAME_LENGTH = 12 * 1024 * 1024; // 12MB list
- int32_t code = 0;
- SRequestObj *pRequest = NULL;
- SCatalogReq catalogReq = {0};
-
+ int32_t code = 0;
+ SRequestObj *pRequest = NULL;
+ SCatalogReq catalogReq = {0};
+
if (NULL == tableNameList) {
return TSDB_CODE_SUCCESS;
}
@@ -911,26 +908,25 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) {
goto _return;
}
- SCatalog* pCtg = NULL;
+ SCatalog *pCtg = NULL;
code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCtg);
if (code != TSDB_CODE_SUCCESS) {
goto _return;
}
- char* sql = "taos_load_table_info";
+ char *sql = "taos_load_table_info";
code = buildRequest(pTscObj, sql, strlen(sql), &pRequest);
if (code != TSDB_CODE_SUCCESS) {
terrno = code;
goto _return;
}
-
+
SSyncQueryParam param = {0};
tsem_init(¶m.sem, 0, 0);
param.pRequest = pRequest;
- SRequestConnInfo conn = {.pTrans = pTscObj->pAppInfo->pTransporter,
- .requestId = pRequest->requestId,
- .requestObjRefId = pRequest->self};
+ SRequestConnInfo conn = {
+ .pTrans = pTscObj->pAppInfo->pTransporter, .requestId = pRequest->requestId, .requestObjRefId = pRequest->self};
conn.mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp);
@@ -951,7 +947,6 @@ _return:
return code;
}
-
TAOS_STMT *taos_stmt_init(TAOS *taos) {
STscObj *pObj = acquireTscObj(*(int64_t *)taos);
if (NULL == pObj) {
diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c
index 593f8c5c0b..9f89d72172 100644
--- a/source/common/src/tdatablock.c
+++ b/source/common/src/tdatablock.c
@@ -1164,7 +1164,7 @@ int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows)
int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) {
int32_t code = 0;
- ASSERT(numOfRows > 0);
+ // ASSERT(numOfRows > 0);
if (numOfRows == 0) {
return TSDB_CODE_SUCCESS;
@@ -1230,6 +1230,32 @@ int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src) {
return 0;
}
+int32_t copyDataBlock(SSDataBlock* dst, const SSDataBlock* src) {
+ ASSERT(src != NULL && dst != NULL);
+
+ blockDataCleanup(dst);
+ int32_t code = blockDataEnsureCapacity(dst, src->info.rows);
+ if (code != TSDB_CODE_SUCCESS) {
+ terrno = code;
+ return code;
+ }
+
+ size_t numOfCols = taosArrayGetSize(src->pDataBlock);
+ for (int32_t i = 0; i < numOfCols; ++i) {
+ SColumnInfoData* pDst = taosArrayGet(dst->pDataBlock, i);
+ SColumnInfoData* pSrc = taosArrayGet(src->pDataBlock, i);
+ if (pSrc->pData == NULL) {
+ continue;
+ }
+
+ colDataAssign(pDst, pSrc, src->info.rows, &src->info);
+ }
+
+ dst->info.rows = src->info.rows;
+ dst->info.window = src->info.window;
+ return TSDB_CODE_SUCCESS;
+}
+
SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) {
if (pDataBlock == NULL) {
return NULL;
@@ -1627,6 +1653,57 @@ void blockDebugShowData(const SArray* dataBlocks, const char* flag) {
}
}
+// for debug
+char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) {
+ int32_t size = 2048;
+ *pDataBuf = taosMemoryCalloc(size, 1);
+ char* dumpBuf = *pDataBuf;
+ char pBuf[128] = {0};
+ int32_t colNum = taosArrayGetSize(pDataBlock->pDataBlock);
+ int32_t rows = pDataBlock->info.rows;
+ int32_t len = 0;
+ len += snprintf(dumpBuf + len, size - len, "\n%s |block type %d |child id %d|\n", flag,
+ (int32_t)pDataBlock->info.type, pDataBlock->info.childId);
+ for (int32_t j = 0; j < rows; j++) {
+ len += snprintf(dumpBuf + len, size - len, "%s |", flag);
+ for (int32_t k = 0; k < colNum; k++) {
+ SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k);
+ void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes);
+ if (colDataIsNull(pColInfoData, rows, j, NULL)) {
+ len += snprintf(dumpBuf + len, size - len, " %15s |", "NULL");
+ continue;
+ }
+ switch (pColInfoData->info.type) {
+ case TSDB_DATA_TYPE_TIMESTAMP:
+ formatTimestamp(pBuf, *(uint64_t*)var, TSDB_TIME_PRECISION_MILLI);
+ len += snprintf(dumpBuf + len, size - len, " %25s |", pBuf);
+ break;
+ case TSDB_DATA_TYPE_INT:
+ len += snprintf(dumpBuf + len, size - len, " %15d |", *(int32_t*)var);
+ break;
+ case TSDB_DATA_TYPE_UINT:
+ len += snprintf(dumpBuf + len, size - len, " %15u |", *(uint32_t*)var);
+ break;
+ case TSDB_DATA_TYPE_BIGINT:
+ len += snprintf(dumpBuf + len, size - len, " %15ld |", *(int64_t*)var);
+ break;
+ case TSDB_DATA_TYPE_UBIGINT:
+ len += snprintf(dumpBuf + len, size - len, " %15lu |", *(uint64_t*)var);
+ break;
+ case TSDB_DATA_TYPE_FLOAT:
+ len += snprintf(dumpBuf + len, size - len, " %15f |", *(float*)var);
+ break;
+ case TSDB_DATA_TYPE_DOUBLE:
+ len += snprintf(dumpBuf + len, size - len, " %15lf |", *(double*)var);
+ break;
+ }
+ }
+ len += snprintf(dumpBuf + len, size - len, "\n");
+ }
+ len += snprintf(dumpBuf + len, size - len, "%s |end\n", flag);
+ return dumpBuf;
+}
+
/**
* @brief TODO: Assume that the final generated result it less than 3M
*
diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c
index 3d1ece0b64..980749ca70 100644
--- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c
+++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c
@@ -247,11 +247,9 @@ int32_t dmAppendVariablesToBlock(SSDataBlock* pBlock, int32_t dnodeId) {
return TSDB_CODE_SUCCESS;
}
-
-
int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
- int32_t size = 0;
- int32_t rowsRead = 0;
+ int32_t size = 0;
+ int32_t rowsRead = 0;
SRetrieveTableReq retrieveReq = {0};
if (tDeserializeSRetrieveTableReq(pMsg->pCont, pMsg->contLen, &retrieveReq) != 0) {
@@ -259,6 +257,11 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
return -1;
}
+ if (strcmp(retrieveReq.user, TSDB_DEFAULT_USER) != 0) {
+ terrno = TSDB_CODE_MND_NO_RIGHTS;
+ return -1;
+ }
+
if (strcasecmp(retrieveReq.tb, TSDB_INS_TABLE_DNODE_VARIABLES)) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
@@ -309,7 +312,6 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
return TSDB_CODE_SUCCESS;
}
-
SArray *dmGetMsgHandles() {
int32_t code = -1;
SArray *pArray = taosArrayInit(16, sizeof(SMgmtHandle));
diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c
index 63d2a65df1..a4745abd5b 100644
--- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c
+++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c
@@ -70,9 +70,9 @@ int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg) {
}
static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
- SDnodeTrans *pTrans = &pDnode->trans;
+ SDnodeTrans * pTrans = &pDnode->trans;
int32_t code = -1;
- SRpcMsg *pMsg = NULL;
+ SRpcMsg * pMsg = NULL;
SMgmtWrapper *pWrapper = NULL;
SDnodeHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(pRpc->msgType)];
@@ -194,11 +194,11 @@ int32_t dmInitMsgHandle(SDnode *pDnode) {
for (EDndNodeType ntype = DNODE; ntype < NODE_END; ++ntype) {
SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype];
- SArray *pArray = (*pWrapper->func.getHandlesFp)();
+ SArray * pArray = (*pWrapper->func.getHandlesFp)();
if (pArray == NULL) return -1;
for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) {
- SMgmtHandle *pMgmt = taosArrayGet(pArray, i);
+ SMgmtHandle * pMgmt = taosArrayGet(pArray, i);
SDnodeHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(pMgmt->msgType)];
if (pMgmt->needCheckVgId) {
pHandle->needCheckVgId = pMgmt->needCheckVgId;
@@ -248,7 +248,14 @@ static inline void dmReleaseHandle(SRpcHandleInfo *pHandle, int8_t type) {
}
}
-static bool rpcRfp(int32_t code) { return code == TSDB_CODE_RPC_REDIRECT; }
+static bool rpcRfp(int32_t code) {
+ if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED ||
+ code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY) {
+ return true;
+ } else {
+ return false;
+ }
+}
int32_t dmInitClient(SDnode *pDnode) {
SDnodeTrans *pTrans = &pDnode->trans;
diff --git a/source/dnode/mnode/impl/inc/mndPrivilege.h b/source/dnode/mnode/impl/inc/mndPrivilege.h
index 15f9e4e6b5..0c56f6f155 100644
--- a/source/dnode/mnode/impl/inc/mndPrivilege.h
+++ b/source/dnode/mnode/impl/inc/mndPrivilege.h
@@ -57,6 +57,8 @@ typedef enum {
MND_OPER_USE_DB,
MND_OPER_WRITE_DB,
MND_OPER_READ_DB,
+ MND_OPER_READ_OR_WRITE_DB,
+ MND_OPER_SHOW_VARIBALES,
} EOperType;
int32_t mndInitPrivilege(SMnode *pMnode);
@@ -64,7 +66,8 @@ void mndCleanupPrivilege(SMnode *pMnode);
int32_t mndCheckOperPrivilege(SMnode *pMnode, const char *user, EOperType operType);
int32_t mndCheckDbPrivilege(SMnode *pMnode, const char *user, EOperType operType, SDbObj *pDb);
-int32_t mndCheckShowPrivilege(SMnode *pMnode, const char *user, int32_t showType);
+int32_t mndCheckDbPrivilegeByName(SMnode *pMnode, const char *user, EOperType operType, const char *dbname);
+int32_t mndCheckShowPrivilege(SMnode *pMnode, const char *user, EShowType showType, const char *dbname);
int32_t mndCheckAlterUserPrivilege(SUserObj *pOperUser, SUserObj *pUser, SAlterUserReq *pAlter);
#ifdef __cplusplus
diff --git a/source/dnode/mnode/impl/inc/mndStream.h b/source/dnode/mnode/impl/inc/mndStream.h
index 69385c3a46..0901e77287 100644
--- a/source/dnode/mnode/impl/inc/mndStream.h
+++ b/source/dnode/mnode/impl/inc/mndStream.h
@@ -33,6 +33,10 @@ SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw);
int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb);
int32_t mndPersistStream(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
+// for sma
+// TODO refactor
+int32_t mndDropStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
+int32_t mndPersistDropStreamLog(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
#ifdef __cplusplus
}
diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c
index 7dc5ee1ea1..5b5de10fba 100644
--- a/source/dnode/mnode/impl/src/mndConsumer.c
+++ b/source/dnode/mnode/impl/src/mndConsumer.c
@@ -431,6 +431,10 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
goto SUBSCRIBE_OVER;
}
+ if (mndCheckDbPrivilegeByName(pMnode, pMsg->info.conn.user, MND_OPER_READ_DB, pTopic->db) != 0) {
+ goto SUBSCRIBE_OVER;
+ }
+
#if 0
// ref topic to prevent drop
// TODO make topic complete
diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c
index 0345f1b345..abd1492d5e 100644
--- a/source/dnode/mnode/impl/src/mndDb.c
+++ b/source/dnode/mnode/impl/src/mndDb.c
@@ -15,9 +15,9 @@
#define _DEFAULT_SOURCE
#include "mndDb.h"
-#include "mndPrivilege.h"
#include "mndDnode.h"
#include "mndOffset.h"
+#include "mndPrivilege.h"
#include "mndShow.h"
#include "mndSma.h"
#include "mndStb.h"
@@ -1336,7 +1336,7 @@ char *buildRetension(SArray *pRetension) {
}
static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, int32_t rows, int64_t numOfTables,
- bool sysDb, ESdbStatus objStatus) {
+ bool sysDb, ESdbStatus objStatus, bool sysinfo) {
int32_t cols = 0;
int32_t bytes = pShow->pMeta->pSchemas[cols].bytes;
@@ -1354,7 +1354,7 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in
char statusB[24] = {0};
STR_WITH_SIZE_TO_VARSTR(statusB, status, strlen(status));
- if (sysDb) {
+ if (sysDb || !sysinfo) {
for (int32_t i = 0; i < pShow->numOfColumns; ++i) {
SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, i);
if (i == 0) {
@@ -1528,17 +1528,21 @@ static int32_t mndRetrieveDbs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc
SDbObj *pDb = NULL;
ESdbStatus objStatus = 0;
+ SUserObj *pUser = mndAcquireUser(pMnode, pReq->info.conn.user);
+ if (pUser == NULL) return 0;
+ bool sysinfo = pUser->sysInfo;
+
// Append the information_schema database into the result.
if (!pShow->sysDbRsp) {
SDbObj infoschemaDb = {0};
setInformationSchemaDbCfg(&infoschemaDb);
- dumpDbInfoData(pBlock, &infoschemaDb, pShow, numOfRows, 14, true, 0);
+ dumpDbInfoData(pBlock, &infoschemaDb, pShow, numOfRows, 14, true, 0, 1);
numOfRows += 1;
SDbObj perfschemaDb = {0};
setPerfSchemaDbCfg(&perfschemaDb);
- dumpDbInfoData(pBlock, &perfschemaDb, pShow, numOfRows, 3, true, 0);
+ dumpDbInfoData(pBlock, &perfschemaDb, pShow, numOfRows, 3, true, 0, 1);
numOfRows += 1;
pShow->sysDbRsp = true;
@@ -1550,16 +1554,19 @@ static int32_t mndRetrieveDbs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc
break;
}
- int32_t numOfTables = 0;
- sdbTraverse(pSdb, SDB_VGROUP, mndGetTablesOfDbFp, &numOfTables, NULL, NULL);
+ if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_READ_OR_WRITE_DB, pDb) == 0) {
+ int32_t numOfTables = 0;
+ sdbTraverse(pSdb, SDB_VGROUP, mndGetTablesOfDbFp, &numOfTables, NULL, NULL);
+
+ dumpDbInfoData(pBlock, pDb, pShow, numOfRows, numOfTables, false, objStatus, sysinfo);
+ numOfRows++;
+ }
- dumpDbInfoData(pBlock, pDb, pShow, numOfRows, numOfTables, false, objStatus);
- numOfRows++;
sdbRelease(pSdb, pDb);
}
pShow->numOfRows += numOfRows;
-
+ mndReleaseUser(pMnode, pUser);
return numOfRows;
}
diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c
index af1d641ebf..5e78d0f434 100644
--- a/source/dnode/mnode/impl/src/mndDnode.c
+++ b/source/dnode/mnode/impl/src/mndDnode.c
@@ -558,7 +558,11 @@ _OVER:
static int32_t mndProcessShowVariablesReq(SRpcMsg *pReq) {
SShowVariablesRsp rsp = {0};
- int32_t code = -1;
+ int32_t code = -1;
+
+ if (mndCheckOperPrivilege(pReq->info.node, pReq->info.conn.user, MND_OPER_SHOW_VARIBALES) != 0) {
+ goto _OVER;
+ }
rsp.variables = taosArrayInit(4, sizeof(SVariablesInfo));
if (NULL == rsp.variables) {
diff --git a/source/dnode/mnode/impl/src/mndPrivilege.c b/source/dnode/mnode/impl/src/mndPrivilege.c
index 478ba2bee4..dc321e38d1 100644
--- a/source/dnode/mnode/impl/src/mndPrivilege.c
+++ b/source/dnode/mnode/impl/src/mndPrivilege.c
@@ -15,6 +15,7 @@
#define _DEFAULT_SOURCE
#include "mndPrivilege.h"
+#include "mndDb.h"
#include "mndUser.h"
int32_t mndInitPrivilege(SMnode *pMnode) { return 0; }
@@ -45,6 +46,7 @@ int32_t mndCheckOperPrivilege(SMnode *pMnode, const char *user, EOperType operTy
case MND_OPER_CONNECT:
case MND_OPER_CREATE_FUNC:
case MND_OPER_DROP_FUNC:
+ case MND_OPER_SHOW_VARIBALES:
break;
default:
terrno = TSDB_CODE_MND_NO_RIGHTS;
@@ -79,7 +81,7 @@ int32_t mndCheckAlterUserPrivilege(SUserObj *pOperUser, SUserObj *pUser, SAlterU
return -1;
}
-int32_t mndCheckShowPrivilege(SMnode *pMnode, const char *user, int32_t showType) {
+int32_t mndCheckShowPrivilege(SMnode *pMnode, const char *user, EShowType showType, const char *dbname) {
int32_t code = 0;
SUserObj *pUser = mndAcquireUser(pMnode, user);
@@ -98,14 +100,34 @@ int32_t mndCheckShowPrivilege(SMnode *pMnode, const char *user, int32_t showType
goto _OVER;
}
- if (!pUser->sysInfo) {
- terrno = TSDB_CODE_MND_NO_RIGHTS;
- code = -1;
+ if (pUser->sysInfo) {
goto _OVER;
}
- terrno = TSDB_CODE_MND_NO_RIGHTS;
- code = -1;
+ switch (showType) {
+ case TSDB_MGMT_TABLE_DB:
+ case TSDB_MGMT_TABLE_STB:
+ case TSDB_MGMT_TABLE_INDEX:
+ case TSDB_MGMT_TABLE_STREAMS:
+ case TSDB_MGMT_TABLE_CONSUMERS:
+ case TSDB_MGMT_TABLE_TOPICS:
+ case TSDB_MGMT_TABLE_SUBSCRIPTIONS:
+ case TSDB_MGMT_TABLE_FUNC:
+ case TSDB_MGMT_TABLE_QUERIES:
+ case TSDB_MGMT_TABLE_CONNS:
+ case TSDB_MGMT_TABLE_APPS:
+ case TSDB_MGMT_TABLE_TRANS:
+ code = 0;
+ break;
+ default:
+ terrno = TSDB_CODE_MND_NO_RIGHTS;
+ code = -1;
+ goto _OVER;
+ }
+
+ if (showType == TSDB_MGMT_TABLE_STB || showType == TSDB_MGMT_TABLE_VGROUP || showType == TSDB_MGMT_TABLE_INDEX) {
+ code = mndCheckDbPrivilegeByName(pMnode, user, MND_OPER_READ_OR_WRITE_DB, dbname);
+ }
_OVER:
mndReleaseUser(pMnode, pUser);
@@ -133,19 +155,11 @@ int32_t mndCheckDbPrivilege(SMnode *pMnode, const char *user, EOperType operType
if (pUser->sysInfo) goto _OVER;
}
- if (operType == MND_OPER_ALTER_DB) {
+ if (operType == MND_OPER_ALTER_DB || operType == MND_OPER_DROP_DB || operType == MND_OPER_COMPACT_DB) {
if (strcmp(pUser->user, pDb->createUser) == 0 && pUser->sysInfo) goto _OVER;
}
- if (operType == MND_OPER_DROP_DB) {
- if (strcmp(pUser->user, pDb->createUser) == 0 && pUser->sysInfo) goto _OVER;
- }
-
- if (operType == MND_OPER_COMPACT_DB) {
- if (strcmp(pUser->user, pDb->createUser) == 0 && pUser->sysInfo) goto _OVER;
- }
-
- if (operType == MND_OPER_USE_DB) {
+ if (operType == MND_OPER_USE_DB || operType == MND_OPER_READ_OR_WRITE_DB) {
if (strcmp(pUser->user, pDb->createUser) == 0) goto _OVER;
if (taosHashGet(pUser->readDbs, pDb->name, strlen(pDb->name) + 1) != NULL) goto _OVER;
if (taosHashGet(pUser->writeDbs, pDb->name, strlen(pDb->name) + 1) != NULL) goto _OVER;
@@ -168,3 +182,12 @@ _OVER:
mndReleaseUser(pMnode, pUser);
return code;
}
+
+int32_t mndCheckDbPrivilegeByName(SMnode *pMnode, const char *user, EOperType operType, const char *dbname) {
+ SDbObj *pDb = mndAcquireDb(pMnode, dbname);
+ if (pDb == NULL) return -1;
+
+ int32_t code = mndCheckDbPrivilege(pMnode, user, operType, pDb);
+ mndReleaseDb(pMnode, pDb);
+ return code;
+}
\ No newline at end of file
diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c
index f2e599b073..38368b4ece 100644
--- a/source/dnode/mnode/impl/src/mndProfile.c
+++ b/source/dnode/mnode/impl/src/mndProfile.c
@@ -254,6 +254,10 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) {
terrstr());
goto _OVER;
}
+
+ if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_READ_OR_WRITE_DB, pDb) != 0) {
+ goto _OVER;
+ }
}
pConn = mndCreateConn(pMnode, pReq->info.conn.user, connReq.connType, pReq->info.conn.clientIp,
diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c
index 3351cff3a3..029a1e6f8c 100644
--- a/source/dnode/mnode/impl/src/mndShow.c
+++ b/source/dnode/mnode/impl/src/mndShow.c
@@ -231,8 +231,14 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
}
mDebug("show:0x%" PRIx64 ", start retrieve data, type:%d", pShow->id, pShow->type);
-
- // if (mndCheckShowPrivilege(pMnode, pReq->info.conn.user, pShow->type) != 0) return -1;
+ if (retrieveReq.user[0] != 0) {
+ memcpy(pReq->info.conn.user, retrieveReq.user, TSDB_USER_LEN);
+ } else {
+ memcpy(pReq->info.conn.user, TSDB_DEFAULT_USER, TSDB_USER_LEN);
+ }
+ if (mndCheckShowPrivilege(pMnode, pReq->info.conn.user, pShow->type, retrieveReq.db) != 0) {
+ return -1;
+ }
int32_t numOfCols = pShow->pMeta->numOfColumns;
SSDataBlock *pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c
index ef24cd0ba4..10eeaba982 100644
--- a/source/dnode/mnode/impl/src/mndSma.c
+++ b/source/dnode/mnode/impl/src/mndSma.c
@@ -15,11 +15,11 @@
#define _DEFAULT_SOURCE
#include "mndSma.h"
-#include "mndPrivilege.h"
#include "mndDb.h"
#include "mndDnode.h"
#include "mndInfoSchema.h"
#include "mndMnode.h"
+#include "mndPrivilege.h"
#include "mndScheduler.h"
#include "mndShow.h"
#include "mndStb.h"
@@ -857,6 +857,23 @@ static int32_t mndDropSma(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SSmaObj *p
mDebug("trans:%d, used to drop sma:%s", pTrans->id, pSma->name);
mndTransSetDbName(pTrans, pDb->name, NULL);
+ SStreamObj *pStream = mndAcquireStream(pMnode, pSma->name);
+ if (pStream == NULL || pStream->smaId != pSma->uid) {
+ sdbRelease(pMnode->pSdb, pStream);
+ goto _OVER;
+ } else {
+ if (mndDropStreamTasks(pMnode, pTrans, pStream) < 0) {
+ mError("stream:%s, failed to drop task since %s", pStream->name, terrstr());
+ sdbRelease(pMnode->pSdb, pStream);
+ goto _OVER;
+ }
+
+ // drop stream
+ if (mndPersistDropStreamLog(pMnode, pTrans, pStream) < 0) {
+ sdbRelease(pMnode->pSdb, pStream);
+ goto _OVER;
+ }
+ }
if (mndSetDropSmaRedoLogs(pMnode, pTrans, pSma) != 0) goto _OVER;
if (mndSetDropSmaVgroupRedoLogs(pMnode, pTrans, pVgroup) != 0) goto _OVER;
if (mndSetDropSmaCommitLogs(pMnode, pTrans, pSma) != 0) goto _OVER;
diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c
index 77b13cd82d..92f85ecd04 100644
--- a/source/dnode/mnode/impl/src/mndStb.c
+++ b/source/dnode/mnode/impl/src/mndStb.c
@@ -15,12 +15,12 @@
#define _DEFAULT_SOURCE
#include "mndStb.h"
-#include "mndPrivilege.h"
#include "mndDb.h"
#include "mndDnode.h"
#include "mndInfoSchema.h"
#include "mndMnode.h"
#include "mndPerfSchema.h"
+#include "mndPrivilege.h"
#include "mndScheduler.h"
#include "mndShow.h"
#include "mndSma.h"
@@ -107,7 +107,7 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) {
int32_t funcNum = taosArrayGetSize(pStb->pFuncs);
SDB_SET_INT32(pRaw, dataPos, funcNum, _OVER)
for (int32_t i = 0; i < funcNum; ++i) {
- char* func = taosArrayGet(pStb->pFuncs, i);
+ char *func = taosArrayGet(pStb->pFuncs, i);
SDB_SET_BINARY(pRaw, dataPos, func, TSDB_FUNC_NAME_LEN, _OVER)
}
@@ -708,7 +708,7 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat
pDst->commentLen = pCreate->commentLen;
pDst->pFuncs = pCreate->pFuncs;
pCreate->pFuncs = NULL;
-
+
if (pDst->commentLen > 0) {
pDst->comment = taosMemoryCalloc(pDst->commentLen + 1, 1);
if (pDst->comment == NULL) {
@@ -1391,12 +1391,11 @@ static int32_t mndBuildStbCfgImp(SDbObj *pDb, SStbObj *pStb, const char *tbName,
if (pStb->pFuncs) {
pRsp->pFuncs = taosArrayDup(pStb->pFuncs);
}
-
+
taosRUnLockLatch(&pStb->lock);
return 0;
}
-
static int32_t mndBuildStbSchema(SMnode *pMnode, const char *dbFName, const char *tbName, STableMetaRsp *pRsp,
int32_t *smaVer) {
char tbFName[TSDB_TABLE_FNAME_LEN] = {0};
@@ -1426,31 +1425,29 @@ static int32_t mndBuildStbSchema(SMnode *pMnode, const char *dbFName, const char
}
static int32_t mndBuildStbCfg(SMnode *pMnode, const char *dbFName, const char *tbName, STableCfgRsp *pRsp) {
- char tbFName[TSDB_TABLE_FNAME_LEN] = {0};
- snprintf(tbFName, sizeof(tbFName), "%s.%s", dbFName, tbName);
+ char tbFName[TSDB_TABLE_FNAME_LEN] = {0};
+ snprintf(tbFName, sizeof(tbFName), "%s.%s", dbFName, tbName);
- SDbObj *pDb = mndAcquireDb(pMnode, dbFName);
- if (pDb == NULL) {
- terrno = TSDB_CODE_MND_DB_NOT_SELECTED;
- return -1;
- }
+ SDbObj *pDb = mndAcquireDb(pMnode, dbFName);
+ if (pDb == NULL) {
+ terrno = TSDB_CODE_MND_DB_NOT_SELECTED;
+ return -1;
+ }
- SStbObj *pStb = mndAcquireStb(pMnode, tbFName);
- if (pStb == NULL) {
- mndReleaseDb(pMnode, pDb);
- terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;
- return -1;
- }
+ SStbObj *pStb = mndAcquireStb(pMnode, tbFName);
+ if (pStb == NULL) {
+ mndReleaseDb(pMnode, pDb);
+ terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;
+ return -1;
+ }
- int32_t code = mndBuildStbCfgImp(pDb, pStb, tbName, pRsp);
-
- mndReleaseDb(pMnode, pDb);
- mndReleaseStb(pMnode, pStb);
- return code;
+ int32_t code = mndBuildStbCfgImp(pDb, pStb, tbName, pRsp);
+
+ mndReleaseDb(pMnode, pDb);
+ mndReleaseStb(pMnode, pStb);
+ return code;
}
-
-
static int32_t mndBuildSMAlterStbRsp(SDbObj *pDb, const SMAlterStbReq *pAlter, SStbObj *pObj, void **pCont,
int32_t *pLen) {
int32_t ret;
@@ -1811,10 +1808,10 @@ _OVER:
}
static int32_t mndProcessTableCfgReq(SRpcMsg *pReq) {
- SMnode *pMnode = pReq->info.node;
- int32_t code = -1;
- STableCfgReq cfgReq = {0};
- STableCfgRsp cfgRsp = {0};
+ SMnode *pMnode = pReq->info.node;
+ int32_t code = -1;
+ STableCfgReq cfgReq = {0};
+ STableCfgRsp cfgRsp = {0};
if (tDeserializeSTableCfgReq(pReq->pCont, pReq->contLen, &cfgReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
@@ -1866,7 +1863,6 @@ _OVER:
return code;
}
-
int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbVersions, int32_t numOfStbs, void **ppRsp,
int32_t *pRspLen) {
SSTbHbRsp hbRsp = {0};
diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c
index b865d0495c..cabbac14f1 100644
--- a/source/dnode/mnode/impl/src/mndStream.c
+++ b/source/dnode/mnode/impl/src/mndStream.c
@@ -14,10 +14,10 @@
*/
#include "mndStream.h"
-#include "mndPrivilege.h"
#include "mndDb.h"
#include "mndDnode.h"
#include "mndMnode.h"
+#include "mndPrivilege.h"
#include "mndScheduler.h"
#include "mndShow.h"
#include "mndStb.h"
@@ -437,10 +437,6 @@ static int32_t mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStre
goto _OVER;
}
- if (mndCheckDbPrivilege(pMnode, user, MND_OPER_WRITE_DB, pDb) != 0) {
- goto _OVER;
- }
-
int32_t numOfStbs = -1;
if (mndGetNumOfStbs(pMnode, pDb->name, &numOfStbs) != 0) {
goto _OVER;
@@ -494,7 +490,7 @@ static int32_t mndPersistTaskDropReq(STrans *pTrans, SStreamTask *pTask) {
return 0;
}
-static int32_t mndDropStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) {
+int32_t mndDropStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) {
int32_t lv = taosArrayGetSize(pStream->tasks);
for (int32_t i = 0; i < lv; i++) {
SArray *pTasks = taosArrayGetP(pStream->tasks, i);
@@ -542,19 +538,6 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
goto _OVER;
}
- // TODO check read auth for source and write auth for target
-#if 0
- pDb = mndAcquireDb(pMnode, createStreamReq.sourceDB);
- if (pDb == NULL) {
- terrno = TSDB_CODE_MND_DB_NOT_SELECTED;
- goto _OVER;
- }
-
- if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) {
- goto _OVER;
- }
-#endif
-
// build stream obj from request
SStreamObj streamObj = {0};
if (mndBuildStreamObjFromCreateReq(pMnode, &streamObj, &createStreamReq) < 0) {
@@ -592,6 +575,16 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
goto _OVER;
}
+ if (mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_READ_DB, streamObj.sourceDb) != 0) {
+ mndTransDrop(pTrans);
+ goto _OVER;
+ }
+
+ if (mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, streamObj.targetDb) != 0) {
+ mndTransDrop(pTrans);
+ goto _OVER;
+ }
+
// execute creation
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
@@ -641,13 +634,9 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) {
}
}
-#if 0
- // todo check auth
- pUser = mndAcquireUser(pMnode, pReq->info.conn.user);
- if (pUser == NULL) {
- goto DROP_STREAM_OVER;
+ if (mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pStream->targetDb) != 0) {
+ return -1;
}
-#endif
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq);
if (pTrans == NULL) {
diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c
index b8c17378c4..8d1220d596 100644
--- a/source/dnode/mnode/impl/src/mndTopic.c
+++ b/source/dnode/mnode/impl/src/mndTopic.c
@@ -14,12 +14,12 @@
*/
#include "mndTopic.h"
-#include "mndPrivilege.h"
#include "mndConsumer.h"
#include "mndDb.h"
#include "mndDnode.h"
#include "mndMnode.h"
#include "mndOffset.h"
+#include "mndPrivilege.h"
#include "mndShow.h"
#include "mndStb.h"
#include "mndSubscribe.h"
@@ -401,6 +401,10 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq *
}
} else if (pCreate->subType == TOPIC_SUB_TYPE__TABLE) {
SStbObj *pStb = mndAcquireStb(pMnode, pCreate->subStbName);
+ if (pStb == NULL) {
+ terrno = TSDB_CODE_MND_STB_NOT_EXIST;
+ return -1;
+ }
topicObj.stbUid = pStb->uid;
}
/*} else if (pCreate->subType == TOPIC_SUB_TYPE__DB) {*/
@@ -480,7 +484,7 @@ static int32_t mndProcessCreateTopicReq(SRpcMsg *pReq) {
goto _OVER;
}
- if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) {
+ if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_READ_DB, pDb) != 0) {
goto _OVER;
}
@@ -571,6 +575,10 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) {
}
#endif
+ if (mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_READ_DB, pTopic->db) != 0) {
+ return -1;
+ }
+
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pReq);
mndTransSetDbName(pTrans, pTopic->db, NULL);
if (pTrans == NULL) {
diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c
index 3a92cba773..b13e654caf 100644
--- a/source/dnode/snode/src/snode.c
+++ b/source/dnode/snode/src/snode.c
@@ -257,7 +257,7 @@ int32_t sndProcessSMsg(SSnode *pSnode, SRpcMsg *pMsg) {
case TDMT_STREAM_TASK_RECOVER_RSP:
return sndProcessTaskRecoverRsp(pSnode, pMsg);
case TDMT_STREAM_RETRIEVE_RSP:
- return sndProcessTaskRecoverRsp(pSnode, pMsg);
+ return sndProcessTaskRetrieveRsp(pSnode, pMsg);
default:
ASSERT(0);
}
diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h
index a32bf0ecdb..858376519d 100644
--- a/source/dnode/vnode/inc/vnode.h
+++ b/source/dnode/vnode/inc/vnode.h
@@ -116,18 +116,18 @@ typedef void *tsdbReaderT;
#define BLOCK_LOAD_TABLE_SEQ_ORDER 2
#define BLOCK_LOAD_TABLE_RR_ORDER 3
-int32_t tsdbSetTableList(tsdbReaderT reader, SArray* tableList);
-tsdbReaderT tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, SArray *tableList, uint64_t qId,
- uint64_t taskId);
-tsdbReaderT tsdbQueryCacheLast(SVnode *pVnode, SQueryTableDataCond *pCond, STableListInfo *groupList, uint64_t qId,
- void *pMemRef);
-int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT *pReader, STableBlockDistInfo *pTableBlockInfo);
-bool isTsdbCacheLastRow(tsdbReaderT *pReader);
-int32_t tsdbGetAllTableList(SMeta *pMeta, uint64_t uid, SArray *list);
-int32_t tsdbGetCtbIdList(SMeta *pMeta, int64_t suid, SArray *list);
-void *tsdbGetIdx(SMeta *pMeta);
-void *tsdbGetIvtIdx(SMeta *pMeta);
-int64_t tsdbGetNumOfRowsInMemTable(tsdbReaderT *pHandle);
+int32_t tsdbSetTableList(tsdbReaderT reader, SArray *tableList);
+tsdbReaderT tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, SArray *tableList, uint64_t qId,
+ uint64_t taskId);
+tsdbReaderT tsdbQueryCacheLast(SVnode *pVnode, SQueryTableDataCond *pCond, STableListInfo *groupList, uint64_t qId,
+ void *pMemRef);
+int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT *pReader, STableBlockDistInfo *pTableBlockInfo);
+bool isTsdbCacheLastRow(tsdbReaderT *pReader);
+int32_t tsdbGetAllTableList(SMeta *pMeta, uint64_t uid, SArray *list);
+int32_t tsdbGetCtbIdList(SMeta *pMeta, int64_t suid, SArray *list);
+void *tsdbGetIdx(SMeta *pMeta);
+void *tsdbGetIvtIdx(SMeta *pMeta);
+int64_t tsdbGetNumOfRowsInMemTable(tsdbReaderT *pHandle);
bool tsdbNextDataBlock(tsdbReaderT pTsdbReadHandle);
void tsdbRetrieveDataBlockInfo(tsdbReaderT *pTsdbReadHandle, SDataBlockInfo *pBlockInfo);
@@ -150,8 +150,7 @@ int32_t tqReadHandleRemoveTbUidList(STqReadHandle *pHandle, const SArray *tbUidL
int32_t tqReadHandleSetMsg(STqReadHandle *pHandle, SSubmitReq *pMsg, int64_t ver);
bool tqNextDataBlock(STqReadHandle *pHandle);
bool tqNextDataBlockFilterOut(STqReadHandle *pHandle, SHashObj *filterOutUids);
-int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle *pHandle, uint64_t *pGroupId, uint64_t *pUid,
- int32_t *pNumOfRows);
+int32_t tqRetrieveDataBlock(SSDataBlock *pBlock, STqReadHandle *pHandle);
// sma
int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days);
@@ -196,6 +195,7 @@ struct SVnodeCfg {
typedef struct {
TSKEY lastKey;
uint64_t uid;
+ uint64_t groupId;
} STableKeyInfo;
struct SMetaEntry {
diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c
index 4a3feca8d0..7109bf1dfc 100644
--- a/source/dnode/vnode/src/meta/metaTable.c
+++ b/source/dnode/vnode/src/meta/metaTable.c
@@ -388,7 +388,7 @@ int metaTtlDropTable(SMeta *pMeta, int64_t ttl, SArray *tbUids) {
}
static void metaBuildTtlIdxKey(STtlIdxKey *ttlKey, const SMetaEntry *pME){
- int32_t ttlDays;
+ int64_t ttlDays;
int64_t ctime;
if (pME->type == TSDB_CHILD_TABLE) {
ctime = pME->ctbEntry.ctime;
diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c
index be8fef1249..afeeeab500 100644
--- a/source/dnode/vnode/src/tq/tqExec.c
+++ b/source/dnode/vnode/src/tq/tqExec.c
@@ -112,7 +112,7 @@ int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkR
tqReadHandleSetMsg(pReader, pReq, 0);
while (tqNextDataBlock(pReader)) {
SSDataBlock block = {0};
- if (tqRetrieveDataBlock(&block, pReader, &block.info.groupId, &block.info.uid, &block.info.rows) < 0) {
+ if (tqRetrieveDataBlock(&block, pReader) < 0) {
if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue;
ASSERT(0);
}
@@ -129,7 +129,7 @@ int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkR
tqReadHandleSetMsg(pReader, pReq, 0);
while (tqNextDataBlockFilterOut(pReader, pExec->execDb.pFilterOutTbUid)) {
SSDataBlock block = {0};
- if (tqRetrieveDataBlock(&block, pReader, &block.info.groupId, &block.info.uid, &block.info.rows) < 0) {
+ if (tqRetrieveDataBlock(&block, pReader) < 0) {
if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue;
ASSERT(0);
}
diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c
index 6184d8f810..cbee639911 100644
--- a/source/dnode/vnode/src/tq/tqRead.c
+++ b/source/dnode/vnode/src/tq/tqRead.c
@@ -109,11 +109,15 @@ int32_t tqReadHandleSetMsg(STqReadHandle* pReadHandle, SSubmitReq* pMsg, int64_t
}
bool tqNextDataBlock(STqReadHandle* pHandle) {
+ if (pHandle->pMsg == NULL) return false;
while (1) {
if (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) < 0) {
return false;
}
- if (pHandle->pBlock == NULL) return false;
+ if (pHandle->pBlock == NULL) {
+ pHandle->pMsg = NULL;
+ return false;
+ }
if (pHandle->tbIdHash == NULL) {
return true;
@@ -142,10 +146,7 @@ bool tqNextDataBlockFilterOut(STqReadHandle* pHandle, SHashObj* filterOutUids) {
return false;
}
-int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle* pHandle, uint64_t* pGroupId, uint64_t* pUid,
- int32_t* pNumOfRows) {
- *pUid = 0;
-
+int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle* pHandle) {
// TODO: cache multiple schema
int32_t sversion = htonl(pHandle->pBlock->sversion);
if (pHandle->cachedSchemaSuid == 0 || pHandle->cachedSchemaVer != sversion ||
@@ -176,7 +177,6 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle* pHandle, uint64_
STSchema* pTschema = pHandle->pSchema;
SSchemaWrapper* pSchemaWrapper = pHandle->pSchemaWrapper;
- *pNumOfRows = pHandle->msgIter.numOfRows;
int32_t colNumNeed = taosArrayGetSize(pHandle->pColIdList);
if (colNumNeed == 0) {
@@ -217,22 +217,22 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle* pHandle, uint64_
}
}
- if (blockDataEnsureCapacity(pBlock, *pNumOfRows) < 0) {
+ if (blockDataEnsureCapacity(pBlock, pHandle->msgIter.numOfRows) < 0) {
goto FAIL;
}
int32_t colActual = blockDataGetNumOfCols(pBlock);
- // TODO in stream shuffle case, fetch groupId
- *pGroupId = 0;
-
STSRowIter iter = {0};
tdSTSRowIterInit(&iter, pTschema);
STSRow* row;
int32_t curRow = 0;
tInitSubmitBlkIter(&pHandle->msgIter, pHandle->pBlock, &pHandle->blkIter);
- *pUid = pHandle->msgIter.uid; // set the uid of table for submit block
+
+ pBlock->info.groupId = 0;
+ pBlock->info.uid = pHandle->msgIter.uid; // set the uid of table for submit block
+ pBlock->info.rows = pHandle->msgIter.numOfRows;
while ((row = tGetSubmitBlkNext(&pHandle->blkIter)) != NULL) {
tdSTSRowIterReset(&iter, row);
diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c
index ef3b205b3e..0bb9918488 100644
--- a/source/dnode/vnode/src/tq/tqSink.c
+++ b/source/dnode/vnode/src/tq/tqSink.c
@@ -18,57 +18,87 @@
SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, bool createTb, int64_t suid,
const char* stbFullName, int32_t vgId) {
SSubmitReq* ret = NULL;
+ SArray* schemaReqs = NULL;
+ SArray* schemaReqSz = NULL;
SArray* tagArray = taosArrayInit(1, sizeof(STagVal));
if (!tagArray) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
- // cal size
- int32_t cap = sizeof(SSubmitReq);
int32_t sz = taosArrayGetSize(pBlocks);
- for (int32_t i = 0; i < sz; i++) {
- SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i);
- int32_t rows = pDataBlock->info.rows;
- // TODO min
- int32_t rowSize = pDataBlock->info.rowSize;
- int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSchema);
- int32_t schemaLen = 0;
- if (createTb) {
- SVCreateTbReq createTbReq = {0};
- char* cname = buildCtbNameByGroupId(stbFullName, pDataBlock->info.groupId);
- createTbReq.name = cname;
- createTbReq.flags = 0;
- createTbReq.type = TSDB_CHILD_TABLE;
- createTbReq.ctb.suid = suid;
-
- STagVal tagVal = {
- .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1,
- .type = TSDB_DATA_TYPE_UBIGINT,
- .i64 = (int64_t)pDataBlock->info.groupId,
+ if (createTb) {
+ schemaReqs = taosArrayInit(sz, sizeof(void*));
+ schemaReqSz = taosArrayInit(sz, sizeof(int32_t));
+ for (int32_t i = 0; i < sz; i++) {
+ SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i);
+ STagVal tagVal = {
+ .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1,
+ .type = TSDB_DATA_TYPE_UBIGINT,
+ .i64 = (int64_t)pDataBlock->info.groupId,
};
STag* pTag = NULL;
taosArrayClear(tagArray);
taosArrayPush(tagArray, &tagVal);
tTagNew(tagArray, 1, false, &pTag);
if (pTag == NULL) {
- tdDestroySVCreateTbReq(&createTbReq);
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
taosArrayDestroy(tagArray);
return NULL;
}
+
+ SVCreateTbReq createTbReq = {0};
+ createTbReq.name = buildCtbNameByGroupId(stbFullName, pDataBlock->info.groupId);
+ createTbReq.flags = 0;
+ createTbReq.type = TSDB_CHILD_TABLE;
+ createTbReq.ctb.suid = suid;
createTbReq.ctb.pTag = (uint8_t*)pTag;
int32_t code;
+ int32_t schemaLen;
tEncodeSize(tEncodeSVCreateTbReq, &createTbReq, schemaLen, code);
-
- tdDestroySVCreateTbReq(&createTbReq);
if (code < 0) {
+ tdDestroySVCreateTbReq(&createTbReq);
taosArrayDestroy(tagArray);
+ taosMemoryFreeClear(ret);
return NULL;
}
- }
+ void* schemaStr = taosMemoryMalloc(schemaLen);
+ if (schemaStr == NULL) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ return NULL;
+ }
+ taosArrayPush(schemaReqs, &schemaStr);
+ taosArrayPush(schemaReqSz, &schemaLen);
+
+ SEncoder encoder = {0};
+ tEncoderInit(&encoder, schemaStr, schemaLen);
+ code = tEncodeSVCreateTbReq(&encoder, &createTbReq);
+ if (code < 0) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ return NULL;
+ }
+ tEncoderClear(&encoder);
+ tdDestroySVCreateTbReq(&createTbReq);
+ }
+ }
+ taosArrayDestroy(tagArray);
+
+ // cal size
+ int32_t cap = sizeof(SSubmitReq);
+ for (int32_t i = 0; i < sz; i++) {
+ SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i);
+ int32_t rows = pDataBlock->info.rows;
+ // TODO min
+ int32_t rowSize = pDataBlock->info.rowSize;
+ int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSchema);
+
+ int32_t schemaLen = 0;
+ if (createTb) {
+ schemaLen = *(int32_t*)taosArrayGet(schemaReqSz, i);
+ }
cap += sizeof(SSubmitBlk) + schemaLen + rows * maxLen;
}
@@ -99,55 +129,13 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo
int32_t schemaLen = 0;
if (createTb) {
- SVCreateTbReq createTbReq = {0};
- char* cname = buildCtbNameByGroupId(stbFullName, pDataBlock->info.groupId);
- createTbReq.name = cname;
- createTbReq.flags = 0;
- createTbReq.type = TSDB_CHILD_TABLE;
- createTbReq.ctb.suid = suid;
-
- STagVal tagVal = {
- .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1,
- .type = TSDB_DATA_TYPE_UBIGINT,
- .i64 = (int64_t)pDataBlock->info.groupId,
- };
- taosArrayClear(tagArray);
- taosArrayPush(tagArray, &tagVal);
- STag* pTag = NULL;
- tTagNew(tagArray, 1, false, &pTag);
- if (pTag == NULL) {
- tdDestroySVCreateTbReq(&createTbReq);
- taosArrayDestroy(tagArray);
- taosMemoryFreeClear(ret);
- return NULL;
- }
- createTbReq.ctb.pTag = (uint8_t*)pTag;
-
- int32_t code;
- tEncodeSize(tEncodeSVCreateTbReq, &createTbReq, schemaLen, code);
- if (code < 0) {
- tdDestroySVCreateTbReq(&createTbReq);
- taosArrayDestroy(tagArray);
- taosMemoryFreeClear(ret);
- return NULL;
- }
-
- SEncoder encoder = {0};
- tEncoderInit(&encoder, blkSchema, schemaLen);
- code = tEncodeSVCreateTbReq(&encoder, &createTbReq);
- tEncoderClear(&encoder);
- tdDestroySVCreateTbReq(&createTbReq);
-
- if (code < 0) {
- taosArrayDestroy(tagArray);
- taosMemoryFreeClear(ret);
- return NULL;
- }
+ schemaLen = *(int32_t*)taosArrayGet(schemaReqSz, i);
+ void* schemaStr = taosArrayGetP(schemaReqs, i);
+ memcpy(blkSchema, schemaStr, schemaLen);
}
blkHead->schemaLen = htonl(schemaLen);
STSRow* rowData = POINTER_SHIFT(blkSchema, schemaLen);
-
for (int32_t j = 0; j < rows; j++) {
SRowBuilder rb = {0};
tdSRowInit(&rb, pTSchema->version);
@@ -175,7 +163,10 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo
}
ret->length = htonl(ret->length);
- taosArrayDestroy(tagArray);
+
+ if (schemaReqs) taosArrayDestroyP(schemaReqs, taosMemoryFree);
+ taosArrayDestroy(schemaReqSz);
+
return ret;
}
diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c
index 540810f876..ab31d65d68 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead.c
@@ -2852,7 +2852,7 @@ int32_t tsdbGetAllTableList(SMeta* pMeta, uint64_t uid, SArray* list) {
break;
}
- STableKeyInfo info = {.lastKey = TSKEY_INITIAL_VAL, uid = id};
+ STableKeyInfo info = {.lastKey = TSKEY_INITIAL_VAL, uid = id, .groupId = 0};
taosArrayPush(list, &info);
}
diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h
index d8d231e952..36f81e86ff 100644
--- a/source/libs/executor/inc/executorimpl.h
+++ b/source/libs/executor/inc/executorimpl.h
@@ -293,6 +293,7 @@ typedef enum EStreamScanMode {
STREAM_SCAN_FROM_RES,
STREAM_SCAN_FROM_UPDATERES,
STREAM_SCAN_FROM_DATAREADER,
+ STREAM_SCAN_FROM_DATAREADER_RETRIEVE,
} EStreamScanMode;
typedef struct SCatchSupporter {
@@ -348,7 +349,9 @@ typedef struct SStreamBlockScanInfo {
SArray* childIds;
SessionWindowSupporter sessionSup;
bool assignBlockUid; // assign block uid to groupId, temporarily used for generating rollup SMA.
- int32_t scanWinIndex;
+ int32_t scanWinIndex; // for state operator
+ int32_t pullDataResIndex;
+ SSDataBlock* pPullDataRes; // pull data SSDataBlock
} SStreamBlockScanInfo;
typedef struct SSysTableScanInfo {
@@ -427,8 +430,13 @@ typedef struct SStreamFinalIntervalOperatorInfo {
STimeWindowAggSupp twAggSup;
SArray* pChildren;
SSDataBlock* pUpdateRes;
+ bool returnUpdate;
SPhysiNode* pPhyNode; // create new child
bool isFinal;
+ SHashObj* pPullDataMap;
+ SArray* pPullWins; // SPullWindowInfo
+ int32_t pullIndex;
+ SSDataBlock* pPullDataRes;
} SStreamFinalIntervalOperatorInfo;
typedef struct SAggOperatorInfo {
@@ -851,6 +859,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex);
int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* groupKey);
+SSDataBlock* createPullDataBlock();
#ifdef __cplusplus
}
diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c
index 5ac5957f2b..374a3a736d 100644
--- a/source/libs/executor/src/executil.c
+++ b/source/libs/executor/src/executil.c
@@ -315,7 +315,7 @@ int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo
}
for (int i = 0; i < taosArrayGetSize(res); i++) {
- STableKeyInfo info = {.lastKey = TSKEY_INITIAL_VAL, .uid = *(uint64_t*)taosArrayGet(res, i)};
+ STableKeyInfo info = {.lastKey = TSKEY_INITIAL_VAL, .uid = *(uint64_t*)taosArrayGet(res, i), .groupId = 0};
taosArrayPush(pListInfo->pTableList, &info);
}
taosArrayDestroy(res);
@@ -336,7 +336,7 @@ int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo
}
}
}else { // Create one table group.
- STableKeyInfo info = {.lastKey = 0, .uid = tableUid};
+ STableKeyInfo info = {.lastKey = 0, .uid = tableUid, .groupId = 0};
taosArrayPush(pListInfo->pTableList, &info);
}
pListInfo->pGroupList = taosArrayInit(4, POINTER_BYTES);
diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c
index f352049810..f18da72f58 100644
--- a/source/libs/executor/src/executorimpl.c
+++ b/source/libs/executor/src/executorimpl.c
@@ -4028,6 +4028,7 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle,
int32_t len = (int32_t)(pStart - (char*)keyBuf);
uint64_t groupId = calcGroupId(keyBuf, len);
taosHashPut(pTableListInfo->map, &(info->uid), sizeof(uint64_t), &groupId, sizeof(uint64_t));
+ info->groupId = groupId;
groupNum++;
nodesClearList(groupNew);
@@ -4126,7 +4127,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
return NULL;
}
} else { // Create one table group.
- STableKeyInfo info = {.lastKey = 0, .uid = pBlockNode->uid};
+ STableKeyInfo info = {.lastKey = 0, .uid = pBlockNode->uid, .groupId = 0};
taosArrayPush(pTableListInfo->pTableList, &info);
}
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 7affac76d2..ac57d3a9da 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -507,20 +507,21 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
STableScanInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
- if(pInfo->currentGroupId == -1){
+ if (pInfo->currentGroupId == -1) {
pInfo->currentGroupId++;
if (pInfo->currentGroupId >= taosArrayGetSize(pTaskInfo->tableqinfoList.pGroupList)) {
setTaskStatus(pTaskInfo, TASK_COMPLETED);
return NULL;
}
- SArray *tableList = taosArrayGetP(pTaskInfo->tableqinfoList.pGroupList, pInfo->currentGroupId);
+ SArray* tableList = taosArrayGetP(pTaskInfo->tableqinfoList.pGroupList, pInfo->currentGroupId);
tsdbCleanupReadHandle(pInfo->dataReader);
- tsdbReaderT* pReader = tsdbReaderOpen(pInfo->readHandle.vnode, &pInfo->cond, tableList, pInfo->queryId, pInfo->taskId);
+ tsdbReaderT* pReader =
+ tsdbReaderOpen(pInfo->readHandle.vnode, &pInfo->cond, tableList, pInfo->queryId, pInfo->taskId);
pInfo->dataReader = pReader;
}
SSDataBlock* result = doTableScanGroup(pOperator);
- if(result){
+ if (result) {
return result;
}
@@ -530,7 +531,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
return NULL;
}
- SArray *tableList = taosArrayGetP(pTaskInfo->tableqinfoList.pGroupList, pInfo->currentGroupId);
+ SArray* tableList = taosArrayGetP(pTaskInfo->tableqinfoList.pGroupList, pInfo->currentGroupId);
tsdbSetTableList(pInfo->dataReader, tableList);
tsdbResetReadHandle(pInfo->dataReader, &pInfo->cond, 0);
@@ -538,7 +539,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
pInfo->scanTimes = 0;
result = doTableScanGroup(pOperator);
- if(result){
+ if (result) {
return result;
}
@@ -776,15 +777,14 @@ static bool isStateWindow(SStreamBlockScanInfo* pInfo) {
return pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE;
}
-static bool prepareDataScan(SStreamBlockScanInfo* pInfo) {
- SSDataBlock* pSDB = pInfo->pUpdateRes;
- STimeWindow win = {
- .skey = INT64_MIN,
- .ekey = INT64_MAX,
+static bool prepareDataScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pSDB, int32_t tsColIndex, int32_t* pRowIndex) {
+ STimeWindow win = {
+ .skey = INT64_MIN,
+ .ekey = INT64_MAX,
};
bool needRead = false;
- if (!isStateWindow(pInfo) && pInfo->updateResIndex < pSDB->info.rows) {
- SColumnInfoData* pColDataInfo = taosArrayGet(pSDB->pDataBlock, pInfo->primaryTsIndex);
+ if (!isStateWindow(pInfo) && (*pRowIndex) < pSDB->info.rows) {
+ SColumnInfoData* pColDataInfo = taosArrayGet(pSDB->pDataBlock, tsColIndex);
TSKEY* tsCols = (TSKEY*)pColDataInfo->pData;
SResultRowInfo dumyInfo;
dumyInfo.cur.pageId = -1;
@@ -793,15 +793,14 @@ static bool prepareDataScan(SStreamBlockScanInfo* pInfo) {
int64_t gap = pInfo->sessionSup.gap;
int32_t winIndex = 0;
SResultWindowInfo* pCurWin =
- getSessionTimeWindow(pAggSup, tsCols[pInfo->updateResIndex], INT64_MIN, pSDB->info.groupId, gap, &winIndex);
+ getSessionTimeWindow(pAggSup, tsCols[(*pRowIndex)], INT64_MIN, pSDB->info.groupId, gap, &winIndex);
win = pCurWin->win;
- pInfo->updateResIndex +=
- updateSessionWindowInfo(pCurWin, tsCols, NULL, pSDB->info.rows, pInfo->updateResIndex, gap, NULL);
+ (*pRowIndex) += updateSessionWindowInfo(pCurWin, tsCols, NULL, pSDB->info.rows, (*pRowIndex), gap, NULL);
} else {
- win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[pInfo->updateResIndex], &pInfo->interval,
- pInfo->interval.precision, NULL);
- pInfo->updateResIndex += getNumOfRowsInTimeWindow(&pSDB->info, tsCols, pInfo->updateResIndex, win.ekey,
- binarySearchForKey, NULL, TSDB_ORDER_ASC);
+ win =
+ getActiveTimeWindow(NULL, &dumyInfo, tsCols[(*pRowIndex)], &pInfo->interval, pInfo->interval.precision, NULL);
+ (*pRowIndex) += getNumOfRowsInTimeWindow(&pSDB->info, tsCols, (*pRowIndex), win.ekey, binarySearchForKey, NULL,
+ TSDB_ORDER_ASC);
}
needRead = true;
} else if (isStateWindow(pInfo)) {
@@ -822,7 +821,10 @@ static bool prepareDataScan(SStreamBlockScanInfo* pInfo) {
STableScanInfo* pTableScanInfo = pInfo->pSnapshotReadOp->info;
pTableScanInfo->cond.twindows[0] = win;
pTableScanInfo->curTWinIdx = 0;
-// tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0);
+ // tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0);
+ // if (!pTableScanInfo->dataReader) {
+ // return false;
+ // }
pTableScanInfo->scanTimes = 0;
pTableScanInfo->currentGroupId = -1;
return true;
@@ -862,12 +864,12 @@ static uint64_t getGroupId(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_
*/
}
-static SSDataBlock* doDataScan(SStreamBlockScanInfo* pInfo) {
+static SSDataBlock* doDataScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pSDB, int32_t tsColIndex, int32_t* pRowIndex) {
while (1) {
SSDataBlock* pResult = NULL;
pResult = doTableScan(pInfo->pSnapshotReadOp);
if (pResult == NULL) {
- if (prepareDataScan(pInfo)) {
+ if (prepareDataScan(pInfo, pSDB, tsColIndex, pRowIndex)) {
// scan next window data
pResult = doTableScan(pInfo->pSnapshotReadOp);
}
@@ -916,7 +918,7 @@ static void setUpdateData(SStreamBlockScanInfo* pInfo, SSDataBlock* pBlock, SSDa
pUpdateBlock->info.rows = i;
pInfo->tsArrayIndex += i;
pUpdateBlock->info.groupId = pInfo->groupId;
- pUpdateBlock->info.type = STREAM_REPROCESS;
+ pUpdateBlock->info.type = STREAM_CLEAR;
blockDataUpdateTsWindow(pUpdateBlock, 0);
}
// all rows have same group id
@@ -970,6 +972,14 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
int32_t current = pInfo->validBlockIndex++;
SSDataBlock* pBlock = taosArrayGetP(pInfo->pBlockLists, current);
blockDataUpdateTsWindow(pBlock, 0);
+ if (pBlock->info.type == STREAM_RETRIEVE) {
+ pInfo->blockType = STREAM_DATA_TYPE_SUBMIT_BLOCK;
+ pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RETRIEVE;
+ copyDataBlock(pInfo->pPullDataRes, pBlock);
+ pInfo->pullDataResIndex = 0;
+ prepareDataScan(pInfo, pInfo->pPullDataRes, 0, &pInfo->pullDataResIndex);
+ updateInfoAddCloseWindowSBF(pInfo->pUpdateInfo);
+ }
return pBlock;
} else if (pInfo->blockType == STREAM_DATA_TYPE_SUBMIT_BLOCK) {
if (pInfo->scanMode == STREAM_SCAN_FROM_RES) {
@@ -979,28 +989,39 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
} else if (pInfo->scanMode == STREAM_SCAN_FROM_UPDATERES) {
pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER;
if (!isStateWindow(pInfo)) {
- prepareDataScan(pInfo);
+ prepareDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex);
}
return pInfo->pUpdateRes;
+ } else if (pInfo->scanMode == STREAM_SCAN_FROM_DATAREADER_RETRIEVE) {
+ SSDataBlock* pSDB = doDataScan(pInfo, pInfo->pPullDataRes, 0, &pInfo->pullDataResIndex);
+ if (pSDB != NULL) {
+ getUpdateDataBlock(pInfo, true, pSDB, NULL);
+ pSDB->info.type = STREAM_PUSH_DATA;
+ return pSDB;
+ }
+ pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER;
} else {
if (isStateWindow(pInfo) && taosArrayGetSize(pInfo->sessionSup.pStreamAggSup->pScanWindow) > 0) {
pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER;
pInfo->updateResIndex = pInfo->pUpdateRes->info.rows;
- prepareDataScan(pInfo);
+ prepareDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex);
}
if (pInfo->scanMode == STREAM_SCAN_FROM_DATAREADER) {
- SSDataBlock* pSDB = doDataScan(pInfo);
+ SSDataBlock* pSDB = doDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex);
if (pSDB == NULL) {
setUpdateData(pInfo, pInfo->pRes, pInfo->pUpdateRes);
if (pInfo->pUpdateRes->info.rows > 0) {
if (!isStateWindow(pInfo)) {
- prepareDataScan(pInfo);
+ // Todo(liuyao) mybe can delete this.
+ bool test = prepareDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex);
+ ASSERT(test == false);
}
return pInfo->pUpdateRes;
} else {
pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE;
}
} else {
+ pSDB->info.type = STREAM_NORMAL;
getUpdateDataBlock(pInfo, true, pSDB, NULL);
return pSDB;
}
@@ -1012,12 +1033,13 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
while (tqNextDataBlock(pInfo->streamBlockReader)) {
SSDataBlock block = {0};
- uint64_t groupId = 0;
- uint64_t uid = 0;
- int32_t numOfRows = 0;
// todo refactor
- int32_t code = tqRetrieveDataBlock(&block, pInfo->streamBlockReader, &groupId, &uid, &numOfRows);
+ int32_t code = tqRetrieveDataBlock(&block, pInfo->streamBlockReader);
+
+ uint64_t groupId = block.info.groupId;
+ uint64_t uid = block.info.uid;
+ int32_t numOfRows = block.info.rows;
if (code != TSDB_CODE_SUCCESS || numOfRows == 0) {
pTaskInfo->code = code;
@@ -1070,10 +1092,12 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
taosArrayDestroy(block.pDataBlock);
if (pInfo->pRes->pDataBlock == NULL) {
// TODO add log
+ updateInfoDestoryColseWinSBF(pInfo->pUpdateInfo);
pOperator->status = OP_EXEC_DONE;
pTaskInfo->code = terrno;
return NULL;
}
+
// currently only the tbname pseudo column
if (pInfo->numOfPseudoExpr > 0) {
addTagPseudoColumnData(&pInfo->readHandle, pInfo->pPseudoExpr, pInfo->numOfPseudoExpr, pInfo->pRes);
@@ -1091,12 +1115,13 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
pOperator->resultInfo.totalRows += pBlockInfo->rows;
if (pBlockInfo->rows == 0) {
+ updateInfoDestoryColseWinSBF(pInfo->pUpdateInfo);
pOperator->status = OP_EXEC_DONE;
} else if (pInfo->pUpdateInfo) {
pInfo->tsArrayIndex = 0;
getUpdateDataBlock(pInfo, true, pInfo->pRes, pInfo->pUpdateRes);
if (pInfo->pUpdateRes->info.rows > 0) {
- if (pInfo->pUpdateRes->info.type == STREAM_REPROCESS) {
+ if (pInfo->pUpdateRes->info.type == STREAM_CLEAR) {
pInfo->updateResIndex = 0;
pInfo->scanMode = STREAM_SCAN_FROM_UPDATERES;
} else if (pInfo->pUpdateRes->info.type == STREAM_INVERT) {
@@ -1130,9 +1155,9 @@ static SArray* extractTableIdList(const STableListInfo* pTableGroupInfo) {
return tableIdList;
}
-SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle,
- STableScanPhysiNode* pTableScanNode, SExecTaskInfo* pTaskInfo,
- STimeWindowAggSupp* pTwSup, uint64_t queryId, uint64_t taskId) {
+SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode,
+ SExecTaskInfo* pTaskInfo, STimeWindowAggSupp* pTwSup, uint64_t queryId,
+ uint64_t taskId) {
SStreamBlockScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamBlockScanInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
@@ -1209,6 +1234,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle,
pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE;
pInfo->sessionSup = (SessionWindowSupporter){.pStreamAggSup = NULL, .gap = -1};
pInfo->groupId = 0;
+ pInfo->pPullDataRes = createPullDataBlock();
pOperator->name = "StreamBlockScanOperator";
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN;
@@ -1718,14 +1744,14 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan
SSDataBlock* pResBlock = createResDataBlock(pDescNode);
int32_t num = 0;
- SArray* colList = extractColMatchInfo(pScanNode->pScanCols, pDescNode, &num, COL_MATCH_FROM_COL_ID);
+ SArray* colList = extractColMatchInfo(pScanNode->pScanCols, pDescNode, &num, COL_MATCH_FROM_COL_ID);
- pInfo->accountId = pScanPhyNode->accountId;
- pInfo->pUser = taosMemoryStrDup((void*) pUser);
+ pInfo->accountId = pScanPhyNode->accountId;
+ pInfo->pUser = taosMemoryStrDup((void*)pUser);
pInfo->showRewrite = pScanPhyNode->showRewrite;
- pInfo->pRes = pResBlock;
- pInfo->pCondition = pScanNode->node.pConditions;
- pInfo->scanCols = colList;
+ pInfo->pRes = pResBlock;
+ pInfo->pCondition = pScanNode->node.pConditions;
+ pInfo->scanCols = colList;
initResultSizeInfo(pOperator, 4096);
@@ -1741,13 +1767,13 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan
pInfo->readHandle = *(SReadHandle*)readHandle;
}
- pOperator->name = "SysTableScanOperator";
+ pOperator->name = "SysTableScanOperator";
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN;
- pOperator->blocking = false;
- pOperator->status = OP_NOT_OPENED;
- pOperator->info = pInfo;
+ pOperator->blocking = false;
+ pOperator->status = OP_NOT_OPENED;
+ pOperator->info = pInfo;
pOperator->exprSupp.numOfExprs = taosArrayGetSize(pResBlock->pDataBlock);
- pOperator->pTaskInfo = pTaskInfo;
+ pOperator->pTaskInfo = pTaskInfo;
pOperator->fpSet =
createOperatorFpSet(operatorDummyOpenFn, doSysTableScan, NULL, NULL, destroySysScanOperator, NULL, NULL, NULL);
@@ -1934,11 +1960,11 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi
goto _error;
}
- pInfo->pTableList = pTableListInfo;
- pInfo->pColMatchInfo = colList;
- pInfo->pRes = createResDataBlock(pDescNode);
- pInfo->readHandle = *pReadHandle;
- pInfo->curPos = 0;
+ pInfo->pTableList = pTableListInfo;
+ pInfo->pColMatchInfo = colList;
+ pInfo->pRes = createResDataBlock(pDescNode);
+ pInfo->readHandle = *pReadHandle;
+ pInfo->curPos = 0;
pOperator->name = "TagScanOperator";
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN;
@@ -1965,7 +1991,10 @@ _error:
typedef struct STableMergeScanInfo {
STableListInfo* tableListInfo;
- int32_t currentGroupId;
+ int32_t tableStartIndex;
+ int32_t tableEndIndex;
+ bool hasGroupId;
+ uint64_t groupId;
SArray* dataReaders; // array of tsdbReaderT*
SReadHandle readHandle;
@@ -2030,6 +2059,22 @@ int32_t createScanTableListInfo(STableScanPhysiNode* pTableScanNode, SReadHandle
return TSDB_CODE_SUCCESS;
}
+int32_t createMultipleDataReaders(SQueryTableDataCond* pQueryCond, SReadHandle* pHandle, STableListInfo* pTableListInfo,
+ int32_t tableStartIdx, int32_t tableEndIdx, SArray* arrayReader, uint64_t queryId,
+ uint64_t taskId) {
+ for (int32_t i = tableStartIdx; i <= tableEndIdx; ++i) {
+ SArray* subTableList = taosArrayInit(1, sizeof(STableKeyInfo));
+ taosArrayPush(subTableList, taosArrayGet(pTableListInfo->pTableList, i));
+
+ tsdbReaderT* pReader = tsdbReaderOpen(pHandle->vnode, pQueryCond, subTableList, queryId, taskId);
+ taosArrayPush(arrayReader, &pReader);
+
+ taosArrayDestroy(subTableList);
+ }
+
+ return TSDB_CODE_SUCCESS;
+}
+
// todo refactor
static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeScanInfo* pTableScanInfo,
int32_t readerIdx, SSDataBlock* pBlock, uint32_t* status) {
@@ -2216,34 +2261,32 @@ SArray* generateSortByTsInfo(int32_t order) {
return pList;
}
-static int32_t createMultipleDataReaders(SQueryTableDataCond* pQueryCond, SReadHandle* pHandle, SArray* tableList, SArray* arrayReader, uint64_t queryId,
- uint64_t taskId) {
- for (int32_t i = 0; i < taosArrayGetSize(tableList); ++i) {
- SArray* tmp = taosArrayInit(1, sizeof(STableKeyInfo));
- taosArrayPush(tmp, taosArrayGet(tableList, i));
-
- tsdbReaderT* pReader = tsdbReaderOpen(pHandle->vnode, pQueryCond, tmp, queryId, taskId);
- taosArrayPush(arrayReader, &pReader);
-
- taosArrayDestroy(tmp);
- }
-
- return TSDB_CODE_SUCCESS;
-}
-
int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) {
STableMergeScanInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
- SArray* tableList = taosArrayGetP(pInfo->tableListInfo->pGroupList, pInfo->currentGroupId);
+ {
+ size_t tableListSize = taosArrayGetSize(pInfo->tableListInfo->pTableList);
+ int32_t i = pInfo->tableStartIndex + 1;
+ for (; i < tableListSize; ++i) {
+ STableKeyInfo* tableKeyInfo = taosArrayGet(pInfo->tableListInfo->pTableList, i);
+ if (tableKeyInfo->groupId != pInfo->groupId) {
+ break;
+ }
+ }
+ pInfo->tableEndIndex = i - 1;
+ }
- createMultipleDataReaders(&pInfo->cond, &pInfo->readHandle, tableList,
+ int32_t tableStartIdx = pInfo->tableStartIndex;
+ int32_t tableEndIdx = pInfo->tableEndIndex;
+
+ STableListInfo* tableListInfo = pInfo->tableListInfo;
+ createMultipleDataReaders(&pInfo->cond, &pInfo->readHandle, tableListInfo, tableStartIdx, tableEndIdx,
pInfo->dataReaders, pInfo->queryId, pInfo->taskId);
// todo the total available buffer should be determined by total capacity of buffer of this task.
// the additional one is reserved for merge result
- int32_t tableLen = taosArrayGetSize(tableList);
- pInfo->sortBufSize = pInfo->bufPageSize * ((tableLen==0?1:tableLen) + 1);
+ pInfo->sortBufSize = pInfo->bufPageSize * (tableEndIdx - tableStartIdx + 1 + 1);
int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize;
pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_MULTISOURCE_MERGE, pInfo->bufPageSize, numOfBufPage,
pInfo->pSortInputBlock, pTaskInfo->id.str);
@@ -2330,43 +2373,38 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) {
if (code != TSDB_CODE_SUCCESS) {
longjmp(pTaskInfo->env, code);
}
+ size_t tableListSize = taosArrayGetSize(pInfo->tableListInfo->pTableList);
+ if (!pInfo->hasGroupId) {
+ pInfo->hasGroupId = true;
- if (pInfo->currentGroupId == -1) {
- pInfo->currentGroupId++;
- if (pInfo->currentGroupId >= taosArrayGetSize(pInfo->tableListInfo->pGroupList)) {
+ if (tableListSize == 0) {
doSetOperatorCompleted(pOperator);
return NULL;
}
+ pInfo->tableStartIndex = 0;
+ pInfo->groupId = ((STableKeyInfo*)taosArrayGet(pInfo->tableListInfo->pTableList, pInfo->tableStartIndex))->groupId;
startGroupTableMergeScan(pOperator);
}
- SSDataBlock* pBlock = getSortedTableMergeScanBlockData(pInfo->pSortHandle, pOperator->resultInfo.capacity, pOperator);
- if (pBlock != NULL) {
- uint64_t* groupId = taosHashGet(pInfo->tableListInfo->map, &(pBlock->info.uid), sizeof(uint64_t));
- if(groupId) pBlock->info.groupId = *groupId;
-
- pOperator->resultInfo.totalRows += pBlock->info.rows;
- return pBlock;
+ SSDataBlock* pBlock = NULL;
+ while (pInfo->tableStartIndex < tableListSize) {
+ pBlock = getSortedTableMergeScanBlockData(pInfo->pSortHandle, pOperator->resultInfo.capacity, pOperator);
+ if (pBlock != NULL) {
+ pBlock->info.groupId = pInfo->groupId;
+ pOperator->resultInfo.totalRows += pBlock->info.rows;
+ return pBlock;
+ } else {
+ stopGroupTableMergeScan(pOperator);
+ if (pInfo->tableEndIndex >= tableListSize - 1) {
+ doSetOperatorCompleted(pOperator);
+ break;
+ }
+ pInfo->tableStartIndex = pInfo->tableEndIndex + 1;
+ pInfo->groupId =
+ ((STableKeyInfo*)taosArrayGet(pInfo->tableListInfo->pTableList, pInfo->tableStartIndex))->groupId;
+ startGroupTableMergeScan(pOperator);
+ }
}
- stopGroupTableMergeScan(pOperator);
- pInfo->currentGroupId++;
- if (pInfo->currentGroupId >= taosArrayGetSize(pInfo->tableListInfo->pGroupList)) {
- doSetOperatorCompleted(pOperator);
- return NULL;
- }
- startGroupTableMergeScan(pOperator);
-
- pBlock = getSortedTableMergeScanBlockData(pInfo->pSortHandle, pOperator->resultInfo.capacity, pOperator);
- if (pBlock != NULL) {
- uint64_t* groupId = taosHashGet(pInfo->tableListInfo->map, &(pBlock->info.uid), sizeof(uint64_t));
- if(groupId) pBlock->info.groupId = *groupId;
-
- pOperator->resultInfo.totalRows += pBlock->info.rows;
- return pBlock;
- }
-
- doSetOperatorCompleted(pOperator);
-
return pBlock;
}
@@ -2403,6 +2441,12 @@ int32_t getTableMergeScanExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExpla
return TSDB_CODE_SUCCESS;
}
+int32_t compareTableKeyInfoByGid(const void* p1, const void* p2) {
+ const STableKeyInfo* info1 = p1;
+ const STableKeyInfo* info2 = p2;
+ return info1->groupId - info2->groupId;
+}
+
SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, STableListInfo* pTableListInfo,
SReadHandle* readHandle, SExecTaskInfo* pTaskInfo, uint64_t queryId,
uint64_t taskId) {
@@ -2411,6 +2455,9 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
if (pInfo == NULL || pOperator == NULL) {
goto _error;
}
+ if (pTableScanNode->pPartitionTags) {
+ taosArraySort(pTableListInfo->pTableList, compareTableKeyInfoByGid);
+ }
SDataBlockDescNode* pDescNode = pTableScanNode->scan.node.pOutputDataBlockDesc;
@@ -2443,7 +2490,6 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
pInfo->dataReaders = taosArrayInit(64, POINTER_BYTES);
pInfo->queryId = queryId;
pInfo->taskId = taskId;
- pInfo->currentGroupId = -1;
pInfo->sortSourceParams = taosArrayInit(64, sizeof(STableMergeScanSortSourceParam));
diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c
index 48b0b1c071..b7d39207d8 100644
--- a/source/libs/executor/src/timewindowoperator.c
+++ b/source/libs/executor/src/timewindowoperator.c
@@ -15,6 +15,7 @@
#include "executorimpl.h"
#include "function.h"
#include "functionMgt.h"
+#include "tcompare.h"
#include "tdatablock.h"
#include "tfill.h"
#include "ttime.h"
@@ -26,6 +27,16 @@ typedef enum SResultTsInterpType {
#define IS_FINAL_OP(op) ((op)->isFinal)
+typedef struct SWinRes {
+ TSKEY ts;
+ uint64_t groupId;
+} SWinRes;
+
+typedef struct SPullWindowInfo {
+ STimeWindow window;
+ uint64_t groupId;
+} SPullWindowInfo;
+
static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator);
static int64_t* extractTsCol(SSDataBlock* pBlock, const SIntervalAggOperatorInfo* pInfo);
@@ -684,11 +695,13 @@ static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t num
}
void printDataBlock(SSDataBlock* pBlock, const char* flag) {
- if (pBlock == NULL) return;
- SArray* blocks = taosArrayInit(1, sizeof(SSDataBlock));
- taosArrayPush(blocks, pBlock);
- blockDebugShowData(blocks, flag);
- taosArrayDestroy(blocks);
+ if (pBlock == NULL){
+ qDebug("======printDataBlock Block is Null");
+ return;
+ }
+ char *pBuf = NULL;
+ qDebug("%s", dumpBlockData(pBlock, flag, &pBuf));
+ taosMemoryFree(pBuf);
}
typedef int64_t (*__get_value_fn_t)(void* data, int32_t index);
@@ -1217,30 +1230,40 @@ void doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf, SExprS
}
}
-void doClearWindow(SAggSupporter* pAggSup, SExprSupp* pSup, char* pData, int16_t bytes, uint64_t groupId,
+bool doClearWindow(SAggSupporter* pAggSup, SExprSupp* pSup, char* pData, int16_t bytes, uint64_t groupId,
int32_t numOfOutput) {
SET_RES_WINDOW_KEY(pAggSup->keyBuf, pData, bytes, groupId);
SResultRowPosition* p1 =
(SResultRowPosition*)taosHashGet(pAggSup->pResultRowHashTable, pAggSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
if (!p1) {
// window has been closed
- return;
+ return false;
}
doClearWindowImpl(p1, pAggSup->pResultBuf, pSup, numOfOutput);
+ return true;
}
static void doClearWindows(SAggSupporter* pAggSup, SExprSupp* pSup1, SInterval* pInterval, int32_t tsIndex,
int32_t numOfOutput, SSDataBlock* pBlock, SArray* pUpWins) {
- SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, tsIndex);
- TSKEY* tsCols = (TSKEY*)pColDataInfo->pData;
+ SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, tsIndex);
+ TSKEY* tsCols = (TSKEY*)pTsCol->pData;
+ uint64_t* pGpDatas = NULL;
+ if (pBlock->info.type == STREAM_RETRIEVE) {
+ SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, 2);
+ pGpDatas = (uint64_t*)pGpCol->pData;
+ }
int32_t step = 0;
for (int32_t i = 0; i < pBlock->info.rows; i += step) {
SResultRowInfo dumyInfo;
dumyInfo.cur.pageId = -1;
STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[i], pInterval, pInterval->precision, NULL);
step = getNumOfRowsInTimeWindow(&pBlock->info, tsCols, i, win.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC);
- doClearWindow(pAggSup, pSup1, (char*)&win.skey, sizeof(TKEY), pBlock->info.groupId, numOfOutput);
- if (pUpWins) {
+ uint64_t groupId = pBlock->info.groupId;
+ if (pGpDatas) {
+ groupId = pGpDatas[i];
+ }
+ bool res = doClearWindow(pAggSup, pSup1, (char*)&win.skey, sizeof(TKEY), groupId, numOfOutput);
+ if (pUpWins && res) {
taosArrayPush(pUpWins, &win);
}
}
@@ -1268,8 +1291,8 @@ bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup) {
return pSup->maxTs != INT64_MIN && pWin->ekey < pSup->maxTs - pSup->waterMark;
}
-static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, SInterval* pInterval,
- SArray* closeWins) {
+static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup,
+ SInterval* pInterval, SHashObj* pPullDataMap, SArray* closeWins) {
void* pIte = NULL;
size_t keyLen = 0;
while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) {
@@ -1280,10 +1303,20 @@ static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup,
SResultRowInfo dumyInfo;
dumyInfo.cur.pageId = -1;
STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, ts, pInterval, pInterval->precision, NULL);
+ SWinRes winRe = {.ts = win.skey, .groupId = groupId,};
+ void* chIds = taosHashGet(pPullDataMap, &winRe, sizeof(SWinRes));
if (isCloseWindow(&win, pSup)) {
- char keyBuf[GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY))];
- SET_RES_WINDOW_KEY(keyBuf, &ts, sizeof(TSKEY), groupId);
- taosHashRemove(pHashMap, keyBuf, keyLen);
+ if (chIds && pPullDataMap) {
+ SArray* chAy = *(SArray**) chIds;
+ int32_t size = taosArrayGetSize(chAy);
+ qInfo("======window %ld wait child size:%d", win.skey ,size);
+ for (int32_t i = 0; i < size; i++) {
+ qInfo("======window %ld wait chid id:%d", win.skey ,*(int32_t*)taosArrayGet(chAy, i));
+ }
+ continue;
+ } else if (pPullDataMap) {
+ qInfo("======close window %ld", win.skey);
+ }
SResultRowPosition* pPos = (SResultRowPosition*)pIte;
if (pSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {
int32_t code = saveResult(ts, pPos->pageId, pPos->offset, groupId, closeWins);
@@ -1291,11 +1324,25 @@ static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup,
return code;
}
}
+ char keyBuf[GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY))];
+ SET_RES_WINDOW_KEY(keyBuf, &ts, sizeof(TSKEY), groupId);
+ taosHashRemove(pHashMap, keyBuf, keyLen);
}
}
return TSDB_CODE_SUCCESS;
}
+static void closeChildIntervalWindow(SArray* pChildren, TSKEY maxTs) {
+ int32_t size = taosArrayGetSize(pChildren);
+ for (int32_t i = 0; i < size; i++) {
+ SOperatorInfo* pChildOp = taosArrayGetP(pChildren, i);
+ SStreamFinalIntervalOperatorInfo* pChInfo = pChildOp->info;
+ ASSERT(pChInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE);
+ pChInfo->twAggSup.maxTs = TMAX(pChInfo->twAggSup.maxTs, maxTs);
+ closeIntervalWindow(pChInfo->aggSup.pResultRowHashTable, &pChInfo->twAggSup, &pChInfo->interval, NULL, NULL);
+ }
+}
+
static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
SIntervalAggOperatorInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
@@ -1324,7 +1371,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
break;
}
- if (pBlock->info.type == STREAM_REPROCESS) {
+ if (pBlock->info.type == STREAM_CLEAR) {
doClearWindows(&pInfo->aggSup, &pOperator->exprSupp, &pInfo->interval, 0, pOperator->exprSupp.numOfExprs, pBlock,
NULL);
qDebug("%s clear existed time window results for updates checked", GET_TASKID(pTaskInfo));
@@ -1345,7 +1392,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);
hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, MAIN_SCAN, pUpdated);
}
- closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, pUpdated);
+ closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, NULL, pUpdated);
finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset);
initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
@@ -1373,6 +1420,11 @@ void destroyStreamFinalIntervalOperatorInfo(void* param, int32_t numOfOutput) {
SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)param;
cleanupBasicInfo(&pInfo->binfo);
cleanupAggSup(&pInfo->aggSup);
+ //it should be empty.
+ taosHashCleanup(pInfo->pPullDataMap);
+ taosArrayDestroy(pInfo->pPullWins);
+ blockDataDestroy(pInfo->pPullDataRes);
+
if (pInfo->pChildren) {
int32_t size = taosArrayGetSize(pInfo->pChildren);
for (int32_t i = 0; i < size; i++) {
@@ -2164,6 +2216,24 @@ bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup) {
return p1 == NULL;
}
+int32_t getNexWindowPos(SInterval* pInterval, SDataBlockInfo* pBlockInfo, TSKEY* tsCols,
+ int32_t startPos, TSKEY eKey, STimeWindow* pNextWin) {
+ int32_t forwardRows = getNumOfRowsInTimeWindow(pBlockInfo, tsCols, startPos,
+ eKey, binarySearchForKey, NULL, TSDB_ORDER_ASC);
+ int32_t prevEndPos = forwardRows - 1 + startPos;
+ return getNextQualifiedWindow(pInterval, pNextWin, pBlockInfo, tsCols, prevEndPos, TSDB_ORDER_ASC);
+}
+
+void addPullWindow(SHashObj* pMap, SWinRes* pWinRes, int32_t size) {
+ SArray* childIds = taosArrayInit(8, sizeof(int32_t));
+ for (int32_t i = 0; i < size; i++) {
+ taosArrayPush(childIds, &i);
+ }
+ taosHashPut(pMap, pWinRes, sizeof(SWinRes), &childIds, sizeof(void*));
+}
+
+static int32_t getChildIndex(SSDataBlock* pBlock) { return pBlock->info.childId; }
+
static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, uint64_t tableGroupId,
SArray* pUpdated) {
SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)pOperatorInfo->info;
@@ -2177,35 +2247,59 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc
SResultRow* pResult = NULL;
int32_t forwardRows = 0;
- if (pSDataBlock->pDataBlock != NULL) {
- SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
- tsCols = (int64_t*)pColDataInfo->pData;
- } else {
- return;
- }
+ ASSERT(pSDataBlock->pDataBlock != NULL);
+ SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
+ tsCols = (int64_t*)pColDataInfo->pData;
int32_t startPos = ascScan ? 0 : (pSDataBlock->info.rows - 1);
TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols);
STimeWindow nextWin = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval,
pInfo->interval.precision, NULL);
while (1) {
- if (IS_FINAL_OP(pInfo) && isCloseWindow(&nextWin, &pInfo->twAggSup) &&
- isDeletedWindow(&nextWin, tableGroupId, &pInfo->aggSup)) {
- SArray* pUpWins = taosArrayInit(8, sizeof(STimeWindow));
- taosArrayPush(pUpWins, &nextWin);
- rebuildIntervalWindow(pInfo, pSup, pUpWins, pInfo->binfo.pRes->info.groupId, pSup->numOfExprs,
- pOperatorInfo->pTaskInfo);
- taosArrayDestroy(pUpWins);
+ if (IS_FINAL_OP(pInfo) && isCloseWindow(&nextWin, &pInfo->twAggSup) && pInfo->pChildren) {
+ bool ignore = true;
+ SWinRes winRes = {.ts = nextWin.skey, .groupId = tableGroupId,};
+ void* chIds = taosHashGet(pInfo->pPullDataMap, &winRes, sizeof(SWinRes));
+ if (isDeletedWindow(&nextWin, tableGroupId, &pInfo->aggSup) && !chIds) {
+ SPullWindowInfo pull = {.window = nextWin, .groupId = tableGroupId};
+ // add pull data request
+ taosArrayPush(pInfo->pPullWins, &pull);
+ addPullWindow(pInfo->pPullDataMap, &winRes, taosArrayGetSize(pInfo->pChildren));
+ } else {
+ int32_t index = -1;
+ SArray* chArray = NULL;
+ if (chIds) {
+ chArray = *(void**) chIds;
+ int32_t chId = getChildIndex(pSDataBlock);
+ index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ);
+ }
+ if (index != -1 && pSDataBlock->info.type == STREAM_PUSH_DATA) {
+ taosArrayRemove(chArray, index);
+ if (taosArrayGetSize(chArray) == 0) {
+ // pull data is over
+ taosHashRemove(pInfo->pPullDataMap, &winRes, sizeof(SWinRes));
+ }
+ }
+ if ( index == -1 || pSDataBlock->info.type == STREAM_PUSH_DATA) {
+ ignore = false;
+ }
+ }
+
+ if (ignore) {
+ startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCols, startPos, nextWin.ekey, &nextWin);
+ if (startPos < 0) {
+ break;
+ }
+ continue;
+ }
}
+
int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, true, &pResult, tableGroupId, pSup->pCtx,
numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
}
- SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t));
- pos->groupId = tableGroupId;
- pos->pos = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset};
- *(int64_t*)pos->key = pResult->win.skey;
+
forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey, NULL,
TSDB_ORDER_ASC);
if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdated) {
@@ -2230,14 +2324,17 @@ static void clearStreamIntervalOperator(SStreamFinalIntervalOperatorInfo* pInfo)
initResultRowInfo(&pInfo->binfo.resultRowInfo);
}
-static void clearUpdateDataBlock(SSDataBlock* pBlock) {
+static void clearSpecialDataBlock(SSDataBlock* pBlock) {
+ if (pBlock->info.rows <= 0) {
+ return;
+ }
blockDataCleanup(pBlock);
}
void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex) {
// ASSERT(pDest->info.capacity >= pSource->info.rows);
blockDataEnsureCapacity(pDest, pSource->info.rows);
- clearUpdateDataBlock(pDest);
+ clearSpecialDataBlock(pDest);
SColumnInfoData* pDestCol = taosArrayGet(pDest->pDataBlock, 0);
SColumnInfoData* pSourceCol = taosArrayGet(pSource->pDataBlock, tsColIndex);
@@ -2254,7 +2351,63 @@ void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsCol
blockDataUpdateTsWindow(pDest, 0);
}
-static int32_t getChildIndex(SSDataBlock* pBlock) { return pBlock->info.childId; }
+static bool needBreak(SStreamFinalIntervalOperatorInfo* pInfo) {
+ int32_t size = taosArrayGetSize(pInfo->pPullWins);
+ if (pInfo->pullIndex < size) {
+ return true;
+ }
+ return false;
+}
+
+static void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pBlock) {
+ clearSpecialDataBlock(pBlock);
+ int32_t size = taosArrayGetSize(array);
+ if (size - (*pIndex) == 0) {
+ return;
+ }
+ blockDataEnsureCapacity(pBlock, size - (*pIndex) );
+ ASSERT(3 <= taosArrayGetSize(pBlock->pDataBlock));
+ for (; (*pIndex) < size; (*pIndex)++) {
+ SPullWindowInfo* pWin = taosArrayGet(array, (*pIndex) );
+ SColumnInfoData* pStartTs = (SColumnInfoData*) taosArrayGet(pBlock->pDataBlock, 0);
+ colDataAppend(pStartTs, pBlock->info.rows, (const char*)&pWin->window.skey, false);
+
+ SColumnInfoData* pEndTs = (SColumnInfoData*) taosArrayGet(pBlock->pDataBlock, 1);
+ colDataAppend(pEndTs, pBlock->info.rows, (const char*)&pWin->window.ekey, false);
+
+ SColumnInfoData* pGroupId = (SColumnInfoData*) taosArrayGet(pBlock->pDataBlock, 2);
+ colDataAppend(pGroupId, pBlock->info.rows, (const char*)&pWin->groupId, false);
+ pBlock->info.rows++;
+ }
+ if ((*pIndex) == size) {
+ *pIndex = 0;
+ taosArrayClear(array);
+ }
+ blockDataUpdateTsWindow(pBlock, 0);
+}
+
+void processPushEmpty(SSDataBlock* pBlock, SHashObj* pMap) {
+ SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, 0);
+ TSKEY* tsData = (TSKEY*)pStartCol->pData;
+ SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, 2);
+ uint64_t* groupIdData = (uint64_t*)pGroupCol->pData;
+ int32_t chId = getChildIndex(pBlock);
+ for (int32_t i = 0; i < pBlock->info.rows; i++) {
+ SWinRes winRes = {.ts = tsData[i], .groupId = groupIdData[i]};
+ void* chIds = taosHashGet(pMap, &winRes, sizeof(SWinRes));
+ if (chIds) {
+ SArray* chArray = *(SArray**) chIds;
+ int32_t index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ);
+ if (index != -1) {
+ taosArrayRemove(chArray, index);
+ if (taosArrayGetSize(chArray) == 0) {
+ // pull data is over
+ taosHashRemove(pMap, &winRes, sizeof(SWinRes));
+ }
+ }
+ }
+ }
+}
static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
SStreamFinalIntervalOperatorInfo* pInfo = pOperator->info;
@@ -2270,28 +2423,50 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
if (pInfo->binfo.pRes->info.rows == 0) {
pOperator->status = OP_EXEC_DONE;
- if (IS_FINAL_OP(pInfo) || pInfo->pUpdateRes->info.rows == 0) {
- if (!IS_FINAL_OP(pInfo)) {
- // semi interval operator clear disk buffer
- clearStreamIntervalOperator(pInfo);
- }
- return NULL;
+ if (!IS_FINAL_OP(pInfo)) {
+ // semi interval operator clear disk buffer
+ clearStreamIntervalOperator(pInfo);
}
+ return NULL;
+ }
+ printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi");
+ return pInfo->binfo.pRes;
+ } else {
+ doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
+ if (pInfo->binfo.pRes->info.rows != 0) {
+ printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi");
+ return pInfo->binfo.pRes;
+ }
+ if (pInfo->pUpdateRes->info.rows != 0 && pInfo->returnUpdate) {
+ pInfo->returnUpdate = false;
+ ASSERT(!IS_FINAL_OP(pInfo));
+ printDataBlock(pInfo->pUpdateRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi");
// process the rest of the data
- pOperator->status = OP_OPENED;
return pInfo->pUpdateRes;
}
- return pInfo->binfo.pRes;
+ doBuildPullDataBlock(pInfo->pPullWins, &pInfo->pullIndex, pInfo->pPullDataRes);
+ if (pInfo->pPullDataRes->info.rows != 0) {
+ // process the rest of the data
+ ASSERT(IS_FINAL_OP(pInfo));
+ printDataBlock(pInfo->pPullDataRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi");
+ return pInfo->pPullDataRes;
+ }
}
while (1) {
SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
if (pBlock == NULL) {
- clearUpdateDataBlock(pInfo->pUpdateRes);
+ clearSpecialDataBlock(pInfo->pUpdateRes);
+ pOperator->status = OP_RES_TO_RETURN;
+ qInfo("Stream Final Interval return data");
break;
}
+ printDataBlock(pBlock, IS_FINAL_OP(pInfo) ? "interval Final recv" : "interval Semi recv");
+ maxTs = TMAX(maxTs, pBlock->info.window.ekey);
- if (pBlock->info.type == STREAM_REPROCESS) {
+ if (pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_PUSH_DATA || pBlock->info.type == STREAM_INVALID) {
+ pInfo->binfo.pRes->info.type = pBlock->info.type;
+ } else if (pBlock->info.type == STREAM_CLEAR) {
SArray* pUpWins = taosArrayInit(8, sizeof(STimeWindow));
doClearWindows(&pInfo->aggSup, pSup, &pInfo->interval, pInfo->primaryTsIndex, pOperator->exprSupp.numOfExprs,
pBlock, pUpWins);
@@ -2310,11 +2485,25 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
}
removeResults(pUpWins, pUpdated);
copyUpdateDataBlock(pInfo->pUpdateRes, pBlock, pInfo->primaryTsIndex);
+ pInfo->returnUpdate = true;
taosArrayDestroy(pUpWins);
break;
} else if (pBlock->info.type == STREAM_GET_ALL && IS_FINAL_OP(pInfo)) {
getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdated);
continue;
+ } else if (pBlock->info.type == STREAM_RETRIEVE && !IS_FINAL_OP(pInfo)) {
+ SArray* pUpWins = taosArrayInit(8, sizeof(STimeWindow));
+ doClearWindows(&pInfo->aggSup, pSup, &pInfo->interval, 0, pOperator->exprSupp.numOfExprs,
+ pBlock, pUpWins);
+ removeResults(pUpWins, pUpdated);
+ taosArrayDestroy(pUpWins);
+ if (taosArrayGetSize(pUpdated) > 0) {
+ break;
+ }
+ continue;
+ } else if (pBlock->info.type == STREAM_PUSH_EMPTY && IS_FINAL_OP(pInfo)) {
+ processPushEmpty(pBlock, pInfo->pPullDataMap);
+ continue;
}
setInputDataBlock(pOperator, pSup->pCtx, pBlock, pInfo->order, MAIN_SCAN, true);
@@ -2334,31 +2523,70 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
SStreamFinalIntervalOperatorInfo* pChInfo = pChildOp->info;
setInputDataBlock(pChildOp, pChildOp->exprSupp.pCtx, pBlock, pChInfo->order, MAIN_SCAN, true);
doHashInterval(pChildOp, pBlock, pBlock->info.groupId, NULL);
- pChInfo->twAggSup.maxTs = TMAX(pChInfo->twAggSup.maxTs, pBlock->info.window.ekey);
+
+ if (needBreak(pInfo)) {
+ break;
+ }
}
- maxTs = TMAX(maxTs, pBlock->info.window.ekey);
}
pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);
if (IS_FINAL_OP(pInfo)) {
- closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, pUpdated);
+ closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup,
+ &pInfo->interval, pInfo->pPullDataMap, pUpdated);
+ closeChildIntervalWindow(pInfo->pChildren, pInfo->twAggSup.maxTs);
}
finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset);
initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
- pOperator->status = OP_RES_TO_RETURN;
- if (pInfo->binfo.pRes->info.rows == 0) {
- pOperator->status = OP_EXEC_DONE;
- if (pInfo->pUpdateRes->info.rows == 0) {
- return NULL;
- }
+ if (pInfo->binfo.pRes->info.rows != 0) {
+ printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi");
+ return pInfo->binfo.pRes;
+ }
+
+ if (pInfo->pUpdateRes->info.rows != 0 && pInfo->returnUpdate) {
+ pInfo->returnUpdate = false;
+ ASSERT(!IS_FINAL_OP(pInfo));
+ printDataBlock(pInfo->pUpdateRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi");
// process the rest of the data
- pOperator->status = OP_OPENED;
return pInfo->pUpdateRes;
}
- return pInfo->binfo.pRes;
+
+ doBuildPullDataBlock(pInfo->pPullWins, &pInfo->pullIndex, pInfo->pPullDataRes);
+ if (pInfo->pPullDataRes->info.rows != 0) {
+ // process the rest of the data
+ ASSERT(IS_FINAL_OP(pInfo));
+ printDataBlock(pInfo->pPullDataRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi");
+ return pInfo->pPullDataRes;
+ }
+ // ASSERT(false);
+ return NULL;
+}
+
+SSDataBlock* createPullDataBlock() {
+ SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
+ pBlock->info.hasVarCol = false;
+ pBlock->info.groupId = 0;
+ pBlock->info.rows = 0;
+ pBlock->info.type = STREAM_RETRIEVE;
+ pBlock->info.rowSize = sizeof(TSKEY) + sizeof(TSKEY) + sizeof(uint64_t);
+
+ pBlock->pDataBlock = taosArrayInit(3, sizeof(SColumnInfoData));
+ SColumnInfoData infoData = {0};
+ infoData.info.type = TSDB_DATA_TYPE_TIMESTAMP;
+ infoData.info.bytes = sizeof(TSKEY);
+ // window start ts
+ taosArrayPush(pBlock->pDataBlock, &infoData);
+ // window end ts
+ taosArrayPush(pBlock->pDataBlock, &infoData);
+
+ infoData.info.type = TSDB_DATA_TYPE_UBIGINT;
+ infoData.info.bytes = sizeof(uint64_t);
+ taosArrayPush(pBlock->pDataBlock, &infoData);
+
+ return pBlock;
}
SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
@@ -2412,23 +2640,30 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
goto _error;
}
}
- // semi interval operator does not catch result
pInfo->pUpdateRes = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
- pInfo->pUpdateRes->info.type = STREAM_REPROCESS;
+ pInfo->pUpdateRes->info.type = STREAM_CLEAR;
blockDataEnsureCapacity(pInfo->pUpdateRes, 128);
+ pInfo->returnUpdate = false;
+
pInfo->pPhyNode = (SPhysiNode*)nodesCloneNode((SNode*)pPhyNode);
if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) {
pInfo->isFinal = true;
pOperator->name = "StreamFinalIntervalOperator";
} else {
+ // semi interval operator does not catch result
pInfo->isFinal = false;
pOperator->name = "StreamSemiIntervalOperator";
}
- if (!IS_FINAL_OP(pInfo)) {
+ if (!IS_FINAL_OP(pInfo) || numOfChild == 0) {
pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;
}
+ pInfo->pPullWins = taosArrayInit(8, sizeof(SPullWindowInfo));
+ pInfo->pullIndex = 0;
+ _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
+ pInfo->pPullDataMap = taosHashInit(64, hashFn, false, HASH_NO_LOCK);
+ pInfo->pPullDataRes = createPullDataBlock();
pOperator->operatorType = pPhyNode->type;
pOperator->blocking = true;
@@ -2811,11 +3046,6 @@ void compactTimeWindow(SStreamSessionAggOperatorInfo* pInfo, int32_t startIndex,
}
}
-typedef struct SWinRes {
- TSKEY ts;
- uint64_t groupId;
-} SWinRes;
-
static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SHashObj* pStUpdated,
SHashObj* pStDeleted, bool hasEndTs) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
@@ -3027,14 +3257,14 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
} else if (pOperator->status == OP_RES_TO_RETURN) {
doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
if (pInfo->pDelRes->info.rows > 0) {
- /*printDataBlock(pInfo->pDelRes, "session del");*/
+ printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo)? "Final Session" : "Single Session");
return pInfo->pDelRes;
}
doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
if (pBInfo->pRes->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) {
doSetOperatorCompleted(pOperator);
}
- /*printDataBlock(pBInfo->pRes, "session insert");*/
+ printDataBlock(pBInfo->pRes, IS_FINAL_OP(pInfo)? "Final Session" : "Single Session");
return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
}
@@ -3048,7 +3278,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
break;
}
- if (pBlock->info.type == STREAM_REPROCESS) {
+ if (pBlock->info.type == STREAM_CLEAR) {
SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo));
doClearSessionWindows(&pInfo->streamAggSup, &pOperator->exprSupp, pBlock, 0, pOperator->exprSupp.numOfExprs,
pInfo->gap, pWins);
@@ -3102,11 +3332,11 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
if (pInfo->pDelRes->info.rows > 0) {
- /*printDataBlock(pInfo->pDelRes, "session del");*/
+ printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo)? "Final Session" : "Single Session");
return pInfo->pDelRes;
}
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
- /*printDataBlock(pBInfo->pRes, "session insert");*/
+ printDataBlock(pBInfo->pRes, IS_FINAL_OP(pInfo)? "Final Session" : "Single Session");
return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
}
@@ -3169,11 +3399,11 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) {
while (1) {
SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
if (pBlock == NULL) {
- clearUpdateDataBlock(pInfo->pUpdateRes);
+ clearSpecialDataBlock(pInfo->pUpdateRes);
break;
}
- if (pBlock->info.type == STREAM_REPROCESS) {
+ if (pBlock->info.type == STREAM_CLEAR) {
SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo));
doClearSessionWindows(&pInfo->streamAggSup, pSup, pBlock, 0, pSup->numOfExprs, pInfo->gap, pWins);
removeSessionResults(pStUpdated, pWins);
@@ -3236,7 +3466,7 @@ SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream
} else {
pInfo->isFinal = false;
pInfo->pUpdateRes = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
- pInfo->pUpdateRes->info.type = STREAM_REPROCESS;
+ pInfo->pUpdateRes->info.type = STREAM_CLEAR;
blockDataEnsureCapacity(pInfo->pUpdateRes, 128);
pOperator->name = "StreamSessionSemiAggOperator";
pOperator->fpSet =
@@ -3554,7 +3784,7 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
break;
}
- if (pBlock->info.type == STREAM_REPROCESS) {
+ if (pBlock->info.type == STREAM_CLEAR) {
doClearStateWindows(&pInfo->streamAggSup, pBlock, pInfo->primaryTsIndex, &pInfo->stateCol, pInfo->stateCol.slotId,
pSeUpdated, pInfo->pSeDeleted);
continue;
@@ -3900,18 +4130,22 @@ _error:
// merge interval operator
typedef struct SMergeIntervalAggOperatorInfo {
SIntervalAggOperatorInfo intervalAggOperatorInfo;
-
- SHashObj* groupIntervalHash;
- void* groupIntervalIter;
+ SList* groupIntervals;
+ SListIter groupIntervalsIter;
bool hasGroupId;
uint64_t groupId;
SSDataBlock* prefetchedBlock;
bool inputBlocksFinished;
} SMergeIntervalAggOperatorInfo;
+typedef struct SGroupTimeWindow {
+ uint64_t groupId;
+ STimeWindow window;
+} SGroupTimeWindow;
+
void destroyMergeIntervalOperatorInfo(void* param, int32_t numOfOutput) {
SMergeIntervalAggOperatorInfo* miaInfo = (SMergeIntervalAggOperatorInfo*)param;
- taosHashCleanup(miaInfo->groupIntervalHash);
+ tdListFree(miaInfo->groupIntervals);
destroyIntervalOperatorInfo(&miaInfo->intervalAggOperatorInfo, numOfOutput);
}
@@ -3940,15 +4174,22 @@ static int32_t outputPrevIntervalResult(SOperatorInfo* pOperatorInfo, uint64_t t
bool ascScan = (iaInfo->order == TSDB_ORDER_ASC);
SExprSupp* pExprSup = &pOperatorInfo->exprSupp;
- STimeWindow* prevWin = taosHashGet(miaInfo->groupIntervalHash, &tableGroupId, sizeof(tableGroupId));
- if (prevWin == NULL) {
- taosHashPut(miaInfo->groupIntervalHash, &tableGroupId, sizeof(tableGroupId), newWin, sizeof(STimeWindow));
- return 0;
- }
+ SGroupTimeWindow groupTimeWindow = {.groupId = tableGroupId, .window = *newWin};
+ tdListAppend(miaInfo->groupIntervals, &groupTimeWindow);
- if ((ascScan && newWin->skey > prevWin->skey || (!ascScan) && newWin->skey < prevWin->skey)) {
- finalizeWindowResult(pOperatorInfo, tableGroupId, prevWin, pResultBlock);
- taosHashPut(miaInfo->groupIntervalHash, &tableGroupId, sizeof(tableGroupId), newWin, sizeof(STimeWindow));
+ SListIter iter = {0};
+ tdListInitIter(miaInfo->groupIntervals, &iter, TD_LIST_FORWARD);
+ SListNode* listNode = NULL;
+ while ((listNode = tdListNext(&iter)) != NULL) {
+ SGroupTimeWindow* prevGrpWin = (SGroupTimeWindow*)listNode->data;
+ if (prevGrpWin->groupId != tableGroupId ) {
+ continue;
+ }
+ STimeWindow* prevWin = &prevGrpWin->window;
+ if ((ascScan && newWin->skey > prevWin->ekey || (!ascScan) && newWin->skey < prevWin->ekey)) {
+ finalizeWindowResult(pOperatorInfo, tableGroupId, prevWin, pResultBlock);
+ tdListPopNode(miaInfo->groupIntervals, listNode);
+ }
}
return 0;
@@ -4075,6 +4316,7 @@ static SSDataBlock* doMergeIntervalAgg(SOperatorInfo* pOperator) {
}
if (pBlock == NULL) {
+ tdListInitIter(miaInfo->groupIntervals, &miaInfo->groupIntervalsIter, TD_LIST_FORWARD);
miaInfo->inputBlocksFinished = true;
break;
}
@@ -4100,14 +4342,12 @@ static SSDataBlock* doMergeIntervalAgg(SOperatorInfo* pOperator) {
}
if (miaInfo->inputBlocksFinished) {
- void* win = taosHashIterate(miaInfo->groupIntervalHash, miaInfo->groupIntervalIter);
- if (win != NULL) {
- miaInfo->groupIntervalIter = win;
+ SListNode* listNode = tdListNext(&miaInfo->groupIntervalsIter);
- size_t len = 0;
- uint64_t* pTableGroupId = taosHashGetKey(win, &len);
- finalizeWindowResult(pOperator, *pTableGroupId, win, pRes);
- pRes->info.groupId = *pTableGroupId;
+ if (listNode != NULL) {
+ SGroupTimeWindow* grpWin = (SGroupTimeWindow*)(listNode->data);
+ finalizeWindowResult(pOperator, grpWin->groupId, &grpWin->window, pRes);
+ pRes->info.groupId = grpWin->groupId;
}
}
@@ -4129,8 +4369,7 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SExprI
goto _error;
}
- miaInfo->groupIntervalHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_NO_LOCK);
- miaInfo->groupIntervalIter = NULL;
+ miaInfo->groupIntervals = tdListNew(sizeof(SGroupTimeWindow));
SIntervalAggOperatorInfo* iaInfo = &miaInfo->intervalAggOperatorInfo;
diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c
index 5c7148932c..d41bc89a5f 100644
--- a/source/libs/function/src/builtins.c
+++ b/source/libs/function/src/builtins.c
@@ -1220,18 +1220,19 @@ static int32_t translateSubstr(SFunctionNode* pFunc, char* pErrBuf, int32_t len)
static int32_t translateCast(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
// The number of parameters has been limited by the syntax definition
- uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
+ //uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
+
// The function return type has been set during syntax parsing
uint8_t para2Type = pFunc->node.resType.type;
- if (para2Type != TSDB_DATA_TYPE_BIGINT && para2Type != TSDB_DATA_TYPE_UBIGINT &&
- para2Type != TSDB_DATA_TYPE_VARCHAR && para2Type != TSDB_DATA_TYPE_NCHAR &&
- para2Type != TSDB_DATA_TYPE_TIMESTAMP) {
- return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
- }
- if ((para2Type == TSDB_DATA_TYPE_TIMESTAMP && IS_VAR_DATA_TYPE(para1Type)) ||
- (para2Type == TSDB_DATA_TYPE_BINARY && para1Type == TSDB_DATA_TYPE_NCHAR)) {
- return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
- }
+ //if (para2Type != TSDB_DATA_TYPE_BIGINT && para2Type != TSDB_DATA_TYPE_UBIGINT &&
+ // para2Type != TSDB_DATA_TYPE_VARCHAR && para2Type != TSDB_DATA_TYPE_NCHAR &&
+ // para2Type != TSDB_DATA_TYPE_TIMESTAMP) {
+ // return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
+ //}
+ //if ((para2Type == TSDB_DATA_TYPE_TIMESTAMP && IS_VAR_DATA_TYPE(para1Type)) ||
+ // (para2Type == TSDB_DATA_TYPE_BINARY && para1Type == TSDB_DATA_TYPE_NCHAR)) {
+ // return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
+ //}
int32_t para2Bytes = pFunc->node.resType.bytes;
if (IS_VAR_DATA_TYPE(para2Type)) {
diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c
index 838071dbf1..983cffe9dc 100644
--- a/source/libs/function/src/udfd.c
+++ b/source/libs/function/src/udfd.c
@@ -12,6 +12,8 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see .
*/
+
+// clang-format off
#include "uv.h"
#include "os.h"
#include "fnLog.h"
@@ -25,6 +27,7 @@
#include "tglobal.h"
#include "tmsg.h"
#include "trpc.h"
+// clang-foramt on
typedef struct SUdfdContext {
uv_loop_t * loop;
@@ -103,12 +106,12 @@ typedef struct SUdfdRpcSendRecvInfo {
uv_sem_t resultSem;
} SUdfdRpcSendRecvInfo;
-static void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet);
+static void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet);
static int32_t udfdFillUdfInfoFromMNode(void *clientRpc, char *udfName, SUdf *udf);
static int32_t udfdConnectToMnode();
static int32_t udfdLoadUdf(char *udfName, SUdf *udf);
-static bool udfdRpcRfp(int32_t code);
-static int initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet);
+static bool udfdRpcRfp(int32_t code);
+static int initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet);
static int32_t udfdOpenClientRpc();
static int32_t udfdCloseClientRpc();
@@ -126,19 +129,19 @@ static void udfdUvHandleError(SUdfdUvConn *conn) { uv_close((uv_handle_t *)conn-
static void udfdPipeRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf);
static void udfdOnNewConnection(uv_stream_t *server, int status);
-static void udfdIntrSignalHandler(uv_signal_t *handle, int signum);
+static void udfdIntrSignalHandler(uv_signal_t *handle, int signum);
static int32_t removeListeningPipe();
-static void udfdPrintVersion();
+static void udfdPrintVersion();
static int32_t udfdParseArgs(int32_t argc, char *argv[]);
static int32_t udfdInitLog();
-static void udfdCtrlAllocBufCb(uv_handle_t *handle, size_t suggested_size, uv_buf_t *buf);
-static void udfdCtrlReadCb(uv_stream_t *q, ssize_t nread, const uv_buf_t *buf);
+static void udfdCtrlAllocBufCb(uv_handle_t *handle, size_t suggested_size, uv_buf_t *buf);
+static void udfdCtrlReadCb(uv_stream_t *q, ssize_t nread, const uv_buf_t *buf);
static int32_t udfdUvInit();
-static void udfdCloseWalkCb(uv_handle_t *handle, void *arg);
+static void udfdCloseWalkCb(uv_handle_t *handle, void *arg);
static int32_t udfdRun();
-static void udfdConnectMnodeThreadFunc(void* args);
+static void udfdConnectMnodeThreadFunc(void *args);
void udfdProcessRequest(uv_work_t *req) {
SUvUdfWork *uvUdf = (SUvUdfWork *)(req->data);
@@ -401,11 +404,11 @@ void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) {
udf->bufSize = pFuncInfo->bufSize;
char path[PATH_MAX] = {0};
- #ifdef WINDOWS
+#ifdef WINDOWS
snprintf(path, sizeof(path), "%s%s.dll", TD_TMP_DIR_PATH, pFuncInfo->name);
- #else
+#else
snprintf(path, sizeof(path), "%s/lib%s.so", TD_TMP_DIR_PATH, pFuncInfo->name);
- #endif
+#endif
TdFilePtr file =
taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC | TD_FILE_AUTO_DEL);
if (file == NULL) {
@@ -544,7 +547,8 @@ int32_t udfdLoadUdf(char *udfName, SUdf *udf) {
return 0;
}
static bool udfdRpcRfp(int32_t code) {
- if (code == TSDB_CODE_RPC_REDIRECT) {
+ if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED ||
+ code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY) {
return true;
} else {
return false;
@@ -652,8 +656,7 @@ void udfdAllocBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) {
buf->base = ctx->inputBuf;
buf->len = ctx->inputCap;
} else {
- fnError("udfd can not allocate enough memory")
- buf->base = NULL;
+ fnError("udfd can not allocate enough memory") buf->base = NULL;
buf->len = 0;
}
} else {
@@ -664,8 +667,7 @@ void udfdAllocBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) {
buf->base = ctx->inputBuf + ctx->inputLen;
buf->len = ctx->inputCap - ctx->inputLen;
} else {
- fnError("udfd can not allocate enough memory")
- buf->base = NULL;
+ fnError("udfd can not allocate enough memory") buf->base = NULL;
buf->len = 0;
}
}
@@ -881,7 +883,7 @@ static int32_t udfdRun() {
return 0;
}
-void udfdConnectMnodeThreadFunc(void* args) {
+void udfdConnectMnodeThreadFunc(void *args) {
int32_t retryMnodeTimes = 0;
int32_t code = 0;
while (retryMnodeTimes++ <= TSDB_MAX_REPLICA) {
@@ -939,7 +941,7 @@ int main(int argc, char *argv[]) {
uv_thread_create(&mnodeConnectThread, udfdConnectMnodeThreadFunc, NULL);
udfdRun();
-
+
removeListeningPipe();
udfdCloseClientRpc();
diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c
index 2ce065ac6a..e50c8d193f 100644
--- a/source/libs/parser/src/parTranslater.c
+++ b/source/libs/parser/src/parTranslater.c
@@ -5044,6 +5044,7 @@ typedef struct SVgroupCreateTableBatch {
static void destroyCreateTbReq(SVCreateTbReq* pReq) {
taosMemoryFreeClear(pReq->name);
+ taosMemoryFreeClear(pReq->comment);
taosMemoryFreeClear(pReq->ntb.schemaRow.pSchema);
}
@@ -5061,6 +5062,7 @@ static int32_t buildNormalTableBatchReq(int32_t acctId, const SCreateTableStmt*
if (pStmt->pOptions->commentNull == false) {
req.comment = strdup(pStmt->pOptions->comment);
if (NULL == req.comment) {
+ destroyCreateTbReq(&req);
return TSDB_CODE_OUT_OF_MEMORY;
}
req.commentLen = strlen(pStmt->pOptions->comment);
@@ -5132,6 +5134,7 @@ static void destroyCreateTbReqBatch(SVgroupCreateTableBatch* pTbBatch) {
for (int32_t i = 0; i < size; ++i) {
SVCreateTbReq* pTableReq = taosArrayGet(pTbBatch->req.pArray, i);
taosMemoryFreeClear(pTableReq->name);
+ taosMemoryFreeClear(pTableReq->comment);
if (pTableReq->type == TSDB_NORMAL_TABLE) {
taosMemoryFreeClear(pTableReq->ntb.schemaRow.pSchema);
diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c
index cf47f38576..face42f114 100644
--- a/source/libs/planner/src/planOptimizer.c
+++ b/source/libs/planner/src/planOptimizer.c
@@ -1107,6 +1107,26 @@ static int32_t partTagsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSub
return code;
}
+//=====================================================================================================================
+// eliminate project optimization
+static bool eliminateProjOptCheckProjColumnNames(SProjectLogicNode* pProjectNode) {
+ SHashObj* pProjColNameHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
+ SNode* pProjection;
+ FOREACH(pProjection, pProjectNode->pProjections) {
+ char* projColumnName = ((SColumnNode*)pProjection)->colName;
+ int32_t* pExist = taosHashGet(pProjColNameHash, projColumnName, strlen(projColumnName));
+ if (NULL != pExist) {
+ taosHashCleanup(pProjColNameHash);
+ return false;
+ } else {
+ int32_t exist = 1;
+ taosHashPut(pProjColNameHash, projColumnName, strlen(projColumnName), &exist, sizeof(exist));
+ }
+ }
+ taosHashCleanup(pProjColNameHash);
+ return true;
+}
+
static bool eliminateProjOptMayBeOptimized(SLogicNode* pNode) {
// TODO: enable this optimization after new mechanising that map projection and targets of project node
if (NULL != pNode->pParent) {
@@ -1122,28 +1142,15 @@ static bool eliminateProjOptMayBeOptimized(SLogicNode* pNode) {
return false;
}
- SHashObj* pProjColNameHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
- SNode* pProjection;
+ SNode* pProjection;
FOREACH(pProjection, pProjectNode->pProjections) {
SExprNode* pExprNode = (SExprNode*)pProjection;
if (QUERY_NODE_COLUMN != nodeType(pExprNode)) {
- taosHashCleanup(pProjColNameHash);
return false;
}
-
- char* projColumnName = ((SColumnNode*)pProjection)->colName;
- int32_t* pExist = taosHashGet(pProjColNameHash, projColumnName, strlen(projColumnName));
- if (NULL != pExist) {
- taosHashCleanup(pProjColNameHash);
- return false;
- } else {
- int32_t exist = 1;
- taosHashPut(pProjColNameHash, projColumnName, strlen(projColumnName), &exist, sizeof(exist));
- }
}
- taosHashCleanup(pProjColNameHash);
- return true;
+ return eliminateProjOptCheckProjColumnNames(pProjectNode);
}
static int32_t eliminateProjOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan,
@@ -1320,15 +1327,56 @@ static int32_t rewriteTailOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogic
return rewriteTailOptimizeImpl(pCxt, pLogicSubplan, pIndef);
}
+static bool eliminateSetOpMayBeOptimized(SLogicNode* pNode) {
+ SLogicNode* pParent = pNode->pParent;
+ if (NULL == pParent ||
+ QUERY_NODE_LOGIC_PLAN_AGG != nodeType(pParent) && QUERY_NODE_LOGIC_PLAN_PROJECT != nodeType(pParent) ||
+ LIST_LENGTH(pParent->pChildren) < 2) {
+ return false;
+ }
+ if (nodeType(pNode) != nodeType(pNode->pParent) || LIST_LENGTH(pNode->pChildren) < 2) {
+ return false;
+ }
+ return true;
+}
+
+static int32_t eliminateSetOpOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan,
+ SLogicNode* pSetOpNode) {
+ SNode* pSibling;
+ FOREACH(pSibling, pSetOpNode->pParent->pChildren) {
+ if (nodesEqualNode(pSibling, (SNode*)pSetOpNode)) {
+ SNode* pChild;
+ FOREACH(pChild, pSetOpNode->pChildren) { ((SLogicNode*)pChild)->pParent = pSetOpNode->pParent; }
+ INSERT_LIST(pSetOpNode->pParent->pChildren, pSetOpNode->pChildren);
+
+ pSetOpNode->pChildren = NULL;
+ ERASE_NODE(pSetOpNode->pParent->pChildren);
+ return TSDB_CODE_SUCCESS;
+ }
+ }
+
+ return TSDB_CODE_PLAN_INTERNAL_ERROR;
+}
+
+static int32_t eliminateSetOpOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) {
+ SLogicNode* pSetOpNode = optFindPossibleNode(pLogicSubplan->pNode, eliminateSetOpMayBeOptimized);
+ if (NULL == pSetOpNode) {
+ return TSDB_CODE_SUCCESS;
+ }
+
+ return eliminateSetOpOptimizeImpl(pCxt, pLogicSubplan, pSetOpNode);
+}
+
// clang-format off
static const SOptimizeRule optimizeRuleSet[] = {
- {.pName = "OptimizeScanData", .optimizeFunc = osdOptimize},
- {.pName = "ConditionPushDown", .optimizeFunc = cpdOptimize},
- {.pName = "OrderByPrimaryKey", .optimizeFunc = opkOptimize},
- {.pName = "SmaIndex", .optimizeFunc = smaOptimize},
- {.pName = "PartitionTags", .optimizeFunc = partTagsOptimize},
- {.pName = "EliminateProject", .optimizeFunc = eliminateProjOptimize},
- {.pName = "RewriteTail", .optimizeFunc = rewriteTailOptimize}
+ {.pName = "OptimizeScanData", .optimizeFunc = osdOptimize},
+ {.pName = "ConditionPushDown", .optimizeFunc = cpdOptimize},
+ {.pName = "OrderByPrimaryKey", .optimizeFunc = opkOptimize},
+ {.pName = "SmaIndex", .optimizeFunc = smaOptimize},
+ {.pName = "PartitionTags", .optimizeFunc = partTagsOptimize},
+ {.pName = "EliminateProject", .optimizeFunc = eliminateProjOptimize},
+ {.pName = "EliminateSetOperator", .optimizeFunc = eliminateSetOpOptimize},
+ {.pName = "RewriteTail", .optimizeFunc = rewriteTailOptimize}
};
// clang-format on
diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c
index c807c1c4cd..bee0461e58 100644
--- a/source/libs/qcom/src/queryUtil.c
+++ b/source/libs/qcom/src/queryUtil.c
@@ -19,6 +19,7 @@
#include "tmsg.h"
#include "trpc.h"
#include "tsched.h"
+// clang-format off
#include "cJSON.h"
#define VALIDNUMOFCOLS(x) ((x) >= TSDB_MIN_COLUMNS && (x) <= TSDB_MAX_COLUMNS)
@@ -147,13 +148,15 @@ int32_t asyncSendMsgToServerExt(void* pTransporter, SEpSet* epSet, int64_t* pTra
}
memcpy(pMsg, pInfo->msgInfo.pData, pInfo->msgInfo.len);
- SRpcMsg rpcMsg = {.msgType = pInfo->msgType,
- .pCont = pMsg,
- .contLen = pInfo->msgInfo.len,
- .info.ahandle = (void*)pInfo,
- .info.handle = pInfo->msgInfo.handle,
- .info.persistHandle = persistHandle,
- .code = 0};
+ SRpcMsg rpcMsg = {
+ .msgType = pInfo->msgType,
+ .pCont = pMsg,
+ .contLen = pInfo->msgInfo.len,
+ .info.ahandle = (void*)pInfo,
+ .info.handle = pInfo->msgInfo.handle,
+ .info.persistHandle = persistHandle,
+ .code = 0
+ };
assert(pInfo->fp != NULL);
TRACE_SET_ROOTID(&rpcMsg.info.traceId, pInfo->requestId);
rpcSendRequestWithCtx(pTransporter, epSet, &rpcMsg, pTransporterId, rpcCtx);
@@ -221,8 +224,9 @@ void destroyQueryExecRes(SQueryExecRes* pRes) {
qError("invalid exec result for request type %d", pRes->msgType);
}
}
+// clang-format on
-int32_t dataConverToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *len) {
+int32_t dataConverToStr(char* str, int type, void* buf, int32_t bufSize, int32_t* len) {
int32_t n = 0;
switch (type) {
@@ -262,7 +266,7 @@ int32_t dataConverToStr(char *str, int type, void *buf, int32_t bufSize, int32_t
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
if (bufSize < 0) {
-// tscError("invalid buf size");
+ // tscError("invalid buf size");
return TSDB_CODE_TSC_INVALID_VALUE;
}
@@ -289,7 +293,7 @@ int32_t dataConverToStr(char *str, int type, void *buf, int32_t bufSize, int32_t
break;
default:
-// tscError("unsupported type:%d", type);
+ // tscError("unsupported type:%d", type);
return TSDB_CODE_TSC_INVALID_VALUE;
}
@@ -332,7 +336,7 @@ char* parseTagDatatoJson(void* p) {
int32_t length = taosUcs4ToMbs((TdUcs4*)pTagVal->pData, pTagVal->nData, tagJsonValue);
if (length < 0) {
qError("charset:%s to %s. val:%s convert json value failed.", DEFAULT_UNICODE_ENCODEC, tsCharset,
- pTagVal->pData);
+ pTagVal->pData);
taosMemoryFree(tagJsonValue);
goto end;
}
@@ -372,7 +376,6 @@ end:
return string;
}
-
int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst) {
if (NULL == pSrc) {
*pDst = NULL;
@@ -393,37 +396,36 @@ int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst) {
*pDst = NULL;
return TSDB_CODE_SUCCESS;
}
-
+
*pDst = taosMemoryMalloc(sizeof(*pSrc));
if (NULL == *pDst) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
memcpy(*pDst, pSrc, sizeof(*pSrc));
if (pSrc->vgHash) {
- (*pDst)->vgHash = taosHashInit(taosHashGetSize(pSrc->vgHash), taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
+ (*pDst)->vgHash = taosHashInit(taosHashGetSize(pSrc->vgHash), taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true,
+ HASH_ENTRY_LOCK);
if (NULL == (*pDst)->vgHash) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
SVgroupInfo* vgInfo = NULL;
- void *pIter = taosHashIterate(pSrc->vgHash, NULL);
+ void* pIter = taosHashIterate(pSrc->vgHash, NULL);
while (pIter) {
vgInfo = pIter;
int32_t* vgId = taosHashGetKey(pIter, NULL);
-
+
if (0 != taosHashPut((*pDst)->vgHash, vgId, sizeof(*vgId), vgInfo, sizeof(*vgInfo))) {
qError("taosHashPut failed, vgId:%d", vgInfo->vgId);
- taosHashCancelIterate(pSrc->vgHash, pIter);
+ taosHashCancelIterate(pSrc->vgHash, pIter);
taosHashCleanup((*pDst)->vgHash);
taosMemoryFreeClear(*pDst);
return TSDB_CODE_CTG_MEM_ERROR;
}
-
+
pIter = taosHashIterate(pSrc->vgHash, pIter);
}
}
-
+
return TSDB_CODE_SUCCESS;
}
-
-
diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c
index d7f0733129..80bebafef2 100644
--- a/source/libs/scalar/src/sclfunc.c
+++ b/source/libs/scalar/src/sclfunc.c
@@ -735,6 +735,60 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
char *input = colDataGetData(pInput[0].columnData, i);
switch(outputType) {
+ case TSDB_DATA_TYPE_TINYINT: {
+ if (inputType == TSDB_DATA_TYPE_BINARY) {
+ *(int8_t *)output = taosStr2Int8(varDataVal(input), NULL, 10);
+ } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
+ char *newBuf = taosMemoryCalloc(1, inputLen);
+ int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf);
+ if (len < 0) {
+ taosMemoryFree(newBuf);
+ return TSDB_CODE_FAILED;
+ }
+ newBuf[len] = 0;
+ *(int8_t *)output = taosStr2Int8(newBuf, NULL, 10);
+ taosMemoryFree(newBuf);
+ } else {
+ GET_TYPED_DATA(*(int8_t *)output, int8_t, inputType, input);
+ }
+ break;
+ }
+ case TSDB_DATA_TYPE_SMALLINT: {
+ if (inputType == TSDB_DATA_TYPE_BINARY) {
+ *(int16_t *)output = taosStr2Int16(varDataVal(input), NULL, 10);
+ } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
+ char *newBuf = taosMemoryCalloc(1, inputLen);
+ int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf);
+ if (len < 0) {
+ taosMemoryFree(newBuf);
+ return TSDB_CODE_FAILED;
+ }
+ newBuf[len] = 0;
+ *(int16_t *)output = taosStr2Int16(newBuf, NULL, 10);
+ taosMemoryFree(newBuf);
+ } else {
+ GET_TYPED_DATA(*(int16_t *)output, int16_t, inputType, input);
+ }
+ break;
+ }
+ case TSDB_DATA_TYPE_INT: {
+ if (inputType == TSDB_DATA_TYPE_BINARY) {
+ *(int32_t *)output = taosStr2Int32(varDataVal(input), NULL, 10);
+ } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
+ char *newBuf = taosMemoryCalloc(1, inputLen);
+ int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf);
+ if (len < 0) {
+ taosMemoryFree(newBuf);
+ return TSDB_CODE_FAILED;
+ }
+ newBuf[len] = 0;
+ *(int32_t *)output = taosStr2Int32(newBuf, NULL, 10);
+ taosMemoryFree(newBuf);
+ } else {
+ GET_TYPED_DATA(*(int32_t *)output, int32_t, inputType, input);
+ }
+ break;
+ }
case TSDB_DATA_TYPE_BIGINT: {
if (inputType == TSDB_DATA_TYPE_BINARY) {
*(int64_t *)output = taosStr2Int64(varDataVal(input), NULL, 10);
@@ -753,6 +807,60 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
}
break;
}
+ case TSDB_DATA_TYPE_UTINYINT: {
+ if (inputType == TSDB_DATA_TYPE_BINARY) {
+ *(uint8_t *)output = taosStr2UInt8(varDataVal(input), NULL, 10);
+ } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
+ char *newBuf = taosMemoryCalloc(1, inputLen);
+ int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf);
+ if (len < 0) {
+ taosMemoryFree(newBuf);
+ return TSDB_CODE_FAILED;
+ }
+ newBuf[len] = 0;
+ *(uint8_t *)output = taosStr2UInt8(newBuf, NULL, 10);
+ taosMemoryFree(newBuf);
+ } else {
+ GET_TYPED_DATA(*(uint8_t *)output, uint8_t, inputType, input);
+ }
+ break;
+ }
+ case TSDB_DATA_TYPE_USMALLINT: {
+ if (inputType == TSDB_DATA_TYPE_BINARY) {
+ *(uint16_t *)output = taosStr2UInt16(varDataVal(input), NULL, 10);
+ } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
+ char *newBuf = taosMemoryCalloc(1, inputLen);
+ int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf);
+ if (len < 0) {
+ taosMemoryFree(newBuf);
+ return TSDB_CODE_FAILED;
+ }
+ newBuf[len] = 0;
+ *(uint16_t *)output = taosStr2UInt16(newBuf, NULL, 10);
+ taosMemoryFree(newBuf);
+ } else {
+ GET_TYPED_DATA(*(uint16_t *)output, uint16_t, inputType, input);
+ }
+ break;
+ }
+ case TSDB_DATA_TYPE_UINT: {
+ if (inputType == TSDB_DATA_TYPE_BINARY) {
+ *(uint32_t *)output = taosStr2UInt32(varDataVal(input), NULL, 10);
+ } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
+ char *newBuf = taosMemoryCalloc(1, inputLen);
+ int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf);
+ if (len < 0) {
+ taosMemoryFree(newBuf);
+ return TSDB_CODE_FAILED;
+ }
+ newBuf[len] = 0;
+ *(uint32_t *)output = taosStr2UInt32(newBuf, NULL, 10);
+ taosMemoryFree(newBuf);
+ } else {
+ GET_TYPED_DATA(*(uint32_t *)output, uint32_t, inputType, input);
+ }
+ break;
+ }
case TSDB_DATA_TYPE_UBIGINT: {
if (inputType == TSDB_DATA_TYPE_BINARY) {
*(uint64_t *)output = taosStr2UInt64(varDataVal(input), NULL, 10);
@@ -771,10 +879,64 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
}
break;
}
+ case TSDB_DATA_TYPE_FLOAT: {
+ if (inputType == TSDB_DATA_TYPE_BINARY) {
+ *(float *)output = taosStr2Float(varDataVal(input), NULL);
+ } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
+ char *newBuf = taosMemoryCalloc(1, inputLen);
+ int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf);
+ if (len < 0) {
+ taosMemoryFree(newBuf);
+ return TSDB_CODE_FAILED;
+ }
+ newBuf[len] = 0;
+ *(float *)output = taosStr2Float(newBuf, NULL);
+ taosMemoryFree(newBuf);
+ } else {
+ GET_TYPED_DATA(*(float *)output, float, inputType, input);
+ }
+ break;
+ }
+ case TSDB_DATA_TYPE_DOUBLE: {
+ if (inputType == TSDB_DATA_TYPE_BINARY) {
+ *(double *)output = taosStr2Double(varDataVal(input), NULL);
+ } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
+ char *newBuf = taosMemoryCalloc(1, inputLen);
+ int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf);
+ if (len < 0) {
+ taosMemoryFree(newBuf);
+ return TSDB_CODE_FAILED;
+ }
+ newBuf[len] = 0;
+ *(double *)output = taosStr2Double(newBuf, NULL);
+ taosMemoryFree(newBuf);
+ } else {
+ GET_TYPED_DATA(*(double *)output, double, inputType, input);
+ }
+ break;
+ }
+ case TSDB_DATA_TYPE_BOOL: {
+ if (inputType == TSDB_DATA_TYPE_BINARY) {
+ *(bool *)output = taosStr2Int8(varDataVal(input), NULL, 10);
+ } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
+ char *newBuf = taosMemoryCalloc(1, inputLen);
+ int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf);
+ if (len < 0) {
+ taosMemoryFree(newBuf);
+ return TSDB_CODE_FAILED;
+ }
+ newBuf[len] = 0;
+ *(bool *)output = taosStr2Int8(newBuf, NULL, 10);
+ taosMemoryFree(newBuf);
+ } else {
+ GET_TYPED_DATA(*(bool *)output, bool, inputType, input);
+ }
+ break;
+ }
case TSDB_DATA_TYPE_TIMESTAMP: {
if (inputType == TSDB_DATA_TYPE_BINARY || inputType == TSDB_DATA_TYPE_NCHAR) {
- //not support
- return TSDB_CODE_FAILED;
+ //convert to 0
+ *(int64_t *)output = 0;
} else {
GET_TYPED_DATA(*(int64_t *)output, int64_t, inputType, input);
}
@@ -789,8 +951,16 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
len = sprintf(varDataVal(output), "%.*s", len, varDataVal(input));
varDataSetLen(output, len);
} else if (inputType == TSDB_DATA_TYPE_NCHAR) {
- //not support
- return TSDB_CODE_FAILED;
+ char *newBuf = taosMemoryCalloc(1, inputLen);
+ int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf);
+ if (len < 0) {
+ taosMemoryFree(newBuf);
+ return TSDB_CODE_FAILED;
+ }
+ len = TMIN(len, outputLen - VARSTR_HEADER_SIZE);
+ memcpy(varDataVal(output), newBuf, len);
+ varDataSetLen(output, len);
+ taosMemoryFree(newBuf);
} else {
char tmp[400] = {0};
NUM_TO_STRING(inputType, input, sizeof(tmp), tmp);
diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c
index f9a9ec0f72..292db3db1f 100644
--- a/source/libs/scalar/src/sclvector.c
+++ b/source/libs/scalar/src/sclvector.c
@@ -468,7 +468,7 @@ double getVectorDoubleValue_JSON(void *src, int32_t index){
}
void* ncharTobinary(void *buf){ // todo need to remove , if tobinary is nchar
- int32_t inputLen = varDataLen(buf);
+ int32_t inputLen = varDataTLen(buf);
void* t = taosMemoryCalloc(1, inputLen);
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(buf), varDataLen(buf), varDataVal(t));
diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c
index 9d92865e47..56d063ae51 100644
--- a/source/libs/stream/src/stream.c
+++ b/source/libs/stream/src/stream.c
@@ -111,7 +111,7 @@ int32_t streamTaskEnqueue(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg*
// enqueue
if (pData != NULL) {
- pData->type = STREAM_DATA_TYPE_SSDATA_BLOCK;
+ pData->type = STREAM_INPUT__DATA_BLOCK;
pData->srcVgId = pReq->dataSrcVgId;
// decode
/*pData->blocks = pReq->data;*/
@@ -146,7 +146,7 @@ int32_t streamTaskEnqueueRetrieve(SStreamTask* pTask, SStreamRetrieveReq* pReq,
// enqueue
if (pData != NULL) {
- pData->type = STREAM_DATA_TYPE_SSDATA_BLOCK;
+ pData->type = STREAM_INPUT__DATA_RETRIEVE;
pData->srcVgId = 0;
// decode
/*pData->blocks = pReq->data;*/
@@ -170,7 +170,7 @@ int32_t streamTaskEnqueueRetrieve(SStreamTask* pTask, SStreamRetrieveReq* pReq,
pCont->rspToTaskId = pReq->srcTaskId;
pCont->rspFromTaskId = pReq->dstTaskId;
pRsp->pCont = buf;
- pRsp->contLen = sizeof(SMsgHead) + sizeof(SStreamDispatchRsp);
+ pRsp->contLen = sizeof(SMsgHead) + sizeof(SStreamRetrieveRsp);
tmsgSendRsp(pRsp);
return status == TASK_INPUT_STATUS__NORMAL ? 0 : -1;
}
diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c
index 1f51a927e7..a5e9b8edd9 100644
--- a/source/libs/stream/src/streamDispatch.c
+++ b/source/libs/stream/src/streamDispatch.c
@@ -115,6 +115,7 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock)
.srcNodeId = pTask->nodeId,
.srcTaskId = pTask->taskId,
.pRetrieve = pRetrieve,
+ .retrieveLen = dataStrLen,
};
int32_t sz = taosArrayGetSize(pTask->childEpInfo);
@@ -146,7 +147,7 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock)
.code = 0,
.msgType = TDMT_STREAM_RETRIEVE,
.pCont = buf,
- .contLen = len,
+ .contLen = sizeof(SMsgHead) + len,
};
if (tmsgSendReq(&pEpInfo->epSet, &rpcMsg) < 0) {
@@ -300,7 +301,7 @@ int32_t streamDispatch(SStreamTask* pTask, SMsgCb* pMsgCb) {
atomic_store_8(&pTask->outputStatus, TASK_OUTPUT_STATUS__NORMAL);
return 0;
}
- ASSERT(pBlock->type == STREAM_DATA_TYPE_SSDATA_BLOCK);
+ ASSERT(pBlock->type == STREAM_INPUT__DATA_BLOCK);
qInfo("stream continue dispatching: task %d", pTask->taskId);
diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c
index cb7dd241c1..c75e6c004a 100644
--- a/source/libs/stream/src/streamExec.c
+++ b/source/libs/stream/src/streamExec.c
@@ -17,6 +17,7 @@
static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) {
void* exec = pTask->exec.executor;
+ bool hasData = false;
// set input
SStreamQueueItem* pItem = (SStreamQueueItem*)data;
@@ -27,7 +28,7 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes)
ASSERT(pTask->isDataScan);
SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)data;
qSetStreamInput(exec, pSubmit->data, STREAM_DATA_TYPE_SUBMIT_BLOCK, false);
- } else if (pItem->type == STREAM_INPUT__DATA_BLOCK) {
+ } else if (pItem->type == STREAM_INPUT__DATA_BLOCK || pItem->type == STREAM_INPUT__DATA_RETRIEVE) {
SStreamDataBlock* pBlock = (SStreamDataBlock*)data;
SArray* blocks = pBlock->blocks;
qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_DATA_TYPE_SSDATA_BLOCK, false);
@@ -43,7 +44,21 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes)
if (qExecTask(exec, &output, &ts) < 0) {
ASSERT(false);
}
- if (output == NULL) break;
+ if (output == NULL) {
+ if (pItem->type == STREAM_INPUT__DATA_RETRIEVE) {
+ //SSDataBlock block = {0};
+ //block.info.type = STREAM_PUSH_EMPTY;
+ //block.info.childId = pTask->selfChildId;
+ SStreamDataBlock* pRetrieveBlock = (SStreamDataBlock*)data;
+ ASSERT(taosArrayGetSize(pRetrieveBlock->blocks) == 1);
+ SSDataBlock* pBlock = createOneDataBlock(taosArrayGet(pRetrieveBlock->blocks, 0), true);
+ pBlock->info.type = STREAM_PUSH_EMPTY;
+ pBlock->info.childId = pTask->selfChildId;
+ taosArrayPush(pRes, pBlock);
+ }
+ break;
+ }
+ hasData = true;
if (output->info.type == STREAM_RETRIEVE) {
if (streamBroadcastToChildren(pTask, output) < 0) {
@@ -99,7 +114,7 @@ static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) {
if (type == STREAM_INPUT__TRIGGER) {
blockDataDestroy(((SStreamTrigger*)data)->pBlock);
taosFreeQitem(data);
- } else if (type == STREAM_INPUT__DATA_BLOCK) {
+ } else if (type == STREAM_INPUT__DATA_BLOCK || type == STREAM_INPUT__DATA_RETRIEVE) {
taosArrayDestroyEx(((SStreamDataBlock*)data)->blocks, (FDelete)tDeleteSSDataBlock);
taosFreeQitem(data);
} else if (type == STREAM_INPUT__DATA_SUBMIT) {
diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c
index ada391b40a..3d64cec8d8 100644
--- a/source/libs/stream/src/streamUpdate.c
+++ b/source/libs/stream/src/streamUpdate.c
@@ -119,6 +119,7 @@ SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t waterma
taosArrayPush(pInfo->pTsBuckets, &dumy);
}
pInfo->numBuckets = DEFAULT_BUCKET_SIZE;
+ pInfo->pCloseWinSBF = NULL;
return pInfo;
}
@@ -154,6 +155,9 @@ bool updateInfoIsUpdated(SUpdateInfo *pInfo, tb_uid_t tableId, TSKEY ts) {
TSKEY maxTs = *(TSKEY *)taosArrayGet(pInfo->pTsBuckets, index);
if (ts < maxTs - pInfo->watermark) {
// this window has been closed.
+ if (pInfo->pCloseWinSBF) {
+ return tScalableBfPut(pInfo->pCloseWinSBF, &ts, sizeof(TSKEY));
+ }
return true;
}
@@ -193,3 +197,19 @@ void updateInfoDestroy(SUpdateInfo *pInfo) {
taosArrayDestroy(pInfo->pTsSBFs);
taosMemoryFree(pInfo);
}
+
+void updateInfoAddCloseWindowSBF(SUpdateInfo *pInfo) {
+ if (pInfo->pCloseWinSBF) {
+ return;
+ }
+ int64_t rows = adjustExpEntries(pInfo->interval * ROWS_PER_MILLISECOND);
+ pInfo->pCloseWinSBF = tScalableBfInit(rows, DEFAULT_FALSE_POSITIVE);
+}
+
+void updateInfoDestoryColseWinSBF(SUpdateInfo *pInfo) {
+ if (!pInfo || !pInfo->pCloseWinSBF) {
+ return;
+ }
+ tScalableBfDestroy(pInfo->pCloseWinSBF);
+ pInfo->pCloseWinSBF = NULL;
+}
diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c
index c5af72c971..3100d0525c 100644
--- a/source/libs/sync/src/syncMain.c
+++ b/source/libs/sync/src/syncMain.c
@@ -914,6 +914,9 @@ void syncNodeStart(SSyncNode* pSyncNode) {
syncNodeBecomeLeader(pSyncNode, "one replica start");
// Raft 3.6.2 Committing entries from previous terms
+ syncNodeAppendNoop(pSyncNode);
+ syncMaybeAdvanceCommitIndex(pSyncNode);
+
return;
}
@@ -1662,6 +1665,12 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde
// change isStandBy to normal (election timeout)
if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) {
syncNodeBecomeLeader(pSyncNode, tmpbuf);
+
+ // Raft 3.6.2 Committing entries from previous terms
+ syncNodeReplicate(pSyncNode);
+ syncNodeAppendNoop(pSyncNode);
+ syncMaybeAdvanceCommitIndex(pSyncNode);
+
} else {
syncNodeBecomeFollower(pSyncNode, tmpbuf);
}
@@ -1807,16 +1816,9 @@ void syncNodeBecomeLeader(SSyncNode* pSyncNode, const char* debugStr) {
// stop elect timer
syncNodeStopElectTimer(pSyncNode);
- // start replicate right now!
- syncNodeReplicate(pSyncNode);
-
// start heartbeat timer
syncNodeStartHeartbeatTimer(pSyncNode);
- // append noop
- syncNodeAppendNoop(pSyncNode);
- syncMaybeAdvanceCommitIndex(pSyncNode); // maybe only one replica
-
// trace log
do {
int32_t debugStrLen = strlen(debugStr);
@@ -1841,9 +1843,9 @@ void syncNodeCandidate2Leader(SSyncNode* pSyncNode) {
syncNodeLog2("==state change syncNodeCandidate2Leader==", pSyncNode);
// Raft 3.6.2 Committing entries from previous terms
-
- // do not use this
- // syncNodeEqNoop(pSyncNode);
+ syncNodeReplicate(pSyncNode);
+ syncNodeAppendNoop(pSyncNode);
+ syncMaybeAdvanceCommitIndex(pSyncNode);
}
void syncNodeFollower2Candidate(SSyncNode* pSyncNode) {
diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h
index bbd9ce3a78..963a85922f 100644
--- a/source/libs/transport/inc/transComm.h
+++ b/source/libs/transport/inc/transComm.h
@@ -105,6 +105,13 @@ typedef SRpcCtxVal STransCtxVal;
typedef SRpcInfo STrans;
typedef SRpcConnInfo STransHandleInfo;
+// ref mgt
+// handle
+typedef struct SExHandle {
+ void* handle;
+ int64_t refId;
+ void* pThrd;
+} SExHandle;
/*convet from fqdn to ip */
typedef struct SCvtAddr {
char ip[TSDB_FQDN_LEN];
@@ -113,14 +120,15 @@ typedef struct SCvtAddr {
} SCvtAddr;
typedef struct {
- SEpSet epSet; // ip list provided by app
- SEpSet origEpSet;
- void* ahandle; // handle provided by app
- tmsg_t msgType; // message type
- int8_t connType; // connection type cli/srv
- int64_t rid; // refId returned by taosAddRef
+ SEpSet epSet; // ip list provided by app
+ SEpSet origEpSet;
+ void* ahandle; // handle provided by app
+ tmsg_t msgType; // message type
+ int8_t connType; // connection type cli/srv
- int8_t retryCount;
+ int8_t retryCnt;
+ int8_t retryLimit;
+ // bool setMaxRetry;
STransCtx appCtx; //
STransMsg* pRsp; // for synchronous API
tsem_t* pSem; // for synchronous API
@@ -239,6 +247,32 @@ int transSendAsync(SAsyncPool* pool, queue* mq);
} \
} \
} while (0)
+
+#define ASYNC_CHECK_HANDLE(exh1, id) \
+ do { \
+ if (id > 0) { \
+ tTrace("handle step1"); \
+ SExHandle* exh2 = transAcquireExHandle(refMgt, id); \
+ if (exh2 == NULL || id != exh2->refId) { \
+ tTrace("handle %p except, may already freed, ignore msg, ref1: %" PRIu64 ", ref2 : %" PRIu64 "", exh1, \
+ exh2 ? exh2->refId : 0, id); \
+ goto _return1; \
+ } \
+ } else if (id == 0) { \
+ tTrace("handle step2"); \
+ SExHandle* exh2 = transAcquireExHandle(refMgt, id); \
+ if (exh2 == NULL || id == exh2->refId) { \
+ tTrace("handle %p except, may already freed, ignore msg, ref1: %" PRIu64 ", ref2 : %" PRIu64 "", exh1, id, \
+ exh2 ? exh2->refId : 0); \
+ goto _return1; \
+ } else { \
+ id = exh1->refId; \
+ } \
+ } else if (id < 0) { \
+ tTrace("handle step3"); \
+ goto _return2; \
+ } \
+ } while (0)
int transInitBuffer(SConnBuffer* buf);
int transClearBuffer(SConnBuffer* buf);
int transDestroyBuffer(SConnBuffer* buf);
@@ -349,21 +383,13 @@ void transDQDestroy(SDelayQueue* queue);
int transDQSched(SDelayQueue* queue, void (*func)(void* arg), void* arg, uint64_t timeoutMs);
-void transPrintEpSet(SEpSet* pEpSet);
+// void transPrintEpSet(SEpSet* pEpSet);
bool transEpSetIsEqual(SEpSet* a, SEpSet* b);
/*
* init global func
*/
void transThreadOnce();
-// ref mgt
-// handle
-typedef struct SExHandle {
- void* handle;
- int64_t refId;
- void* pThrd;
-} SExHandle;
-
void transInitEnv();
int32_t transOpenExHandleMgt(int size);
void transCloseExHandleMgt(int32_t mgt);
diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c
index 1ec96f4a7a..4f7b19b539 100644
--- a/source/libs/transport/src/trans.c
+++ b/source/libs/transport/src/trans.c
@@ -79,6 +79,7 @@ void* rpcOpen(const SRpcInit* pInit) {
return pRpc;
}
void rpcClose(void* arg) {
+ tInfo("start to close rpc");
SRpcInfo* pRpc = (SRpcInfo*)arg;
(*taosCloseHandle[pRpc->connType])(pRpc->tcphandle);
transCloseExHandleMgt(pRpc->refMgt);
diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c
index e563bf0ddd..7374d1fffc 100644
--- a/source/libs/transport/src/transCli.c
+++ b/source/libs/transport/src/transCli.c
@@ -1,4 +1,4 @@
-/* * Copyright (c) 2019 TAOS Data, Inc.
+/** Copyright (c) 2019 TAOS Data, Inc.
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
@@ -25,7 +25,6 @@ typedef struct SCliConn {
uv_write_t writeReq;
void* hostThrd;
- int hThrdIdx;
SConnBuffer readBuf;
STransQueue cliMsgs;
@@ -36,6 +35,7 @@ typedef struct SCliConn {
bool broken; // link broken or not
ConnStatus status; //
+ int64_t refId;
char* ip;
uint32_t port;
@@ -54,7 +54,7 @@ typedef struct SCliMsg {
int sent; //(0: no send, 1: alread sent)
} SCliMsg;
-typedef struct SCliThrdObj {
+typedef struct SCliThrd {
TdThread thread; // tid
int64_t pid; // pid
uv_loop_t* loop;
@@ -72,13 +72,13 @@ typedef struct SCliThrdObj {
SCvtAddr cvtAddr;
bool quit;
-} SCliThrdObj;
+} SCliThrd;
typedef struct SCliObj {
- char label[TSDB_LABEL_LEN];
- int32_t index;
- int numOfThreads;
- SCliThrdObj** pThreadObj;
+ char label[TSDB_LABEL_LEN];
+ int32_t index;
+ int numOfThreads;
+ SCliThrd** pThreadObj;
} SCliObj;
typedef struct SConnList {
@@ -106,11 +106,18 @@ static void cliAsyncCb(uv_async_t* handle);
static int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg);
-static SCliConn* cliCreateConn(SCliThrdObj* thrd);
+static SCliConn* cliCreateConn(SCliThrd* thrd);
static void cliDestroyConn(SCliConn* pConn, bool clear /*clear tcp handle or not*/);
static void cliDestroy(uv_handle_t* handle);
static void cliSend(SCliConn* pConn);
+static bool cliIsEpsetUpdated(int32_t code, STransConnCtx* pCtx) {
+ if (code != 0) return false;
+ if (pCtx->retryCnt == 0) return false;
+ if (transEpSetIsEqual(&pCtx->epSet, &pCtx->origEpSet)) return false;
+ return true;
+}
+
void cliMayCvtFqdnToIp(SEpSet* pEpSet, SCvtAddr* pCvtAddr);
/*
* set TCP connection timeout per-socket level
@@ -122,14 +129,14 @@ static void cliHandleResp(SCliConn* conn);
static void cliHandleExcept(SCliConn* conn);
// handle req from app
-static void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd);
-static void cliHandleQuit(SCliMsg* pMsg, SCliThrdObj* pThrd);
-static void cliHandleRelease(SCliMsg* pMsg, SCliThrdObj* pThrd);
-static void cliHandleUpdate(SCliMsg* pMsg, SCliThrdObj* pThrd);
-static void (*cliAsyncHandle[])(SCliMsg* pMsg, SCliThrdObj* pThrd) = {cliHandleReq, cliHandleQuit, cliHandleRelease,
- NULL, cliHandleUpdate};
+static void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd);
+static void cliHandleQuit(SCliMsg* pMsg, SCliThrd* pThrd);
+static void cliHandleRelease(SCliMsg* pMsg, SCliThrd* pThrd);
+static void cliHandleUpdate(SCliMsg* pMsg, SCliThrd* pThrd);
+static void (*cliAsyncHandle[])(SCliMsg* pMsg, SCliThrd* pThrd) = {cliHandleReq, cliHandleQuit, cliHandleRelease, NULL,
+ cliHandleUpdate};
-static void cliSendQuit(SCliThrdObj* thrd);
+static void cliSendQuit(SCliThrd* thrd);
static void destroyUserdata(STransMsg* userdata);
static int cliRBChoseIdx(STrans* pTransInst);
@@ -137,8 +144,8 @@ static int cliRBChoseIdx(STrans* pTransInst);
static void destroyCmsg(SCliMsg* cmsg);
static void transDestroyConnCtx(STransConnCtx* ctx);
// thread obj
-static SCliThrdObj* createThrdObj();
-static void destroyThrdObj(SCliThrdObj* pThrd);
+static SCliThrd* createThrdObj();
+static void destroyThrdObj(SCliThrd* pThrd);
static void cliWalkCb(uv_handle_t* handle, void* arg);
@@ -154,7 +161,6 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) {
destroyCmsg(pMsg);
}
}
-
#define CLI_RELEASE_UV(loop) \
do { \
uv_walk(loop, cliWalkCb, NULL); \
@@ -168,17 +174,23 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) {
snprintf(key, sizeof(key), "%s:%d", ip, (int)port); \
} while (0)
-#define CONN_HOST_THREAD_IDX(conn) (conn ? ((SCliConn*)conn)->hThrdIdx : -1)
+#define CONN_HOST_THREAD_IDX1(idx, exh, refId, pThrd) \
+ do { \
+ if (exh == NULL) { \
+ idx = -1; \
+ } else { \
+ ASYNC_CHECK_HANDLE((exh), refId); \
+ pThrd = (SCliThrd*)(exh)->pThrd; \
+ } \
+ } while (0)
#define CONN_PERSIST_TIME(para) (para * 1000 * 10)
#define CONN_GET_HOST_THREAD(conn) (conn ? ((SCliConn*)conn)->hostThrd : NULL)
-#define CONN_GET_INST_LABEL(conn) (((STrans*)(((SCliThrdObj*)(conn)->hostThrd)->pTransInst))->label)
+#define CONN_GET_INST_LABEL(conn) (((STrans*)(((SCliThrd*)(conn)->hostThrd)->pTransInst))->label)
#define CONN_SHOULD_RELEASE(conn, head) \
do { \
if ((head)->release == 1 && (head->msgLen) == sizeof(*head)) { \
- int connStatus = conn->status; \
uint64_t ahandle = head->ahandle; \
CONN_GET_MSGCTX_BY_AHANDLE(conn, ahandle); \
- conn->status = ConnRelease; \
transClearBuffer(&conn->readBuf); \
transFreeMsg(transContFromHead((char*)head)); \
tDebug("%s conn %p receive release request, ref: %d", CONN_GET_INST_LABEL(conn), conn, T_REF_VAL_GET(conn)); \
@@ -187,9 +199,7 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) {
} \
destroyCmsg(pMsg); \
cliReleaseUnfinishedMsg(conn); \
- if (connStatus != ConnInPool) { \
- addConnToPool(((SCliThrdObj*)conn->hostThrd)->pool, conn); \
- } \
+ addConnToPool(((SCliThrd*)conn->hostThrd)->pool, conn); \
return; \
} \
} while (0)
@@ -255,8 +265,25 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) {
#define REQUEST_PERSIS_HANDLE(msg) ((msg)->info.persistHandle == 1)
#define REQUEST_RELEASE_HANDLE(cmsg) ((cmsg)->type == Release)
+#define EPSET_GET_SIZE(epSet) (epSet)->numOfEps
#define EPSET_GET_INUSE_IP(epSet) ((epSet)->eps[(epSet)->inUse].fqdn)
#define EPSET_GET_INUSE_PORT(epSet) ((epSet)->eps[(epSet)->inUse].port)
+#define EPSET_FORWARD_INUSE(epSet) \
+ do { \
+ (epSet)->inUse = (++((epSet)->inUse)) % ((epSet)->numOfEps); \
+ } while (0)
+#define EPSET_DEBUG_STR(epSet, tbuf) \
+ do { \
+ int len = snprintf(tbuf, sizeof(tbuf), "epset:{"); \
+ for (int i = 0; i < (epSet)->numOfEps; i++) { \
+ if (i == (epSet)->numOfEps - 1) { \
+ len += snprintf(tbuf + len, sizeof(tbuf) - len, "%d. %s:%d", i, (epSet)->eps[i].fqdn, (epSet)->eps[i].port); \
+ } else { \
+ len += snprintf(tbuf + len, sizeof(tbuf) - len, "%d. %s:%d, ", i, (epSet)->eps[i].fqdn, (epSet)->eps[i].port); \
+ } \
+ } \
+ len += snprintf(tbuf + len, sizeof(tbuf) - len, "}, inUse:%d", (epSet)->inUse); \
+ } while (0);
static void* cliWorkThread(void* arg);
@@ -271,8 +298,8 @@ _RETURN:
return false;
}
void cliHandleResp(SCliConn* conn) {
- SCliThrdObj* pThrd = conn->hostThrd;
- STrans* pTransInst = pThrd->pTransInst;
+ SCliThrd* pThrd = conn->hostThrd;
+ STrans* pTransInst = pThrd->pTransInst;
STransMsgHead* pHead = (STransMsgHead*)(conn->readBuf.buf);
pHead->code = htonl(pHead->code);
@@ -292,17 +319,9 @@ void cliHandleResp(SCliConn* conn) {
if (CONN_NO_PERSIST_BY_APP(conn)) {
pMsg = transQueuePop(&conn->cliMsgs);
- pCtx = pMsg ? pMsg->ctx : NULL;
- if (pMsg == NULL && !CONN_NO_PERSIST_BY_APP(conn)) {
- transMsg.info.ahandle = transCtxDumpVal(&conn->ctx, transMsg.msgType);
- if (transMsg.info.ahandle == NULL) {
- transMsg.info.ahandle = transCtxDumpBrokenlinkVal(&conn->ctx, (int32_t*)&(transMsg.msgType));
- }
- tDebug("%s conn %p construct ahandle %p, persist: 0", CONN_GET_INST_LABEL(conn), conn, transMsg.info.ahandle);
- } else {
- transMsg.info.ahandle = pCtx ? pCtx->ahandle : NULL;
- tDebug("%s conn %p get ahandle %p, persist: 0", CONN_GET_INST_LABEL(conn), conn, transMsg.info.ahandle);
- }
+ pCtx = pMsg->ctx;
+ transMsg.info.ahandle = pCtx->ahandle;
+ tDebug("%s conn %p get ahandle %p, persist: 0", CONN_GET_INST_LABEL(conn), conn, transMsg.info.ahandle);
} else {
uint64_t ahandle = (uint64_t)pHead->ahandle;
CONN_GET_MSGCTX_BY_AHANDLE(conn, ahandle);
@@ -324,26 +343,22 @@ void cliHandleResp(SCliConn* conn) {
}
// buf's mem alread translated to transMsg.pCont
transClearBuffer(&conn->readBuf);
-
if (!CONN_NO_PERSIST_BY_APP(conn)) {
- transMsg.info.handle = conn;
+ transMsg.info.handle = (void*)conn->refId;
tDebug("%s conn %p ref by app", CONN_GET_INST_LABEL(conn), conn);
}
- // char buf[64] = {0};
- // TRACE_TO_STR(&transMsg.info.traceId, buf);
+
STraceId* trace = &transMsg.info.traceId;
- tGTrace("conn %p %s received from %s:%d, local info: %s:%d, msg size: %d, code: %d", conn, TMSG_INFO(pHead->msgType),
- taosInetNtoa(conn->addr.sin_addr), ntohs(conn->addr.sin_port), taosInetNtoa(conn->localAddr.sin_addr),
- ntohs(conn->localAddr.sin_port), transMsg.contLen, transMsg.code);
+ tGTrace("%s conn %p %s received from %s:%d, local info: %s:%d, msg size: %d, code: %d", CONN_GET_INST_LABEL(conn),
+ conn, TMSG_INFO(pHead->msgType), taosInetNtoa(conn->addr.sin_addr), ntohs(conn->addr.sin_port),
+ taosInetNtoa(conn->localAddr.sin_addr), ntohs(conn->localAddr.sin_port), transMsg.contLen, transMsg.code);
if (pCtx == NULL && CONN_NO_PERSIST_BY_APP(conn)) {
- tDebug("%s except, server continue send while cli ignore it", CONN_GET_INST_LABEL(conn));
- // transUnrefCliHandle(conn);
+ tDebug("%s except, conn %p read while cli ignore it", CONN_GET_INST_LABEL(conn), conn);
return;
}
if (CONN_RELEASE_BY_SERVER(conn) && transMsg.info.ahandle == NULL) {
- tDebug("%s except, server continue send while cli ignore it", CONN_GET_INST_LABEL(conn));
- // transUnrefCliHandle(conn);
+ tDebug("%s except, conn %p read while cli ignore it", CONN_GET_INST_LABEL(conn), conn);
return;
}
@@ -375,9 +390,9 @@ void cliHandleExcept(SCliConn* pConn) {
return;
}
}
- SCliThrdObj* pThrd = pConn->hostThrd;
- STrans* pTransInst = pThrd->pTransInst;
- bool once = false;
+ SCliThrd* pThrd = pConn->hostThrd;
+ STrans* pTransInst = pThrd->pTransInst;
+ bool once = false;
do {
SCliMsg* pMsg = transQueuePop(&pConn->cliMsgs);
if (pMsg == NULL && once) {
@@ -389,7 +404,6 @@ void cliHandleExcept(SCliConn* pConn) {
transMsg.code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
transMsg.msgType = pMsg ? pMsg->msg.msgType + 1 : 0;
transMsg.info.ahandle = NULL;
- transMsg.info.handle = pConn;
if (pMsg == NULL && !CONN_NO_PERSIST_BY_APP(pConn)) {
transMsg.info.ahandle = transCtxDumpVal(&pConn->ctx, transMsg.msgType);
@@ -414,15 +428,15 @@ void cliHandleExcept(SCliConn* pConn) {
return;
}
destroyCmsg(pMsg);
- tTrace("%s conn %p start to destroy", CONN_GET_INST_LABEL(pConn), pConn);
+ tTrace("%s conn %p start to destroy, ref:%d", CONN_GET_INST_LABEL(pConn), pConn, T_REF_VAL_GET(pConn));
} while (!transQueueEmpty(&pConn->cliMsgs));
transUnrefCliHandle(pConn);
}
void cliTimeoutCb(uv_timer_t* handle) {
- SCliThrdObj* pThrd = handle->data;
- STrans* pTransInst = pThrd->pTransInst;
- int64_t currentTime = pThrd->nextTimeout;
+ SCliThrd* pThrd = handle->data;
+ STrans* pTransInst = pThrd->pTransInst;
+ int64_t currentTime = pThrd->nextTimeout;
tTrace("%s conn timeout, try to remove expire conn from conn pool", pTransInst->label);
SConnList* p = taosHashIterate((SHashObj*)pThrd->pool, NULL);
@@ -487,10 +501,26 @@ static SCliConn* getConnFromPool(void* pool, char* ip, uint32_t port) {
assert(h == &conn->conn);
return conn;
}
+static void allocConnRef(SCliConn* conn, bool update) {
+ if (update) {
+ transRemoveExHandle(refMgt, conn->refId);
+ }
+ SExHandle* exh = taosMemoryCalloc(1, sizeof(SExHandle));
+ exh->handle = conn;
+ exh->pThrd = conn->hostThrd;
+ exh->refId = transAddExHandle(refMgt, exh);
+ conn->refId = exh->refId;
+}
static void addConnToPool(void* pool, SCliConn* conn) {
- SCliThrdObj* thrd = conn->hostThrd;
+ if (conn->status == ConnInPool) {
+ // assert(0);
+ return;
+ }
+ SCliThrd* thrd = conn->hostThrd;
CONN_HANDLE_THREAD_QUIT(thrd);
+ allocConnRef(conn, true);
+
STrans* pTransInst = thrd->pTransInst;
conn->expireTime = taosGetTimestampMs() + CONN_PERSIST_TIME(pTransInst->idleTime);
transQueueClear(&conn->cliMsgs);
@@ -499,7 +529,7 @@ static void addConnToPool(void* pool, SCliConn* conn) {
char key[128] = {0};
CONN_CONSTRUCT_HASH_KEY(key, conn->ip, conn->port);
- tTrace("%s conn %p added to conn pool, read buf cap: %d", CONN_GET_INST_LABEL(conn), conn, conn->readBuf.cap);
+ tTrace("%s conn %p added to conn pool, read buf cap:%d", CONN_GET_INST_LABEL(conn), conn, conn->readBuf.cap);
SConnList* plist = taosHashGet((SHashObj*)pool, key, strlen(key));
// list already create before
@@ -540,13 +570,14 @@ static void cliRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) {
return;
}
if (nread < 0) {
- tError("%s conn %p read error: %s", CONN_GET_INST_LABEL(conn), conn, uv_err_name(nread));
+ tError("%s conn %p read error: %s, ref: %d", CONN_GET_INST_LABEL(conn), conn, uv_err_name(nread),
+ T_REF_VAL_GET(conn));
conn->broken = true;
cliHandleExcept(conn);
}
}
-static SCliConn* cliCreateConn(SCliThrdObj* pThrd) {
+static SCliConn* cliCreateConn(SCliThrd* pThrd) {
SCliConn* conn = taosMemoryCalloc(1, sizeof(SCliConn));
// read/write stream handle
conn->stream = (uv_stream_t*)taosMemoryMalloc(sizeof(uv_tcp_t));
@@ -562,11 +593,16 @@ static SCliConn* cliCreateConn(SCliThrdObj* pThrd) {
conn->status = ConnNormal;
conn->broken = 0;
transRefCliHandle(conn);
+
+ allocConnRef(conn, false);
+
return conn;
}
static void cliDestroyConn(SCliConn* conn, bool clear) {
tTrace("%s conn %p remove from conn pool", CONN_GET_INST_LABEL(conn), conn);
QUEUE_REMOVE(&conn->conn);
+ QUEUE_INIT(&conn->conn);
+ transRemoveExHandle(refMgt, conn->refId);
if (clear) {
uv_close((uv_handle_t*)conn->stream, cliDestroy);
}
@@ -593,7 +629,7 @@ static bool cliHandleNoResp(SCliConn* conn) {
}
if (res == true) {
if (cliMaySendCachedMsg(conn) == false) {
- SCliThrdObj* thrd = conn->hostThrd;
+ SCliThrd* thrd = conn->hostThrd;
addConnToPool(thrd->pool, conn);
}
}
@@ -629,8 +665,8 @@ void cliSend(SCliConn* pConn) {
STransConnCtx* pCtx = pCliMsg->ctx;
- SCliThrdObj* pThrd = pConn->hostThrd;
- STrans* pTransInst = pThrd->pTransInst;
+ SCliThrd* pThrd = pConn->hostThrd;
+ STrans* pTransInst = pThrd->pTransInst;
STransMsg* pMsg = (STransMsg*)(&pCliMsg->msg);
if (pMsg->pCont == 0) {
@@ -651,12 +687,10 @@ void cliSend(SCliConn* pConn) {
uv_buf_t wb = uv_buf_init((char*)pHead, msgLen);
- // char buf[64] = {0};
- // TRACE_TO_STR(&pMsg->info.traceId, buf);
STraceId* trace = &pMsg->info.traceId;
- tGTrace("conn %p %s is sent to %s:%d, local info %s:%d", pConn, TMSG_INFO(pHead->msgType),
- taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), taosInetNtoa(pConn->localAddr.sin_addr),
- ntohs(pConn->localAddr.sin_port));
+ tGTrace("%s conn %p %s is sent to %s:%d, local info %s:%d", CONN_GET_INST_LABEL(pConn), pConn,
+ TMSG_INFO(pHead->msgType), taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port),
+ taosInetNtoa(pConn->localAddr.sin_addr), ntohs(pConn->localAddr.sin_port));
if (pHead->persist == 1) {
CONN_SET_PERSIST_BY_APP(pConn);
@@ -664,7 +698,6 @@ void cliSend(SCliConn* pConn) {
pConn->writeReq.data = pConn;
uv_write(&pConn->writeReq, (uv_stream_t*)pConn->stream, &wb, 1, cliSendCb);
-
return;
_RETURN:
return;
@@ -690,19 +723,24 @@ void cliConnCb(uv_connect_t* req, int status) {
cliSend(pConn);
}
-static void cliHandleQuit(SCliMsg* pMsg, SCliThrdObj* pThrd) {
+static void cliHandleQuit(SCliMsg* pMsg, SCliThrd* pThrd) {
+ pThrd->quit = true;
tDebug("cli work thread %p start to quit", pThrd);
destroyCmsg(pMsg);
destroyConnPool(pThrd->pool);
uv_timer_stop(&pThrd->timer);
uv_walk(pThrd->loop, cliWalkCb, NULL);
- pThrd->quit = true;
-
// uv_stop(pThrd->loop);
}
-static void cliHandleRelease(SCliMsg* pMsg, SCliThrdObj* pThrd) {
- SCliConn* conn = pMsg->msg.info.handle;
+static void cliHandleRelease(SCliMsg* pMsg, SCliThrd* pThrd) {
+ int64_t refId = (int64_t)(pMsg->msg.info.handle);
+ SExHandle* exh = transAcquireExHandle(refMgt, refId);
+ if (exh == NULL) {
+ tDebug("%" PRId64 " already release", refId);
+ }
+
+ SCliConn* conn = exh->handle;
tDebug("%s conn %p start to release to inst", CONN_GET_INST_LABEL(conn), conn);
if (T_REF_VAL_GET(conn) == 2) {
@@ -711,33 +749,37 @@ static void cliHandleRelease(SCliMsg* pMsg, SCliThrdObj* pThrd) {
return;
}
cliSend(conn);
- } else {
- // conn already broken down
- transUnrefCliHandle(conn);
}
}
-static void cliHandleUpdate(SCliMsg* pMsg, SCliThrdObj* pThrd) {
+static void cliHandleUpdate(SCliMsg* pMsg, SCliThrd* pThrd) {
STransConnCtx* pCtx = pMsg->ctx;
-
pThrd->cvtAddr = pCtx->cvtAddr;
destroyCmsg(pMsg);
}
-SCliConn* cliGetConn(SCliMsg* pMsg, SCliThrdObj* pThrd) {
+SCliConn* cliGetConn(SCliMsg* pMsg, SCliThrd* pThrd, bool* ignore) {
SCliConn* conn = NULL;
- if (pMsg->msg.info.handle != NULL) {
- conn = (SCliConn*)(pMsg->msg.info.handle);
- if (conn != NULL) {
- tTrace("%s conn %p reused", CONN_GET_INST_LABEL(conn), conn);
- }
- } else {
- STransConnCtx* pCtx = pMsg->ctx;
- conn = getConnFromPool(pThrd->pool, EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet));
- if (conn != NULL) {
- tTrace("%s conn %p get from conn pool", CONN_GET_INST_LABEL(conn), conn);
+ int64_t refId = (int64_t)(pMsg->msg.info.handle);
+ if (refId != 0) {
+ SExHandle* exh = transAcquireExHandle(refMgt, refId);
+ if (exh == NULL) {
+ *ignore = true;
+ destroyCmsg(pMsg);
+ return NULL;
+ // assert(0);
} else {
- tTrace("%s not found conn in conn pool %p", ((STrans*)pThrd->pTransInst)->label, pThrd->pool);
+ conn = exh->handle;
+ transReleaseExHandle(refMgt, refId);
}
+ return conn;
+ };
+
+ STransConnCtx* pCtx = pMsg->ctx;
+ conn = getConnFromPool(pThrd->pool, EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet));
+ if (conn != NULL) {
+ tTrace("%s conn %p get from conn pool:%p", CONN_GET_INST_LABEL(conn), conn, pThrd->pool);
+ } else {
+ tTrace("%s not found conn in conn pool:%p", ((STrans*)pThrd->pTransInst)->label, pThrd->pool);
}
return conn;
}
@@ -752,22 +794,19 @@ void cliMayCvtFqdnToIp(SEpSet* pEpSet, SCvtAddr* pCvtAddr) {
}
}
}
-void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) {
- uint64_t et = taosGetTimestampUs();
- uint64_t el = et - pMsg->st;
- // tTrace("%s cli msg tran time cost: %" PRIu64 "us", ((STrans*)pThrd->pTransInst)->label, el);
-
+void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) {
STransConnCtx* pCtx = pMsg->ctx;
STrans* pTransInst = pThrd->pTransInst;
cliMayCvtFqdnToIp(&pCtx->epSet, &pThrd->cvtAddr);
- transPrintEpSet(&pCtx->epSet);
-
- SCliConn* conn = cliGetConn(pMsg, pThrd);
+ // transPrintEpSet(&pCtx->epSet);
+ bool ignore = false;
+ SCliConn* conn = cliGetConn(pMsg, pThrd, &ignore);
+ if (ignore == true) {
+ return;
+ }
if (conn != NULL) {
- conn->hThrdIdx = pCtx->hThrdIdx;
-
transCtxMerge(&conn->ctx, &pCtx->appCtx);
transQueuePush(&conn->cliMsgs, pMsg);
cliSend(conn);
@@ -776,7 +815,6 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) {
transCtxMerge(&conn->ctx, &pCtx->appCtx);
transQueuePush(&conn->cliMsgs, pMsg);
- conn->hThrdIdx = pCtx->hThrdIdx;
conn->ip = strdup(EPSET_GET_INUSE_IP(&pCtx->epSet));
conn->port = EPSET_GET_INUSE_PORT(&pCtx->epSet);
@@ -784,7 +822,7 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) {
if (ret) {
tError("%s conn %p failed to set conn option, errmsg %s", transLabel(pTransInst), conn, uv_err_name(ret));
}
- int fd = taosCreateSocketWithTimeOutOpt(TRANS_CONN_TIMEOUT);
+ int32_t fd = taosCreateSocketWithTimeout(TRANS_CONN_TIMEOUT);
if (fd == -1) {
tTrace("%s conn %p failed to create socket", transLabel(pTransInst), conn);
cliHandleExcept(conn);
@@ -807,9 +845,9 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) {
}
}
static void cliAsyncCb(uv_async_t* handle) {
- SAsyncItem* item = handle->data;
- SCliThrdObj* pThrd = item->pThrd;
- SCliMsg* pMsg = NULL;
+ SAsyncItem* item = handle->data;
+ SCliThrd* pThrd = item->pThrd;
+ SCliMsg* pMsg = NULL;
// batch process to avoid to lock/unlock frequently
queue wq;
@@ -835,7 +873,7 @@ static void cliAsyncCb(uv_async_t* handle) {
}
static void* cliWorkThread(void* arg) {
- SCliThrdObj* pThrd = (SCliThrdObj*)arg;
+ SCliThrd* pThrd = (SCliThrd*)arg;
pThrd->pid = taosGetSelfPthreadId();
setThreadName("trans-cli-work");
uv_run(pThrd->loop, UV_RUN_DEFAULT);
@@ -848,10 +886,10 @@ void* transInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads,
STrans* pTransInst = shandle;
memcpy(cli->label, label, strlen(label));
cli->numOfThreads = numOfThreads;
- cli->pThreadObj = (SCliThrdObj**)taosMemoryCalloc(cli->numOfThreads, sizeof(SCliThrdObj*));
+ cli->pThreadObj = (SCliThrd**)taosMemoryCalloc(cli->numOfThreads, sizeof(SCliThrd*));
for (int i = 0; i < cli->numOfThreads; i++) {
- SCliThrdObj* pThrd = createThrdObj();
+ SCliThrd* pThrd = createThrdObj();
pThrd->nextTimeout = taosGetTimestampMs() + CONN_PERSIST_TIME(pTransInst->idleTime);
pThrd->pTransInst = shandle;
@@ -885,8 +923,8 @@ static void destroyCmsg(SCliMsg* pMsg) {
taosMemoryFree(pMsg);
}
-static SCliThrdObj* createThrdObj() {
- SCliThrdObj* pThrd = (SCliThrdObj*)taosMemoryCalloc(1, sizeof(SCliThrdObj));
+static SCliThrd* createThrdObj() {
+ SCliThrd* pThrd = (SCliThrd*)taosMemoryCalloc(1, sizeof(SCliThrd));
QUEUE_INIT(&pThrd->msg);
taosThreadMutexInit(&pThrd->msgMtx, NULL);
@@ -904,7 +942,7 @@ static SCliThrdObj* createThrdObj() {
pThrd->quit = false;
return pThrd;
}
-static void destroyThrdObj(SCliThrdObj* pThrd) {
+static void destroyThrdObj(SCliThrd* pThrd) {
if (pThrd == NULL) {
return;
}
@@ -925,7 +963,7 @@ static void transDestroyConnCtx(STransConnCtx* ctx) {
taosMemoryFree(ctx);
}
-void cliSendQuit(SCliThrdObj* thrd) {
+void cliSendQuit(SCliThrd* thrd) {
// cli can stop gracefully
SCliMsg* msg = taosMemoryCalloc(1, sizeof(SCliMsg));
msg->type = Quit;
@@ -938,7 +976,10 @@ void cliWalkCb(uv_handle_t* handle, void* arg) {
}
int cliRBChoseIdx(STrans* pTransInst) {
- int64_t index = pTransInst->index;
+ int8_t index = pTransInst->index;
+ if (pTransInst->numOfThreads == 0) {
+ return -1;
+ }
if (pTransInst->index++ >= pTransInst->numOfThreads) {
pTransInst->index = 0;
}
@@ -946,82 +987,71 @@ int cliRBChoseIdx(STrans* pTransInst) {
}
static void doDelayTask(void* param) {
STaskArg* arg = param;
-
- SCliMsg* pMsg = arg->param1;
- SCliThrdObj* pThrd = arg->param2;
- cliHandleReq(pMsg, pThrd);
-
+ SCliMsg* pMsg = arg->param1;
+ SCliThrd* pThrd = arg->param2;
taosMemoryFree(arg);
+
+ cliHandleReq(pMsg, pThrd);
+}
+
+static void cliSchedMsgToNextNode(SCliMsg* pMsg, SCliThrd* pThrd) {
+ STransConnCtx* pCtx = pMsg->ctx;
+
+ STraceId* trace = &pMsg->msg.info.traceId;
+ char tbuf[256] = {0};
+ EPSET_DEBUG_STR(&pCtx->epSet, tbuf);
+ tGTrace("%s retry on next node, use %s, retryCnt:%d, limit:%d", transLabel(pThrd->pTransInst), tbuf,
+ pCtx->retryCnt + 1, pCtx->retryLimit);
+
+ STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg));
+ arg->param1 = pMsg;
+ arg->param2 = pThrd;
+ transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL);
+}
+
+void cliCompareAndSwap(int8_t* val, int8_t exp, int8_t newVal) {
+ if (*val != exp) {
+ *val = newVal;
+ }
}
int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) {
- SCliThrdObj* pThrd = pConn->hostThrd;
- STrans* pTransInst = pThrd->pTransInst;
+ SCliThrd* pThrd = pConn->hostThrd;
+ STrans* pTransInst = pThrd->pTransInst;
if (pMsg == NULL || pMsg->ctx == NULL) {
tTrace("%s conn %p handle resp", pTransInst->label, pConn);
pTransInst->cfp(pTransInst->parent, pResp, NULL);
return 0;
}
-
- STransConnCtx* pCtx = pMsg->ctx;
- SEpSet* pEpSet = &pCtx->epSet;
-
- if (pCtx->retryCount == 0) {
- pCtx->origEpSet = pCtx->epSet;
- }
/*
- * upper layer handle retry if code equal TSDB_CODE_RPC_NETWORK_UNAVAIL
+ * no retry
+ * 1. query conn
+ * 2. rpc thread already receive quit msg
*/
- tmsg_t msgType = pCtx->msgType;
- if ((pTransInst->retry != NULL && pEpSet->numOfEps > 1 && (pTransInst->retry(pResp->code))) ||
- (pResp->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || pResp->code == TSDB_CODE_APP_NOT_READY ||
- pResp->code == TSDB_CODE_NODE_NOT_DEPLOYED || pResp->code == TSDB_CODE_SYN_NOT_LEADER)) {
+ STransConnCtx* pCtx = pMsg->ctx;
+ int32_t code = pResp->code;
+ if (pTransInst->retry != NULL && pTransInst->retry(code)) {
pMsg->sent = 0;
- tTrace("try to send req to next node");
- pMsg->st = taosGetTimestampUs();
- pCtx->retryCount += 1;
- if (pResp->code == TSDB_CODE_RPC_NETWORK_UNAVAIL && pCtx->setMaxRetry == false) {
- if (pCtx->retryCount < pEpSet->numOfEps * 3) {
- pEpSet->inUse = (++pEpSet->inUse) % pEpSet->numOfEps;
- if (pThrd->quit == false) {
- STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg));
- arg->param1 = pMsg;
- arg->param2 = pThrd;
- transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL);
- transPrintEpSet(pEpSet);
- tTrace("%s use local epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse,
- pCtx->retryCount + 1, pEpSet->numOfEps * 3);
-
- transUnrefCliHandle(pConn);
- return -1;
- }
+ pCtx->retryCnt += 1;
+ if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL) {
+ cliCompareAndSwap(&pCtx->retryLimit, TRANS_RETRY_COUNT_LIMIT, EPSET_GET_SIZE(&pCtx->epSet) * 3);
+ if (pCtx->retryCnt < pCtx->retryLimit) {
+ transUnrefCliHandle(pConn);
+ EPSET_FORWARD_INUSE(&pCtx->epSet);
+ cliSchedMsgToNextNode(pMsg, pThrd);
+ return -1;
}
- } else if (pCtx->retryCount < TRANS_RETRY_COUNT_LIMIT) {
- pCtx->setMaxRetry = true;
- if (pResp->contLen == 0) {
- pEpSet->inUse = (++pEpSet->inUse) % pEpSet->numOfEps;
- transPrintEpSet(&pCtx->epSet);
- tTrace("%s use local epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse,
- pCtx->retryCount + 1, TRANS_RETRY_COUNT_LIMIT);
- } else {
- SEpSet epSet = {0};
- tDeserializeSEpSet(pResp->pCont, pResp->contLen, &epSet);
- pCtx->epSet = epSet;
-
- transPrintEpSet(&pCtx->epSet);
- tTrace("%s use remote epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse,
- pCtx->retryCount + 1, TRANS_RETRY_COUNT_LIMIT);
- }
- if (pThrd->quit == false) {
- if (pResp->code != TSDB_CODE_RPC_NETWORK_UNAVAIL) {
- if (pConn->status != ConnInPool) addConnToPool(pThrd->pool, pConn);
+ } else {
+ cliCompareAndSwap(&pCtx->retryLimit, TRANS_RETRY_COUNT_LIMIT, TRANS_RETRY_COUNT_LIMIT);
+ if (pCtx->retryCnt < pCtx->retryLimit) {
+ addConnToPool(pThrd->pool, pConn);
+ if (pResp->contLen == 0) {
+ EPSET_FORWARD_INUSE(&pCtx->epSet);
} else {
- transUnrefCliHandle(pConn);
+ tDeserializeSEpSet(pResp->pCont, pResp->contLen, &pCtx->epSet);
}
- STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg));
- arg->param1 = pMsg;
- arg->param2 = pThrd;
- transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL);
+ transFreeMsg(pResp->pCont);
+ cliSchedMsgToNextNode(pMsg, pThrd);
return -1;
}
}
@@ -1029,20 +1059,20 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) {
STraceId* trace = &pResp->info.traceId;
if (pCtx->pSem != NULL) {
- tGTrace("conn %p(sync) handle resp", pConn);
+ tGTrace("%s conn %p(sync) handle resp", CONN_GET_INST_LABEL(pConn), pConn);
if (pCtx->pRsp == NULL) {
- tGTrace("conn %p(sync) failed to resp, ignore", pConn);
+ tGTrace("%s conn %p(sync) failed to resp, ignore", CONN_GET_INST_LABEL(pConn), pConn);
} else {
memcpy((char*)pCtx->pRsp, (char*)pResp, sizeof(*pResp));
}
tsem_post(pCtx->pSem);
pCtx->pRsp = NULL;
} else {
- tGTrace("conn %p handle resp", pConn);
- if (pResp->code != 0 || pCtx->retryCount == 0 || transEpSetIsEqual(&pCtx->epSet, &pCtx->origEpSet)) {
+ tGTrace("%s conn %p handle resp", CONN_GET_INST_LABEL(pConn), pConn);
+ if (!cliIsEpsetUpdated(code, pCtx)) {
pTransInst->cfp(pTransInst->parent, pResp, NULL);
} else {
- pTransInst->cfp(pTransInst->parent, pResp, pEpSet);
+ pTransInst->cfp(pTransInst->parent, pResp, &pCtx->epSet);
}
}
return 0;
@@ -1074,38 +1104,58 @@ void transUnrefCliHandle(void* handle) {
return;
}
int ref = T_REF_DEC((SCliConn*)handle);
- tTrace("%s conn %p ref %d", CONN_GET_INST_LABEL((SCliConn*)handle), handle, ref);
+ tTrace("%s conn %p ref:%d", CONN_GET_INST_LABEL((SCliConn*)handle), handle, ref);
if (ref == 0) {
cliDestroyConn((SCliConn*)handle, true);
}
}
+SCliThrd* transGetWorkThrdFromHandle(int64_t handle) {
+ SCliThrd* pThrd = NULL;
+ SExHandle* exh = transAcquireExHandle(refMgt, handle);
+ if (exh == NULL) {
+ return NULL;
+ }
+ pThrd = exh->pThrd;
+ transReleaseExHandle(refMgt, handle);
+ return pThrd;
+}
+SCliThrd* transGetWorkThrd(STrans* trans, int64_t handle) {
+ if (handle == 0) {
+ int idx = cliRBChoseIdx(trans);
+ if (idx < 0) return NULL;
+ return ((SCliObj*)trans->tcphandle)->pThreadObj[idx];
+ }
+ return transGetWorkThrdFromHandle(handle);
+}
void transReleaseCliHandle(void* handle) {
- SCliThrdObj* thrd = CONN_GET_HOST_THREAD(handle);
- if (thrd == NULL) {
+ int idx = -1;
+ SCliThrd* pThrd = transGetWorkThrdFromHandle((int64_t)handle);
+ if (pThrd == NULL) {
return;
}
-
STransMsg tmsg = {.info.handle = handle};
SCliMsg* cmsg = taosMemoryCalloc(1, sizeof(SCliMsg));
cmsg->msg = tmsg;
cmsg->type = Release;
- transSendAsync(thrd->asyncPool, &cmsg->q);
+ transSendAsync(pThrd->asyncPool, &cmsg->q);
+ return;
}
void transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransCtx* ctx) {
- STrans* pTransInst = (STrans*)shandle;
- int idx = CONN_HOST_THREAD_IDX((SCliConn*)pReq->info.handle);
- if (idx == -1) {
- idx = cliRBChoseIdx(pTransInst);
+ STrans* pTransInst = (STrans*)shandle;
+ SCliThrd* pThrd = transGetWorkThrd(pTransInst, (int64_t)pReq->info.handle);
+ if (pThrd == NULL) {
+ transFreeMsg(pReq->pCont);
+ return;
}
+
TRACE_SET_MSGID(&pReq->info.traceId, tGenIdPI64());
STransConnCtx* pCtx = taosMemoryCalloc(1, sizeof(STransConnCtx));
pCtx->epSet = *pEpSet;
pCtx->ahandle = pReq->info.ahandle;
pCtx->msgType = pReq->msgType;
- pCtx->hThrdIdx = idx;
if (ctx != NULL) {
pCtx->appCtx = *ctx;
@@ -1118,19 +1168,19 @@ void transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STra
cliMsg->st = taosGetTimestampUs();
cliMsg->type = Normal;
- SCliThrdObj* thrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[idx];
-
STraceId* trace = &pReq->info.traceId;
- tGTrace("%s send request at thread:%08" PRId64 ", dst: %s:%d, app:%p", transLabel(pTransInst), thrd->pid,
+ tGTrace("%s send request at thread:%08" PRId64 ", dst: %s:%d, app:%p", transLabel(pTransInst), pThrd->pid,
EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet), pReq->info.ahandle);
- ASSERT(transSendAsync(thrd->asyncPool, &(cliMsg->q)) == 0);
+ ASSERT(transSendAsync(pThrd->asyncPool, &(cliMsg->q)) == 0);
+ return;
}
void transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp) {
- STrans* pTransInst = (STrans*)shandle;
- int idx = CONN_HOST_THREAD_IDX(pReq->info.handle);
- if (idx == -1) {
- idx = cliRBChoseIdx(pTransInst);
+ STrans* pTransInst = (STrans*)shandle;
+ SCliThrd* pThrd = transGetWorkThrd(pTransInst, (int64_t)pReq->info.handle);
+ if (pThrd == NULL) {
+ transFreeMsg(pReq->pCont);
+ return;
}
tsem_t* sem = taosMemoryCalloc(1, sizeof(tsem_t));
tsem_init(sem, 0, 0);
@@ -1139,9 +1189,9 @@ void transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransM
STransConnCtx* pCtx = taosMemoryCalloc(1, sizeof(STransConnCtx));
pCtx->epSet = *pEpSet;
+ pCtx->origEpSet = *pEpSet;
pCtx->ahandle = pReq->info.ahandle;
pCtx->msgType = pReq->msgType;
- pCtx->hThrdIdx = idx;
pCtx->pSem = sem;
pCtx->pRsp = pRsp;
@@ -1151,22 +1201,21 @@ void transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransM
cliMsg->st = taosGetTimestampUs();
cliMsg->type = Normal;
- SCliThrdObj* thrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[idx];
-
STraceId* trace = &pReq->info.traceId;
- tGTrace("%s send request at thread:%08" PRId64 ", dst: %s:%d, app:%p", transLabel(pTransInst), thrd->pid,
+ tGTrace("%s send request at thread:%08" PRId64 ", dst: %s:%d, app:%p", transLabel(pTransInst), pThrd->pid,
EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet), pReq->info.ahandle);
- transSendAsync(thrd->asyncPool, &(cliMsg->q));
+ transSendAsync(pThrd->asyncPool, &(cliMsg->q));
tsem_wait(sem);
tsem_destroy(sem);
taosMemoryFree(sem);
+ return;
}
/*
*
**/
-void transSetDefaultAddr(void* ahandle, const char* ip, const char* fqdn) {
- STrans* pTransInst = ahandle;
+void transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn) {
+ STrans* pTransInst = shandle;
SCvtAddr cvtAddr = {0};
if (ip != NULL && fqdn != NULL) {
@@ -1176,14 +1225,13 @@ void transSetDefaultAddr(void* ahandle, const char* ip, const char* fqdn) {
}
for (int i = 0; i < pTransInst->numOfThreads; i++) {
STransConnCtx* pCtx = taosMemoryCalloc(1, sizeof(STransConnCtx));
- pCtx->hThrdIdx = i;
pCtx->cvtAddr = cvtAddr;
SCliMsg* cliMsg = taosMemoryCalloc(1, sizeof(SCliMsg));
cliMsg->ctx = pCtx;
cliMsg->type = Update;
- SCliThrdObj* thrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[i];
+ SCliThrd* thrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[i];
tDebug("%s update epset at thread:%08" PRId64 "", pTransInst->label, thrd->pid);
transSendAsync(thrd->asyncPool, &(cliMsg->q));
diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c
index 8cd7f9d827..bff7d79bd3 100644
--- a/source/libs/transport/src/transComm.c
+++ b/source/libs/transport/src/transComm.c
@@ -455,16 +455,16 @@ void transPrintEpSet(SEpSet* pEpSet) {
return;
}
char buf[512] = {0};
- int len = snprintf(buf, sizeof(buf), "epset { ");
+ int len = snprintf(buf, sizeof(buf), "epset:{");
for (int i = 0; i < pEpSet->numOfEps; i++) {
if (i == pEpSet->numOfEps - 1) {
- len += snprintf(buf + len, sizeof(buf) - len, "%d. %s:%d ", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port);
+ len += snprintf(buf + len, sizeof(buf) - len, "%d. %s:%d", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port);
} else {
len += snprintf(buf + len, sizeof(buf) - len, "%d. %s:%d, ", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port);
}
}
len += snprintf(buf + len, sizeof(buf) - len, "}");
- tTrace("%s, inUse: %d", buf, pEpSet->inUse);
+ tTrace("%s, inUse:%d", buf, pEpSet->inUse);
}
bool transEpSetIsEqual(SEpSet* a, SEpSet* b) {
if (a->numOfEps != b->numOfEps || a->inUse != b->inUse) {
diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c
index 593a790a21..892d32696e 100644
--- a/source/libs/transport/src/transSvr.c
+++ b/source/libs/transport/src/transSvr.c
@@ -65,7 +65,7 @@ typedef struct SSvrMsg {
STransMsgType type;
} SSvrMsg;
-typedef struct SWorkThrdObj {
+typedef struct SWorkThrd {
TdThread thread;
uv_connect_t connect_req;
uv_pipe_t* pipe;
@@ -78,7 +78,7 @@ typedef struct SWorkThrdObj {
queue conn;
void* pTransInst;
bool quit;
-} SWorkThrdObj;
+} SWorkThrd;
typedef struct SServerObj {
TdThread thread;
@@ -86,10 +86,10 @@ typedef struct SServerObj {
uv_loop_t* loop;
// work thread info
- int workerIdx;
- int numOfThreads;
- int numOfWorkerReady;
- SWorkThrdObj** pThreadObj;
+ int workerIdx;
+ int numOfThreads;
+ int numOfWorkerReady;
+ SWorkThrd** pThreadObj;
uv_pipe_t pipeListen;
uv_pipe_t** pipe;
@@ -133,14 +133,14 @@ static SSvrConn* createConn(void* hThrd);
static void destroyConn(SSvrConn* conn, bool clear /*clear handle or not*/);
static void destroyConnRegArg(SSvrConn* conn);
-static int reallocConnRefHandle(SSvrConn* conn);
+static int reallocConnRef(SSvrConn* conn);
-static void uvHandleQuit(SSvrMsg* msg, SWorkThrdObj* thrd);
-static void uvHandleRelease(SSvrMsg* msg, SWorkThrdObj* thrd);
-static void uvHandleResp(SSvrMsg* msg, SWorkThrdObj* thrd);
-static void uvHandleRegister(SSvrMsg* msg, SWorkThrdObj* thrd);
-static void (*transAsyncHandle[])(SSvrMsg* msg, SWorkThrdObj* thrd) = {uvHandleResp, uvHandleQuit, uvHandleRelease,
- uvHandleRegister, NULL};
+static void uvHandleQuit(SSvrMsg* msg, SWorkThrd* thrd);
+static void uvHandleRelease(SSvrMsg* msg, SWorkThrd* thrd);
+static void uvHandleResp(SSvrMsg* msg, SWorkThrd* thrd);
+static void uvHandleRegister(SSvrMsg* msg, SWorkThrd* thrd);
+static void (*transAsyncHandle[])(SSvrMsg* msg, SWorkThrd* thrd) = {uvHandleResp, uvHandleQuit, uvHandleRelease,
+ uvHandleRegister, NULL};
static int32_t exHandlesMgt;
@@ -160,7 +160,7 @@ static void* transWorkerThread(void* arg);
static void* transAcceptThread(void* arg);
// add handle loop
-static bool addHandleToWorkloop(SWorkThrdObj* pThrd, char* pipeName);
+static bool addHandleToWorkloop(SWorkThrd* pThrd, char* pipeName);
static bool addHandleToAcceptloop(void* arg);
#define CONN_SHOULD_RELEASE(conn, head) \
@@ -176,7 +176,7 @@ static bool addHandleToAcceptloop(void* arg);
srvMsg->msg = tmsg; \
srvMsg->type = Release; \
srvMsg->pConn = conn; \
- reallocConnRefHandle(conn); \
+ reallocConnRef(conn); \
if (!transQueuePush(&conn->srvMsgs, srvMsg)) { \
return; \
} \
@@ -206,32 +206,6 @@ static bool addHandleToAcceptloop(void* arg);
} \
} while (0)
-#define ASYNC_CHECK_HANDLE(exh1, refId) \
- do { \
- if (refId > 0) { \
- tTrace("handle step1"); \
- SExHandle* exh2 = transAcquireExHandle(refMgt, refId); \
- if (exh2 == NULL || refId != exh2->refId) { \
- tTrace("handle %p except, may already freed, ignore msg, ref1: %" PRIu64 ", ref2 : %" PRIu64 "", exh1, \
- exh2 ? exh2->refId : 0, refId); \
- goto _return1; \
- } \
- } else if (refId == 0) { \
- tTrace("handle step2"); \
- SExHandle* exh2 = transAcquireExHandle(refMgt, refId); \
- if (exh2 == NULL || refId != exh2->refId) { \
- tTrace("handle %p except, may already freed, ignore msg, ref1: %" PRIu64 ", ref2 : %" PRIu64 "", exh1, refId, \
- exh2 ? exh2->refId : 0); \
- goto _return1; \
- } else { \
- refId = exh1->refId; \
- } \
- } else if (refId < 0) { \
- tTrace("handle step3"); \
- goto _return2; \
- } \
- } while (0)
-
void uvAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) {
SSvrConn* conn = handle->data;
SConnBuffer* pBuf = &conn->readBuf;
@@ -259,7 +233,7 @@ static void uvHandleReq(SSvrConn* pConn) {
// wreq->data = pConn;
// uv_read_stop((uv_stream_t*)pConn->pTcp);
// transRefSrvHandle(pConn);
- // uv_queue_work(((SWorkThrdObj*)pConn->hostThrd)->loop, wreq, uvWorkDoTask, uvWorkAfterTask);
+ // uv_queue_work(((SWorkThrd*)pConn->hostThrd)->loop, wreq, uvWorkDoTask, uvWorkAfterTask);
CONN_SHOULD_RELEASE(pConn, pHead);
@@ -379,7 +353,7 @@ void uvOnSendCb(uv_write_t* req, int status) {
// if (msg->type == Release && conn->status != ConnNormal) {
// conn->status = ConnNormal;
// transUnrefSrvHandle(conn);
- // reallocConnRefHandle(conn);
+ // reallocConnRef(conn);
// destroySmsg(msg);
// transQueueClear(&conn->srvMsgs);
// return;
@@ -448,6 +422,8 @@ static void uvPrepareSendData(SSvrMsg* smsg, uv_buf_t* wb) {
transUnrefSrvHandle(pConn);
} else {
pHead->msgType = pMsg->msgType;
+ if (pHead->msgType == 0 && transMsgLenFromCont(pMsg->contLen) == sizeof(STransMsgHead))
+ pHead->msgType = pConn->inType + 1;
}
}
@@ -504,7 +480,7 @@ static void destroySmsg(SSvrMsg* smsg) {
transFreeMsg(smsg->msg.pCont);
taosMemoryFree(smsg);
}
-static void destroyAllConn(SWorkThrdObj* pThrd) {
+static void destroyAllConn(SWorkThrd* pThrd) {
tTrace("thread %p destroy all conn ", pThrd);
while (!QUEUE_IS_EMPTY(&pThrd->conn)) {
queue* h = QUEUE_HEAD(&pThrd->conn);
@@ -519,10 +495,10 @@ static void destroyAllConn(SWorkThrdObj* pThrd) {
}
}
void uvWorkerAsyncCb(uv_async_t* handle) {
- SAsyncItem* item = handle->data;
- SWorkThrdObj* pThrd = item->pThrd;
- SSvrConn* conn = NULL;
- queue wq;
+ SAsyncItem* item = handle->data;
+ SWorkThrd* pThrd = item->pThrd;
+ SSvrConn* conn = NULL;
+ queue wq;
// batch process to avoid to lock/unlock frequently
taosThreadMutexLock(&item->mtx);
@@ -650,7 +626,7 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) {
assert(buf->base[0] == notify[0]);
taosMemoryFree(buf->base);
- SWorkThrdObj* pThrd = q->data;
+ SWorkThrd* pThrd = q->data;
uv_pipe_t* pipe = (uv_pipe_t*)q;
if (!uv_pipe_pending_count(pipe)) {
@@ -718,10 +694,10 @@ void uvOnPipeConnectionCb(uv_connect_t* connect, int status) {
if (status != 0) {
return;
}
- SWorkThrdObj* pThrd = container_of(connect, SWorkThrdObj, connect_req);
+ SWorkThrd* pThrd = container_of(connect, SWorkThrd, connect_req);
uv_read_start((uv_stream_t*)pThrd->pipe, uvAllocConnBufferCb, uvOnConnectionCb);
}
-static bool addHandleToWorkloop(SWorkThrdObj* pThrd, char* pipeName) {
+static bool addHandleToWorkloop(SWorkThrd* pThrd, char* pipeName) {
pThrd->loop = (uv_loop_t*)taosMemoryMalloc(sizeof(uv_loop_t));
if (0 != uv_loop_init(pThrd->loop)) {
return false;
@@ -774,14 +750,14 @@ static bool addHandleToAcceptloop(void* arg) {
}
void* transWorkerThread(void* arg) {
setThreadName("trans-worker");
- SWorkThrdObj* pThrd = (SWorkThrdObj*)arg;
+ SWorkThrd* pThrd = (SWorkThrd*)arg;
uv_run(pThrd->loop, UV_RUN_DEFAULT);
return NULL;
}
static SSvrConn* createConn(void* hThrd) {
- SWorkThrdObj* pThrd = hThrd;
+ SWorkThrd* pThrd = hThrd;
SSvrConn* pConn = (SSvrConn*)taosMemoryCalloc(1, sizeof(SSvrConn));
QUEUE_INIT(&pConn->queue);
@@ -826,7 +802,7 @@ static void destroyConnRegArg(SSvrConn* conn) {
conn->regArg.init = 0;
}
}
-static int reallocConnRefHandle(SSvrConn* conn) {
+static int reallocConnRef(SSvrConn* conn) {
transReleaseExHandle(refMgt, conn->refId);
transRemoveExHandle(refMgt, conn->refId);
// avoid app continue to send msg on invalid handle
@@ -844,7 +820,7 @@ static void uvDestroyConn(uv_handle_t* handle) {
if (conn == NULL) {
return;
}
- SWorkThrdObj* thrd = conn->hostThrd;
+ SWorkThrd* thrd = conn->hostThrd;
transReleaseExHandle(refMgt, conn->refId);
transRemoveExHandle(refMgt, conn->refId);
@@ -889,7 +865,7 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads,
srv->numOfThreads = numOfThreads;
srv->workerIdx = 0;
srv->numOfWorkerReady = 0;
- srv->pThreadObj = (SWorkThrdObj**)taosMemoryCalloc(srv->numOfThreads, sizeof(SWorkThrdObj*));
+ srv->pThreadObj = (SWorkThrd**)taosMemoryCalloc(srv->numOfThreads, sizeof(SWorkThrd*));
srv->pipe = (uv_pipe_t**)taosMemoryCalloc(srv->numOfThreads, sizeof(uv_pipe_t*));
srv->ip = ip;
srv->port = port;
@@ -914,7 +890,7 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads,
assert(0 == uv_listen((uv_stream_t*)&srv->pipeListen, SOMAXCONN, uvPipeListenCb));
for (int i = 0; i < srv->numOfThreads; i++) {
- SWorkThrdObj* thrd = (SWorkThrdObj*)taosMemoryCalloc(1, sizeof(SWorkThrdObj));
+ SWorkThrd* thrd = (SWorkThrd*)taosMemoryCalloc(1, sizeof(SWorkThrd));
thrd->pTransInst = shandle;
thrd->quit = false;
srv->pThreadObj[i] = thrd;
@@ -959,7 +935,7 @@ End:
return NULL;
}
-void uvHandleQuit(SSvrMsg* msg, SWorkThrdObj* thrd) {
+void uvHandleQuit(SSvrMsg* msg, SWorkThrd* thrd) {
thrd->quit = true;
if (QUEUE_IS_EMPTY(&thrd->conn)) {
uv_walk(thrd->loop, uvWalkCb, NULL);
@@ -968,10 +944,10 @@ void uvHandleQuit(SSvrMsg* msg, SWorkThrdObj* thrd) {
}
taosMemoryFree(msg);
}
-void uvHandleRelease(SSvrMsg* msg, SWorkThrdObj* thrd) {
+void uvHandleRelease(SSvrMsg* msg, SWorkThrd* thrd) {
SSvrConn* conn = msg->pConn;
if (conn->status == ConnAcquire) {
- reallocConnRefHandle(conn);
+ reallocConnRef(conn);
if (!transQueuePush(&conn->srvMsgs, msg)) {
return;
}
@@ -982,12 +958,12 @@ void uvHandleRelease(SSvrMsg* msg, SWorkThrdObj* thrd) {
}
destroySmsg(msg);
}
-void uvHandleResp(SSvrMsg* msg, SWorkThrdObj* thrd) {
+void uvHandleResp(SSvrMsg* msg, SWorkThrd* thrd) {
// send msg to client
tDebug("%s conn %p start to send resp (2/2)", transLabel(thrd->pTransInst), msg->pConn);
uvStartSendResp(msg);
}
-void uvHandleRegister(SSvrMsg* msg, SWorkThrdObj* thrd) {
+void uvHandleRegister(SSvrMsg* msg, SWorkThrd* thrd) {
SSvrConn* conn = msg->pConn;
tDebug("%s conn %p register brokenlink callback", transLabel(thrd->pTransInst), conn);
if (conn->status == ConnAcquire) {
@@ -1008,7 +984,7 @@ void uvHandleRegister(SSvrMsg* msg, SWorkThrdObj* thrd) {
taosMemoryFree(msg);
}
}
-void destroyWorkThrd(SWorkThrdObj* pThrd) {
+void destroyWorkThrd(SWorkThrd* pThrd) {
if (pThrd == NULL) {
return;
}
@@ -1019,7 +995,7 @@ void destroyWorkThrd(SWorkThrdObj* pThrd) {
taosMemoryFree(pThrd->loop);
taosMemoryFree(pThrd);
}
-void sendQuitToWorkThrd(SWorkThrdObj* pThrd) {
+void sendQuitToWorkThrd(SWorkThrd* pThrd) {
SSvrMsg* msg = taosMemoryCalloc(1, sizeof(SSvrMsg));
msg->type = Quit;
tDebug("server send quit msg to work thread");
@@ -1055,8 +1031,6 @@ void transCloseServer(void* arg) {
int ref = atomic_sub_fetch_32(&tranSSvrInst, 1);
if (ref == 0) {
- // TdThreadOnce tmpInit = PTHREAD_ONCE_INIT;
- // memcpy(&transModuleInit, &tmpInit, sizeof(TdThreadOnce));
transCloseExHandleMgt(refMgt);
}
}
@@ -1086,7 +1060,7 @@ void transReleaseSrvHandle(void* handle) {
ASYNC_CHECK_HANDLE(exh, refId);
- SWorkThrdObj* pThrd = exh->pThrd;
+ SWorkThrd* pThrd = exh->pThrd;
ASYNC_ERR_JRET(pThrd);
STransMsg tmsg = {.code = 0, .info.handle = exh, .info.ahandle = NULL, .info.refId = refId};
@@ -1116,7 +1090,7 @@ void transSendResponse(const STransMsg* msg) {
STransMsg tmsg = *msg;
tmsg.info.refId = refId;
- SWorkThrdObj* pThrd = exh->pThrd;
+ SWorkThrd* pThrd = exh->pThrd;
ASYNC_ERR_JRET(pThrd);
SSvrMsg* m = taosMemoryCalloc(1, sizeof(SSvrMsg));
@@ -1146,7 +1120,7 @@ void transRegisterMsg(const STransMsg* msg) {
STransMsg tmsg = *msg;
tmsg.info.refId = refId;
- SWorkThrdObj* pThrd = exh->pThrd;
+ SWorkThrd* pThrd = exh->pThrd;
ASYNC_ERR_JRET(pThrd);
SSvrMsg* m = taosMemoryCalloc(1, sizeof(SSvrMsg));
diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c
index 682afbb785..2de0fea9ac 100644
--- a/source/libs/wal/src/walRead.c
+++ b/source/libs/wal/src/walRead.c
@@ -104,6 +104,8 @@ static int32_t walReadChangeFile(SWalReadHandle *pRead, int64_t fileFirstVer) {
return -1;
}
+ pRead->pReadLogTFile = pLogTFile;
+
walBuildIdxName(pRead->pWal, fileFirstVer, fnameStr);
TdFilePtr pIdxTFile = taosOpenFile(fnameStr, TD_FILE_READ);
if (pIdxTFile == NULL) {
@@ -112,7 +114,6 @@ static int32_t walReadChangeFile(SWalReadHandle *pRead, int64_t fileFirstVer) {
return -1;
}
- pRead->pReadLogTFile = pLogTFile;
pRead->pReadIdxTFile = pIdxTFile;
return 0;
}
diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c
index 1d169a0891..d1295667af 100644
--- a/source/libs/wal/src/walWrite.c
+++ b/source/libs/wal/src/walWrite.c
@@ -42,6 +42,9 @@ int32_t walRestoreFromSnapshot(SWal *pWal, int64_t ver) {
char fnameStr[WAL_FILE_LEN];
walBuildLogName(pWal, pFileInfo->firstVer, fnameStr);
taosRemoveFile(fnameStr);
+
+ walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr);
+ taosRemoveFile(fnameStr);
}
}
walRemoveMeta(pWal);
@@ -105,7 +108,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
}
walBuildIdxName(pWal, walGetCurFileFirstVer(pWal), fnameStr);
- TdFilePtr pIdxTFile = taosOpenFile(fnameStr, TD_FILE_WRITE | TD_FILE_READ);
+ TdFilePtr pIdxTFile = taosOpenFile(fnameStr, TD_FILE_WRITE | TD_FILE_READ | TD_FILE_APPEND);
if (pIdxTFile == NULL) {
taosThreadMutexUnlock(&pWal->mutex);
@@ -126,7 +129,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
ASSERT(entry.ver == ver);
walBuildLogName(pWal, walGetCurFileFirstVer(pWal), fnameStr);
- TdFilePtr pLogTFile = taosOpenFile(fnameStr, TD_FILE_WRITE | TD_FILE_READ);
+ TdFilePtr pLogTFile = taosOpenFile(fnameStr, TD_FILE_WRITE | TD_FILE_READ | TD_FILE_APPEND);
if (pLogTFile == NULL) {
// TODO
taosThreadMutexUnlock(&pWal->mutex);
@@ -307,8 +310,8 @@ int walRoll(SWal *pWal) {
static int walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) {
SWalIdxEntry entry = {.ver = ver, .offset = offset};
- /*int64_t idxOffset = taosLSeekFile(pWal->pWriteIdxTFile, 0, SEEK_CUR);*/
- /*wDebug("write index: ver: %ld, offset: %ld, at %ld", ver, offset, idxOffset);*/
+ int64_t idxOffset = taosLSeekFile(pWal->pWriteIdxTFile, 0, SEEK_END);
+ wDebug("write index: ver: %ld, offset: %ld, at %ld", ver, offset, idxOffset);
int64_t size = taosWriteFile(pWal->pWriteIdxTFile, &entry, sizeof(SWalIdxEntry));
if (size != sizeof(SWalIdxEntry)) {
terrno = TAOS_SYSTEM_ERROR(errno);
diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c
index 05b7498cc0..0c6cd80f44 100644
--- a/source/os/src/osFile.c
+++ b/source/os/src/osFile.c
@@ -671,7 +671,7 @@ void taosFprintfFile(TdFilePtr pFile, const char *format, ...) {
fflush(pFile->fp);
}
-bool taosValidFile(TdFilePtr pFile) { return pFile != NULL; }
+bool taosValidFile(TdFilePtr pFile) { return pFile != NULL && pFile->fd > 0; }
int32_t taosUmaskFile(int32_t maskVal) {
#ifdef WINDOWS
diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c
index 4d61e7036d..b0e07ff010 100644
--- a/source/os/src/osSocket.c
+++ b/source/os/src/osSocket.c
@@ -946,7 +946,7 @@ int32_t taosGetFqdn(char *fqdn) {
#endif // __APPLE__
int32_t ret = getaddrinfo(hostname, NULL, &hints, &result);
if (!result) {
- fprintf(stderr,"failed to get fqdn, code:%d, reason:%s", ret, gai_strerror(ret));
+ fprintf(stderr, "failed to get fqdn, code:%d, reason:%s", ret, gai_strerror(ret));
return -1;
}
@@ -1073,7 +1073,7 @@ int32_t taosCloseEpoll(TdEpollPtr *ppEpoll) {
* Set TCP connection timeout per-socket level.
* ref [https://github.com/libuv/help/issues/54]
*/
-int taosCreateSocketWithTimeOutOpt(uint32_t conn_timeout_sec) {
+int32_t taosCreateSocketWithTimeout(uint32_t timeout) {
#if defined(WINDOWS)
SOCKET fd;
#else
@@ -1083,11 +1083,11 @@ int taosCreateSocketWithTimeOutOpt(uint32_t conn_timeout_sec) {
return -1;
}
#if defined(WINDOWS)
- if (0 != setsockopt(fd, IPPROTO_TCP, TCP_MAXRT, (char *)&conn_timeout_sec, sizeof(conn_timeout_sec))) {
+ if (0 != setsockopt(fd, IPPROTO_TCP, TCP_MAXRT, (char *)&timeout, sizeof(timeout))) {
return -1;
}
#else // Linux like systems
- uint32_t conn_timeout_ms = conn_timeout_sec * 1000;
+ uint32_t conn_timeout_ms = timeout * 1000;
if (0 != setsockopt(fd, IPPROTO_TCP, TCP_USER_TIMEOUT, (char *)&conn_timeout_ms, sizeof(conn_timeout_ms))) {
return -1;
}
diff --git a/tests/pytest/util/cluster.py b/tests/pytest/util/cluster.py
index 6eb78f0771..21cad4c5bb 100644
--- a/tests/pytest/util/cluster.py
+++ b/tests/pytest/util/cluster.py
@@ -24,7 +24,7 @@ class ClusterDnodes(TDDnodes):
class ConfigureyCluster:
- """This will create defined number of dnodes and create a cluset.
+ """This will create defined number of dnodes and create a cluster.
at the same time, it will return TDDnodes list: dnodes, """
hostname= socket.gethostname()
@@ -85,8 +85,8 @@ class ConfigureyCluster:
count+=1
time.sleep(1)
else:
- tdLog.debug("create cluster with %d dnode but check dnode not ready within 5s ! "%self.dnodeNums)
- return -1
+ tdLog.exit("create cluster with %d dnode but check dnode not ready within 5s ! "%self.dnodeNums)
+
cluster = ConfigureyCluster()
\ No newline at end of file
diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py
index be3454f78f..a38b14a52d 100644
--- a/tests/pytest/util/dnodes.py
+++ b/tests/pytest/util/dnodes.py
@@ -508,7 +508,6 @@ class TDDnode:
def stoptaosd(self):
if (not self.remoteIP == ""):
- print("123")
self.remoteExec(self.cfgDict, "tdDnodes.dnodes[%d].running=1\ntdDnodes.dnodes[%d].stop()"%(self.index-1,self.index-1))
tdLog.info("stop dnode%d"%self.index)
return
@@ -518,18 +517,21 @@ class TDDnode:
toBeKilled = "valgrind.bin"
if self.running != 0:
- psCmd = "ps -ef|grep -w %s| grep dnode%d|grep -v grep | awk '{print $2}'" % (toBeKilled,self.index)
- processID = subprocess.check_output(
- psCmd, shell=True).decode("utf-8")
-
- while(processID):
- killCmd = "kill -INT %s > /dev/null 2>&1" % processID
- os.system(killCmd)
- time.sleep(1)
+ if platform.system().lower() == 'windows':
+ os.system("wmic process where \"name='taosd.exe' and CommandLine like '%%dnode%d%%'\" get processId | xargs echo | awk '{print $2}' | xargs taskkill -f -pid"%self.index)
+ else:
+ psCmd = "ps -ef|grep -w %s| grep dnode%d|grep -v grep | awk '{print $2}'" % (toBeKilled,self.index)
processID = subprocess.check_output(
psCmd, shell=True).decode("utf-8")
- if self.valgrind:
- time.sleep(2)
+
+ while(processID):
+ killCmd = "kill -INT %s > /dev/null 2>&1" % processID
+ os.system(killCmd)
+ time.sleep(1)
+ processID = subprocess.check_output(
+ psCmd, shell=True).decode("utf-8")
+ if self.valgrind:
+ time.sleep(2)
self.running = 0
tdLog.debug("dnode:%d is stopped by kill -INT" % (self.index))
diff --git a/tests/script/test-all.bat b/tests/script/test-all.bat
new file mode 100644
index 0000000000..7a1a4bc7fa
--- /dev/null
+++ b/tests/script/test-all.bat
@@ -0,0 +1,58 @@
+@echo off
+SETLOCAL EnableDelayedExpansion
+for /F "tokens=1,2 delims=#" %%a in ('"prompt #$H#$E# & echo on & for %%b in (1) do rem"') do ( set "DEL=%%a")
+set /a a=0
+echo Windows Taosd Full Test
+set /a exitNum=0
+rm -rf failed.txt
+set caseFile="jenkins\\basic.txt"
+if not "%2" == "" (
+ set caseFile="%2"
+)
+for /F "usebackq tokens=*" %%i in (!caseFile!) do (
+ set line=%%i
+ if "!line:~,9!" == "./test.sh" (
+ set /a a+=1
+ echo !a! Processing %%i
+ call :GetTimeSeconds !time!
+ set time1=!_timeTemp!
+ echo Start at !time!
+ call !line:./test.sh=wtest.bat! > result_!a!.txt 2>error_!a!.txt
+ if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && set /a exitNum=8 && echo %%i >>failed.txt ) else ( call :colorEcho 0a "Success" &echo. )
+ )
+)
+exit !exitNum!
+
+:colorEcho
+set timeNow=%time%
+call :GetTimeSeconds %timeNow%
+set time2=%_timeTemp%
+set /a interTime=%time2% - %time1%
+echo End at %timeNow% , cast %interTime%s
+echo off
+ "%~2"
+findstr /v /a:%1 /R "^$" "%~2" nul
+del "%~2" > nul 2>&1i
+goto :eof
+
+:GetTimeSeconds
+set tt=%1
+set tt=%tt:.= %
+set tt=%tt::= %
+set tt=%tt: 0= %
+set /a index=1
+for %%a in (%tt%) do (
+ if !index! EQU 1 (
+ set /a hh=%%a
+ )^
+ else if !index! EQU 2 (
+ set /a mm=%%a
+
+ )^
+ else if !index! EQU 3 (
+ set /a ss=%%a
+ )
+ set /a index=index+1
+)
+set /a _timeTemp=(%hh%*60+%mm%)*60+%ss%
+goto :eof
diff --git a/tests/script/tsim/stream/distributeInterval0.sim b/tests/script/tsim/stream/distributeInterval0.sim
index 91ce49bc8c..8936655974 100644
--- a/tests/script/tsim/stream/distributeInterval0.sim
+++ b/tests/script/tsim/stream/distributeInterval0.sim
@@ -10,6 +10,30 @@ sql create dnode $hostname2 port 7200
system sh/exec.sh -n dnode2 -s start
+print ===== step1
+$x = 0
+step1:
+ $x = $x + 1
+ sleep 1000
+ if $x == 10 then
+ print ====> dnode not ready!
+ return -1
+ endi
+sql show dnodes
+print ===> $data00 $data01 $data02 $data03 $data04 $data05
+print ===> $data10 $data11 $data12 $data13 $data14 $data15
+if $rows != 2 then
+ return -1
+endi
+if $data(1)[4] != ready then
+ goto step1
+endi
+if $data(2)[4] != ready then
+ goto step1
+endi
+
+print ===== step2
+
sql create database test vgroups 4;
sql use test;
sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int);
@@ -80,17 +104,17 @@ endi
if $data03 != 4 then
print ======$data03
- return -1
+ goto loop1
endi
if $data04 != 52 then
print ======$data04
- return -1
+ goto loop1
endi
if $data05 != 13 then
print ======$data05
- return -1
+ goto loop1
endi
# row 1
@@ -179,7 +203,7 @@ sql use test1;
sql create stable st(ts timestamp, a int, b int , c int) tags(ta int,tb int,tc int);
sql create table ts1 using st tags(1,1,1);
sql create table ts2 using st tags(2,2,2);
-sql create stream stream_t2 trigger at_once into streamtST1 as select _wstartts, count(*) c1, count(a) c2 , sum(a) c3 , max(b) c5, min(c) c6 from st interval(10s) ;
+sql create stream stream_t2 trigger at_once watermark 20s into streamtST1 as select _wstartts, count(*) c1, count(a) c2 , sum(a) c3 , max(b) c5, min(c) c6 from st interval(10s) ;
sql insert into ts1 values(1648791211000,1,2,3);
sql insert into ts1 values(1648791222001,2,2,3);
diff --git a/tests/script/tsim/stream/partitionby.sim b/tests/script/tsim/stream/partitionby.sim
index b84a01eb4a..c634ad85ee 100644
--- a/tests/script/tsim/stream/partitionby.sim
+++ b/tests/script/tsim/stream/partitionby.sim
@@ -74,7 +74,7 @@ sql create stable st(ts timestamp,a int,b int,c int,id int) tags(ta int,tb int,t
sql create table ts1 using st tags(1,1,1);
sql create table ts2 using st tags(2,2,2);
-sql create stream stream_t2 trigger at_once into streamtST as select _wstartts, count(*) c1, count(a) c2 , sum(a) c3 , max(b) c5, min(c) c6, max(id) c7 from st partition by ta interval(10s) ;
+sql create stream stream_t2 trigger at_once watermark 20s into streamtST as select _wstartts, count(*) c1, count(a) c2 , sum(a) c3 , max(b) c5, min(c) c6, max(id) c7 from st partition by ta interval(10s) ;
sql insert into ts1 values(1648791211000,1,2,3,1);
sql insert into ts1 values(1648791222001,2,2,3,2);
sql insert into ts2 values(1648791211000,1,2,3,3);
@@ -83,16 +83,16 @@ sql insert into ts2 values(1648791222001,2,2,3,4);
sql insert into ts2 values(1648791222002,2,2,3,5);
sql insert into ts2 values(1648791222002,2,2,3,6);
-sql insert into ts1 values(1648791211000,1,2,3,1);
-sql insert into ts1 values(1648791222001,2,2,3,2);
-sql insert into ts2 values(1648791211000,1,2,3,3);
-sql insert into ts2 values(1648791222001,2,2,3,4);
+sql insert into ts1 values(1648791211000,1,2,3,7);
+sql insert into ts1 values(1648791222001,2,2,3,8);
+sql insert into ts2 values(1648791211000,1,2,3,9);
+sql insert into ts2 values(1648791222001,2,2,3,10);
$loop_count = 0
loop2:
sleep 300
-sql select * from streamtST;
+sql select * from streamtST order by c7 asc;
$loop_count = $loop_count + 1
if $loop_count == 10 then
@@ -104,8 +104,18 @@ print =====data01=$data01
goto loop2
endi
-if $data02 != 1 then
-print =====data02=$data02
+if $data11 != 1 then
+print =====data11=$data11
+goto loop2
+endi
+
+if $data21 != 1 then
+print =====data21=$data21
+goto loop2
+endi
+
+if $data31 != 2 then
+print =====data31=$data31
goto loop2
endi
@@ -114,8 +124,18 @@ print =====data03=$data03
goto loop2
endi
-if $data04 != 2 then
-print =====data04=$data04
+if $data13 != 2 then
+print =====data13=$data13
+goto loop2
+endi
+
+if $data23 != 1 then
+print =====data23=$data23
+goto loop2
+endi
+
+if $data33 != 4 then
+print =====data33=$data33
goto loop2
endi
diff --git a/tests/script/tsim/stream/schedSnode.sim b/tests/script/tsim/stream/schedSnode.sim
index dbf714a96f..dbdaaf65d0 100644
--- a/tests/script/tsim/stream/schedSnode.sim
+++ b/tests/script/tsim/stream/schedSnode.sim
@@ -79,17 +79,17 @@ endi
if $data03 != 4 then
print ======$data03
- return -1
+ goto loop1
endi
if $data04 != 52 then
print ======$data04
- return -1
+ goto loop1
endi
if $data05 != 13 then
print ======$data05
- return -1
+ goto loop1
endi
# row 1
diff --git a/tests/script/tsim/user/privilege_sysinfo.sim b/tests/script/tsim/user/privilege_sysinfo.sim
index ea3294765c..10c31dc288 100644
--- a/tests/script/tsim/user/privilege_sysinfo.sim
+++ b/tests/script/tsim/user/privilege_sysinfo.sim
@@ -8,14 +8,12 @@ sql create user sysinfo0 pass 'taosdata'
sql create user sysinfo1 pass 'taosdata'
sql alter user sysinfo0 sysinfo 0
sql alter user sysinfo1 sysinfo 1
+sql create database db
print user sysinfo0 login
sql close
sql connect sysinfo0
-system sh/exec.sh -n dnode1 -s stop
-return
-
print =============== check oper
sql_error create user u1 pass 'u1'
sql_error drop user sysinfo1
@@ -42,6 +40,24 @@ sql_error kill connection 1
sql_error kill query 1
print =============== check db
-sql_error create database db
+sql_error create database d1
+sql_error drop database db
+sql_error use db
+sql_error alter database db replica 1;
+sql_error show db.vgroups
+sql_error show db.stables
+sql_error show db.tables
+
+print =============== check show
+sql_error show users
+sql_error show cluster
+sql_error show dnodes
+sql_error show mnodes
+sql_error show snodes
+sql_error show qnodes
+sql_error show bnodes
+sql_error show grants
+sql_error show dnode 1 variables;
+sql show variables;
system sh/exec.sh -n dnode1 -s stop -x SIGINT
\ No newline at end of file
diff --git a/tests/system-test/1-insert/table_param_ttl.py b/tests/system-test/1-insert/table_param_ttl.py
new file mode 100644
index 0000000000..49d6476d9c
--- /dev/null
+++ b/tests/system-test/1-insert/table_param_ttl.py
@@ -0,0 +1,79 @@
+###################################################################
+# Copyright (c) 2016 by TAOS Technologies, Inc.
+# All rights reserved.
+#
+# This file is proprietary and confidential to TAOS Technologies.
+# No part of this file may be reproduced, stored, transmitted,
+# disclosed or used in any form or by any means other than as
+# expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.common import *
+
+class TDTestCase:
+ updatecfgDict = {'ttlUnit':5,'ttlPushInterval':3}
+ def init(self, conn, logSql):
+ tdLog.debug("start to execute %s" % __file__)
+ tdSql.init(conn.cursor())
+ self.ntbname = 'ntb'
+ self.stbname = 'stb'
+ self.tbnum = 10
+ self.ttl_param = 1
+ self.default_ttl = 100
+ self.modify_ttl = 1
+ def ttl_check_ntb(self):
+ tdSql.prepare()
+
+ for i in range(self.tbnum):
+ tdSql.execute(f'create table {self.ntbname}_{i} (ts timestamp,c0 int) ttl {self.ttl_param}')
+ tdSql.query(f'show tables')
+ tdSql.checkRows(self.tbnum)
+ sleep(self.updatecfgDict['ttlUnit']*self.ttl_param+self.updatecfgDict['ttlPushInterval'])
+ tdSql.query(f'show tables')
+ tdSql.checkRows(0)
+ for i in range(self.tbnum):
+ tdSql.execute(f'create table {self.ntbname}_{i} (ts timestamp,c0 int) ttl {self.default_ttl}')
+ for i in range(int(self.tbnum/2)):
+ tdSql.execute(f'alter table {self.ntbname}_{i} ttl {self.modify_ttl}')
+ sleep(self.updatecfgDict['ttlUnit']*self.modify_ttl+self.updatecfgDict['ttlPushInterval'])
+ tdSql.query(f'show tables')
+ tdSql.checkRows(self.tbnum - int(self.tbnum/2))
+ tdSql.execute('drop database db')
+ def ttl_check_ctb(self):
+ tdSql.prepare()
+ tdSql.execute(f'create table {self.stbname} (ts timestamp,c0 int) tags(t0 int)')
+
+ for i in range(self.tbnum):
+ tdSql.execute(f'create table {self.stbname}_{i} using {self.stbname} tags({i}) ttl {self.ttl_param}')
+ tdSql.query(f'show tables')
+ tdSql.checkRows(self.tbnum)
+ sleep(self.updatecfgDict['ttlUnit']*self.ttl_param+self.updatecfgDict['ttlPushInterval'])
+ tdSql.query(f'show tables')
+ tdSql.checkRows(0)
+ for i in range(self.tbnum):
+ tdSql.execute(f'create table {self.stbname}_{i} using {self.stbname} tags({i}) ttl {self.default_ttl}')
+ tdSql.query(f'show tables')
+ tdSql.checkRows(self.tbnum)
+ for i in range(int(self.tbnum/2)):
+ tdSql.execute(f'alter table {self.stbname}_{i} ttl {self.modify_ttl}')
+ sleep(self.updatecfgDict['ttlUnit']*self.modify_ttl+self.updatecfgDict['ttlPushInterval'])
+ tdSql.query(f'show tables')
+ tdSql.checkRows(self.tbnum - int(self.tbnum/2))
+ tdSql.execute('drop database db')
+
+ def run(self):
+ self.ttl_check_ntb()
+ self.ttl_check_ctb()
+
+ def stop(self):
+ tdSql.close()
+ tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
diff --git a/tests/system-test/2-query/bottom.py b/tests/system-test/2-query/bottom.py
index 1037b0a8f3..a95daf22f4 100644
--- a/tests/system-test/2-query/bottom.py
+++ b/tests/system-test/2-query/bottom.py
@@ -17,100 +17,140 @@ from util.log import *
from util.cases import *
from util.sql import *
from util.common import *
-
+from util.sqlset import *
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor())
-
+ self.dbname = 'db_test'
+ self.setsql = TDSetSql()
+ self.ntbname = 'ntb'
self.rowNum = 10
self.tbnum = 20
self.ts = 1537146000000
self.binary_str = 'taosdata'
self.nchar_str = '涛思数据'
- def bottom_check_base(self):
- tdSql.prepare()
- tdSql.execute('''create table stb(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 tinyint unsigned, col6 smallint unsigned,
- col7 int unsigned, col8 bigint unsigned, col9 float, col10 double, col11 bool, col12 binary(20), col13 nchar(20)) tags(loc nchar(20))''')
- tdSql.execute("create table stb_1 using stb tags('beijing')")
- column_list = ['col1','col2','col3','col4','col5','col6','col7','col8']
- error_column_list = ['col11','col12','col13']
- error_param_list = [0,101]
- for i in range(self.rowNum):
- tdSql.execute(f"insert into stb_1 values(%d, %d, %d, %d, %d, %d, %d, %d, %d, %f, %f, %d, '{self.binary_str}%d', '{self.nchar_str}%d')"
- % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1))
+ self.column_dict = {
+ 'ts' : 'timestamp',
+ 'col1': 'tinyint',
+ 'col2': 'smallint',
+ 'col3': 'int',
+ 'col4': 'bigint',
+ 'col5': 'tinyint unsigned',
+ 'col6': 'smallint unsigned',
+ 'col7': 'int unsigned',
+ 'col8': 'bigint unsigned',
+ 'col9': 'float',
+ 'col10': 'double',
+ 'col11': 'bool',
+ 'col12': 'binary(20)',
+ 'col13': 'nchar(20)'
+ }
- for i in column_list:
- tdSql.query(f'select bottom({i},2) from stb_1')
- tdSql.checkRows(2)
- tdSql.checkEqual(tdSql.queryResult,[(2,),(1,)])
- for j in error_param_list:
- tdSql.error(f'select bottom({i},{j}) from stb_1')
- for i in error_column_list:
- tdSql.error(f'select bottom({i},10) from stb_1')
- tdSql.query("select ts,bottom(col1, 2),ts from stb_1 group by tbname")
- tdSql.checkRows(2)
- tdSql.query('select bottom(col2,1) from stb_1 interval(1y) order by col2')
- tdSql.checkData(0,0,1)
-
- tdSql.error('select * from stb_1 where bottom(col2,1)=1')
- tdSql.execute('drop database db')
- def bottom_check_distribute(self):
- # prepare data for vgroup 4
- dbname = tdCom.getLongName(5, "letters")
+ self.param_list = [1,100]
+ def insert_data(self,column_dict,tbname,row_num):
+ sql = ''
+ for k, v in column_dict.items():
+ if v.lower() == 'timestamp' or v.lower() == 'tinyint' or v.lower() == 'smallint' or v.lower() == 'int' or v.lower() == 'bigint' or \
+ v.lower() == 'tinyint unsigned' or v.lower() == 'smallint unsigned' or v.lower() == 'int unsigned' or v.lower() == 'bigint unsigned' or v.lower() == 'bool':
+ sql += '%d,'
+ elif v.lower() == 'float' or v.lower() == 'double':
+ sql += '%f,'
+ elif 'binary' in v.lower():
+ sql += f'"{self.binary_str}%d",'
+ elif 'nchar' in v.lower():
+ sql += f'"{self.nchar_str}%d",'
+ insert_sql = f'insert into {tbname} values({sql[:-1]})'
+ for i in range(row_num):
+ insert_list = []
+ for k, v in column_dict.items():
+ if v.lower() in[ 'tinyint' , 'smallint' , 'int', 'bigint' , 'tinyint unsigned' , 'smallint unsigned' , 'int unsigned' , 'bigint unsigned'] or\
+ 'binary' in v.lower() or 'nchar' in v.lower():
+ insert_list.append(0 + i)
+ elif v.lower() == 'float' or v.lower() == 'double':
+ insert_list.append(0.1 + i)
+ elif v.lower() == 'bool':
+ insert_list.append(i % 2)
+ elif v.lower() == 'timestamp':
+ insert_list.append(self.ts + i)
+ tdSql.execute(insert_sql%(tuple(insert_list)))
+ def bottom_check_data(self,tbname,tb_type):
+ new_column_dict = {}
+ for param in self.param_list:
+ for k,v in self.column_dict.items():
+ if v.lower() in ['tinyint','smallint','int','bigint','tinyint unsigned','smallint unsigned','int unsigned','bigint unsigned']:
+ tdSql.query(f'select bottom({k},{param}) from {tbname} order by {k}')
+ if param >= self.rowNum:
+ if tb_type in ['normal_table','child_table']:
+ tdSql.checkRows(self.rowNum)
+ values_list = []
+ for i in range(self.rowNum):
+ tp = (i,)
+ values_list.append(tp)
+ tdSql.checkEqual(tdSql.queryResult,values_list)
+ elif tb_type == 'stable':
+ tdSql.checkRows(param)
+ elif param < self.rowNum:
+ if tb_type in ['normal_table','child_table']:
+ tdSql.checkRows(param)
+ values_list = []
+ for i in range(param):
+ tp = (i,)
+ values_list.append(tp)
+ tdSql.checkEqual(tdSql.queryResult,values_list)
+ elif tb_type == 'stable':
+ tdSql.checkRows(param)
+ for i in [self.param_list[0]-1,self.param_list[-1]+1]:
+ tdSql.error(f'select top({k},{i}) from {tbname}')
+ new_column_dict.update({k:v})
+ elif v.lower() == 'bool' or 'binary' in v.lower() or 'nchar' in v.lower():
+ tdSql.error(f'select top({k},{param}) from {tbname}')
+ tdSql.error(f'select * from {tbname} where top({k},{param})=1')
+ pass
+ def bottom_check_ntb(self):
+ tdSql.execute(f'create database if not exists {self.dbname} vgroups 1')
+ tdSql.execute(f'use {self.dbname}')
+ tdSql.execute(self.setsql.set_create_normaltable_sql(self.ntbname,self.column_dict))
+ self.insert_data(self.column_dict,self.ntbname,self.rowNum)
+ self.bottom_check_data(self.ntbname,'normal_table')
+ tdSql.execute(f'drop database {self.dbname}')
+ def bottom_check_stb(self):
stbname = tdCom.getLongName(5, "letters")
- vgroup_num = 2
- child_table_num = 20
- tdSql.execute(f"create database if not exists {dbname} vgroups {vgroup_num}")
- tdSql.execute(f'use {dbname}')
- # build 20 child tables,every table insert 10 rows
- tdSql.execute(f'''create table {stbname}(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 tinyint unsigned, col6 smallint unsigned,
- col7 int unsigned, col8 bigint unsigned, col9 float, col10 double, col11 bool, col12 binary(20), col13 nchar(20)) tags(loc nchar(20))''')
- for i in range(child_table_num):
- tdSql.execute(f"create table {stbname}_{i} using {stbname} tags('beijing')")
- tdSql.execute(f"insert into {stbname}_{i}(ts) values(%d)" % (self.ts - 1-i))
- column_list = ['col1','col2','col3','col4','col5','col6','col7','col8']
- error_column_list = ['col11','col12','col13']
- error_param_list = [0,101]
- for i in [f'{stbname}', f'{dbname}.{stbname}']:
- for j in column_list:
- tdSql.query(f"select bottom({j},1) from {i}")
- tdSql.checkRows(0)
+ tag_dict = {
+ 't0':'int'
+ }
+ tag_values = [
+ f'1'
+ ]
+ tdSql.execute(f"create database if not exists {self.dbname} vgroups 2")
+ tdSql.execute(f'use {self.dbname}')
+ tdSql.execute(self.setsql.set_create_stable_sql(stbname,self.column_dict,tag_dict))
+ for i in range(self.tbnum):
+ tdSql.execute(f"create table {stbname}_{i} using {stbname} tags({tag_values[0]})")
+ tdSql.execute(self.insert_data(self.column_dict,f'{stbname}_{i}',self.rowNum))
tdSql.query('show tables')
vgroup_list = []
for i in range(len(tdSql.queryResult)):
vgroup_list.append(tdSql.queryResult[i][6])
vgroup_list_set = set(vgroup_list)
-
for i in vgroup_list_set:
vgroups_num = vgroup_list.count(i)
- if vgroups_num >=2:
+ if vgroups_num >= 2:
tdLog.info(f'This scene with {vgroups_num} vgroups is ok!')
- continue
else:
- tdLog.exit(f'This scene does not meet the requirements with {vgroups_num} vgroup!\n')
- for i in range(self.rowNum):
- for j in range(child_table_num):
- tdSql.execute(f"insert into {stbname}_{j} values(%d, %d, %d, %d, %d, %d, %d, %d, %d, %f, %f, %d, '{self.binary_str}%d', '{self.nchar_str}%d')"
- % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1))
- for i in column_list:
- tdSql.query(f'select bottom({i},2) from {stbname}')
- tdSql.checkRows(2)
- tdSql.checkEqual(tdSql.queryResult,[(1,),(1,)])
- for j in error_param_list:
- tdSql.error(f'select bottom({i},{j}) from {stbname}')
- for i in error_column_list:
- tdSql.error(f'select bottom({i},10) from {stbname}')
-
- tdSql.execute(f'drop database {dbname}')
+ tdLog.exit(
+ 'This scene does not meet the requirements with {vgroups_num} vgroup!\n')
+ for i in range(self.tbnum):
+ self.bottom_check_data(f'{stbname}_{i}','child_table')
+ self.bottom_check_data(f'{stbname}','stable')
+ tdSql.execute(f'drop database {self.dbname}')
+
def run(self):
-
- self.bottom_check_base()
- self.bottom_check_distribute()
+ self.bottom_check_ntb()
+ self.bottom_check_stb()
-
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
diff --git a/tests/system-test/2-query/cast.py b/tests/system-test/2-query/cast.py
index 387170991f..934bbbd7b4 100644
--- a/tests/system-test/2-query/cast.py
+++ b/tests/system-test/2-query/cast.py
@@ -630,27 +630,27 @@ class TDTestCase:
( tdSql.checkData(i, 0, '12') for i in range(tdSql.queryRows) )
tdLog.printNoPrefix("==========step40: error cast condition, should return error ")
- tdSql.error("select cast(c1 as int) as b from ct4")
- tdSql.error("select cast(c1 as bool) as b from ct4")
- tdSql.error("select cast(c1 as tinyint) as b from ct4")
- tdSql.error("select cast(c1 as smallint) as b from ct4")
- tdSql.error("select cast(c1 as float) as b from ct4")
- tdSql.error("select cast(c1 as double) as b from ct4")
- tdSql.error("select cast(c1 as tinyint unsigned) as b from ct4")
- tdSql.error("select cast(c1 as smallint unsigned) as b from ct4")
- tdSql.error("select cast(c1 as int unsigned) as b from ct4")
+ #tdSql.error("select cast(c1 as int) as b from ct4")
+ #tdSql.error("select cast(c1 as bool) as b from ct4")
+ #tdSql.error("select cast(c1 as tinyint) as b from ct4")
+ #tdSql.error("select cast(c1 as smallint) as b from ct4")
+ #tdSql.error("select cast(c1 as float) as b from ct4")
+ #tdSql.error("select cast(c1 as double) as b from ct4")
+ #tdSql.error("select cast(c1 as tinyint unsigned) as b from ct4")
+ #tdSql.error("select cast(c1 as smallint unsigned) as b from ct4")
+ #tdSql.error("select cast(c1 as int unsigned) as b from ct4")
- tdSql.error("select cast(c2 as int) as b from ct4")
- tdSql.error("select cast(c3 as bool) as b from ct4")
- tdSql.error("select cast(c4 as tinyint) as b from ct4")
- tdSql.error("select cast(c5 as smallint) as b from ct4")
- tdSql.error("select cast(c6 as float) as b from ct4")
- tdSql.error("select cast(c7 as double) as b from ct4")
- tdSql.error("select cast(c8 as tinyint unsigned) as b from ct4")
+ #tdSql.error("select cast(c2 as int) as b from ct4")
+ #tdSql.error("select cast(c3 as bool) as b from ct4")
+ #tdSql.error("select cast(c4 as tinyint) as b from ct4")
+ #tdSql.error("select cast(c5 as smallint) as b from ct4")
+ #tdSql.error("select cast(c6 as float) as b from ct4")
+ #tdSql.error("select cast(c7 as double) as b from ct4")
+ #tdSql.error("select cast(c8 as tinyint unsigned) as b from ct4")
- tdSql.error("select cast(c8 as timestamp ) as b from ct4")
- tdSql.error("select cast(c9 as timestamp ) as b from ct4")
- tdSql.error("select cast(c9 as binary(64) ) as b from ct4")
+ #tdSql.error("select cast(c8 as timestamp ) as b from ct4")
+ #tdSql.error("select cast(c9 as timestamp ) as b from ct4")
+ #tdSql.error("select cast(c9 as binary(64) ) as b from ct4")
pass
def run(self):
diff --git a/tests/system-test/2-query/json_tag.py b/tests/system-test/2-query/json_tag.py
index 2ef1b8dad2..9b96b6ebd0 100644
--- a/tests/system-test/2-query/json_tag.py
+++ b/tests/system-test/2-query/json_tag.py
@@ -11,12 +11,14 @@
# -*- coding: utf-8 -*-
+import imp
import sys
import taos
from util.log import tdLog
from util.cases import tdCases
from util.sql import tdSql
import json
+import os
class TDTestCase:
@@ -29,6 +31,9 @@ class TDTestCase:
return
def init(self, conn, logSql):
+ self.testcasePath = os.path.split(__file__)[0]
+ self.testcaseFilename = os.path.split(__file__)[-1]
+ os.system("rm -rf %s/%s.sql" % (self.testcasePath,self.testcaseFilename))
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
@@ -557,6 +562,103 @@ class TDTestCase:
tdSql.checkRows(3)
tdSql.query("select round(dataint) from jsons1 where jtag->'tag1'>1")
tdSql.checkRows(3)
+
+ #math function
+ tdSql.query("select sin(dataint) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select cos(dataint) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select tan(dataint) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select asin(dataint) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select acos(dataint) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select atan(dataint) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select ceil(dataint) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select floor(dataint) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select round(dataint) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select abs(dataint) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select pow(dataint,5) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select log(dataint,10) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select sqrt(dataint) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select HISTOGRAM(dataint,'user_input','[1, 33, 555, 7777]',1) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select csum(dataint) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select mavg(dataint,1) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select statecount(dataint,'GE',10) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select stateduration(dataint,'GE',0) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select sample(dataint,3) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select HYPERLOGLOG(dataint) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(1)
+ tdSql.query("select twa(dataint) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(1)
+
+ # function not ready
+ # tdSql.query("select tail(dataint,1) from jsons1 where jtag->'tag1'>1;")
+ # tdSql.checkRows(3)
+ # tdSql.query("select unique(dataint) from jsons1 where jtag->'tag1'>1;")
+ # tdSql.checkRows(3)
+ # tdSql.query("select mode(dataint) from jsons1 where jtag->'tag1'>1;")
+ # tdSql.checkRows(3)
+ # tdSql.query("select irate(dataint) from jsons1 where jtag->'tag1'>1;")
+ # tdSql.checkRows(1)
+
+ #str function
+ tdSql.query("select upper(dataStr) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select ltrim(dataStr) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select lower(dataStr) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select rtrim(dataStr) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select LENGTH(dataStr) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select CHAR_LENGTH(dataStr) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select SUBSTR(dataStr,5) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select CONCAT(dataStr,dataStrBin) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select CONCAT_ws('adad!@!@%$^$%$^$%^a',dataStr,dataStrBin) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select CAST(dataStr as bigint) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+
+ #time function
+ tdSql.query("select now() from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select today() from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select TIMEZONE() from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select TO_ISO8601(ts) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select TO_UNIXTIMESTAMP(datastr) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select TIMETRUNCATE(ts,1u) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select TIMEDIFF(ts,_c0) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select TIMEDIFF(ts,1u) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select ELAPSED(ts,1h) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(1)
+
#
# #test TD-12077
tdSql.execute("insert into jsons1_16 using jsons1 tags('{\"tag1\":\"收到货\",\"tag2\":\"\",\"tag3\":-2.111}') values(1591062628000, 2, NULL, '你就会', 'dws')")
diff --git a/tests/system-test/6-cluster/5dnode1mnode.py b/tests/system-test/6-cluster/5dnode1mnode.py
index 75134224db..5f4ab7357b 100644
--- a/tests/system-test/6-cluster/5dnode1mnode.py
+++ b/tests/system-test/6-cluster/5dnode1mnode.py
@@ -20,7 +20,7 @@ class MyDnodes(TDDnodes):
self.simDeployed = False
class TDTestCase:
-
+ noConn = True
def init(self,conn ,logSql):
tdLog.debug(f"start to excute {__file__}")
self.TDDnodes = None
@@ -40,7 +40,7 @@ class TDTestCase:
projPath = selfPath[:selfPath.find("tests")]
for root, dirs, files in os.walk(projPath):
- if ("taosd" in files):
+ if ("taosd" in files or "taosd.exe" in files):
rootRealPath = os.path.dirname(os.path.realpath(root))
if ("packaging" not in rootRealPath):
buildPath = root[:len(root) - len("/build/bin")]
@@ -81,7 +81,7 @@ class TDTestCase:
dnode_id = dnode.cfgDict["fqdn"] + ":" +dnode.cfgDict["serverPort"]
dnode_first_host = dnode.cfgDict["firstEp"].split(":")[0]
dnode_first_port = dnode.cfgDict["firstEp"].split(":")[-1]
- cmd = f" taos -h {dnode_first_host} -P {dnode_first_port} -s ' create dnode \"{dnode_id} \" ' ;"
+ cmd = f"{self.getBuildPath()}/build/bin/taos -h {dnode_first_host} -P {dnode_first_port} -s \"create dnode \\\"{dnode_id}\\\"\""
print(cmd)
os.system(cmd)
diff --git a/tests/system-test/6-cluster/5dnode2mnode.py b/tests/system-test/6-cluster/5dnode2mnode.py
index d3cde987c6..e08e738be6 100644
--- a/tests/system-test/6-cluster/5dnode2mnode.py
+++ b/tests/system-test/6-cluster/5dnode2mnode.py
@@ -20,6 +20,7 @@ class MyDnodes(TDDnodes):
self.simDeployed = False
class TDTestCase:
+ noConn = True
def init(self,conn ,logSql):
tdLog.debug(f"start to excute {__file__}")
@@ -40,7 +41,7 @@ class TDTestCase:
projPath = selfPath[:selfPath.find("tests")]
for root, dirs, files in os.walk(projPath):
- if ("taosd" in files):
+ if ("taosd" in files or "taosd.exe" in files):
rootRealPath = os.path.dirname(os.path.realpath(root))
if ("packaging" not in rootRealPath):
buildPath = root[:len(root) - len("/build/bin")]
@@ -85,7 +86,7 @@ class TDTestCase:
dnode_id = dnode.cfgDict["fqdn"] + ":" +dnode.cfgDict["serverPort"]
dnode_first_host = dnode.cfgDict["firstEp"].split(":")[0]
dnode_first_port = dnode.cfgDict["firstEp"].split(":")[-1]
- cmd = f" taos -h {dnode_first_host} -P {dnode_first_port} -s ' create dnode \"{dnode_id} \" ' ;"
+ cmd = f"{self.getBuildPath()}/build/bin/taos -h {dnode_first_host} -P {dnode_first_port} -s \"create dnode \\\"{dnode_id}\\\"\""
print(cmd)
os.system(cmd)
diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopCreateDb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopCreateDb.py
new file mode 100644
index 0000000000..59fe1c0b16
--- /dev/null
+++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopCreateDb.py
@@ -0,0 +1,181 @@
+from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE
+import taos
+import sys
+import time
+import os
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import TDDnodes
+from util.dnodes import TDDnode
+from util.cluster import *
+sys.path.append("./6-cluster")
+from clusterCommonCreate import *
+from clusterCommonCheck import clusterComCheck
+
+import time
+import socket
+import subprocess
+from multiprocessing import Process
+import threading
+import time
+import inspect
+import ctypes
+
+class TDTestCase:
+
+ def init(self,conn ,logSql):
+ tdLog.debug(f"start to excute {__file__}")
+ self.TDDnodes = None
+ tdSql.init(conn.cursor())
+ self.host = socket.gethostname()
+
+
+ def getBuildPath(self):
+ selfPath = os.path.dirname(os.path.realpath(__file__))
+
+ if ("community" in selfPath):
+ projPath = selfPath[:selfPath.find("community")]
+ else:
+ projPath = selfPath[:selfPath.find("tests")]
+
+ for root, dirs, files in os.walk(projPath):
+ if ("taosd" in files):
+ rootRealPath = os.path.dirname(os.path.realpath(root))
+ if ("packaging" not in rootRealPath):
+ buildPath = root[:len(root) - len("/build/bin")]
+ break
+ return buildPath
+
+ def _async_raise(self, tid, exctype):
+ """raises the exception, performs cleanup if needed"""
+ if not inspect.isclass(exctype):
+ exctype = type(exctype)
+ res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, ctypes.py_object(exctype))
+ if res == 0:
+ raise ValueError("invalid thread id")
+ elif res != 1:
+ # """if it returns a number greater than one, you're in trouble,
+ # and you should call it again with exc=NULL to revert the effect"""
+ ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None)
+ raise SystemError("PyThreadState_SetAsyncExc failed")
+
+ def stopThread(self,thread):
+ self._async_raise(thread.ident, SystemExit)
+
+
+ def insertData(self,countstart,countstop):
+ # fisrt add data : db\stable\childtable\general table
+
+ for couti in range(countstart,countstop):
+ tdLog.debug("drop database if exists db%d" %couti)
+ tdSql.execute("drop database if exists db%d" %couti)
+ print("create database if not exists db%d replica 1 duration 300" %couti)
+ tdSql.execute("create database if not exists db%d replica 1 duration 300" %couti)
+ tdSql.execute("use db%d" %couti)
+ tdSql.execute(
+ '''create table stb1
+ (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
+ tags (t1 int)
+ '''
+ )
+ tdSql.execute(
+ '''
+ create table t1
+ (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
+ '''
+ )
+ for i in range(4):
+ tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )')
+
+
+ def fiveDnodeThreeMnode(self,dnodenumbers,mnodeNums,restartNumber):
+ tdLog.printNoPrefix("======== test case 1: ")
+ paraDict = {'dbName': 'db',
+ 'dropFlag': 1,
+ 'event': '',
+ 'vgroups': 4,
+ 'replica': 1,
+ 'stbName': 'stb',
+ 'colPrefix': 'c',
+ 'tagPrefix': 't',
+ 'colSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+ 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+ 'ctbPrefix': 'ctb',
+ 'ctbNum': 1,
+ 'rowsPerTbl': 10000,
+ 'batchNum': 10,
+ 'startTs': 1640966400000, # 2022-01-01 00:00:00.000
+ 'pollDelay': 10,
+ 'showMsg': 1,
+ 'showRow': 1}
+ dnodenumbers=int(dnodenumbers)
+ mnodeNums=int(mnodeNums)
+ dbNumbers = int(dnodenumbers * restartNumber)
+
+ tdLog.info("first check dnode and mnode")
+ tdSql.query("show dnodes;")
+ tdSql.checkData(0,1,'%s:6030'%self.host)
+ tdSql.checkData(4,1,'%s:6430'%self.host)
+ clusterComCheck.checkDnodes(dnodenumbers)
+ clusterComCheck.checkMnodeStatus(1)
+
+ # fisr add three mnodes;
+ tdLog.info("fisr add three mnodes and check mnode status")
+ tdSql.execute("create mnode on dnode 2")
+ clusterComCheck.checkMnodeStatus(2)
+ tdSql.execute("create mnode on dnode 3")
+ clusterComCheck.checkMnodeStatus(3)
+
+ # add some error operations and
+ tdLog.info("Confirm the status of the dnode again")
+ tdSql.error("create mnode on dnode 2")
+ tdSql.query("show dnodes;")
+ print(tdSql.queryResult)
+ clusterComCheck.checkDnodes(dnodenumbers)
+
+ tdLog.info("Take turns stopping all dnodes ")
+ # seperate vnode and mnode in different dnodes.
+ # create database and stable
+ tdDnodes=cluster.dnodes
+ stopcount =0
+ while stopcount < restartNumber:
+ for i in range(dnodenumbers):
+ # threads=[]
+ # threads = MyThreadFunc(self.insert_data(i*2,i*2+2))
+ paraDict["dbName"]= 'db%d%d'%(stopcount,i)
+ threads=threading.Thread(target=clusterComCreate.create_database, args=(tdSql, paraDict["dbName"],paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica']))
+ threads.start()
+ tdDnodes[i].stoptaosd()
+ # sleep(10)
+ tdDnodes[i].starttaosd()
+ # sleep(10)
+
+ if clusterComCheck.checkDnodes(dnodenumbers):
+ # threads.join()
+ tdLog.info("first restart loop")
+ else:
+ print("456")
+ threads.join()
+ self.stopThread(threads)
+ tdLog.exit("one or more of dnodes failed to start ")
+ # self.check3mnode()
+ stopcount+=1
+ threads.join()
+ clusterComCheck.checkDnodes(dnodenumbers)
+ clusterComCheck.checkDbRows(dbNumbers)
+ for i in range(restartNumber):
+ clusterComCheck.checkDb(dnodenumbers,'db%d'%i)
+
+
+ def run(self):
+ # print(self.master_dnode.cfgDict)
+ self.fiveDnodeThreeMnode(5,3,1)
+
+ def stop(self):
+ tdSql.close()
+ tdLog.success(f"{__file__} successfully executed")
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())
\ No newline at end of file
diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py
new file mode 100644
index 0000000000..cf608f6480
--- /dev/null
+++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py
@@ -0,0 +1,181 @@
+from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE
+import taos
+import sys
+import time
+import os
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import TDDnodes
+from util.dnodes import TDDnode
+from util.cluster import *
+sys.path.append("./6-cluster")
+from clusterCommonCreate import *
+from clusterCommonCheck import clusterComCheck
+
+import time
+import socket
+import subprocess
+from multiprocessing import Process
+import threading
+import time
+import inspect
+import ctypes
+
+class TDTestCase:
+
+ def init(self,conn ,logSql):
+ tdLog.debug(f"start to excute {__file__}")
+ self.TDDnodes = None
+ tdSql.init(conn.cursor())
+ self.host = socket.gethostname()
+
+
+ def getBuildPath(self):
+ selfPath = os.path.dirname(os.path.realpath(__file__))
+
+ if ("community" in selfPath):
+ projPath = selfPath[:selfPath.find("community")]
+ else:
+ projPath = selfPath[:selfPath.find("tests")]
+
+ for root, dirs, files in os.walk(projPath):
+ if ("taosd" in files):
+ rootRealPath = os.path.dirname(os.path.realpath(root))
+ if ("packaging" not in rootRealPath):
+ buildPath = root[:len(root) - len("/build/bin")]
+ break
+ return buildPath
+
+ def _async_raise(self, tid, exctype):
+ """raises the exception, performs cleanup if needed"""
+ if not inspect.isclass(exctype):
+ exctype = type(exctype)
+ res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, ctypes.py_object(exctype))
+ if res == 0:
+ raise ValueError("invalid thread id")
+ elif res != 1:
+ # """if it returns a number greater than one, you're in trouble,
+ # and you should call it again with exc=NULL to revert the effect"""
+ ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None)
+ raise SystemError("PyThreadState_SetAsyncExc failed")
+
+ def stopThread(self,thread):
+ self._async_raise(thread.ident, SystemExit)
+
+
+ def insertData(self,countstart,countstop):
+ # fisrt add data : db\stable\childtable\general table
+
+ for couti in range(countstart,countstop):
+ tdLog.debug("drop database if exists db%d" %couti)
+ tdSql.execute("drop database if exists db%d" %couti)
+ print("create database if not exists db%d replica 1 duration 300" %couti)
+ tdSql.execute("create database if not exists db%d replica 1 duration 300" %couti)
+ tdSql.execute("use db%d" %couti)
+ tdSql.execute(
+ '''create table stb1
+ (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
+ tags (t1 int)
+ '''
+ )
+ tdSql.execute(
+ '''
+ create table t1
+ (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
+ '''
+ )
+ for i in range(4):
+ tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )')
+
+
+ def fiveDnodeThreeMnode(self,dnodenumbers,mnodeNums,restartNumber):
+ tdLog.printNoPrefix("======== test case 1: ")
+ paraDict = {'dbName': 'db',
+ 'dropFlag': 1,
+ 'event': '',
+ 'vgroups': 4,
+ 'replica': 1,
+ 'stbName': 'stb',
+ 'colPrefix': 'c',
+ 'tagPrefix': 't',
+ 'colSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+ 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+ 'ctbPrefix': 'ctb',
+ 'ctbNum': 1,
+ 'rowsPerTbl': 10000,
+ 'batchNum': 10,
+ 'startTs': 1640966400000, # 2022-01-01 00:00:00.000
+ 'pollDelay': 10,
+ 'showMsg': 1,
+ 'showRow': 1}
+ dnodenumbers=int(dnodenumbers)
+ mnodeNums=int(mnodeNums)
+ dbNumbers = int(mnodeNums * restartNumber)
+ tdLog.info("first check dnode and mnode")
+ tdSql.query("show dnodes;")
+ tdSql.checkData(0,1,'%s:6030'%self.host)
+ tdSql.checkData(4,1,'%s:6430'%self.host)
+ clusterComCheck.checkDnodes(dnodenumbers)
+ clusterComCheck.checkMnodeStatus(1)
+
+ # fisr add three mnodes;
+ tdLog.info("fisr add three mnodes and check mnode status")
+ tdSql.execute("create mnode on dnode 2")
+ clusterComCheck.checkMnodeStatus(2)
+ tdSql.execute("create mnode on dnode 3")
+ clusterComCheck.checkMnodeStatus(3)
+
+ # add some error operations and
+ tdLog.info("Confirm the status of the dnode again")
+ tdSql.error("create mnode on dnode 2")
+ tdSql.query("show dnodes;")
+ print(tdSql.queryResult)
+ clusterComCheck.checkDnodes(dnodenumbers)
+
+ tdLog.info("Take turns stopping Mnodes ")
+ # seperate vnode and mnode in different dnodes.
+ # create database and stable
+ tdDnodes=cluster.dnodes
+ stopcount =0
+ while stopcount < restartNumber:
+ tdLog.info("first restart loop")
+ for i in range(mnodeNums):
+ # threads=[]
+ # threads = MyThreadFunc(self.insert_data(i*2,i*2+2))
+ paraDict["dbName"]= 'db%d%d'%(stopcount,i)
+ threads=threading.Thread(target=clusterComCreate.create_database, args=(tdSql, paraDict["dbName"],paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica']))
+ threads.start()
+ tdDnodes[i].stoptaosd()
+ # sleep(10)
+ tdDnodes[i].starttaosd()
+ # sleep(10)
+
+ if clusterComCheck.checkDnodes(dnodenumbers):
+ # threads.join()
+ tdLog.info("123")
+ else:
+ print("456")
+ threads.join()
+ self.stopThread(threads)
+ tdLog.exit("one or more of dnodes failed to start ")
+ # self.check3mnode()
+ stopcount+=1
+ threads.join()
+ clusterComCheck.checkDnodes(dnodenumbers)
+ clusterComCheck.checkDbRows(dbNumbers)
+ for i in range(restartNumber):
+ clusterComCheck.checkDb(mnodeNums,'db%d'%i)
+
+
+ def run(self):
+ # print(self.master_dnode.cfgDict)
+ self.fiveDnodeThreeMnode(5,3,1)
+
+ def stop(self):
+ tdSql.close()
+ tdLog.success(f"{__file__} successfully executed")
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())
\ No newline at end of file
diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py
new file mode 100644
index 0000000000..2d2322fada
--- /dev/null
+++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py
@@ -0,0 +1,181 @@
+from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE
+import taos
+import sys
+import time
+import os
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import TDDnodes
+from util.dnodes import TDDnode
+from util.cluster import *
+sys.path.append("./6-cluster")
+from clusterCommonCreate import *
+from clusterCommonCheck import clusterComCheck
+
+import time
+import socket
+import subprocess
+from multiprocessing import Process
+import threading
+import time
+import inspect
+import ctypes
+
+class TDTestCase:
+
+ def init(self,conn ,logSql):
+ tdLog.debug(f"start to excute {__file__}")
+ self.TDDnodes = None
+ tdSql.init(conn.cursor())
+ self.host = socket.gethostname()
+
+
+ def getBuildPath(self):
+ selfPath = os.path.dirname(os.path.realpath(__file__))
+
+ if ("community" in selfPath):
+ projPath = selfPath[:selfPath.find("community")]
+ else:
+ projPath = selfPath[:selfPath.find("tests")]
+
+ for root, dirs, files in os.walk(projPath):
+ if ("taosd" in files):
+ rootRealPath = os.path.dirname(os.path.realpath(root))
+ if ("packaging" not in rootRealPath):
+ buildPath = root[:len(root) - len("/build/bin")]
+ break
+ return buildPath
+
+ def _async_raise(self, tid, exctype):
+ """raises the exception, performs cleanup if needed"""
+ if not inspect.isclass(exctype):
+ exctype = type(exctype)
+ res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, ctypes.py_object(exctype))
+ if res == 0:
+ raise ValueError("invalid thread id")
+ elif res != 1:
+ # """if it returns a number greater than one, you're in trouble,
+ # and you should call it again with exc=NULL to revert the effect"""
+ ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None)
+ raise SystemError("PyThreadState_SetAsyncExc failed")
+
+ def stopThread(self,thread):
+ self._async_raise(thread.ident, SystemExit)
+
+
+ def insertData(self,countstart,countstop):
+ # fisrt add data : db\stable\childtable\general table
+
+ for couti in range(countstart,countstop):
+ tdLog.debug("drop database if exists db%d" %couti)
+ tdSql.execute("drop database if exists db%d" %couti)
+ print("create database if not exists db%d replica 1 duration 300" %couti)
+ tdSql.execute("create database if not exists db%d replica 1 duration 300" %couti)
+ tdSql.execute("use db%d" %couti)
+ tdSql.execute(
+ '''create table stb1
+ (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
+ tags (t1 int)
+ '''
+ )
+ tdSql.execute(
+ '''
+ create table t1
+ (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
+ '''
+ )
+ for i in range(4):
+ tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )')
+
+
+ def fiveDnodeThreeMnode(self,dnodenumbers,mnodeNums,restartNumber):
+ tdLog.printNoPrefix("======== test case 1: ")
+ paraDict = {'dbName': 'db',
+ 'dropFlag': 1,
+ 'event': '',
+ 'vgroups': 4,
+ 'replica': 1,
+ 'stbName': 'stb',
+ 'colPrefix': 'c',
+ 'tagPrefix': 't',
+ 'colSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+ 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+ 'ctbPrefix': 'ctb',
+ 'ctbNum': 1,
+ 'rowsPerTbl': 10000,
+ 'batchNum': 10,
+ 'startTs': 1640966400000, # 2022-01-01 00:00:00.000
+ 'pollDelay': 10,
+ 'showMsg': 1,
+ 'showRow': 1}
+ dnodenumbers=int(dnodenumbers)
+ mnodeNums=int(mnodeNums)
+ vnodeNumbers = int(dnodenumbers-mnodeNums)
+ dbNumbers = int(vnodeNumbers * restartNumber)
+ tdLog.info("first check dnode and mnode")
+ tdSql.query("show dnodes;")
+ tdSql.checkData(0,1,'%s:6030'%self.host)
+ tdSql.checkData(4,1,'%s:6430'%self.host)
+ clusterComCheck.checkDnodes(dnodenumbers)
+ clusterComCheck.checkMnodeStatus(1)
+
+ # fisr add three mnodes;
+ tdLog.info("fisr add three mnodes and check mnode status")
+ tdSql.execute("create mnode on dnode 2")
+ clusterComCheck.checkMnodeStatus(2)
+ tdSql.execute("create mnode on dnode 3")
+ clusterComCheck.checkMnodeStatus(3)
+
+ # add some error operations and
+ tdLog.info("Confirm the status of the dnode again")
+ tdSql.error("create mnode on dnode 2")
+ tdSql.query("show dnodes;")
+ print(tdSql.queryResult)
+ clusterComCheck.checkDnodes(dnodenumbers)
+
+ tdLog.info("Take turns stopping Vnodes ")
+ # seperate vnode and mnode in different dnodes.
+ # create database and stable
+ tdDnodes=cluster.dnodes
+ stopcount =0
+ while stopcount < restartNumber:
+ for i in range(vnodeNumbers):
+ # threads=[]
+ # threads = MyThreadFunc(self.insert_data(i*2,i*2+2))
+ paraDict["dbName"]= 'db%d%d'%(stopcount,i)
+ threads=threading.Thread(target=clusterComCreate.create_database, args=(tdSql, paraDict["dbName"],paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica']))
+ threads.start()
+ tdDnodes[mnodeNums+i].stoptaosd()
+ # sleep(10)
+ tdDnodes[mnodeNums+i].starttaosd()
+ # sleep(10)
+
+ if clusterComCheck.checkDnodes(vnodeNumbers):
+ # threads.join()
+ tdLog.info("first restart loop")
+ else:
+ print("456")
+ threads.join()
+ self.stopThread(threads)
+ tdLog.exit("one or more of dnodes failed to start ")
+ # self.check3mnode()
+ stopcount+=1
+ threads.join()
+ clusterComCheck.checkDnodes(dnodenumbers)
+ clusterComCheck.checkDbRows(dbNumbers)
+ for i in range(restartNumber):
+ clusterComCheck.checkDb(vnodeNumbers,'db%d'%i)
+
+
+ def run(self):
+ # print(self.master_dnode.cfgDict)
+ self.fiveDnodeThreeMnode(5,3,1)
+
+ def stop(self):
+ tdSql.close()
+ tdLog.success(f"{__file__} successfully executed")
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())
\ No newline at end of file
diff --git a/tests/system-test/6-cluster/5dnode3mnodeSeperate1VnodeStopInsert.py b/tests/system-test/6-cluster/5dnode3mnodeSeperate1VnodeStopInsert.py
index 1739db09af..aa1d7ecc29 100644
--- a/tests/system-test/6-cluster/5dnode3mnodeSeperate1VnodeStopInsert.py
+++ b/tests/system-test/6-cluster/5dnode3mnodeSeperate1VnodeStopInsert.py
@@ -9,6 +9,11 @@ from util.sql import *
from util.cases import *
from util.dnodes import TDDnodes
from util.dnodes import TDDnode
+from util.cluster import *
+from util.common import *
+sys.path.append("./7-tmq")
+from tmqCommon import *
+
import time
import socket
import subprocess
@@ -17,26 +22,13 @@ import threading
import time
import inspect
import ctypes
-class MyDnodes(TDDnodes):
- def __init__(self ,dnodes_lists):
- super(MyDnodes,self).__init__()
- self.dnodes = dnodes_lists # dnode must be TDDnode instance
- self.simDeployed = False
-
class TDTestCase:
def init(self,conn ,logSql):
tdLog.debug(f"start to excute {__file__}")
- self.TDDnodes = None
-
- def buildcluster(self,dnodenumber):
- self.depoly_cluster(dnodenumber)
- self.master_dnode = self.TDDnodes.dnodes[0]
- self.host=self.master_dnode.cfgDict["fqdn"]
- conn1 = taos.connect(self.master_dnode.cfgDict["fqdn"] , config=self.master_dnode.cfgDir)
- tdSql.init(conn1.cursor())
-
+ # tdSql.init(conn.cursor())
+ # self.host = socket.gethostname()
def getBuildPath(self):
selfPath = os.path.dirname(os.path.realpath(__file__))
@@ -106,52 +98,6 @@ class TDTestCase:
tdSql.checkData(0,0,rowsPerSTable)
return
- def depoly_cluster(self ,dnodes_nums=5,independent=True):
-
- testCluster = False
- valgrind = 0
- hostname = socket.gethostname()
- dnodes = []
- start_port = 6030
- start_port_sec = 6130
- for num in range(1, dnodes_nums+1):
- dnode = TDDnode(num)
- dnode.addExtraCfg("firstEp", f"{hostname}:{start_port}")
- dnode.addExtraCfg("fqdn", f"{hostname}")
- dnode.addExtraCfg("serverPort", f"{start_port + (num-1)*100}")
- dnode.addExtraCfg("monitorFqdn", hostname)
- dnode.addExtraCfg("monitorPort", 7043)
- dnode.addExtraCfg("secondEp", f"{hostname}:{start_port_sec}")
- # configure three dnoe don't support vnodes
- if independent and (num < 4):
- dnode.addExtraCfg("supportVnodes", 0)
-
- dnodes.append(dnode)
-
- self.TDDnodes = MyDnodes(dnodes)
- self.TDDnodes.init("")
- self.TDDnodes.setTestCluster(testCluster)
- self.TDDnodes.setValgrind(valgrind)
- self.TDDnodes.stopAll()
- for dnode in self.TDDnodes.dnodes:
- self.TDDnodes.deploy(dnode.index,{})
-
- for dnode in self.TDDnodes.dnodes:
- self.TDDnodes.starttaosd(dnode.index)
-
- # create cluster
- for dnode in self.TDDnodes.dnodes[1:]:
- # print(dnode.cfgDict)
- dnode_id = dnode.cfgDict["fqdn"] + ":" +dnode.cfgDict["serverPort"]
- dnode_first_host = dnode.cfgDict["firstEp"].split(":")[0]
- dnode_first_port = dnode.cfgDict["firstEp"].split(":")[-1]
- cmd = f" taos -h {dnode_first_host} -P {dnode_first_port} -s ' create dnode \"{dnode_id} \" ' ;"
- print(cmd)
- os.system(cmd)
-
- time.sleep(2)
- tdLog.info(" create cluster with %d dnode done! " %dnodes_nums)
-
def checkdnodes(self,dnodenumber):
count=0
while count < 100:
@@ -305,6 +251,14 @@ class TDTestCase:
tdSql.checkData(2,2,'offline')
tdSql.checkData(2,3,'ready')
+
+ def check5dnode(self):
+ tdSql.query("show dnodes;")
+ tdSql.checkData(0,1,'%s:6030'%self.host)
+ tdSql.checkData(4,1,'%s:6430'%self.host)
+ tdSql.checkData(0,4,'ready')
+ tdSql.checkData(4,4,'ready')
+
def five_dnode_three_mnode(self,dnodenumber):
tdSql.query("show dnodes;")
tdSql.checkData(0,1,'%s:6030'%self.host)
@@ -346,6 +300,7 @@ class TDTestCase:
threads.join()
else:
print("456")
+ threads.join()
self.stop_thread(threads)
assert 1 == 2 ,"some dnode started failed"
return False
@@ -357,16 +312,8 @@ class TDTestCase:
self.check3mnode()
- def getConnection(self, dnode):
- host = dnode.cfgDict["fqdn"]
- port = dnode.cfgDict["serverPort"]
- config_dir = dnode.cfgDir
- return taos.connect(host=host, port=int(port), config=config_dir)
-
-
def run(self):
# print(self.master_dnode.cfgDict)
- self.buildcluster(5)
self.five_dnode_three_mnode(5)
def stop(self):
diff --git a/tests/system-test/6-cluster/5dnode3mnodeStop.py b/tests/system-test/6-cluster/5dnode3mnodeStop.py
index 3a85207af6..5311d29846 100644
--- a/tests/system-test/6-cluster/5dnode3mnodeStop.py
+++ b/tests/system-test/6-cluster/5dnode3mnodeStop.py
@@ -12,7 +12,10 @@ from util.dnodes import TDDnodes
from util.dnodes import TDDnode
from util.cluster import *
from test import tdDnodes
+sys.path.append("./6-cluster")
+from clusterCommonCreate import *
+from clusterCommonCheck import *
import time
import socket
import subprocess
@@ -216,64 +219,88 @@ class TDTestCase:
else:
tdLog.exit("create cluster with %d dnode but check dnode not ready within 5s ! "%dnodeNumbers)
- def five_dnode_three_mnode(self,dnodenumber):
- self.check_dnodes_status(5)
- tdSql.query("show mnodes;")
- tdLog.debug(self.host)
- tdSql.checkRows(1)
+ def fiveDnodeThreeMnode(self,dnodenumbers,mnodeNums,restartNumber):
+ tdLog.printNoPrefix("======== test case 1: ")
+ paraDict = {'dbName': 'db',
+ 'dropFlag': 1,
+ 'event': '',
+ 'vgroups': 4,
+ 'replica': 1,
+ 'stbName': 'stb',
+ 'colPrefix': 'c',
+ 'tagPrefix': 't',
+ 'colSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+ 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+ 'ctbPrefix': 'ctb',
+ 'ctbNum': 1,
+ 'rowsPerTbl': 10000,
+ 'batchNum': 10,
+ 'startTs': 1640966400000, # 2022-01-01 00:00:00.000
+ 'pollDelay': 10,
+ 'showMsg': 1,
+ 'showRow': 1}
+ dnodenumbers=int(dnodenumbers)
+ mnodeNums=int(mnodeNums)
+ dbNumbers = int(dnodenumbers * restartNumber)
+
+ tdLog.info("first check dnode and mnode")
+ tdSql.query("show dnodes;")
tdSql.checkData(0,1,'%s:6030'%self.host)
- tdSql.checkData(0,2,'leader')
- tdSql.checkData(0,3,'ready')
+ tdSql.checkData(4,1,'%s:6430'%self.host)
+ clusterComCheck.checkDnodes(dnodenumbers)
+ clusterComCheck.checkMnodeStatus(1)
# fisr add three mnodes;
+ tdLog.info("fisr add three mnodes and check mnode status")
tdSql.execute("create mnode on dnode 2")
- time.sleep(10)
+ clusterComCheck.checkMnodeStatus(2)
tdSql.execute("create mnode on dnode 3")
+ clusterComCheck.checkMnodeStatus(3)
- # fisrt check statut ready
- self.check3mnode()
-
-
+ # add some error operations and
+ tdLog.info("Confirm the status of the dnode again")
tdSql.error("create mnode on dnode 2")
-
tdSql.query("show dnodes;")
- # tdLog.debug(tdSql.queryResult)
-
- tdLog.debug("stop and follower of mnode")
+ print(tdSql.queryResult)
+ clusterComCheck.checkDnodes(dnodenumbers)
+ # restart all taosd
tdDnodes=cluster.dnodes
- # tdLog.debug(tdDnodes[0])
tdDnodes[1].stoptaosd()
- self.check3mnode2off()
+ clusterComCheck.check3mnodeoff(2,3)
tdDnodes[1].starttaosd()
- self.check3mnode()
+ clusterComCheck.checkMnodeStatus(3)
tdDnodes[2].stoptaosd()
- self.check3mnode3off()
+ clusterComCheck.check3mnodeoff(3,3)
tdDnodes[2].starttaosd()
- self.check3mnode()
+ clusterComCheck.checkMnodeStatus(3)
tdDnodes[0].stoptaosd()
- self.check3mnode1off()
+ clusterComCheck.check3mnodeoff(1,3)
tdDnodes[0].starttaosd()
- self.check3mnode()
+ clusterComCheck.checkMnodeStatus(3)
- self.check3mnode()
+ tdLog.info("Take turns stopping all dnodes ")
+ # seperate vnode and mnode in different dnodes.
+ # create database and stable
stopcount =0
while stopcount <= 2:
- for i in range(dnodenumber):
+ tdLog.info("first restart loop")
+ for i in range(dnodenumbers):
tdDnodes[i].stoptaosd()
tdDnodes[i].starttaosd()
- # self.check3mnode()
stopcount+=1
- self.check3mnode()
+ clusterComCheck.checkDnodes(dnodenumbers)
+ clusterComCheck.checkMnodeStatus(3)
def run(self):
- self.five_dnode_three_mnode(5)
+ # print(self.master_dnode.cfgDict)
+ self.fiveDnodeThreeMnode(5,3,1)
def stop(self):
tdSql.close()
tdLog.success(f"{__file__} successfully executed")
tdCases.addLinux(__file__, TDTestCase())
-tdCases.addWindows(__file__, TDTestCase())
\ No newline at end of file
+tdCases.addWindows(__file__, TDTestCase())
diff --git a/tests/system-test/6-cluster/clusterCommonCheck.py b/tests/system-test/6-cluster/clusterCommonCheck.py
new file mode 100644
index 0000000000..d030008963
--- /dev/null
+++ b/tests/system-test/6-cluster/clusterCommonCheck.py
@@ -0,0 +1,211 @@
+###################################################################
+# Copyright (c) 2016 by TAOS Technologies, Inc.
+# All rights reserved.
+#
+# This file is proprietary and confidential to TAOS Technologies.
+# No part of this file may be reproduced, stored, transmitted,
+# disclosed or used in any form or by any means other than as
+# expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+from collections import defaultdict
+import random
+import string
+import threading
+import requests
+import time
+# import socketfrom
+
+import taos
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+from util.common import *
+
+# class actionType(Enum):
+# CREATE_DATABASE = 0
+# CREATE_STABLE = 1
+# CREATE_CTABLE = 2
+# INSERT_DATA = 3
+
+class ClusterComCheck:
+ def init(self, conn, logSql):
+ tdSql.init(conn.cursor())
+ # tdSql.init(conn.cursor(), logSql) # output sql.txt file
+
+ def checkDnodes(self,dnodeNumbers):
+ count=0
+ while count < 5:
+ tdSql.query("show dnodes")
+ # tdLog.debug(tdSql.queryResult)
+ status=0
+ for i in range(dnodeNumbers):
+ if tdSql.queryResult[i][4] == "ready":
+ status+=1
+ tdLog.info(status)
+
+ if status == dnodeNumbers:
+ tdLog.success("it find cluster with %d dnodes and check that all cluster dnodes are ready within 5s! " %dnodeNumbers)
+ return True
+ count+=1
+ time.sleep(1)
+ else:
+ tdLog.debug(tdSql.queryResult)
+ tdLog.exit("it find cluster with %d dnodes but check that there dnodes are not ready within 5s ! "%dnodeNumbers)
+
+ def checkDbRows(self,dbNumbers):
+ dbNumbers=int(dbNumbers)
+ count=0
+ while count < 5:
+ tdSql.query("show databases;")
+ if tdSql.checkRows(dbNumbers+2):
+ tdLog.success("we find %d databases and expect %d in clusters! " %(tdSql.queryRows,dbNumbers+2))
+ return True
+ else:
+ continue
+ else :
+ tdLog.debug(tdSql.queryResult)
+ tdLog.exit("we find %d databases but expect %d in clusters! " %(tdSql.queryRows,dbNumbers))
+
+ def checkDb(self,dbNumbers,dbindex):
+ count=0
+ while count < 5:
+ query_status=0
+ for i in range(dbNumbers):
+ for j in range(dbNumbers):
+ tdSql.query("show databases;")
+ if "%s%d"%(dbindex,j) == tdSql.queryResult[i+2][0] :
+ if tdSql.queryResult[i+2][19] == "ready":
+ query_status+=1
+ else:
+ continue
+ # print(query_status)
+ count+=1
+ if query_status == dbNumbers:
+ tdLog.success("we find cluster with %d dnode and check all databases are ready within 5s! " %dbNumbers)
+ return True
+ else:
+ tdLog.debug(tdSql.queryResult)
+ tdLog.exit("database is not ready within 5s")
+
+ def checkData(self,dbname,stbname,stableCount,CtableCount,rowsPerSTable,):
+ tdSql.execute("use %s"%dbname)
+ tdSql.query("show stables")
+ tdSql.checkRows(stableCount)
+ tdSql.query("show tables")
+ tdSql.checkRows(CtableCount)
+ for i in range(stableCount):
+ tdSql.query("select count(*) from %s%d"%(stbname,i))
+ tdSql.checkData(0,0,rowsPerSTable)
+ return
+
+ def checkMnodeStatus(self,mnodeNums):
+ self.mnodeNums=int(mnodeNums)
+ # self.leaderDnode=int(leaderDnode)
+
+ count=0
+
+ while count < 10:
+ time.sleep(1)
+ tdSql.query("show mnodes;")
+ if tdSql.checkRows(self.mnodeNums) :
+ tdLog.success("cluster has %d mnodes" %self.mnodeNums )
+
+ if self.mnodeNums == 1:
+ if tdSql.queryResult[0][2]== 'leader' and tdSql.queryResult[0][3]== 'ready' :
+ tdLog.success("%d mnodes is ready in 10s"%self.mnodeNums)
+ return True
+ count+=1
+ elif self.mnodeNums == 3 :
+ if tdSql.queryResult[0][2]=='leader' and tdSql.queryResult[0][3]== 'ready' :
+ if tdSql.queryResult[1][2]=='follower' and tdSql.queryResult[1][3]== 'ready' :
+ if tdSql.queryResult[2][2]=='follower' and tdSql.queryResult[2][3]== 'ready' :
+ tdLog.success("%d mnodes is ready in 10s"%self.mnodeNums)
+ return True
+ elif tdSql.queryResult[1][2]=='leader' and tdSql.queryResult[1][3]== 'ready' :
+ if tdSql.queryResult[0][2]=='follower' and tdSql.queryResult[0][3]== 'ready' :
+ if tdSql.queryResult[2][2]=='follower' and tdSql.queryResult[2][3]== 'ready' :
+ tdLog.success("%d mnodes is ready in 10s"%self.mnodeNums)
+ return True
+ elif tdSql.queryResult[2][2]=='leader' and tdSql.queryResult[2][3]== 'ready' :
+ if tdSql.queryResult[0][2]=='follower' and tdSql.queryResult[0][3]== 'ready' :
+ if tdSql.queryResult[1][2]=='follower' and tdSql.queryResult[1][3]== 'ready' :
+ tdLog.success("%d mnodes is ready in 10s"%self.mnodeNums)
+ return True
+ count+=1
+ elif self.mnodeNums == 2 :
+ if tdSql.queryResult[0][2]=='leader' and tdSql.queryResult[0][3]== 'ready' :
+ if tdSql.queryResult[1][2]=='follower' and tdSql.queryResult[1][3]== 'ready' :
+ tdLog.success("%d mnodes is ready in 10s"%self.mnodeNums)
+ return True
+ elif tdSql.queryResult[1][2]=='leader' and tdSql.queryResult[1][3]== 'ready' :
+ if tdSql.queryResult[0][2]=='follower' and tdSql.queryResult[0][3]== 'ready' :
+ tdLog.success("%d mnodes is ready in 10s"%self.mnodeNums)
+ return True
+ count+=1
+ else:
+ tdLog.debug(tdSql.queryResult)
+ tdLog.exit("cluster of %d mnodes is not ready in 10s " %self.mnodeNums)
+
+
+
+
+ def check3mnodeoff(self,offlineDnodeNo,mnodeNums=3):
+ count=0
+ while count < 10:
+ time.sleep(1)
+ tdSql.query("show mnodes;")
+ if tdSql.checkRows(mnodeNums) :
+ tdLog.success("cluster has %d mnodes" %self.mnodeNums )
+ else:
+ tdLog.exit("mnode number is correct")
+ if offlineDnodeNo == 1:
+ if tdSql.queryResult[0][2]=='offline' :
+ if tdSql.queryResult[1][2]=='leader' and tdSql.queryResult[1][3]== 'ready' :
+ if tdSql.queryResult[2][2]=='follower' and tdSql.queryResult[2][3]== 'ready' :
+ tdLog.success("stop mnodes on dnode %d successfully in 10s" %offlineDnodeNo)
+ return True
+ elif tdSql.queryResult[1][2]=='follower' and tdSql.queryResult[1][3]== 'ready' :
+ if tdSql.queryResult[2][2]=='leader' and tdSql.queryResult[2][3]== 'ready' :
+ tdLog.debug("stop mnodes on dnode %d successfully in 10s" %offlineDnodeNo)
+ return True
+ count+=1
+ elif offlineDnodeNo == 2:
+ if tdSql.queryResult[1][2]=='offline' :
+ if tdSql.queryResult[0][2]=='leader' and tdSql.queryResult[0][3]== 'ready' :
+ if tdSql.queryResult[2][2]=='follower' and tdSql.queryResult[2][3]== 'ready' :
+ tdLog.debug("stop mnodes on dnode %d successfully in 10s" %offlineDnodeNo)
+ return True
+ elif tdSql.queryResult[0][2]=='follower' and tdSql.queryResult[0][3]== 'ready' :
+ if tdSql.queryResult[2][2]=='leader' and tdSql.queryResult[2][3]== 'ready' :
+ tdLog.debug("stop mnodes on dnode %d successfully in 10s" %offlineDnodeNo)
+ return True
+ count+=1
+ elif offlineDnodeNo == 3:
+ if tdSql.queryResult[2][2]=='offline' :
+ if tdSql.queryResult[0][2]=='leader' and tdSql.queryResult[0][3]== 'ready' :
+ if tdSql.queryResult[1][2]=='follower' and tdSql.queryResult[1][3]== 'ready' :
+ tdLog.debug("stop mnodes on dnode %d successfully in 10s" %offlineDnodeNo)
+ return True
+ elif tdSql.queryResult[0][2]=='follower' and tdSql.queryResult[0][3]== 'ready' :
+ if tdSql.queryResult[1][2]=='leader' and tdSql.queryResult[1][3]== 'ready' :
+ tdLog.debug("stop mnodes on dnode %d successfully in 10s" %offlineDnodeNo)
+ return True
+ count+=1
+ else:
+ tdLog.debug(tdSql.queryResult)
+ tdLog.exit("stop mnodes on dnode %d failed in 10s ")
+
+
+
+
+
+
+ def close(self):
+ self.cursor.close()
+
+clusterComCheck = ClusterComCheck()
diff --git a/tests/system-test/6-cluster/clusterCommonCreate.py b/tests/system-test/6-cluster/clusterCommonCreate.py
new file mode 100644
index 0000000000..b3107d8537
--- /dev/null
+++ b/tests/system-test/6-cluster/clusterCommonCreate.py
@@ -0,0 +1,298 @@
+###################################################################
+# Copyright (c) 2016 by TAOS Technologies, Inc.
+# All rights reserved.
+#
+# This file is proprietary and confidential to TAOS Technologies.
+# No part of this file may be reproduced, stored, transmitted,
+# disclosed or used in any form or by any means other than as
+# expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+from collections import defaultdict
+import random
+import string
+import threading
+import requests
+import time
+# import socketfrom
+
+import taos
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+from util.common import *
+
+# class actionType(Enum):
+# CREATE_DATABASE = 0
+# CREATE_STABLE = 1
+# CREATE_CTABLE = 2
+# INSERT_DATA = 3
+
+class ClusterComCreate:
+ def init(self, conn, logSql):
+ tdSql.init(conn.cursor())
+ # tdSql.init(conn.cursor(), logSql) # output sql.txt file
+
+ def initConsumerTable(self,cdbName='cdb'):
+ tdLog.info("create consume database, and consume info table, and consume result table")
+ tdSql.query("create database if not exists %s vgroups 1"%(cdbName))
+ tdSql.query("drop table if exists %s.consumeinfo "%(cdbName))
+ tdSql.query("drop table if exists %s.consumeresult "%(cdbName))
+ tdSql.query("drop table if exists %s.notifyinfo "%(cdbName))
+
+ tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)"%cdbName)
+ tdSql.query("create table %s.consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)"%cdbName)
+ tdSql.query("create table %s.notifyinfo (ts timestamp, cmdid int, consumerid int)"%cdbName)
+
+ def initConsumerInfoTable(self,cdbName='cdb'):
+ tdLog.info("drop consumeinfo table")
+ tdSql.query("drop table if exists %s.consumeinfo "%(cdbName))
+ tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)"%cdbName)
+
+ def insertConsumerInfo(self,consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifmanualcommit,cdbName='cdb'):
+ sql = "insert into %s.consumeinfo values "%cdbName
+ sql += "(now, %d, '%s', '%s', %d, %d, %d)"%(consumerId, topicList, keyList, expectrowcnt, ifcheckdata, ifmanualcommit)
+ tdLog.info("consume info sql: %s"%sql)
+ tdSql.query(sql)
+
+ def selectConsumeResult(self,expectRows,cdbName='cdb'):
+ resultList=[]
+ while 1:
+ tdSql.query("select * from %s.consumeresult"%cdbName)
+ #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3))
+ if tdSql.getRows() == expectRows:
+ break
+ else:
+ time.sleep(5)
+
+ for i in range(expectRows):
+ tdLog.info ("consume id: %d, consume msgs: %d, consume rows: %d"%(tdSql.getData(i , 1), tdSql.getData(i , 2), tdSql.getData(i , 3)))
+ resultList.append(tdSql.getData(i , 3))
+
+ return resultList
+
+ def startTmqSimProcess(self,pollDelay,dbName,showMsg=1,showRow=1,cdbName='cdb',valgrind=0):
+ buildPath = tdCom.getBuildPath()
+ cfgPath = tdCom.getClientCfgPath()
+ if valgrind == 1:
+ logFile = cfgPath + '/../log/valgrind-tmq.log'
+ shellCmd = 'nohup valgrind --log-file=' + logFile
+ shellCmd += '--tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all --num-callers=20 -v --workaround-gcc296-bugs=yes '
+
+ if (platform.system().lower() == 'windows'):
+ shellCmd = 'mintty -h never -w hide ' + buildPath + '\\build\\bin\\tmq_sim.exe -c ' + cfgPath
+ shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName)
+ shellCmd += "> nul 2>&1 &"
+ else:
+ shellCmd = 'nohup ' + buildPath + '/build/bin/tmq_sim -c ' + cfgPath
+ shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName)
+ shellCmd += "> /dev/null 2>&1 &"
+ tdLog.info(shellCmd)
+ os.system(shellCmd)
+
+ def getStartConsumeNotifyFromTmqsim(self,cdbName='cdb'):
+ while 1:
+ tdSql.query("select * from %s.notifyinfo"%cdbName)
+ #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3))
+ if (tdSql.getRows() == 1) and (tdSql.getData(0, 1) == 0):
+ break
+ else:
+ time.sleep(0.1)
+ return
+
+ def getStartCommitNotifyFromTmqsim(self,cdbName='cdb'):
+ while 1:
+ tdSql.query("select * from %s.notifyinfo"%cdbName)
+ #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3))
+ if tdSql.getRows() == 2 :
+ print(tdSql.getData(0, 1), tdSql.getData(1, 1))
+ if tdSql.getData(1, 1) == 1:
+ break
+ time.sleep(0.1)
+ return
+
+ def create_database(self,tsql, dbName,dropFlag=1,vgroups=4,replica=1):
+ if dropFlag == 1:
+ tsql.execute("drop database if exists %s"%(dbName))
+
+ tsql.execute("create database if not exists %s vgroups %d replica %d"%(dbName, vgroups, replica))
+ tdLog.debug("complete to create database %s"%(dbName))
+ return
+
+ def create_stable(self,tsql, dbName,stbName):
+ tsql.execute("create table if not exists %s.%s (ts timestamp, c1 int, c2 int, c3 binary(16)) tags(t1 int, t2 binary(32))"%(dbName, stbName))
+ tdLog.debug("complete to create %s.%s" %(dbName, stbName))
+ return
+
+ def create_ctable(self,tsql=None, dbName='dbx',stbName='stb',ctbPrefix='ctb',ctbNum=1):
+ tsql.execute("use %s" %dbName)
+ pre_create = "create table"
+ sql = pre_create
+ #tdLog.debug("doing create one stable %s and %d child table in %s ..." %(stbname, count ,dbname))
+ for i in range(ctbNum):
+ tagValue = 'beijing'
+ if (i % 2 == 0):
+ tagValue = 'shanghai'
+
+ sql += " %s%d using %s tags(%d, '%s')"%(ctbPrefix,i,stbName,i+1, tagValue)
+ if (i > 0) and (i%100 == 0):
+ tsql.execute(sql)
+ sql = pre_create
+ if sql != pre_create:
+ tsql.execute(sql)
+
+ tdLog.debug("complete to create %d child tables in %s.%s" %(ctbNum, dbName, stbName))
+ return
+
+ def insert_data(self,tsql,dbName,stbName,ctbNum,rowsPerTbl,batchNum,startTs=None):
+ tdLog.debug("start to insert data ............")
+ tsql.execute("use %s" %dbName)
+ pre_insert = "insert into "
+ sql = pre_insert
+
+ if startTs is None:
+ t = time.time()
+ startTs = int(round(t * 1000))
+ #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows))
+ for i in range(ctbNum):
+ sql += " %s%d values "%(stbName,i)
+ for j in range(rowsPerTbl):
+ sql += "(%d, %d, 'tmqrow_%d') "%(startTs + j, j, j)
+ if (j > 0) and ((j%batchNum == 0) or (j == rowsPerTbl - 1)):
+ tsql.execute(sql)
+ if j < rowsPerTbl - 1:
+ sql = "insert into %s%d values " %(stbName,i)
+ else:
+ sql = "insert into "
+ #end sql
+ if sql != pre_insert:
+ #print("insert sql:%s"%sql)
+ tsql.execute(sql)
+ tdLog.debug("insert data ............ [OK]")
+ return
+
+ def insert_data_1(self,tsql,dbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs):
+ tdLog.debug("start to insert data ............")
+ tsql.execute("use %s" %dbName)
+ pre_insert = "insert into "
+ sql = pre_insert
+
+ t = time.time()
+ startTs = int(round(t * 1000))
+ #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows))
+ for i in range(ctbNum):
+ sql += " %s%d values "%(ctbPrefix,i)
+ for j in range(rowsPerTbl):
+ if (j % 2 == 0):
+ sql += "(%d, %d, %d, 'tmqrow_%d') "%(startTs + j, j, j, j)
+ else:
+ sql += "(%d, %d, %d, 'tmqrow_%d') "%(startTs + j, j, -j, j)
+ if (j > 0) and ((j%batchNum == 0) or (j == rowsPerTbl - 1)):
+ tsql.execute(sql)
+ if j < rowsPerTbl - 1:
+ sql = "insert into %s%d values " %(ctbPrefix,i)
+ else:
+ sql = "insert into "
+ #end sql
+ if sql != pre_insert:
+ #print("insert sql:%s"%sql)
+ tsql.execute(sql)
+ tdLog.debug("insert data ............ [OK]")
+ return
+
+ def insert_data_interlaceByMultiTbl(self,tsql,dbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs=0):
+ tdLog.debug("start to insert data ............")
+ tsql.execute("use %s" %dbName)
+ pre_insert = "insert into "
+ sql = pre_insert
+
+ if startTs == 0:
+ t = time.time()
+ startTs = int(round(t * 1000))
+
+ ctbDict = {}
+ for i in range(ctbNum):
+ ctbDict[i] = 0
+
+ #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows))
+ rowsOfCtb = 0
+ while rowsOfCtb < rowsPerTbl:
+ for i in range(ctbNum):
+ sql += " %s.%s_%d values "%(dbName,ctbPrefix,i)
+ for k in range(batchNum):
+ sql += "(%d, %d, 'tmqrow_%d') "%(startTs + ctbDict[i], ctbDict[i], ctbDict[i])
+ ctbDict[i] += 1
+ if (0 == ctbDict[i]%batchNum) or (ctbDict[i] == rowsPerTbl):
+ tsql.execute(sql)
+ sql = "insert into "
+ break
+ rowsOfCtb = ctbDict[0]
+
+ tdLog.debug("insert data ............ [OK]")
+ return
+
+ def insert_data_with_autoCreateTbl(self,tsql,dbName,stbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs=0):
+ tdLog.debug("start to insert data wiht auto create child table ............")
+ tsql.execute("use %s" %dbName)
+ pre_insert = "insert into "
+ sql = pre_insert
+
+ if startTs == 0:
+ t = time.time()
+ startTs = int(round(t * 1000))
+
+ #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows))
+ rowsOfSql = 0
+ for i in range(ctbNum):
+ sql += " %s.%s_%d using %s.%s tags (%d) values "%(dbName,ctbPrefix,i,dbName,stbName,i)
+ for j in range(rowsPerTbl):
+ sql += "(%d, %d, 'tmqrow_%d') "%(startTs + j, j, j)
+ rowsOfSql += 1
+ if (j > 0) and ((rowsOfSql == batchNum) or (j == rowsPerTbl - 1)):
+ tsql.execute(sql)
+ rowsOfSql = 0
+ if j < rowsPerTbl - 1:
+ sql = "insert into %s.%s_%d using %s.%s tags (%d) values " %(dbName,ctbPrefix,i,dbName,stbName,i)
+ else:
+ sql = "insert into "
+ #end sql
+ if sql != pre_insert:
+ #print("insert sql:%s"%sql)
+ tsql.execute(sql)
+ tdLog.debug("insert data ............ [OK]")
+ return
+
+ def syncCreateDbStbCtbInsertData(self, tsql, paraDict):
+ tdCom.create_database(tsql, paraDict["dbName"],paraDict["dropFlag"])
+ tdCom.create_stable(tsql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema'])
+ tdCom.create_ctable(tsql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix'])
+ if "event" in paraDict and type(paraDict['event']) == type(threading.Event()):
+ paraDict["event"].set()
+
+ ctbPrefix = paraDict['ctbPrefix']
+ ctbNum = paraDict["ctbNum"]
+ for i in range(ctbNum):
+ tbName = '%s%s'%(ctbPrefix,i)
+ tdCom.insert_rows(tsql,dbname=paraDict["dbName"],tbname=tbName,start_ts_value=paraDict['startTs'],count=paraDict['rowsPerTbl'])
+ return
+
+ def threadFunction(self, **paraDict):
+ # create new connector for new tdSql instance in my thread
+ newTdSql = tdCom.newTdSql()
+ self.syncCreateDbStbCtbInsertData(self, newTdSql, paraDict)
+ return
+
+ def asyncCreateDbStbCtbInsertData(self, paraDict):
+ pThread = threading.Thread(target=self.threadFunction, kwargs=paraDict)
+ pThread.start()
+ return pThread
+
+
+ def close(self):
+ self.cursor.close()
+
+clusterComCreate = ClusterComCreate()
diff --git a/tests/system-test/7-tmq/tmqCommon.py b/tests/system-test/7-tmq/tmqCommon.py
index b8aa78e3ac..788ae3474c 100644
--- a/tests/system-test/7-tmq/tmqCommon.py
+++ b/tests/system-test/7-tmq/tmqCommon.py
@@ -86,7 +86,13 @@ class TMQCom:
shellCmd += '--tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all --num-callers=20 -v --workaround-gcc296-bugs=yes '
if (platform.system().lower() == 'windows'):
- shellCmd = 'mintty -h never -w hide ' + buildPath + '\\build\\bin\\tmq_sim.exe -c ' + cfgPath
+ processorName = buildPath + '\\build\\bin\\tmq_sim.exe'
+ if alias != 0:
+ processorNameNew = buildPath + '\\build\\bin\\tmq_sim_new.exe'
+ shellCmd = 'cp %s %s'%(processorName, processorNameNew)
+ os.system(shellCmd)
+ processorName = processorNameNew
+ shellCmd = 'mintty -h never ' + processorName + ' -c ' + cfgPath
shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName)
shellCmd += "> nul 2>&1 &"
else:
diff --git a/tests/system-test/7-tmq/tmqError.py b/tests/system-test/7-tmq/tmqError.py
index 5b5658d528..bd8ec565d8 100644
--- a/tests/system-test/7-tmq/tmqError.py
+++ b/tests/system-test/7-tmq/tmqError.py
@@ -288,7 +288,10 @@ class TDTestCase:
tdLog.exit("tmq consume rows error!")
tdSql.query("drop topic %s"%topicFromStb1)
- os.system('pkill tmq_sim')
+ if (platform.system().lower() == 'windows'):
+ os.system("TASKKILL /F /IM tmq_sim.exe")
+ else:
+ os.system('pkill tmq_sim')
tdLog.printNoPrefix("======== test case 1 end ...... ")
diff --git a/tests/system-test/7-tmq/tmqShow.py b/tests/system-test/7-tmq/tmqShow.py
new file mode 100644
index 0000000000..6b7e7375ff
--- /dev/null
+++ b/tests/system-test/7-tmq/tmqShow.py
@@ -0,0 +1,158 @@
+
+import taos
+import sys
+import time
+import socket
+import os
+import threading
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+from util.common import *
+sys.path.append("./7-tmq")
+from tmqCommon import *
+
+class TDTestCase:
+ def init(self, conn, logSql):
+ tdLog.debug(f"start to excute {__file__}")
+ tdSql.init(conn.cursor())
+ #tdSql.init(conn.cursor(), logSql) # output sql.txt file
+
+ def tmqCase1(self):
+ tdLog.printNoPrefix("======== test case 1: ")
+ paraDict = {'dbName': 'db1',
+ 'dropFlag': 1,
+ 'event': '',
+ 'vgroups': 4,
+ 'stbName': 'stb',
+ 'colPrefix': 'c',
+ 'tagPrefix': 't',
+ 'colSchema': [{'type': 'INT', 'count':2}, {'type': 'binary', 'len':20, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
+ 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+ 'ctbPrefix': 'ctb',
+ 'ctbNum': 10,
+ 'rowsPerTbl': 4000,
+ 'batchNum': 15,
+ 'startTs': 1640966400000, # 2022-01-01 00:00:00.000
+ 'pollDelay': 20,
+ 'showMsg': 1,
+ 'showRow': 1}
+
+ topicNameList = ['topic1', 'topic2', 'topic3', 'topic4']
+ consumeGroupIdList = ['cgrp1', 'cgrp1', 'cgrp3', 'cgrp4']
+ consumerIdList = [0, 1, 2, 3]
+ tmqCom.initConsumerTable()
+ tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict['vgroups'],replica=1)
+ tdLog.info("create stb")
+ tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema'])
+ tdLog.info("create ctb")
+ tdCom.create_ctable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix'])
+ # tdLog.info("insert data")
+ # tmqCom.insert_data(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"])
+
+ tdLog.info("create 4 topics")
+ sqlString = "create topic %s as database %s" %(topicNameList[0], paraDict['dbName'])
+ tdLog.info("create topic sql: %s"%sqlString)
+ tdSql.execute(sqlString)
+
+ sqlString = "create topic %s as stable %s.%s" %(topicNameList[1], paraDict['dbName'], paraDict['stbName'])
+ tdLog.info("create topic sql: %s"%sqlString)
+ tdSql.execute(sqlString)
+
+ queryString = "select * from %s.%s where c1 %% 7 == 0" %(paraDict['dbName'], paraDict['stbName'])
+ sqlString = "create topic %s as %s" %(topicNameList[2], queryString)
+ tdLog.info("create topic sql: %s"%sqlString)
+ tdSql.execute(sqlString)
+
+ queryString = "select ts, log(c1), ceil(pow(c1,3)) from %s.%s where c1 %% 7 == 0" %(paraDict['dbName'], paraDict['stbName'])
+ sqlString = "create topic %s as %s " %(topicNameList[3], queryString)
+ tdLog.info("create topic sql: %s"%sqlString)
+ tdSql.execute(sqlString)
+
+ tdSql.query("show topics")
+ tdLog.debug(tdSql.queryResult)
+ rows = tdSql.getRows()
+ if rows != len(consumerIdList):
+ tdLog.exit("topic rows error")
+
+ for i in range (rows):
+ topicName = tdSql.getData(i,0)
+ matchFlag = 0
+ while matchFlag == 0:
+ for j in range(len(topicNameList)):
+ if topicName == topicNameList[j]:
+ matchFlag = 1
+ break
+ if matchFlag == 0:
+ tdLog.exit("topic name: %s is error", topicName)
+
+ # init consume info, and start tmq_sim, then check consume result
+ tdLog.info("insert consume info to consume processor")
+ expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"]
+ topicList = topicNameList[0]
+ ifcheckdata = 0
+ ifManualCommit = 0
+ keyList = 'group.id:%s, enable.auto.commit:false, auto.commit.interval.ms:6000, auto.offset.reset:earliest'%consumeGroupIdList[0]
+ tmqCom.insertConsumerInfo(consumerIdList[0], expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+
+ topicList = topicNameList[1]
+ keyList = 'group.id:%s, enable.auto.commit:false, auto.commit.interval.ms:6000, auto.offset.reset:earliest'%consumeGroupIdList[1]
+ tmqCom.insertConsumerInfo(consumerIdList[1], expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+
+ topicList = topicNameList[2]
+ keyList = 'group.id:%s, enable.auto.commit:false, auto.commit.interval.ms:6000, auto.offset.reset:earliest'%consumeGroupIdList[2]
+ tmqCom.insertConsumerInfo(consumerIdList[2], expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+
+ topicList = topicNameList[3]
+ keyList = 'group.id:%s, enable.auto.commit:false, auto.commit.interval.ms:6000, auto.offset.reset:earliest'%consumeGroupIdList[3]
+ tmqCom.insertConsumerInfo(consumerIdList[3], expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+
+ tdLog.info("start consume processor")
+ tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow'])
+
+ tdLog.info("async insert data")
+ pThread = tmqCom.asyncInsertData(paraDict)
+
+ time.sleep(5)
+ tdLog.info("check show consumers")
+ tdSql.query("show consumers")
+ # tdLog.info(tdSql.queryResult)
+ rows = tdSql.getRows()
+ tdLog.info("show consumers rows: %d"%rows)
+ if rows != len(topicNameList):
+ tdLog.exit("show consumers rows error")
+
+ tdLog.info("check show subscriptions")
+ tdSql.query("show subscriptions")
+ # tdLog.debug(tdSql.queryResult)
+ rows = tdSql.getRows()
+ tdLog.info("show subscriptions rows: %d"%rows)
+ if rows != paraDict['vgroups'] * len(topicNameList):
+ tdLog.exit("show subscriptions rows error")
+
+ pThread.join()
+
+ tdLog.info("insert process end, and start to check consume result")
+ expectRows = len(consumerIdList)
+ _ = tmqCom.selectConsumeResult(expectRows)
+
+ time.sleep(10)
+ for i in range(len(topicNameList)):
+ tdSql.query("drop topic %s"%topicNameList[i])
+
+ tdLog.printNoPrefix("======== test case 1 end ...... ")
+
+ def run(self):
+ tdSql.prepare()
+ self.tmqCase1()
+
+ def stop(self):
+ tdSql.close()
+ tdLog.success(f"{__file__} successfully executed")
+
+event = threading.Event()
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())
diff --git a/tests/system-test/99-TDcase/TD-16821.py b/tests/system-test/99-TDcase/TD-16821.py
new file mode 100644
index 0000000000..ef74515792
--- /dev/null
+++ b/tests/system-test/99-TDcase/TD-16821.py
@@ -0,0 +1,186 @@
+
+import taos
+import sys
+import time
+import socket
+import os
+import threading
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+from util.common import *
+sys.path.append("./7-tmq")
+from tmqCommon import *
+
+class TDTestCase:
+ def init(self, conn, logSql):
+ tdLog.debug(f"start to excute {__file__}")
+ tdSql.init(conn.cursor())
+ #tdSql.init(conn.cursor(), logSql) # output sql.txt file
+
+ def checkFileContent(self, consumerId, queryString):
+ buildPath = tdCom.getBuildPath()
+ cfgPath = tdCom.getClientCfgPath()
+ dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId)
+ cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile)
+ tdLog.info(cmdStr)
+ os.system(cmdStr)
+
+ consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId)
+ tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile))
+
+ consumeFile = open(consumeRowsFile, mode='r')
+ queryFile = open(dstFile, mode='r')
+
+ # skip first line for it is schema
+ queryFile.readline()
+
+ while True:
+ dst = queryFile.readline()
+ src = consumeFile.readline()
+
+ if dst:
+ if dst != src:
+ tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId)
+ else:
+ break
+ return
+
+ def tmqCase1(self):
+ tdLog.printNoPrefix("======== test case 1: ")
+ paraDict = {'dbName': 'db1',
+ 'dropFlag': 1,
+ 'event': '',
+ 'vgroups': 4,
+ 'stbName': 'stb',
+ 'colPrefix': 'c',
+ 'tagPrefix': 't',
+ 'colSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+ 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+ 'ctbPrefix': 'ctb',
+ 'ctbNum': 1,
+ 'rowsPerTbl': 10000,
+ 'batchNum': 10,
+ 'startTs': 1640966400000, # 2022-01-01 00:00:00.000
+ 'pollDelay': 10,
+ 'showMsg': 1,
+ 'showRow': 1}
+
+ topicNameList = ['topic1', 'topic2', 'topic3']
+ expectRowsList = []
+ tmqCom.initConsumerTable()
+ tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1)
+ tdLog.info("create stb")
+ tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema'])
+ tdLog.info("create ctb")
+ tdCom.create_ctable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix'])
+ tdLog.info("insert data")
+ tmqCom.insert_data(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"])
+
+ tdLog.info("create topics from stb with filter")
+ # queryString = "select ts, log(c1), ceil(pow(c1,3)) from %s.%s where c1 %% 7 == 0" %(paraDict['dbName'], paraDict['stbName'])
+ # queryString = "select ts, c1, c2 from %s.%s" %(paraDict['dbName'], paraDict['stbName'])
+ queryString = "select * from %s.%s" %(paraDict['dbName'], paraDict['stbName'])
+ sqlString = "create topic %s as stable %s.%s" %(topicNameList[0], paraDict["dbName"],paraDict["stbName"])
+ # sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
+ tdLog.info("create topic sql: %s"%sqlString)
+ tdSql.execute(sqlString)
+ # queryString = 'select * from %s.%s'%(paraDict["dbName"],paraDict["stbName"])
+ tdSql.query(queryString)
+ expectRowsList.append(tdSql.getRows())
+
+ # init consume info, and start tmq_sim, then check consume result
+ tdLog.info("insert consume info to consume processor")
+ consumerId = 0
+ expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"]
+ topicList = topicNameList[0]
+ ifcheckdata = 1
+ ifManualCommit = 1
+ keyList = 'group.id:cgrp1, enable.auto.commit:false, auto.commit.interval.ms:6000, auto.offset.reset:earliest'
+ tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+
+ tdLog.info("start consume processor")
+ tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow'])
+
+ tdLog.info("wait the consume result")
+ expectRows = 1
+ resultList = tmqCom.selectConsumeResult(expectRows)
+
+ if expectRowsList[0] != resultList[0]:
+ tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0]))
+ tdLog.exit("0 tmq consume rows error!")
+
+ self.checkFileContent(consumerId, queryString)
+
+ # reinit consume info, and start tmq_sim, then check consume result
+ tmqCom.initConsumerTable()
+
+ queryString = "select ts, log(c1), cos(c1) from %s.%s where c1 > 3169" %(paraDict['dbName'], paraDict['stbName'])
+ sqlString = "create topic %s as %s" %(topicNameList[1], queryString)
+ tdLog.info("create topic sql: %s"%sqlString)
+ tdSql.execute(sqlString)
+ tdSql.query(queryString)
+ expectRowsList.append(tdSql.getRows())
+
+ consumerId = 1
+ topicList = topicNameList[1]
+ tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+
+ tdLog.info("start consume processor")
+ tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow'])
+
+ tdLog.info("wait the consume result")
+ expectRows = 1
+ resultList = tmqCom.selectConsumeResult(expectRows)
+ if expectRowsList[1] != resultList[0]:
+ tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[1], resultList[0]))
+ tdLog.exit("1 tmq consume rows error!")
+
+ self.checkFileContent(consumerId, queryString)
+
+ # reinit consume info, and start tmq_sim, then check consume result
+ tmqCom.initConsumerTable()
+
+ queryString = "select ts, log(c1), atan(c1) from %s.%s where ts >= %d" %(paraDict['dbName'], paraDict['stbName'], paraDict["startTs"]+6137)
+ sqlString = "create topic %s as %s" %(topicNameList[2], queryString)
+ tdLog.info("create topic sql: %s"%sqlString)
+ tdSql.execute(sqlString)
+ tdSql.query(queryString)
+ expectRowsList.append(tdSql.getRows())
+
+ consumerId = 2
+ topicList = topicNameList[2]
+ tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+
+ tdLog.info("start consume processor")
+ tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow'])
+
+ tdLog.info("wait the consume result")
+ expectRows = 1
+ resultList = tmqCom.selectConsumeResult(expectRows)
+ # if expectRowsList[2] != resultList[0]:
+ # tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[2], resultList[0]))
+ # tdLog.exit("2 tmq consume rows error!")
+
+ # self.checkFileContent(consumerId, queryString)
+
+ time.sleep(10)
+ for i in range(len(topicNameList)):
+ tdSql.query("drop topic %s"%topicNameList[i])
+
+ tdLog.printNoPrefix("======== test case 1 end ...... ")
+
+ def run(self):
+ tdSql.prepare()
+ self.tmqCase1()
+
+ def stop(self):
+ tdSql.close()
+ tdLog.success(f"{__file__} successfully executed")
+
+event = threading.Event()
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())
diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh
index ef217b828f..571bb166e1 100755
--- a/tests/system-test/fulltest.sh
+++ b/tests/system-test/fulltest.sh
@@ -23,6 +23,7 @@ python3 ./test.py -f 1-insert/alter_stable.py
python3 ./test.py -f 1-insert/alter_table.py
python3 ./test.py -f 1-insert/insertWithMoreVgroup.py
python3 ./test.py -f 1-insert/table_comment.py
+python3 ./test.py -f 1-insert/table_param_ttl.py
python3 ./test.py -f 2-query/between.py
python3 ./test.py -f 2-query/distinct.py
python3 ./test.py -f 2-query/varchar.py
@@ -116,6 +117,10 @@ python3 ./test.py -f 2-query/function_null.py
python3 ./test.py -f 6-cluster/5dnode1mnode.py
python3 ./test.py -f 6-cluster/5dnode2mnode.py
python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3
+python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 5 -M 3
+# BUG python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopCreateDb.py -N 5 -M 3
+# BUG python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -N 5 -M 3
+
# python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5
# BUG python3 ./test.py -f 6-cluster/5dnode3mnodeStopInsert.py
@@ -139,3 +144,4 @@ python3 ./test.py -f 7-tmq/tmqCheckData.py
python3 ./test.py -f 7-tmq/tmqUdf.py
#python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 5
python3 ./test.py -f 7-tmq/tmqConsumerGroup.py
+python3 ./test.py -f 7-tmq/tmqShow.py
diff --git a/tests/system-test/test-all.bat b/tests/system-test/test-all.bat
index 275cbeebbb..adc9e0ce28 100644
--- a/tests/system-test/test-all.bat
+++ b/tests/system-test/test-all.bat
@@ -2,7 +2,7 @@
SETLOCAL EnableDelayedExpansion
for /F "tokens=1,2 delims=#" %%a in ('"prompt #$H#$E# & echo on & for %%b in (1) do rem"') do ( set "DEL=%%a")
set /a a=0
-if %1 == full (
+if "%1" == "full" (
echo Windows Taosd Full Test
set /a exitNum=0
del /Q /F failed.txt
diff --git a/tests/system-test/test.py b/tests/system-test/test.py
index 8a8356449c..76b83da348 100644
--- a/tests/system-test/test.py
+++ b/tests/system-test/test.py
@@ -22,8 +22,6 @@ import json
import platform
import socket
import threading
-from distutils.log import warn as printf
-from fabric2 import Connection
sys.path.append("../pytest")
from util.log import *
from util.dnodes import *
@@ -187,9 +185,9 @@ if __name__ == "__main__":
tdLog.info("Procedures for tdengine deployed in %s" % (host))
if platform.system().lower() == 'windows':
+ fileName = fileName.replace("/", os.sep)
if (masterIp == "" and not fileName[0:12] == "0-others\\udf"):
threading.Thread(target=checkRunTimeError,daemon=True).start()
- tdCases.logSql(logSql)
tdLog.info("Procedures for testing self-deployment")
tdDnodes.init(deployPath, masterIp)
tdDnodes.setTestCluster(testCluster)
@@ -208,18 +206,46 @@ if __name__ == "__main__":
uModule = importlib.import_module(moduleName)
try:
ucase = uModule.TDTestCase()
- if ((json.dumps(updateCfgDict) == '{}') and (ucase.updatecfgDict is not None)):
+ if ((json.dumps(updateCfgDict) == '{}') and hasattr(ucase, 'updatecfgDict')):
updateCfgDict = ucase.updatecfgDict
updateCfgDictStr = "-d %s"%base64.b64encode(json.dumps(updateCfgDict).encode()).decode()
except Exception as r:
print(r)
else:
pass
- tdDnodes.deploy(1,updateCfgDict)
- tdDnodes.start(1)
- conn = taos.connect(
- host="%s"%(host),
- config=tdDnodes.sim.getCfgDir())
+ if dnodeNums == 1 :
+ tdDnodes.deploy(1,updateCfgDict)
+ tdDnodes.start(1)
+ tdCases.logSql(logSql)
+ else :
+ tdLog.debug("create an cluster with %s nodes and make %s dnode as independent mnode"%(dnodeNums,mnodeNums))
+ dnodeslist = cluster.configure_cluster(dnodeNums=dnodeNums,mnodeNums=mnodeNums)
+ tdDnodes = ClusterDnodes(dnodeslist)
+ tdDnodes.init(deployPath, masterIp)
+ tdDnodes.setTestCluster(testCluster)
+ tdDnodes.setValgrind(valgrind)
+ tdDnodes.stopAll()
+ for dnode in tdDnodes.dnodes:
+ tdDnodes.deploy(dnode.index,{})
+ for dnode in tdDnodes.dnodes:
+ tdDnodes.starttaosd(dnode.index)
+ tdCases.logSql(logSql)
+ conn = taos.connect(
+ host,
+ config=tdDnodes.getSimCfgPath())
+ print(tdDnodes.getSimCfgPath(),host)
+ cluster.create_dnode(conn)
+ try:
+ if cluster.check_dnode(conn) :
+ print("check dnode ready")
+ except Exception as r:
+ print(r)
+ if ucase is not None and hasattr(ucase, 'noConn') and ucase.noConn == True:
+ conn = None
+ else:
+ conn = taos.connect(
+ host="%s"%(host),
+ config=tdDnodes.sim.getCfgDir())
if is_test_framework:
tdCases.runOneWindows(conn, fileName)
else:
@@ -307,4 +333,5 @@ if __name__ == "__main__":
tdCases.runOneLinux(conn, sp[0] + "_" + "restart.py")
else:
tdLog.info("not need to query")
- conn.close()
+ if conn is not None:
+ conn.close()
diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c
index c2fba52e6e..81fa72d15a 100644
--- a/tests/test/c/tmqSim.c
+++ b/tests/test/c/tmqSim.c
@@ -635,8 +635,9 @@ void loop_consume(SThreadInfo* pInfo) {
}
}
+ int32_t consumeDelay = g_stConfInfo.consumeDelay == -1 ? -1 : (g_stConfInfo.consumeDelay * 1000);
while (running) {
- TAOS_RES* tmqMsg = tmq_consumer_poll(pInfo->tmq, g_stConfInfo.consumeDelay * 1000);
+ TAOS_RES* tmqMsg = tmq_consumer_poll(pInfo->tmq, consumeDelay);
if (tmqMsg) {
if (0 != g_stConfInfo.showMsgFlag) {
totalRows += msg_process(tmqMsg, pInfo, totalMsgs);
diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c
index 8a017d378d..7b275c5a15 100644
--- a/tools/shell/src/shellEngine.c
+++ b/tools/shell/src/shellEngine.c
@@ -156,6 +156,7 @@ void shellRunSingleCommandImp(char *command) {
}
fname = sptr + 2;
+ while (*fname == ' ') fname++;
*sptr = '\0';
}
@@ -858,9 +859,7 @@ void shellGetGrantInfo() {
int32_t code = taos_errno(tres);
if (code != TSDB_CODE_SUCCESS) {
- if (code == TSDB_CODE_OPS_NOT_SUPPORT) {
- fprintf(stdout, "Server is Community Edition, %s\n\n", sinfo);
- } else {
+ if (code != TSDB_CODE_OPS_NOT_SUPPORT && code != TSDB_CODE_MND_NO_RIGHTS) {
fprintf(stderr, "Failed to check Server Edition, Reason:0x%04x:%s\n\n", code, taos_errstr(tres));
}
return;
diff --git a/tools/taos-tools b/tools/taos-tools
index 28a49b447f..a875a057d1 160000
--- a/tools/taos-tools
+++ b/tools/taos-tools
@@ -1 +1 @@
-Subproject commit 28a49b447f71c4f014ebbac858b7215b897d57fd
+Subproject commit a875a057d1225d85c6323b9edaccc2b1a9641987